diff --git a/dvc/command/status.py b/dvc/command/status.py index 712e65181..cce4c3f57 100644 --- a/dvc/command/status.py +++ b/dvc/command/status.py @@ -9,8 +9,7 @@ class CmdDataStatus(CmdDataBase): def _normalize(self, s): s+=':' - if len(s) >= self.STATUS_LEN: - return s + assert len(s) < self.STATUS_LEN return s + (self.STATUS_LEN - len(s))*' ' def _show(self, status, indent=0): diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 1bf79dbc9..6d52b33bf 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -11,3 +11,9 @@ class DvcException(Exception): except Exception: pass super(DvcException, self).__init__(msg) + + +class UnsupportedRemoteError(DvcException): + def __init__(self, config, cause=None): + msg = "Remote '{}' is not supported.".format(config) + super(UnsupportedRemoteError, self).__init__(msg, cause=cause) diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index a6537c921..7fec8f094 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -6,7 +6,7 @@ from dvc.remote.ssh import RemoteSSH from dvc.remote.azure import RemoteAzure from dvc.config import Config -from dvc.exceptions import DvcException +from dvc.exceptions import UnsupportedRemoteError REMOTES = [RemoteHDFS, RemoteSSH, RemoteS3, RemoteGS, RemoteAzure, RemoteLOCAL] @@ -24,4 +24,4 @@ def Remote(project, config): for r in REMOTES: if r.supported(config): return r(project, config) - raise DvcException('Remote \'{}\' is not supported.'.format(config)) + raise UnsupportedRemoteError(str(config)) diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index daca6d0bb..b36c5aadf 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -68,68 +68,71 @@ class RemoteAzure(RemoteBase): self.__blob_service.create_container(self.bucket) return self.__blob_service - def _get_etag(self, bucket, key): - try: - blob = self.blob_service.get_blob_properties(bucket, key) - return blob.properties.etag - except Exception: - return None - - def save_info(self, path_info): - if path_info['scheme'] != self.scheme: - raise NotImplementedError - - return {self.PARAM_ETAG: self._get_etag( - path_info['bucket'], path_info['key'])} - - def save(self, path_info): - if path_info['scheme'] != self.scheme: - raise NotImplementedError - - etag = self._get_etag(path_info['bucket'], path_info['key']) - dest_key = '{}/{}'.format(etag[0:2], etag[2:]) - - self._copy_blob(to_bucket=self.bucket, - to_key=dest_key, - from_bucket=path_info['bucket'], - from_key=path_info['key']) - - return {self.PARAM_ETAG: etag} - - def _copy_blob(self, to_bucket, to_key, from_bucket, from_key): - source = self.blob_service.make_blob_url(from_bucket, from_key) - - copy = self.blob_service.copy_blob(to_bucket, to_key, source) - - if self.COPY_POLL_SECONDS <= 0: - return - - while copy.status != 'success': - time.sleep(self.COPY_POLL_SECONDS) - copy = self.blob_service.get_blob_properties( - to_bucket, to_key).properties.copy - - def checkout(self, path_info, checksum_info): - if path_info['scheme'] != self.scheme: - raise NotImplementedError - - etag = checksum_info.get(self.PARAM_ETAG, None) - if not etag: - return - - self._copy_blob(to_bucket=path_info['bucket'], - to_key=path_info['key'], - from_bucket=self.bucket, - from_key='{}/{}'.format(etag[0:2], etag[2:])) - - def remove(self, path_info): - if path_info['scheme'] != self.scheme: - raise NotImplementedError - - Logger.debug('Removing azure://{}/{}'.format(path_info['bucket'], - path_info['key'])) - - self.blob_service.delete_blob(path_info['bucket'], path_info['key']) +# FIXME: temporarily disabled because of the lack of test for external azure +# dependencies/outputs/cache. +# +# def _get_etag(self, bucket, key): +# try: +# blob = self.blob_service.get_blob_properties(bucket, key) +# return blob.properties.etag +# except Exception: +# return None +# +# def save_info(self, path_info): +# if path_info['scheme'] != self.scheme: +# raise NotImplementedError +# +# return {self.PARAM_ETAG: self._get_etag( +# path_info['bucket'], path_info['key'])} +# +# def save(self, path_info): +# if path_info['scheme'] != self.scheme: +# raise NotImplementedError +# +# etag = self._get_etag(path_info['bucket'], path_info['key']) +# dest_key = '{}/{}'.format(etag[0:2], etag[2:]) +# +# self._copy_blob(to_bucket=self.bucket, +# to_key=dest_key, +# from_bucket=path_info['bucket'], +# from_key=path_info['key']) +# +# return {self.PARAM_ETAG: etag} +# +# def _copy_blob(self, to_bucket, to_key, from_bucket, from_key): +# source = self.blob_service.make_blob_url(from_bucket, from_key) +# +# copy = self.blob_service.copy_blob(to_bucket, to_key, source) +# +# if self.COPY_POLL_SECONDS <= 0: +# return +# +# while copy.status != 'success': +# time.sleep(self.COPY_POLL_SECONDS) +# copy = self.blob_service.get_blob_properties( +# to_bucket, to_key).properties.copy +# +# def checkout(self, path_info, checksum_info): +# if path_info['scheme'] != self.scheme: +# raise NotImplementedError +# +# etag = checksum_info.get(self.PARAM_ETAG, None) +# if not etag: +# return +# +# self._copy_blob(to_bucket=path_info['bucket'], +# to_key=path_info['key'], +# from_bucket=self.bucket, +# from_key='{}/{}'.format(etag[0:2], etag[2:])) +# +# def remove(self, path_info): +# if path_info['scheme'] != self.scheme: +# raise NotImplementedError +# +# Logger.debug('Removing azure://{}/{}'.format(path_info['bucket'], +# path_info['key'])) +# +# self.blob_service.delete_blob(path_info['bucket'], path_info['key']) def md5s_to_path_infos(self, md5s): return [{ @@ -217,17 +220,20 @@ class RemoteAzure(RemoteBase): if not no_progress_bar: progress.finish_target(name) - def gc(self, checksum_infos): - used_etags = [info[self.PARAM_ETAG] for info in checksum_infos['azure']] - used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']] - - all_blobs = self.blob_service.list_blobs(self.bucket) - - for blob in all_blobs: - etag = blob.properties.etag - if etag in used_etags: - continue - path_info = {'scheme': self.scheme, - 'key': blob.name, - 'bucket': self.bucket} - self.remove(path_info) +# FIXME: temporarily disabled because of the lack of test for external azure +# dependencies/outputs/cache. +# +# def gc(self, checksum_infos): +# used_etags = [info[self.PARAM_ETAG] for info in checksum_infos['azure']] +# used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']] +# +# all_blobs = self.blob_service.list_blobs(self.bucket) +# +# for blob in all_blobs: +# etag = blob.properties.etag +# if etag in used_etags: +# continue +# path_info = {'scheme': self.scheme, +# 'key': blob.name, +# 'bucket': self.bucket} +# self.remove(path_info) diff --git a/tests/test_add.py b/tests/test_add.py index 0acf3194d..59894b40a 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -6,6 +6,7 @@ import filecmp from dvc.main import main from dvc.utils import file_md5 from dvc.stage import Stage +from dvc.exceptions import DvcException from dvc.output.base import OutputAlreadyTrackedError from dvc.output.base import OutputDoesNotExistError, OutputIsNotFileOrDirError from dvc.command.add import CmdAdd @@ -27,10 +28,10 @@ class TestAdd(TestDvc): self.assertEqual(stage.outs[0].info['md5'], md5) -class TestAddNonExistentFile(TestDvc): +class TestAddUnupportedFile(TestDvc): def test(self): - with self.assertRaises(OutputDoesNotExistError) as cx: - self.dvc.add('non_existent_file') + with self.assertRaises(DvcException) as cx: + self.dvc.add('unsupported://unsupported') class TestAddDirectory(TestDvc): diff --git a/tests/test_import.py b/tests/test_import.py index f1f3630d2..fc75f536e 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -5,6 +5,7 @@ import filecmp from dvc.main import main from dvc.command.imp import CmdImport +from dvc.exceptions import DvcException from tests.basic_env import TestDvc @@ -18,3 +19,7 @@ class TestCmdImport(TestDvc): ret = main(['import', 'non-existing-file', 'import']) self.assertNotEqual(ret, 0) + + def test_unsupported(self): + ret = main(['import', 'unsupported://path', 'import_unsupported']) + self.assertNotEqual(ret, 0) diff --git a/tests/test_remote.py b/tests/test_remote.py index d8c5c9219..31884b8ee 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,5 +1,8 @@ from dvc.main import main from dvc.command.remote import CmdRemoteAdd +from dvc.exceptions import UnsupportedRemoteError +from dvc.remote import Remote +from dvc.config import Config from tests.basic_env import TestDvc @@ -34,3 +37,7 @@ class TestRemote(TestDvc): cmd.configobj.write = None ret = cmd.run() self.assertNotEqual(ret, 0) + + def test_unsupported(self): + with self.assertRaises(UnsupportedRemoteError): + Remote(self.dvc, {Config.SECTION_REMOTE_URL: 'unsupported://url'}) diff --git a/tests/test_repro.py b/tests/test_repro.py index d7112c82b..d1c79453a 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -247,6 +247,9 @@ class TestReproMissingMd5InStageFile(TestRepro): class TestCmdRepro(TestRepro): def test(self): + ret = main(['status']) + self.assertEqual(ret, 0) + ret = main(['repro', self.file1_stage]) self.assertEqual(ret, 0)