azure: support external file dependencies (#3852)

* azure: support external dependencies and outputs

- Implement get_file_checksum with get_etag
- Implement a DependencyAzure and OutputAzure classes
- Add AzureOutput to OUTS, OUTS_MAP and CHECKSUMS_SCHEMA
- Add AzureDependency to DEPS and DEPS_MAP
- Add unit tests for AzureOutput and AzureDependency

Fixes #3540.

* Use content_md5 instead of etag

The etag is different for the same file in different locations.

* Use etag and remove external outputs for Azure

Due to inconsistencies with Azure's Content-MD5 property, it is not
stable enough to be used at the moment. Using the etag offers enough
reliability to support external dependencies. See also:
https://discordapp.com/channels/485586884165107732/565699007037571084/713462619469643796

dvc/dependency/__init__.py

@@ -2,6 +2,7 @@ from collections import defaultdict
 from urllib.parse import urlparse
 
 import dvc.output as output
+from dvc.dependency.azure import AzureDependency
 from dvc.dependency.gs import GSDependency
 from dvc.dependency.hdfs import HDFSDependency
 from dvc.dependency.http import HTTPDependency
@@ -17,6 +18,7 @@ from dvc.scheme import Schemes
 from .repo import RepoDependency
 
 DEPS = [
+    AzureDependency,
     GSDependency,
     HDFSDependency,
     HTTPDependency,
@@ -30,6 +32,7 @@ DEP_MAP = {
     Schemes.LOCAL: LocalDependency,
     Schemes.SSH: SSHDependency,
     Schemes.S3: S3Dependency,
+    Schemes.AZURE: AzureDependency,
     Schemes.GS: GSDependency,
     Schemes.HDFS: HDFSDependency,
     Schemes.HTTP: HTTPDependency,

dvc/dependency/azure.py

@@ -0,0 +1,7 @@
+from dvc.dependency.base import BaseDependency
+from dvc.output.base import BaseOutput
+from dvc.remote.azure import AzureRemote
+
+
+class AzureDependency(BaseDependency, BaseOutput):
+    REMOTE = AzureRemote

dvc/remote/azure.py

@@ -29,10 +29,8 @@ class AzureRemote(BaseRemote):
         self.path_info = self.path_cls(url)
 
         if not self.path_info.bucket:
-            self.path_info = self.path_cls.from_parts(
-                scheme=self.scheme,
-                netloc=os.getenv("AZURE_STORAGE_CONTAINER_NAME"),
-            )
+            container = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
+            self.path_info = self.path_cls(f"azure://{container}")
 
         self.connection_string = config.get("connection_string") or os.getenv(
             "AZURE_STORAGE_CONNECTION_STRING"
@@ -58,6 +56,15 @@ class AzureRemote(BaseRemote):
             blob_service.create_container(self.path_info.bucket)
         return blob_service
 
+    def get_etag(self, path_info):
+        etag = self.blob_service.get_blob_properties(
+            path_info.bucket, path_info.path
+        ).properties.etag
+        return etag.strip('"')
+
+    def get_file_checksum(self, path_info):
+        return self.get_etag(path_info)
+
     def remove(self, path_info):
         if path_info.scheme != self.scheme:
             raise NotImplementedError

tests/unit/dependency/test_azure.py

@@ -0,0 +1,7 @@
+from dvc.dependency.azure import AzureDependency
+from tests.unit.dependency.test_local import TestLocalDependency
+
+
+class TestAzureDependency(TestLocalDependency):
+    def _get_cls(self):
+        return AzureDependency