From f9fbe975099f8f4ca66d345ac007031aed7de924 Mon Sep 17 00:00:00 2001
From: Steffan Sluis <steffansluis@gmail.com>
Date: Sat, 23 May 2020 17:53:52 +0200
Subject: [PATCH] azure: support external file dependencies (#3852)

* azure: support external dependencies and outputs

- Implement get_file_checksum with get_etag
- Implement a DependencyAzure and OutputAzure classes
- Add AzureOutput to OUTS, OUTS_MAP and CHECKSUMS_SCHEMA
- Add AzureDependency to DEPS and DEPS_MAP
- Add unit tests for AzureOutput and AzureDependency

Fixes #3540.

* Use content_md5 instead of etag

The etag is different for the same file in different locations.

* Use etag and remove external outputs for Azure

Due to inconsistencies with Azure's Content-MD5 property, it is not
stable enough to be used at the moment. Using the etag offers enough
reliability to support external dependencies. See also:
https://discordapp.com/channels/485586884165107732/565699007037571084/713462619469643796
---
 dvc/dependency/__init__.py          |  3 +++
 dvc/dependency/azure.py             |  7 +++++++
 dvc/remote/azure.py                 | 15 +++++++++++----
 tests/unit/dependency/test_azure.py |  7 +++++++
 4 files changed, 28 insertions(+), 4 deletions(-)
 create mode 100644 dvc/dependency/azure.py
 create mode 100644 tests/unit/dependency/test_azure.py

diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py
index 45de5bf57..1fa2073e2 100644
--- a/dvc/dependency/__init__.py
+++ b/dvc/dependency/__init__.py
@@ -2,6 +2,7 @@ from collections import defaultdict
 from urllib.parse import urlparse
 
 import dvc.output as output
+from dvc.dependency.azure import AzureDependency
 from dvc.dependency.gs import GSDependency
 from dvc.dependency.hdfs import HDFSDependency
 from dvc.dependency.http import HTTPDependency
@@ -17,6 +18,7 @@ from dvc.scheme import Schemes
 from .repo import RepoDependency
 
 DEPS = [
+    AzureDependency,
     GSDependency,
     HDFSDependency,
     HTTPDependency,
@@ -30,6 +32,7 @@ DEP_MAP = {
     Schemes.LOCAL: LocalDependency,
     Schemes.SSH: SSHDependency,
     Schemes.S3: S3Dependency,
+    Schemes.AZURE: AzureDependency,
     Schemes.GS: GSDependency,
     Schemes.HDFS: HDFSDependency,
     Schemes.HTTP: HTTPDependency,
diff --git a/dvc/dependency/azure.py b/dvc/dependency/azure.py
new file mode 100644
index 000000000..c119d15ef
--- /dev/null
+++ b/dvc/dependency/azure.py
@@ -0,0 +1,7 @@
+from dvc.dependency.base import BaseDependency
+from dvc.output.base import BaseOutput
+from dvc.remote.azure import AzureRemote
+
+
+class AzureDependency(BaseDependency, BaseOutput):
+    REMOTE = AzureRemote
diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py
index 90d76dbf8..e23d8ed2d 100644
--- a/dvc/remote/azure.py
+++ b/dvc/remote/azure.py
@@ -29,10 +29,8 @@ class AzureRemote(BaseRemote):
         self.path_info = self.path_cls(url)
 
         if not self.path_info.bucket:
-            self.path_info = self.path_cls.from_parts(
-                scheme=self.scheme,
-                netloc=os.getenv("AZURE_STORAGE_CONTAINER_NAME"),
-            )
+            container = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
+            self.path_info = self.path_cls(f"azure://{container}")
 
         self.connection_string = config.get("connection_string") or os.getenv(
             "AZURE_STORAGE_CONNECTION_STRING"
@@ -58,6 +56,15 @@ class AzureRemote(BaseRemote):
             blob_service.create_container(self.path_info.bucket)
         return blob_service
 
+    def get_etag(self, path_info):
+        etag = self.blob_service.get_blob_properties(
+            path_info.bucket, path_info.path
+        ).properties.etag
+        return etag.strip('"')
+
+    def get_file_checksum(self, path_info):
+        return self.get_etag(path_info)
+
     def remove(self, path_info):
         if path_info.scheme != self.scheme:
             raise NotImplementedError
diff --git a/tests/unit/dependency/test_azure.py b/tests/unit/dependency/test_azure.py
new file mode 100644
index 000000000..fff791061
--- /dev/null
+++ b/tests/unit/dependency/test_azure.py
@@ -0,0 +1,7 @@
+from dvc.dependency.azure import AzureDependency
+from tests.unit.dependency.test_local import TestLocalDependency
+
+
+class TestAzureDependency(TestLocalDependency):
+    def _get_cls(self):
+        return AzureDependency