From 3009a25ae2f41607cb4e5bb38b97188ada556233 Mon Sep 17 00:00:00 2001
From: "Mr. Outis" <mroutis@protonmail.com>
Date: Sun, 27 Jan 2019 23:56:43 -0600
Subject: [PATCH] remote http: use Content-MD5 header if there is not ETag
 available

Fix #1412
---
 dvc/remote/http.py   |  8 ++++++--
 tests/test_repro.py  | 10 ++++++++++
 tests/utils/httpd.py | 18 ++++++++++++++++--
 3 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/dvc/remote/http.py b/dvc/remote/http.py
index fc502fc46..1e74d6bf7 100644
--- a/dvc/remote/http.py
+++ b/dvc/remote/http.py
@@ -107,10 +107,14 @@ class RemoteHTTP(RemoteBase):
         return self._request('HEAD', url).headers.get('Content-Length')
 
     def _etag(self, url):
-        etag = self._request('HEAD', url).headers.get('ETag')
+        etag = (self._request('HEAD', url).headers.get('ETag')
+                or self._request('HEAD', url).headers.get('Content-MD5'))
 
         if not etag:
-            raise DvcException("could not find an ETag for '{}'".format(url))
+            raise DvcException(
+                "could not find an ETag or Content-MD5 header for '{url}'"
+                .format(url=url)
+            )
 
         if etag.startswith('W/'):
             raise DvcException(
diff --git a/tests/test_repro.py b/tests/test_repro.py
index dc23db423..ff6327116 100644
--- a/tests/test_repro.py
+++ b/tests/test_repro.py
@@ -1131,6 +1131,16 @@ class TestReproExternalHTTP(TestReproExternalBase):
         self.assertTrue(os.path.exists(import_output))
         self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
 
+        self.dvc.remove('imported_file.dvc')
+
+        with StaticFileServer(handler='Content-MD5'):
+            import_url = urljoin(self.remote, self.FOO)
+            import_output = 'imported_file'
+            import_stage = self.dvc.imp(import_url, import_output)
+
+        self.assertTrue(os.path.exists(import_output))
+        self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
+
         # Run --deps
         with StaticFileServer():
             run_dependency = urljoin(self.remote, self.BAR)
diff --git a/tests/utils/httpd.py b/tests/utils/httpd.py
index afe6eefd5..be5c44cf0 100644
--- a/tests/utils/httpd.py
+++ b/tests/utils/httpd.py
@@ -17,12 +17,26 @@ class ETagHandler(SimpleHTTPRequestHandler):
         SimpleHTTPRequestHandler.end_headers(self)
 
 
+class ContentMD5Handler(SimpleHTTPRequestHandler):
+    def end_headers(self):
+        file = self.translate_path(self.path)
+
+        if not os.path.isdir(file) and os.path.exists(file):
+            with open(file, 'r') as fd:
+                md5 = hashlib.md5(fd.read().encode('utf8')).hexdigest()
+                self.send_header('Content-MD5', md5)
+
+        SimpleHTTPRequestHandler.end_headers(self)
+
+
 class StaticFileServer:
     __server_lock = threading.Lock()
 
-    def __init__(self):
+    def __init__(self, handler='etag'):
+        handler_class = ETagHandler if handler == 'etag' else ContentMD5Handler
+
         self.__server_lock.acquire()
-        self.httpd = HTTPServer(('localhost', 8000), ETagHandler)
+        self.httpd = HTTPServer(('localhost', 8000), handler_class)
 
     def __enter__(self):
         self.server_thread = threading.Thread(target=self.httpd.serve_forever)