remote http: use Content-MD5 header if there is not ETag available

Fix #1412

dvc/remote/http.py

@@ -107,10 +107,14 @@ class RemoteHTTP(RemoteBase):
         return self._request('HEAD', url).headers.get('Content-Length')
 
     def _etag(self, url):
-        etag = self._request('HEAD', url).headers.get('ETag')
+        etag = (self._request('HEAD', url).headers.get('ETag')
+                or self._request('HEAD', url).headers.get('Content-MD5'))
 
         if not etag:
-            raise DvcException("could not find an ETag for '{}'".format(url))
+            raise DvcException(
+                "could not find an ETag or Content-MD5 header for '{url}'"
+                .format(url=url)
+            )
 
         if etag.startswith('W/'):
             raise DvcException(

tests/test_repro.py

@@ -1131,6 +1131,16 @@ class TestReproExternalHTTP(TestReproExternalBase):
         self.assertTrue(os.path.exists(import_output))
         self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
 
+        self.dvc.remove('imported_file.dvc')
+
+        with StaticFileServer(handler='Content-MD5'):
+            import_url = urljoin(self.remote, self.FOO)
+            import_output = 'imported_file'
+            import_stage = self.dvc.imp(import_url, import_output)
+
+        self.assertTrue(os.path.exists(import_output))
+        self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
+
         # Run --deps
         with StaticFileServer():
             run_dependency = urljoin(self.remote, self.BAR)

tests/utils/httpd.py

@@ -17,12 +17,26 @@ class ETagHandler(SimpleHTTPRequestHandler):
         SimpleHTTPRequestHandler.end_headers(self)
 
 
+class ContentMD5Handler(SimpleHTTPRequestHandler):
+    def end_headers(self):
+        file = self.translate_path(self.path)
+
+        if not os.path.isdir(file) and os.path.exists(file):
+            with open(file, 'r') as fd:
+                md5 = hashlib.md5(fd.read().encode('utf8')).hexdigest()
+                self.send_header('Content-MD5', md5)
+
+        SimpleHTTPRequestHandler.end_headers(self)
+
+
 class StaticFileServer:
     __server_lock = threading.Lock()
 
-    def __init__(self):
+    def __init__(self, handler='etag'):
+        handler_class = ETagHandler if handler == 'etag' else ContentMD5Handler
+
         self.__server_lock.acquire()
-        self.httpd = HTTPServer(('localhost', 8000), ETagHandler)
+        self.httpd = HTTPServer(('localhost', 8000), handler_class)
 
     def __enter__(self):
         self.server_thread = threading.Thread(target=self.httpd.serve_forever)