From ef944e650988a9684c486f8a3d72a5f43a6fddea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pawe=C5=82=20Redzy=C5=84ski?= <pawelredzynski@gmail.com>
Date: Tue, 18 Aug 2020 11:45:06 +0200
Subject: [PATCH] http: replace HEAD with streamed GET (#4390)

---
 dvc/tree/http.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/dvc/tree/http.py b/dvc/tree/http.py
index e530c119d..d818c5116 100644
--- a/dvc/tree/http.py
+++ b/dvc/tree/http.py
@@ -122,12 +122,27 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
         except requests.exceptions.RequestException:
             raise DvcException(f"could not perform a {method} request")
 
+    def _head(self, url):
+        response = self.request("HEAD", url)
+        if response.ok:
+            return response
+
+        # Sometimes servers are configured to forbid HEAD requests
+        # Context: https://github.com/iterative/dvc/issues/4131
+        with self.request("GET", url, stream=True) as r:
+            if r.ok:
+                return r
+
+        return response
+
     def exists(self, path_info, use_dvcignore=True):
-        return bool(self.request("HEAD", path_info.url))
+        return bool(self._head(path_info.url))
 
     def get_file_hash(self, path_info):
         url = path_info.url
-        headers = self.request("HEAD", url).headers
+
+        headers = self._head(url).headers
+
         etag = headers.get("ETag") or headers.get("Content-MD5")
 
         if not etag: