http: raise error on unsuccessful http statuses on exists (#4785)

* http: added check for HTTP status code 404

Fixes #4288

* http: add unit test and minor change

In tests_exists, changed to raise a HTTPError from dvc instead.

Fixes #4288

* http: removed fixtures in test for HTTPTree.exists

Fixes #4288

* test: add comments on why `raw` is required on the mock

Co-authored-by: Saugat Pachhai (सौगात) <[email protected]>

dvc/tree/http.py

@@ -141,7 +141,12 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
         return response
 
     def exists(self, path_info, use_dvcignore=True):
-        return bool(self._head(path_info.url))
+        res = self._head(path_info.url)
+        if res.status_code == 404:
+            return False
+        if bool(res):
+            return True
+        raise HTTPError(res.status_code, res.reason)
 
     def get_file_hash(self, path_info):
         url = path_info.url

tests/unit/remote/test_http.py

@@ -125,3 +125,31 @@ def test_http_method(dvc):
     assert tree._auth_method() == auth
     assert tree.method == "PUT"
     assert isinstance(tree._auth_method(), HTTPBasicAuth)
+
+
+def test_exists(mocker):
+    import io
+
+    import requests
+
+    from dvc.path_info import URLInfo
+
+    res = requests.Response()
+    # need to add `raw`, as `exists()` fallbacks to a streaming GET requests
+    # on HEAD request failure.
+    res.raw = io.StringIO("foo")
+
+    tree = HTTPTree(None, {})
+    mocker.patch.object(tree, "request", return_value=res)
+
+    url = URLInfo("https://example.org/file.txt")
+
+    res.status_code = 200
+    assert tree.exists(url) is True
+
+    res.status_code = 404
+    assert tree.exists(url) is False
+
+    res.status_code = 403
+    with pytest.raises(HTTPError):
+        tree.exists(url)