http: fix info() if there is no content-length header (#5602)

dvc/fs/http.py

@@ -1,6 +1,7 @@
 import logging
 import os.path
 import threading
+from typing import Optional
 
 from funcy import cached_property, memoize, wrap_prop, wrap_with
 
@@ -146,10 +147,9 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         raise HTTPError(res.status_code, res.reason)
 
     def info(self, path_info):
-        url = path_info.url
-        headers = self._head(url).headers
-        etag = headers.get("ETag") or headers.get("Content-MD5")
-        size = int(headers.get("Content-Length"))
+        resp = self._head(path_info.url)
+        etag = resp.headers.get("ETag") or resp.headers.get("Content-MD5")
+        size = self._content_length(resp)
         return {"etag": etag, "size": size}
 
     def _upload_fobj(self, fobj, to_info):
@@ -195,6 +195,6 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
 
     @staticmethod
-    def _content_length(response):
+    def _content_length(response) -> Optional[int]:
         res = response.headers.get("Content-Length")
         return int(res) if res else None

tests/unit/remote/test_http.py

@@ -1,7 +1,11 @@
+import io
+
 import pytest
+import requests
 
 from dvc.exceptions import HTTPError
 from dvc.fs.http import HTTPFileSystem
+from dvc.path_info import URLInfo
 
 
 def test_download_fails_on_error_code(dvc, http):
@@ -128,12 +132,6 @@ def test_http_method(dvc):
 
 
 def test_exists(mocker):
-    import io
-
-    import requests
-
-    from dvc.path_info import URLInfo
-
     res = requests.Response()
     # need to add `raw`, as `exists()` fallbacks to a streaming GET requests
     # on HEAD request failure.
@@ -153,3 +151,20 @@ def test_exists(mocker):
     res.status_code = 403
     with pytest.raises(HTTPError):
         fs.exists(url)
+
+
+@pytest.mark.parametrize(
+    "headers, expected_size", [({"Content-Length": "3"}, 3), ({}, None)]
+)
+def test_content_length(mocker, headers, expected_size):
+    res = requests.Response()
+    res.headers.update(headers)
+    res.status_code = 200
+
+    fs = HTTPFileSystem(None, {})
+    mocker.patch.object(fs, "request", return_value=res)
+
+    url = URLInfo("https://example.org/file.txt")
+
+    assert fs.info(url) == {"etag": None, "size": expected_size}
+    assert fs._content_length(res) == expected_size