diff --git a/dvc/fs/http.py b/dvc/fs/http.py index a843ad1ce..461b8ea12 100644 --- a/dvc/fs/http.py +++ b/dvc/fs/http.py @@ -1,6 +1,7 @@ import logging import os.path import threading +from typing import Optional from funcy import cached_property, memoize, wrap_prop, wrap_with @@ -146,10 +147,9 @@ class HTTPFileSystem(BaseFileSystem): # pylint:disable=abstract-method raise HTTPError(res.status_code, res.reason) def info(self, path_info): - url = path_info.url - headers = self._head(url).headers - etag = headers.get("ETag") or headers.get("Content-MD5") - size = int(headers.get("Content-Length")) + resp = self._head(path_info.url) + etag = resp.headers.get("ETag") or resp.headers.get("Content-MD5") + size = self._content_length(resp) return {"etag": etag, "size": size} def _upload_fobj(self, fobj, to_info): @@ -195,6 +195,6 @@ class HTTPFileSystem(BaseFileSystem): # pylint:disable=abstract-method ) @staticmethod - def _content_length(response): + def _content_length(response) -> Optional[int]: res = response.headers.get("Content-Length") return int(res) if res else None diff --git a/tests/unit/remote/test_http.py b/tests/unit/remote/test_http.py index 830c8e4e2..7b76284fd 100644 --- a/tests/unit/remote/test_http.py +++ b/tests/unit/remote/test_http.py @@ -1,7 +1,11 @@ +import io + import pytest +import requests from dvc.exceptions import HTTPError from dvc.fs.http import HTTPFileSystem +from dvc.path_info import URLInfo def test_download_fails_on_error_code(dvc, http): @@ -128,12 +132,6 @@ def test_http_method(dvc): def test_exists(mocker): - import io - - import requests - - from dvc.path_info import URLInfo - res = requests.Response() # need to add `raw`, as `exists()` fallbacks to a streaming GET requests # on HEAD request failure. @@ -153,3 +151,20 @@ def test_exists(mocker): res.status_code = 403 with pytest.raises(HTTPError): fs.exists(url) + + +@pytest.mark.parametrize( + "headers, expected_size", [({"Content-Length": "3"}, 3), ({}, None)] +) +def test_content_length(mocker, headers, expected_size): + res = requests.Response() + res.headers.update(headers) + res.status_code = 200 + + fs = HTTPFileSystem(None, {}) + mocker.patch.object(fs, "request", return_value=res) + + url = URLInfo("https://example.org/file.txt") + + assert fs.info(url) == {"etag": None, "size": expected_size} + assert fs._content_length(res) == expected_size