[6478] Fix 401 unauthorized for directories in basic auth protected http remote (#6479)

* added auth to iter_url and created custom open for http fs

* added default value to auth

* Update dvc/utils/http.py

Co-authored-by: Ruslan Kuprieiev <[email protected]>

* Update dvc/fs/http.py

Co-authored-by: Ruslan Kuprieiev <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* pr fixes

Co-authored-by: Ruslan Kuprieiev <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

dvc/fs/base.py

@@ -9,7 +9,6 @@ from dvc.path_info import URLInfo
 from dvc.progress import Tqdm
 from dvc.utils import tmp_fname
 from dvc.utils.fs import makedirs, move
-from dvc.utils.http import open_url
 
 logger = logging.getLogger(__name__)
 
@@ -126,12 +125,6 @@ class BaseFileSystem:
         )
 
     def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
-        if hasattr(self, "_generate_download_url"):
-            # pylint:disable=no-member
-            func = self._generate_download_url  # type: ignore[attr-defined]
-            get_url = partial(func, path_info)
-            return open_url(get_url, mode=mode, encoding=encoding, **kwargs)
-
         raise RemoteActionNotImplemented("open", self.scheme)
 
     def exists(self, path_info) -> bool:

dvc/fs/http.py

@@ -65,9 +65,6 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 self.headers.update({self.custom_auth_header: self.password})
         return None
 
-    def _generate_download_url(self, path_info):
-        return path_info.url
-
     @wrap_prop(threading.Lock())
     @cached_property
     def _session(self):
@@ -190,6 +187,17 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 desc=name or to_info.url,
             )
 
+    def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
+        from dvc.utils.http import open_url
+
+        return open_url(
+            path_info.url,
+            mode=mode,
+            encoding=encoding,
+            auth=self._auth_method(path_info),
+            **kwargs,
+        )
+
     @staticmethod
     def _content_length(response) -> Optional[int]:
         res = response.headers.get("Content-Length")

dvc/utils/http.py

@@ -24,13 +24,15 @@ def open_url(url, mode="r", encoding=None, **iter_opts):
 
 
 @contextmanager
-def iter_url(url, chunk_size=io.DEFAULT_BUFFER_SIZE):
+def iter_url(url, chunk_size=io.DEFAULT_BUFFER_SIZE, auth=None):
     """Iterate over chunks requested from url."""
     import requests
 
     def request(headers=None):
         the_url = url() if callable(url) else url
-        response = requests.get(the_url, stream=True, headers=headers)
+        response = requests.get(
+            the_url, stream=True, headers=headers, auth=auth
+        )
         if response.status_code == 404:
             raise FileNotFoundError(f"Can't open {the_url}")
         response.raise_for_status()