From ba40d91de67ff869f87c403d098c8500733560be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Wed, 22 Dec 2021 19:54:21 +0545 Subject: [PATCH] fs.hdfs: skip port if not available --- dvc/fs/hdfs.py | 6 +++--- tests/unit/fs/test_hdfs.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 tests/unit/fs/test_hdfs.py diff --git a/dvc/fs/hdfs.py b/dvc/fs/hdfs.py index 2f7b8f56d..e77fdf9dd 100644 --- a/dvc/fs/hdfs.py +++ b/dvc/fs/hdfs.py @@ -27,9 +27,9 @@ class HDFSFileSystem(CallbackMixin, FSSpecWrapper): def unstrip_protocol(self, path: str) -> str: host = self.fs_args["host"] - port = self.fs_args["port"] - path = path.lstrip("/") - return f"hdfs://{host}:{port}/{path}" + port = self.fs_args.get("port") + netloc = host + (f":{port}" if port else "") + return "hdfs://" + netloc + "/" + path.lstrip("/") @staticmethod def _get_kwargs_from_urls(urlpath): diff --git a/tests/unit/fs/test_hdfs.py b/tests/unit/fs/test_hdfs.py new file mode 100644 index 000000000..fee6b735e --- /dev/null +++ b/tests/unit/fs/test_hdfs.py @@ -0,0 +1,16 @@ +import pytest + +from dvc.fs import HDFSFileSystem + + +@pytest.mark.parametrize( + "fs_args, expected_url", + [ + ({"host": "example.com"}, "hdfs://example.com"), + ({"host": "example.com", "port": None}, "hdfs://example.com"), + ({"host": "example.com", "port": 8020}, "hdfs://example.com:8020"), + ], +) +def test_hdfs_unstrip_protocol(fs_args, expected_url): + fs = HDFSFileSystem(**fs_args) + assert fs.unstrip_protocol("/path") == expected_url + "/path"