pyarrow support for Python3.9 (#5391)

setup.py

@@ -101,7 +101,7 @@ oss = ["oss2==2.6.1", "pycryptodome<3.9.9"]
 ssh = ["paramiko[invoke]>=2.7.0"]
 
 # Remove the env marker if/when pyarrow is available for Python3.9
-hdfs = ["pyarrow>=2.0.0;  python_version < '3.9'"]
+hdfs = ["pyarrow>=2.0.0"]
 webhdfs = ["hdfs==2.5.8"]
 webdav = ["webdavclient3>=3.14.5"]
 # gssapi should not be included in all_remotes, because it doesn't have wheels

tests/__init__.py

@@ -1,11 +1,6 @@
 import os
 import sys
 
-# FIXME: Search and replace these from the tests if pyarrow wheel is available
-PY39 = sys.version_info >= (3, 9, 0)
-PYARROW_NOT_AVAILABLE = "pyarrow not available yet for Python3.9"
-
-
 # Increasing fd ulimit for tests
 if os.name == "nt":
     import subprocess

tests/func/test_output.py

@@ -2,13 +2,8 @@ import pytest
 
 from dvc.output import OUTS_MAP, _get
 from dvc.stage import Stage
-from tests import PY39, PYARROW_NOT_AVAILABLE
-
-MARKERS = [
-    pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE),
-    pytest.mark.hdfs,
-]
 
+MARKERS = [pytest.mark.hdfs]
 TESTS = [
     ("s3://bucket/path", "s3"),
     ("gs://bucket/path", "gs"),

tests/remotes/hdfs.py

@@ -10,7 +10,6 @@ from tempfile import TemporaryDirectory
 import pytest
 
 from dvc.path_info import URLInfo
-from tests import PY39, PYARROW_NOT_AVAILABLE
 
 from .base import Base
 
@@ -85,9 +84,6 @@ def hadoop(test_config):
     if platform.system() != "Linux":
         pytest.skip("only supported on Linux")
 
-    if PY39:
-        pytest.skip(PYARROW_NOT_AVAILABLE)
-
     import wget
     from appdirs import user_cache_dir
 
@@ -252,9 +248,6 @@ class FakeHadoopFileSystem:
 
 @pytest.fixture
 def hdfs(mocker):
-    if PY39:
-        pytest.skip(PYARROW_NOT_AVAILABLE)
-
     # Windows might not have Visual C++ Redistributable for Visual Studio
     # 2015 installed, which will result in the following error:
     # "The pyarrow installation is not built with support for

tests/unit/dependency/test_hdfs.py

@@ -1,11 +1,9 @@
 import pytest
 
 from dvc.dependency.hdfs import HDFSDependency
-from tests import PY39, PYARROW_NOT_AVAILABLE
 from tests.unit.dependency.test_local import TestLocalDependency
 
 
-@pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE)
 @pytest.mark.hdfs
 class TestHDFSDependency(TestLocalDependency):
     def _get_cls(self):

tests/unit/output/test_hdfs.py

@@ -1,11 +1,9 @@
 import pytest
 
 from dvc.output.hdfs import HDFSOutput
-from tests import PY39, PYARROW_NOT_AVAILABLE
 from tests.unit.output.test_local import TestLocalOutput
 
 
-@pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE)
 @pytest.mark.hdfs
 class TestHDFSOutput(TestLocalOutput):
     def _get_cls(self):