add support for Python 3.10 (#6422)

* add support for Python 3.10

pyarrow is not available, so we are skipping tests for it.
And, restricting `hdfs` extras to be not installed in Python3.10.

Pathlib removed `_init` method and `init` parameter, so just working
around that. It's not a big change.

* skip windows on 3.10; update pip/wheel/setuptools during install

pywin32 is not yet available for 3.10 Python, so skipping those for now

* install libgit2 with Python3.10 as pygit2 wheel is not available

* install libgit2

* only run on macOS, libgit2-1.1 not available on ubuntu

.github/workflows/tests.yaml

@@ -42,13 +42,14 @@ jobs:
       matrix:
         os: [ubuntu-18.04, windows-2019, macos-10.15]
         pyv: ["3.6", "3.7", "3.8", "3.9"]
-# macos 11.0 workers take too long to start up
-#        include:
-#          - os: "macos-11.0"
-#            pyv: "3.9"
+        include:
+          - os: "macos-10.15"
+            pyv: "3.10.0-rc.1"
         exclude:
           - os: macos-10.15
             pyv: "3.9"
+          - os: macos-10.15
+            pyv: "3.7"
 
     steps:
     - uses: actions/[email protected]
@@ -68,10 +69,14 @@ jobs:
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
         restore-keys: |
           ${{ runner.os }}-pip-
+    - name: install libgit2 on Python3.10 in macOS
+      if: matrix.pyv == '3.10.0-rc.1' && matrix.os == 'macos-10.15'
+      run: |
+        brew install libgit2
     - name: install
       if: steps.cache.pip-cache-dir.cache-hit != 'true'
       run: |
-        pip install wheel
+        pip install --upgrade pip setuptools wheel
         pip install -e ".[all,tests]"
     - name: install temporary dependencies
       run: |

setup.py

@@ -71,7 +71,7 @@ install_requires = [
     "zc.lockfile>=1.2.1",
     "flufl.lock>=3.2,<4",
     "win-unicode-console>=0.5; sys_platform == 'win32'",
-    "pywin32>=225; sys_platform == 'win32'",
+    "pywin32>=225; sys_platform == 'win32' and python_version < '3.10'",
     "networkx>=2.5",
     "psutil>=5.8.0",
     "pydot>=1.2.4",
@@ -99,12 +99,10 @@ gs = ["gcsfs==2021.7.0"]
 gdrive = ["pydrive2[fsspec]>=1.9.0"]
 s3 = ["s3fs==2021.7.0", "aiobotocore[boto3]>1.0.1"]
 azure = ["adlfs==2021.7.1", "azure-identity>=1.4.0", "knack"]
-# https://github.com/Legrandin/pycryptodome/issues/465
 oss = ["ossfs==2021.7.5"]
 ssh = ["sshfs>=2021.7.1"]
 
-# Remove the env marker if/when pyarrow is available for Python3.9
-hdfs = ["pyarrow>=2.0.0"]
+hdfs = ["pyarrow>=2.0.0; python_version < '3.10'"]
 webhdfs = ["hdfs==2.5.8"]
 webdav = ["webdav4>=0.9.0"]
 # gssapi should not be included in all_remotes, because it doesn't have wheels

tests/dir_helpers.py

@@ -47,6 +47,7 @@ from global repo template to creating everything inplace, which:
 
 import os
 import pathlib
+import sys
 from contextlib import contextmanager
 from textwrap import dedent
 
@@ -81,12 +82,17 @@ class TmpDir(pathlib.Path):
             cls = (  # pylint: disable=self-cls-assignment
                 WindowsTmpDir if os.name == "nt" else PosixTmpDir
             )
-        self = cls._from_parts(args, init=False)
+        # init parameter and `_init` method has been removed in Python 3.10.
+        kw = {"init": False} if sys.version_info < (3, 10) else {}
+        self = cls._from_parts(  # pylint: disable=unexpected-keyword-arg
+            args, **kw
+        )
         if not self._flavour.is_supported:
             raise NotImplementedError(
                 f"cannot instantiate {cls.__name__!r} on your system"
             )
-        self._init()
+        if sys.version_info < (3, 10):
+            self._init()  # pylint: disable=no-member
         return self
 
     def init(self, *, scm=False, dvc=False, subdir=False):

tests/remotes/hdfs.py

@@ -2,6 +2,7 @@ import locale
 import os
 import platform
 import subprocess
+import sys
 import uuid
 from contextlib import contextmanager
 from pathlib import Path
@@ -74,6 +75,9 @@ class HDFS(Base, URLInfo):  # pylint: disable=abstract-method
 def hadoop(test_config):
     test_config.requires("hdfs")
 
+    if sys.version_info >= (3, 10):
+        pytest.skip("pyarrow is not available for 3.10 yet.")
+
     if platform.system() != "Linux":
         pytest.skip("only supported on Linux")
 
@@ -240,11 +244,16 @@ class FakeHadoopFileSystem:
 
 
 @pytest.fixture
-def hdfs(mocker):
+def hdfs(test_config, mocker):
     # Windows might not have Visual C++ Redistributable for Visual Studio
     # 2015 installed, which will result in the following error:
     # "The pyarrow installation is not built with support for
     # 'HadoopFileSystem'"
+    test_config.requires("hdfs")
+
+    if sys.version_info >= (3, 10):
+        pytest.skip("pyarrow is not available for 3.10 yet.")
+
     mocker.patch("pyarrow.fs._not_imported", [])
     mocker.patch(
         "pyarrow.fs.HadoopFileSystem", FakeHadoopFileSystem, create=True