Run CI for Python3.9 (#4647)

* Run CI for Python3.9, skips hdfs tests for 3.9

Also, `dvc[hdfs]` install will be a no-op till we have
a pyarrow wheel for 3.9.

Note that this does not add an entry to the classifier
claiming it supports 3.9, that will be added when 3.9
gets released (or, wait for pyarrow to be released).

* Skip HDFS related tests on Python3.9

Also, fix the python version regex for `3.9.0b5` like versions.

* Exclude python3.9 for Windows

libxml does not have wheels for Python39.
We do compile them on Linux and Mac

* Update tests/unit/test_info.py

* Use realpath for gitignore

Windows tests for Python3.8 was failing
intermittently.

Note: Also removed the realpath wrap around self.root_dir as it's already a
realpath when `Git` is instantiated.

Co-authored-by: Ruslan Kuprieiev <[email protected]>

.github/workflows/tests.yaml

@@ -28,16 +28,21 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-18.04, windows-2019, macos-10.15]
-        pyv: [3.6, 3.7, 3.8]
+        pyv: [3.6, 3.7, 3.8, "3.9-dev"]
         exclude:
           - os: windows-2019
-            pyv: 3.8
+            pyv: "3.9-dev"
     steps:
     - uses: actions/[email protected]
     - name: Set up Python
       uses: actions/[email protected]
       with:
         python-version: ${{ matrix.pyv }}
+    - name: Install libxml2 and libxslt
+      if: matrix.pyv == '3.9-dev' && matrix.os == 'ubuntu-18.04'
+      # need to install for building lxml, a dependency of webdavclient3
+      run: |
+        sudo apt install libxml2 libxslt-dev
     - name: install
       run: |
         pip install ".[all,tests]"
@@ -46,12 +51,10 @@ jobs:
         git config --global user.email "[email protected]"
         git config --global user.name "DVC Tester"
     - name: run tests
-      run: python -m tests --cov-report=xml
+      run: python -m tests -ra --cov-report=xml
     - name: upload coverage report
       if: github.event.name == 'push'
       uses: codecov/[email protected]
       with:
         file: ./coverage.xml
         fail_ci_if_error: true
-
-

dvc/scm/git.py

@@ -177,7 +177,7 @@ class Git(Base):
 
         gitignore = os.path.join(ignore_file_dir, self.GITIGNORE)
 
-        if not path_isin(gitignore, os.path.realpath(self.root_dir)):
+        if not path_isin(os.path.realpath(gitignore), self.root_dir):
             raise FileNotInRepoError(path)
 
         return entry, gitignore

setup.py

@@ -1,6 +1,5 @@
 import importlib.util
 import os
-import sys
 
 from setuptools import find_packages, setup
 from setuptools.command.build_py import build_py as _build_py
@@ -93,7 +92,9 @@ s3 = ["boto3>=1.9.201"]
 azure = ["azure-storage-blob>=12.0", "knack"]
 oss = ["oss2==2.6.1"]
 ssh = ["paramiko[invoke]>=2.7.0"]
-hdfs = ["pyarrow>=0.17.0"]
+
+# Remove the env marker if/when pyarrow is available for Python3.9
+hdfs = ["pyarrow>=0.17.0;  python_version < '3.9'"]
 webdav = ["webdavclient3>=3.14.5"]
 # gssapi should not be included in all_remotes, because it doesn't have wheels
 # for linux and mac, so it will fail to compile if user doesn't have all the
@@ -138,11 +139,9 @@ tests_requirements = [
     "pylint-plugin-utils",
     "wget",
     "filelock",
+    "black==19.10b0",
 ]
 
-if (sys.version_info) >= (3, 6):
-    tests_requirements.append("black==19.10b0")
-
 setup(
     name="dvc",
     version=version,

tests/__init__.py

@@ -1,4 +1,10 @@
 import os
+import sys
+
+# FIXME: Search and replace these from the tests if pyarrow wheel is available
+PY39 = sys.version_info >= (3, 9, 0)
+PYARROW_NOT_AVAILABLE = "pyarrow not available yet for Python3.9"
+
 
 # Increasing fd ulimit for tests
 if os.name == "nt":

tests/func/test_output.py

@@ -2,12 +2,14 @@ import pytest
 
 from dvc.output import OUTS_MAP, _get
 from dvc.stage import Stage
+from tests import PY39, PYARROW_NOT_AVAILABLE
+
+MARKERS = [pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE)]
 
 TESTS = [
     ("s3://bucket/path", "s3"),
     ("gs://bucket/path", "gs"),
     ("ssh://example.com:/dir/path", "ssh"),
-    ("hdfs://example.com/dir/path", "hdfs"),
     ("path/to/file", "local"),
     ("path\\to\\file", "local"),
     ("file", "local"),
@@ -16,6 +18,8 @@ TESTS = [
     ("../file", "local"),
     ("..\\file", "local"),
     ("unknown://path", "local"),
+    pytest.param("hdfs://example.com/dir/path", "hdfs", marks=MARKERS),
+    pytest.param("hdfs://example.com/dir/path", "hdfs", marks=MARKERS),
 ]
 
 

tests/func/test_version.py

@@ -1,13 +1,14 @@
 import re
 
 from dvc.main import main
+from tests.unit.test_info import PYTHON_VERSION_REGEX
 
 
 def test_(tmp_dir, dvc, scm, caplog):
     assert main(["version"]) == 0
 
     assert re.search(r"DVC version: \d+\.\d+\.\d+.*", caplog.text)
-    assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", caplog.text)
+    assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", caplog.text)
     assert re.search(r"Supports: .*", caplog.text)
     assert re.search(r"Cache types: .*", caplog.text)
     assert "Repo: dvc, git" in caplog.text

tests/remotes/hdfs.py

@@ -7,6 +7,7 @@ from contextlib import contextmanager
 import pytest
 
 from dvc.path_info import URLInfo
+from tests import PY39, PYARROW_NOT_AVAILABLE
 
 from .base import Base
 
@@ -70,12 +71,15 @@ class HDFS(Base, URLInfo):  # pylint: disable=abstract-method
 
 @pytest.fixture(scope="session")
 def hadoop():
-    import wget
-    from appdirs import user_cache_dir
-
     if platform.system() != "Linux":
         pytest.skip("only supported on Linux")
 
+    if PY39:
+        pytest.skip(PYARROW_NOT_AVAILABLE)
+
+    import wget
+    from appdirs import user_cache_dir
+
     hadoop_name = "hadoop-2.7.2.tar.gz"
     java_name = "openjdk-7u75-b13-linux-x64-18_dec_2014.tar.gz"
 

tests/unit/dependency/test_hdfs.py

@@ -1,7 +1,11 @@
+import pytest
+
 from dvc.dependency.hdfs import HDFSDependency
+from tests import PY39, PYARROW_NOT_AVAILABLE
 from tests.unit.dependency.test_local import TestLocalDependency
 
 
+@pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE)
 class TestHDFSDependency(TestLocalDependency):
     def _get_cls(self):
         return HDFSDependency

tests/unit/output/test_hdfs.py

@@ -1,7 +1,11 @@
+import pytest
+
 from dvc.output.hdfs import HDFSOutput
+from tests import PY39, PYARROW_NOT_AVAILABLE
 from tests.unit.output.test_local import TestLocalOutput
 
 
+@pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE)
 class TestHDFSOutput(TestLocalOutput):
     def _get_cls(self):
         return HDFSOutput

tests/unit/test_info.py

@@ -6,6 +6,12 @@ import pytest
 
 from dvc.info import get_dvc_info, psutil
 
+# Python's version is in the shape of:
+# <major>.<minor>.<patch>[{a|b|rc}N][.postN][.devN]
+# `patch` is more than enough for the tests.
+# Refer PEP-0440 for complete regex just in-case.
+PYTHON_VERSION_REGEX = r"Python \d\.\d+\.\d+\S*"
+
 
 @pytest.mark.parametrize("scm_init", [True, False])
 def test_info_in_repo(scm_init, tmp_dir):
@@ -16,7 +22,7 @@ def test_info_in_repo(scm_init, tmp_dir):
     dvc_info = get_dvc_info()
 
     assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info)
-    assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", dvc_info)
+    assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", dvc_info)
     assert re.search(r"Supports: .*", dvc_info)
     assert re.search(r"Cache types: .*", dvc_info)
 
@@ -60,7 +66,7 @@ def test_info_outside_of_repo(tmp_dir, caplog):
     dvc_info = get_dvc_info()
 
     assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info)
-    assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", dvc_info)
+    assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", dvc_info)
     assert re.search(r"Supports: .*", dvc_info)
     assert not re.search(r"Cache types: .*", dvc_info)
     assert "Repo:" not in dvc_info
@@ -70,5 +76,5 @@ def test_info_outside_of_repo(tmp_dir, caplog):
 def test_fs_info_outside_of_repo(tmp_dir, caplog):
     dvc_info = get_dvc_info()
     assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info)
-    assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", dvc_info)
+    assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", dvc_info)
     assert re.search(r"Supports: .*", dvc_info)