From fc6999ad17dc219d1f0db6da6d1650b3c2f918a0 Mon Sep 17 00:00:00 2001 From: Saugat Pachhai Date: Thu, 1 Oct 2020 13:26:39 +0545 Subject: [PATCH] Run CI for Python3.9 (#4647) * Run CI for Python3.9, skips hdfs tests for 3.9 Also, `dvc[hdfs]` install will be a no-op till we have a pyarrow wheel for 3.9. Note that this does not add an entry to the classifier claiming it supports 3.9, that will be added when 3.9 gets released (or, wait for pyarrow to be released). * Skip HDFS related tests on Python3.9 Also, fix the python version regex for `3.9.0b5` like versions. * Exclude python3.9 for Windows libxml does not have wheels for Python39. We do compile them on Linux and Mac * Update tests/unit/test_info.py * Use realpath for gitignore Windows tests for Python3.8 was failing intermittently. Note: Also removed the realpath wrap around self.root_dir as it's already a realpath when `Git` is instantiated. Co-authored-by: Ruslan Kuprieiev --- .github/workflows/tests.yaml | 13 ++++++++----- dvc/scm/git.py | 2 +- setup.py | 9 ++++----- tests/__init__.py | 6 ++++++ tests/func/test_output.py | 6 +++++- tests/func/test_version.py | 3 ++- tests/remotes/hdfs.py | 10 +++++++--- tests/unit/dependency/test_hdfs.py | 4 ++++ tests/unit/output/test_hdfs.py | 4 ++++ tests/unit/test_info.py | 12 +++++++++--- 10 files changed, 50 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 4582e6683..9c8097d1a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -28,16 +28,21 @@ jobs: fail-fast: false matrix: os: [ubuntu-18.04, windows-2019, macos-10.15] - pyv: [3.6, 3.7, 3.8] + pyv: [3.6, 3.7, 3.8, "3.9-dev"] exclude: - os: windows-2019 - pyv: 3.8 + pyv: "3.9-dev" steps: - uses: actions/checkout@v2.3.2 - name: Set up Python uses: actions/setup-python@v2.1.2 with: python-version: ${{ matrix.pyv }} + - name: Install libxml2 and libxslt + if: matrix.pyv == '3.9-dev' && matrix.os == 'ubuntu-18.04' + # need to install for building lxml, a dependency of webdavclient3 + run: | + sudo apt install libxml2 libxslt-dev - name: install run: | pip install ".[all,tests]" @@ -46,12 +51,10 @@ jobs: git config --global user.email "dvctester@example.com" git config --global user.name "DVC Tester" - name: run tests - run: python -m tests --cov-report=xml + run: python -m tests -ra --cov-report=xml - name: upload coverage report if: github.event.name == 'push' uses: codecov/codecov-action@v1.0.13 with: file: ./coverage.xml fail_ci_if_error: true - - diff --git a/dvc/scm/git.py b/dvc/scm/git.py index d5d72c1b8..09f56b5ef 100644 --- a/dvc/scm/git.py +++ b/dvc/scm/git.py @@ -177,7 +177,7 @@ class Git(Base): gitignore = os.path.join(ignore_file_dir, self.GITIGNORE) - if not path_isin(gitignore, os.path.realpath(self.root_dir)): + if not path_isin(os.path.realpath(gitignore), self.root_dir): raise FileNotInRepoError(path) return entry, gitignore diff --git a/setup.py b/setup.py index fe226c805..10196cb57 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ import importlib.util import os -import sys from setuptools import find_packages, setup from setuptools.command.build_py import build_py as _build_py @@ -93,7 +92,9 @@ s3 = ["boto3>=1.9.201"] azure = ["azure-storage-blob>=12.0", "knack"] oss = ["oss2==2.6.1"] ssh = ["paramiko[invoke]>=2.7.0"] -hdfs = ["pyarrow>=0.17.0"] + +# Remove the env marker if/when pyarrow is available for Python3.9 +hdfs = ["pyarrow>=0.17.0; python_version < '3.9'"] webdav = ["webdavclient3>=3.14.5"] # gssapi should not be included in all_remotes, because it doesn't have wheels # for linux and mac, so it will fail to compile if user doesn't have all the @@ -138,11 +139,9 @@ tests_requirements = [ "pylint-plugin-utils", "wget", "filelock", + "black==19.10b0", ] -if (sys.version_info) >= (3, 6): - tests_requirements.append("black==19.10b0") - setup( name="dvc", version=version, diff --git a/tests/__init__.py b/tests/__init__.py index 343dc6fa0..2b28b65af 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,10 @@ import os +import sys + +# FIXME: Search and replace these from the tests if pyarrow wheel is available +PY39 = sys.version_info >= (3, 9, 0) +PYARROW_NOT_AVAILABLE = "pyarrow not available yet for Python3.9" + # Increasing fd ulimit for tests if os.name == "nt": diff --git a/tests/func/test_output.py b/tests/func/test_output.py index 619323add..2baf7b7b0 100644 --- a/tests/func/test_output.py +++ b/tests/func/test_output.py @@ -2,12 +2,14 @@ import pytest from dvc.output import OUTS_MAP, _get from dvc.stage import Stage +from tests import PY39, PYARROW_NOT_AVAILABLE + +MARKERS = [pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE)] TESTS = [ ("s3://bucket/path", "s3"), ("gs://bucket/path", "gs"), ("ssh://example.com:/dir/path", "ssh"), - ("hdfs://example.com/dir/path", "hdfs"), ("path/to/file", "local"), ("path\\to\\file", "local"), ("file", "local"), @@ -16,6 +18,8 @@ TESTS = [ ("../file", "local"), ("..\\file", "local"), ("unknown://path", "local"), + pytest.param("hdfs://example.com/dir/path", "hdfs", marks=MARKERS), + pytest.param("hdfs://example.com/dir/path", "hdfs", marks=MARKERS), ] diff --git a/tests/func/test_version.py b/tests/func/test_version.py index ebeeaafc5..4d63a2c2d 100644 --- a/tests/func/test_version.py +++ b/tests/func/test_version.py @@ -1,13 +1,14 @@ import re from dvc.main import main +from tests.unit.test_info import PYTHON_VERSION_REGEX def test_(tmp_dir, dvc, scm, caplog): assert main(["version"]) == 0 assert re.search(r"DVC version: \d+\.\d+\.\d+.*", caplog.text) - assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", caplog.text) + assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", caplog.text) assert re.search(r"Supports: .*", caplog.text) assert re.search(r"Cache types: .*", caplog.text) assert "Repo: dvc, git" in caplog.text diff --git a/tests/remotes/hdfs.py b/tests/remotes/hdfs.py index 2c804e4b0..7a765f080 100644 --- a/tests/remotes/hdfs.py +++ b/tests/remotes/hdfs.py @@ -7,6 +7,7 @@ from contextlib import contextmanager import pytest from dvc.path_info import URLInfo +from tests import PY39, PYARROW_NOT_AVAILABLE from .base import Base @@ -70,12 +71,15 @@ class HDFS(Base, URLInfo): # pylint: disable=abstract-method @pytest.fixture(scope="session") def hadoop(): - import wget - from appdirs import user_cache_dir - if platform.system() != "Linux": pytest.skip("only supported on Linux") + if PY39: + pytest.skip(PYARROW_NOT_AVAILABLE) + + import wget + from appdirs import user_cache_dir + hadoop_name = "hadoop-2.7.2.tar.gz" java_name = "openjdk-7u75-b13-linux-x64-18_dec_2014.tar.gz" diff --git a/tests/unit/dependency/test_hdfs.py b/tests/unit/dependency/test_hdfs.py index c78ee872f..10118908c 100644 --- a/tests/unit/dependency/test_hdfs.py +++ b/tests/unit/dependency/test_hdfs.py @@ -1,7 +1,11 @@ +import pytest + from dvc.dependency.hdfs import HDFSDependency +from tests import PY39, PYARROW_NOT_AVAILABLE from tests.unit.dependency.test_local import TestLocalDependency +@pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE) class TestHDFSDependency(TestLocalDependency): def _get_cls(self): return HDFSDependency diff --git a/tests/unit/output/test_hdfs.py b/tests/unit/output/test_hdfs.py index 8ac702705..ec81081c4 100644 --- a/tests/unit/output/test_hdfs.py +++ b/tests/unit/output/test_hdfs.py @@ -1,7 +1,11 @@ +import pytest + from dvc.output.hdfs import HDFSOutput +from tests import PY39, PYARROW_NOT_AVAILABLE from tests.unit.output.test_local import TestLocalOutput +@pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE) class TestHDFSOutput(TestLocalOutput): def _get_cls(self): return HDFSOutput diff --git a/tests/unit/test_info.py b/tests/unit/test_info.py index fe5dc2fb0..133cd386f 100644 --- a/tests/unit/test_info.py +++ b/tests/unit/test_info.py @@ -6,6 +6,12 @@ import pytest from dvc.info import get_dvc_info, psutil +# Python's version is in the shape of: +# ..[{a|b|rc}N][.postN][.devN] +# `patch` is more than enough for the tests. +# Refer PEP-0440 for complete regex just in-case. +PYTHON_VERSION_REGEX = r"Python \d\.\d+\.\d+\S*" + @pytest.mark.parametrize("scm_init", [True, False]) def test_info_in_repo(scm_init, tmp_dir): @@ -16,7 +22,7 @@ def test_info_in_repo(scm_init, tmp_dir): dvc_info = get_dvc_info() assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info) - assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", dvc_info) + assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", dvc_info) assert re.search(r"Supports: .*", dvc_info) assert re.search(r"Cache types: .*", dvc_info) @@ -60,7 +66,7 @@ def test_info_outside_of_repo(tmp_dir, caplog): dvc_info = get_dvc_info() assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info) - assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", dvc_info) + assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", dvc_info) assert re.search(r"Supports: .*", dvc_info) assert not re.search(r"Cache types: .*", dvc_info) assert "Repo:" not in dvc_info @@ -70,5 +76,5 @@ def test_info_outside_of_repo(tmp_dir, caplog): def test_fs_info_outside_of_repo(tmp_dir, caplog): dvc_info = get_dvc_info() assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info) - assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", dvc_info) + assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", dvc_info) assert re.search(r"Supports: .*", dvc_info)