diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 4582e6683..9c8097d1a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -28,16 +28,21 @@ jobs: fail-fast: false matrix: os: [ubuntu-18.04, windows-2019, macos-10.15] - pyv: [3.6, 3.7, 3.8] + pyv: [3.6, 3.7, 3.8, "3.9-dev"] exclude: - os: windows-2019 - pyv: 3.8 + pyv: "3.9-dev" steps: - uses: actions/checkout@v2.3.2 - name: Set up Python uses: actions/setup-python@v2.1.2 with: python-version: ${{ matrix.pyv }} + - name: Install libxml2 and libxslt + if: matrix.pyv == '3.9-dev' && matrix.os == 'ubuntu-18.04' + # need to install for building lxml, a dependency of webdavclient3 + run: | + sudo apt install libxml2 libxslt-dev - name: install run: | pip install ".[all,tests]" @@ -46,12 +51,10 @@ jobs: git config --global user.email "dvctester@example.com" git config --global user.name "DVC Tester" - name: run tests - run: python -m tests --cov-report=xml + run: python -m tests -ra --cov-report=xml - name: upload coverage report if: github.event.name == 'push' uses: codecov/codecov-action@v1.0.13 with: file: ./coverage.xml fail_ci_if_error: true - - diff --git a/dvc/scm/git.py b/dvc/scm/git.py index d5d72c1b8..09f56b5ef 100644 --- a/dvc/scm/git.py +++ b/dvc/scm/git.py @@ -177,7 +177,7 @@ class Git(Base): gitignore = os.path.join(ignore_file_dir, self.GITIGNORE) - if not path_isin(gitignore, os.path.realpath(self.root_dir)): + if not path_isin(os.path.realpath(gitignore), self.root_dir): raise FileNotInRepoError(path) return entry, gitignore diff --git a/setup.py b/setup.py index fe226c805..10196cb57 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ import importlib.util import os -import sys from setuptools import find_packages, setup from setuptools.command.build_py import build_py as _build_py @@ -93,7 +92,9 @@ s3 = ["boto3>=1.9.201"] azure = ["azure-storage-blob>=12.0", "knack"] oss = ["oss2==2.6.1"] ssh = ["paramiko[invoke]>=2.7.0"] -hdfs = ["pyarrow>=0.17.0"] + +# Remove the env marker if/when pyarrow is available for Python3.9 +hdfs = ["pyarrow>=0.17.0; python_version < '3.9'"] webdav = ["webdavclient3>=3.14.5"] # gssapi should not be included in all_remotes, because it doesn't have wheels # for linux and mac, so it will fail to compile if user doesn't have all the @@ -138,11 +139,9 @@ tests_requirements = [ "pylint-plugin-utils", "wget", "filelock", + "black==19.10b0", ] -if (sys.version_info) >= (3, 6): - tests_requirements.append("black==19.10b0") - setup( name="dvc", version=version, diff --git a/tests/__init__.py b/tests/__init__.py index 343dc6fa0..2b28b65af 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,10 @@ import os +import sys + +# FIXME: Search and replace these from the tests if pyarrow wheel is available +PY39 = sys.version_info >= (3, 9, 0) +PYARROW_NOT_AVAILABLE = "pyarrow not available yet for Python3.9" + # Increasing fd ulimit for tests if os.name == "nt": diff --git a/tests/func/test_output.py b/tests/func/test_output.py index 619323add..2baf7b7b0 100644 --- a/tests/func/test_output.py +++ b/tests/func/test_output.py @@ -2,12 +2,14 @@ import pytest from dvc.output import OUTS_MAP, _get from dvc.stage import Stage +from tests import PY39, PYARROW_NOT_AVAILABLE + +MARKERS = [pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE)] TESTS = [ ("s3://bucket/path", "s3"), ("gs://bucket/path", "gs"), ("ssh://example.com:/dir/path", "ssh"), - ("hdfs://example.com/dir/path", "hdfs"), ("path/to/file", "local"), ("path\\to\\file", "local"), ("file", "local"), @@ -16,6 +18,8 @@ TESTS = [ ("../file", "local"), ("..\\file", "local"), ("unknown://path", "local"), + pytest.param("hdfs://example.com/dir/path", "hdfs", marks=MARKERS), + pytest.param("hdfs://example.com/dir/path", "hdfs", marks=MARKERS), ] diff --git a/tests/func/test_version.py b/tests/func/test_version.py index ebeeaafc5..4d63a2c2d 100644 --- a/tests/func/test_version.py +++ b/tests/func/test_version.py @@ -1,13 +1,14 @@ import re from dvc.main import main +from tests.unit.test_info import PYTHON_VERSION_REGEX def test_(tmp_dir, dvc, scm, caplog): assert main(["version"]) == 0 assert re.search(r"DVC version: \d+\.\d+\.\d+.*", caplog.text) - assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", caplog.text) + assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", caplog.text) assert re.search(r"Supports: .*", caplog.text) assert re.search(r"Cache types: .*", caplog.text) assert "Repo: dvc, git" in caplog.text diff --git a/tests/remotes/hdfs.py b/tests/remotes/hdfs.py index 2c804e4b0..7a765f080 100644 --- a/tests/remotes/hdfs.py +++ b/tests/remotes/hdfs.py @@ -7,6 +7,7 @@ from contextlib import contextmanager import pytest from dvc.path_info import URLInfo +from tests import PY39, PYARROW_NOT_AVAILABLE from .base import Base @@ -70,12 +71,15 @@ class HDFS(Base, URLInfo): # pylint: disable=abstract-method @pytest.fixture(scope="session") def hadoop(): - import wget - from appdirs import user_cache_dir - if platform.system() != "Linux": pytest.skip("only supported on Linux") + if PY39: + pytest.skip(PYARROW_NOT_AVAILABLE) + + import wget + from appdirs import user_cache_dir + hadoop_name = "hadoop-2.7.2.tar.gz" java_name = "openjdk-7u75-b13-linux-x64-18_dec_2014.tar.gz" diff --git a/tests/unit/dependency/test_hdfs.py b/tests/unit/dependency/test_hdfs.py index c78ee872f..10118908c 100644 --- a/tests/unit/dependency/test_hdfs.py +++ b/tests/unit/dependency/test_hdfs.py @@ -1,7 +1,11 @@ +import pytest + from dvc.dependency.hdfs import HDFSDependency +from tests import PY39, PYARROW_NOT_AVAILABLE from tests.unit.dependency.test_local import TestLocalDependency +@pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE) class TestHDFSDependency(TestLocalDependency): def _get_cls(self): return HDFSDependency diff --git a/tests/unit/output/test_hdfs.py b/tests/unit/output/test_hdfs.py index 8ac702705..ec81081c4 100644 --- a/tests/unit/output/test_hdfs.py +++ b/tests/unit/output/test_hdfs.py @@ -1,7 +1,11 @@ +import pytest + from dvc.output.hdfs import HDFSOutput +from tests import PY39, PYARROW_NOT_AVAILABLE from tests.unit.output.test_local import TestLocalOutput +@pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE) class TestHDFSOutput(TestLocalOutput): def _get_cls(self): return HDFSOutput diff --git a/tests/unit/test_info.py b/tests/unit/test_info.py index fe5dc2fb0..133cd386f 100644 --- a/tests/unit/test_info.py +++ b/tests/unit/test_info.py @@ -6,6 +6,12 @@ import pytest from dvc.info import get_dvc_info, psutil +# Python's version is in the shape of: +# ..[{a|b|rc}N][.postN][.devN] +# `patch` is more than enough for the tests. +# Refer PEP-0440 for complete regex just in-case. +PYTHON_VERSION_REGEX = r"Python \d\.\d+\.\d+\S*" + @pytest.mark.parametrize("scm_init", [True, False]) def test_info_in_repo(scm_init, tmp_dir): @@ -16,7 +22,7 @@ def test_info_in_repo(scm_init, tmp_dir): dvc_info = get_dvc_info() assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info) - assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", dvc_info) + assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", dvc_info) assert re.search(r"Supports: .*", dvc_info) assert re.search(r"Cache types: .*", dvc_info) @@ -60,7 +66,7 @@ def test_info_outside_of_repo(tmp_dir, caplog): dvc_info = get_dvc_info() assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info) - assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", dvc_info) + assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", dvc_info) assert re.search(r"Supports: .*", dvc_info) assert not re.search(r"Cache types: .*", dvc_info) assert "Repo:" not in dvc_info @@ -70,5 +76,5 @@ def test_info_outside_of_repo(tmp_dir, caplog): def test_fs_info_outside_of_repo(tmp_dir, caplog): dvc_info = get_dvc_info() assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info) - assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", dvc_info) + assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", dvc_info) assert re.search(r"Supports: .*", dvc_info)