dvc: support Python3.8 (#3463)

* py38: make wheels

- ci: set ARROW_LIBHDFS_DIR to load native library
- deps: pin pyarrow to 0.15.1 for py<=3.7
- pin deps for python<3.8
- pin deps for python==3.8 as well
- add windows Py38 on travis
- do not use speedcopy for Python3.8 and Windows

* use realpath to resolve symbolic links

* do not build pkgs for Windows and Linux on Python3.8

* reorder

* pyarrow: document regression fix

Co-authored-by: Casper da Costa-Luis <[email protected]>

.travis.yml

@@ -37,6 +37,15 @@ jobs:
     stage: test
     os: windows           # Windows 10.0.17134 N/A Build 17134
     language: shell       # 'language: python' is an error on Travis CI Windows
+    env:
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
+  - name: "3.8 on Windows"
+    os: windows           # Windows 10.0.17134 N/A Build 17134
+    language: shell       # 'language: python' is an error on Travis CI Windows
+    env:
+      - PYTHON_VERSION=3.8.0
+      - PATH=/c/Python38:/c/Python38/Scripts:$PATH
   - name: "3.5 on Linux"
     language: python
     python: 3.5
@@ -46,6 +55,9 @@ jobs:
   - name: "3.7 on Linux"
     language: python
     python: 3.7
+  - name: "3.8 on Linux"
+    language: python
+    python: 3.8
   - name: "3.7 on Mac"
     os: osx
     osx_image: xcode11.3
@@ -63,6 +75,9 @@ jobs:
     language: shell
     install:
     script: ./scripts/build_windows.cmd
+    env:
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
   - name: "Linux pkgs"
     language: python
     python: 3.7
@@ -71,11 +86,11 @@ jobs:
     script: ./scripts/build_posix.sh
   - name: "PyPI pkgs"
     language: python
-    python: 3.7
+    python: 3.8
     script: ./scripts/build_package.sh
   - name: Snapcraft snap
     language: python
-    python: 3.7
+    python: 3.8
     addons:
      snaps:
      - name: snapcraft

dvc/system.py

@@ -3,13 +3,14 @@ import logging
 import os
 import platform
 import shutil
+import sys
 
 from dvc.compat import fspath
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
-if platform.system() == "Windows":
+if platform.system() == "Windows" and sys.version_info < (3, 8):
     try:
         import speedcopy
 

scripts/ci/before_install.sh

@@ -44,8 +44,8 @@ if [[ "$TRAVIS_BUILD_STAGE_NAME" == "Test" ]]; then
 fi
 
 if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
-  $scriptdir/retry.sh choco install python --version 3.7.5
-  echo 'PATH="/c/Python37:/c/Python37/Scripts:$PATH"' >>env.sh
+  $scriptdir/retry.sh choco install python --version $PYTHON_VERSION
+  echo "PATH='$PATH'" >>env.sh
 elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
   ln -s -f /usr/local/bin/python3 /usr/local/bin/python
   ln -s -f /usr/local/bin/pip3 /usr/local/bin/pip

scripts/ci/install_hadoop.sh

@@ -23,6 +23,14 @@ echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >>env.sh
 echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >>env.sh
 echo "export YARN_HOME=/usr/local/hadoop" >>env.sh
 echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >>env.sh
+
+# PyArrow==0.16.0 regression https://issues.apache.org/jira/browse/ARROW-7841
+# retrieves native library from $HADOOP_HOME directory instead of
+# `$HADOOP_HOME/lib/native`.
+# Fix: force search for `libhdfs.so` inside `$HADOOP_HOME/lib/native`.
+# Note: not needed for PyArrow==0.17.0.
+echo "export ARROW_LIBHDFS_DIR=/usr/local/hadoop/lib/native" >> env.sh
+
 echo "export JAVA_HOME=/usr/" >>env.sh
 echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >>env.sh
 

setup.py

@@ -77,7 +77,7 @@ install_requires = [
     "pywin32>=225; sys_platform == 'win32'",
     "networkx>=2.1,<2.4",
     "pydot>=1.2.4",
-    "speedcopy>=2.0.1; sys_platform == 'win32'",
+    "speedcopy>=2.0.1; python_version < '3.8' and sys_platform == 'win32'",
     "flatten_json>=0.1.6",
     "texttable>=0.5.2",
     "pygtrie==2.3.2",
@@ -92,7 +92,12 @@ s3 = ["boto3>=1.9.201"]
 azure = ["azure-storage-blob==2.1.0"]
 oss = ["oss2==2.6.1"]
 ssh = ["paramiko>=2.5.0"]
-hdfs = ["pyarrow==0.15.1"]
+hdfs = [
+    # pyarrow-0.16.0 import fails on 3.5 and 3.7 (works on 3.6 though)
+    # due to: https://issues.apache.org/jira/browse/ARROW-7852
+    "pyarrow==0.15.1; python_version < '3.8'",
+    "pyarrow==0.16.0; python_version == '3.8'",
+]
 # gssapi should not be included in all_remotes, because it doesn't have wheels
 # for linux and mac, so it will fail to compile if user doesn't have all the
 # requirements, including kerberos itself. Once all the wheels are available,
@@ -160,6 +165,7 @@ setup(
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
     ],
     packages=find_packages(exclude=["tests"]),
     include_package_data=True,

tests/func/test_checkout.py

@@ -441,10 +441,10 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
         self.assertEqual(ret, 0)
 
         self.assertTrue(System.is_symlink(self.FOO))
-        old_foo_link = os.readlink(self.FOO)
+        old_foo_link = os.path.realpath(self.FOO)
 
         self.assertTrue(System.is_symlink(self.DATA))
-        old_data_link = os.readlink(self.DATA)
+        old_data_link = os.path.realpath(self.DATA)
 
         old_cache_dir = self.dvc.cache.local.cache_dir
         new_cache_dir = old_cache_dir + "_new"
@@ -457,10 +457,10 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
         self.assertEqual(ret, 0)
 
         self.assertTrue(System.is_symlink(self.FOO))
-        new_foo_link = os.readlink(self.FOO)
+        new_foo_link = os.path.realpath(self.FOO)
 
         self.assertTrue(System.is_symlink(self.DATA))
-        new_data_link = os.readlink(self.DATA)
+        new_data_link = os.path.realpath(self.DATA)
 
         self.assertEqual(
             relpath(old_foo_link, old_cache_dir),