test: fix hdfs/azurite/oss tests (#3864)

* travis: fix typo

* tests: azure: fix expected etag format

* test: ssh: increase the number of tries

* travis: don't install docker

* dvc: update pyarrow

* tests: ssh: don't leak connections

* remote: don't fail when cache/remote doesn't have prefix subdir

dvc/remote/hdfs.py

@@ -165,6 +165,8 @@ class HDFSRemote(BaseRemote):
         dirs = deque([root])
 
         with self.hdfs(self.path_info) as hdfs:
+            if prefix and not hdfs.exists(root):
+                return
             while dirs:
                 try:
                     entries = hdfs.ls(dirs.pop(), detail=True)

dvc/remote/local.py

@@ -98,6 +98,8 @@ class LocalRemote(BaseRemote):
         assert self.path_info is not None
         if prefix:
             path_info = self.path_info / prefix[:2]
+            if not self.exists(path_info):
+                return
         else:
             path_info = self.path_info
         if progress_callback:

dvc/remote/ssh/__init__.py

@@ -265,6 +265,8 @@ class SSHRemote(BaseRemote):
         else:
             root = self.path_info.path
         with self.ssh(self.path_info) as ssh:
+            if prefix and not ssh.exists(root):
+                return
             # If we simply return an iterator then with above closes instantly
             if progress_callback:
                 for path in ssh.walk_files(root):

scripts/ci/before_install.sh

@@ -15,7 +15,7 @@ fi
 
 echo >env.sh
 
-if [[ "$TRAVIS_BUILD_STAGE_NAME" == "Test" ]]; then
+if [[ "$TRAVIS_BUILD_STAGE_NAME" == "test" ]]; then
   if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then
     # NOTE: ssh keys for ssh test to be able to ssh to the localhost
     ssh-keygen -t rsa -N "" -f mykey

scripts/ci/install_azurite.sh

@@ -3,13 +3,6 @@
 set -e
 set -x
 
-# install docker
-export DEBIAN_FRONTEND=noninteractive
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-sudo apt-get update
-sudo apt-get install -y docker-ce
-
 # run azurite
 sudo docker run -d --restart always -e executable=blob -p 10000:10000 \
   --tmpfs /opt/azurite/folder \

setup.py

@@ -92,12 +92,7 @@ s3 = ["boto3>=1.9.201"]
 azure = ["azure-storage-blob==2.1.0"]
 oss = ["oss2==2.6.1"]
 ssh = ["paramiko>=2.5.0"]
-hdfs = [
-    # pyarrow-0.16.0 import fails on 3.7 (works on 3.6 though)
-    # due to: https://issues.apache.org/jira/browse/ARROW-7852
-    "pyarrow==0.15.1; python_version < '3.8'",
-    "pyarrow==0.16.0; python_version == '3.8'",
-]
+hdfs = ["pyarrow>=0.17.0"]
 # gssapi should not be included in all_remotes, because it doesn't have wheels
 # for linux and mac, so it will fail to compile if user doesn't have all the
 # requirements, including kerberos itself. Once all the wheels are available,

tests/func/test_repro.py

@@ -1070,7 +1070,7 @@ class TestReproExternalHDFS(HDFS, TestReproExternalBase):
         self.assertEqual(p.returncode, 0)
 
 
-@flaky(max_runs=3, min_passes=1)
+@flaky(max_runs=5, min_passes=1)
 class TestReproExternalSSH(SSH, TestReproExternalBase):
     _dir = None
     cache_type = "copy"
@@ -1095,23 +1095,33 @@ class TestReproExternalSSH(SSH, TestReproExternalBase):
     def write(self, bucket, key, body):
         path = posixpath.join(self._dir, key)
 
-        ssh = paramiko.SSHClient()
-        ssh.load_system_host_keys()
-        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        ssh.connect("127.0.0.1")
-
-        sftp = ssh.open_sftp()
+        ssh = None
+        sftp = None
         try:
-            sftp.stat(path)
-            sftp.remove(path)
-        except OSError:
-            pass
+            ssh = paramiko.SSHClient()
+            ssh.load_system_host_keys()
+            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+            ssh.connect("127.0.0.1")
 
-        stdin, stdout, stderr = ssh.exec_command(f"mkdir -p $(dirname {path})")
-        self.assertEqual(stdout.channel.recv_exit_status(), 0)
+            sftp = ssh.open_sftp()
+            try:
+                sftp.stat(path)
+                sftp.remove(path)
+            except OSError:
+                pass
 
-        with sftp.open(path, "w+") as fobj:
-            fobj.write(body)
+            stdin, stdout, stderr = ssh.exec_command(
+                f"mkdir -p $(dirname {path})"
+            )
+            self.assertEqual(stdout.channel.recv_exit_status(), 0)
+
+            with sftp.open(path, "w+") as fobj:
+                fobj.write(body)
+        finally:
+            if sftp:
+                sftp.close()
+            if ssh:
+                ssh.close()
 
 
 class TestReproExternalLOCAL(Local, TestReproExternalBase):

tests/unit/remote/test_azure.py

@@ -42,5 +42,7 @@ def test_get_file_checksum(tmp_dir):
     to_info = remote.path_cls(Azure.get_url())
     remote.upload(PathInfo("foo"), to_info)
     assert remote.exists(to_info)
-    # make sure the checksum is a hex number
-    int(remote.get_file_checksum(to_info), 16)
+    checksum = remote.get_file_checksum(to_info)
+    assert checksum
+    assert isinstance(checksum, str)
+    assert checksum.strip("'").strip('"') == checksum