From 0fc83be9ff019342edadb3cdfc6f2c7e6a69b7c9 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Sat, 31 Aug 2019 03:23:35 -0500 Subject: [PATCH] tests: ssh hardlink optimization --- dvc/path_info.py | 2 +- dvc/remote/ssh/__init__.py | 4 +++- tests/conftest.py | 10 ++++++++++ tests/unit/remote/ssh/test_connection.py | 9 --------- tests/unit/remote/ssh/test_ssh.py | 23 +++++++++++++++++++++++ 5 files changed, 37 insertions(+), 11 deletions(-) diff --git a/dvc/path_info.py b/dvc/path_info.py index c7d0051d8..596c5071b 100644 --- a/dvc/path_info.py +++ b/dvc/path_info.py @@ -7,7 +7,7 @@ import posixpath from funcy import cached_property from dvc.utils.compat import str, builtin_str, basestring, is_py2 -from dvc.utils.compat import pathlib, urlparse +from dvc.utils.compat import pathlib, urlparse, urljoin # On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs, diff --git a/dvc/remote/ssh/__init__.py b/dvc/remote/ssh/__init__.py index 2f8483abc..d0082d0e6 100644 --- a/dvc/remote/ssh/__init__.py +++ b/dvc/remote/ssh/__init__.py @@ -193,7 +193,9 @@ class RemoteSSH(RemoteBASE): # See dvc/remote/local/__init__.py - hardlink() if self.getsize(from_info) == 0: - self.open(to_info, "w").close() + + with self.ssh(to_info) as ssh: + ssh.sftp.open(to_info.path, "w").close() logger.debug( "Created empty file: {src} -> {dest}".format( diff --git a/tests/conftest.py b/tests/conftest.py index 3356a4a0b..051d2b7a9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -177,3 +177,13 @@ def _close_pools(): yield close_pools() + + +@pytest.fixture +def chdir_tmp(tmp_path): + cwd = os.getcwd() + try: + os.chdir(tmp_path) + yield + finally: + os.chdir(cwd) diff --git a/tests/unit/remote/ssh/test_connection.py b/tests/unit/remote/ssh/test_connection.py index 42371bc8f..ae79d4d92 100644 --- a/tests/unit/remote/ssh/test_connection.py +++ b/tests/unit/remote/ssh/test_connection.py @@ -9,15 +9,6 @@ import filecmp from dvc.system import System -@pytest.fixture -def chdir_tmp(tmp_path): - cwd = os.getcwd() - try: - os.chdir(tmp_path) - yield - finally: - os.chdir(cwd) - @pytest.fixture def foo(): with open("foo", "w") as fobj: diff --git a/tests/unit/remote/ssh/test_ssh.py b/tests/unit/remote/ssh/test_ssh.py index 0dee1ac3e..009e12a53 100644 --- a/tests/unit/remote/ssh/test_ssh.py +++ b/tests/unit/remote/ssh/test_ssh.py @@ -1,6 +1,7 @@ import getpass import os import sys +import posixpath from unittest import TestCase @@ -8,6 +9,8 @@ from mock import patch, mock_open import pytest from dvc.remote.ssh import RemoteSSH +from dvc.path_info import URLInfo +from dvc.system import System class TestRemoteSSH(TestCase): @@ -182,3 +185,23 @@ def test_ssh_gss_auth(mock_file, mock_exists, config, expected_gss_auth): mock_exists.assert_called_with(RemoteSSH.ssh_config_filename()) mock_file.assert_called_with(RemoteSSH.ssh_config_filename()) assert remote.gss_auth == expected_gss_auth + +def test_hardlink_optimization(chdir_tmp, ssh_server): + config = { + "url": "ssh://localhost", + "hostname": ssh_server.test_creds["host"], + "port": ssh_server.test_creds["port"], + "user": ssh_server.test_creds["username"], + "keyfile": ssh_server.test_creds["key_filename"], + } + remote = RemoteSSH(None, config) + + open("empty", "w").close() + + def urlpath(path): + netloc = remote.path_info.netloc + path = posixpath.abspath(path) + return URLInfo("ssh://{}/{}".format(netloc, path)) + + remote.hardlink(urlpath("empty"), urlpath("bar")) + assert not System.is_hardlink("bar")