From 16e162941ea90e78ba310bfdd57d6df5a251c3ec Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 13 Jun 2019 00:20:21 +0300 Subject: [PATCH] system: handle PurePath ------------------------------------------------------------ Traceback (most recent call last): File "/Users/efiop/git/dvc/dvc/remote/local/__init__.py", line 280, in upload no_progress_bar=no_progress_bar, File "/Users/efiop/git/dvc/dvc/utils/__init__.py", line 129, in copyfile System.reflink(src, dest) File "/Users/efiop/git/dvc/dvc/system.py", line 125, in reflink ret = System._reflink_darwin(source, link_name) File "/Users/efiop/git/dvc/dvc/system.py", line 89, in _reflink_darwin ctypes.c_char_p(src.encode("utf-8")), AttributeError: 'PosixPathInfo' object has no attribute 'encode' ------------------------------------------------------------ Signed-off-by: Ruslan Kuprieiev --- dvc/state.py | 8 ++++---- dvc/system.py | 22 +++++++++++++++++----- tests/func/test_state.py | 10 +++------- tests/func/test_system.py | 2 +- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/dvc/state.py b/dvc/state.py index ed9656717..2999bf7e3 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -10,7 +10,7 @@ from dvc.config import Config from dvc.utils import remove, current_timestamp from dvc.exceptions import DvcException from dvc.utils.fs import get_mtime_and_size, get_inode -from dvc.utils.compat import fspath +from dvc.utils.compat import fspath_py35 logger = logging.getLogger(__name__) @@ -353,7 +353,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes assert path_info.scheme == "local" assert checksum is not None - path = fspath(path_info) + path = fspath_py35(path_info) assert os.path.exists(path) actual_mtime, actual_size = get_mtime_and_size(path) @@ -382,7 +382,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes doesn't exist in the state database. """ assert path_info.scheme == "local" - path = fspath(path_info) + path = fspath_py35(path_info) if not os.path.exists(path): return None @@ -409,7 +409,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes path_info (dict): path info to add to the list of links. """ assert path_info.scheme == "local" - path = fspath(path_info) + path = fspath_py35(path_info) if not os.path.exists(path): return diff --git a/dvc/system.py b/dvc/system.py index 4750bb30e..b445ec01a 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -from dvc.utils.compat import str, open +from dvc.utils.compat import str, open, fspath import os import errno @@ -16,6 +16,8 @@ class System(object): import ctypes from dvc.exceptions import DvcException + source, link_name = fspath(source), fspath(link_name) + if System.is_unix(): try: os.link(source, link_name) @@ -40,6 +42,8 @@ class System(object): import ctypes from dvc.exceptions import DvcException + source, link_name = fspath(source), fspath(link_name) + if System.is_unix(): try: os.symlink(source, link_name) @@ -117,6 +121,8 @@ class System(object): import platform from dvc.exceptions import DvcException + source, link_name = fspath(source), fspath(link_name) + system = platform.system() try: if system == "Windows": @@ -134,7 +140,7 @@ class System(object): raise DvcException("reflink is not supported") @staticmethod - def getdirinfo(path): + def _getdirinfo(path): import ctypes from ctypes import c_void_p, c_wchar_p, Structure, WinError, POINTER from ctypes.wintypes import DWORD, HANDLE, BOOL @@ -206,6 +212,8 @@ class System(object): @staticmethod def inode(path): + path = fspath(path) + if System.is_unix(): import ctypes @@ -216,7 +224,7 @@ class System(object): inode = ctypes.c_ulong(inode).value else: # getdirinfo from ntfsutils works on both files and dirs - info = System.getdirinfo(path) + info = System._getdirinfo(path) inode = abs( hash( ( @@ -267,6 +275,8 @@ class System(object): @staticmethod def is_symlink(path): + path = fspath(path) + if System.is_unix(): return os.path.islink(path) @@ -275,14 +285,16 @@ class System(object): from winnt import FILE_ATTRIBUTE_REPARSE_POINT if os.path.lexists(path): - info = System.getdirinfo(path) + info = System._getdirinfo(path) return info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT return False @staticmethod def is_hardlink(path): + path = fspath(path) + if System.is_unix(): return os.stat(path).st_nlink > 1 - info = System.getdirinfo(path) + info = System._getdirinfo(path) return info.nNumberOfLinks > 1 diff --git a/tests/func/test_state.py b/tests/func/test_state.py index ccac2139f..edc2012bf 100644 --- a/tests/func/test_state.py +++ b/tests/func/test_state.py @@ -6,7 +6,6 @@ from dvc.utils.compat import str from dvc.state import State from dvc.utils import file_md5 from dvc.main import main -from dvc.utils.fs import get_inode from dvc.path_info import PathInfo from tests.basic_env import TestDvc @@ -58,12 +57,9 @@ class TestStateOverflow(TestDvc): class TestGetStateRecordForInode(TestDvc): @staticmethod - def mock_get_inode(special_path, special_value): + def mock_get_inode(inode): def get_inode_mocked(path): - if path == special_path: - return special_value - else: - return get_inode(path) + return inode return get_inode_mocked @@ -75,7 +71,7 @@ class TestGetStateRecordForInode(TestDvc): path = os.path.join(self.dvc.root_dir, self.FOO) md5 = file_md5(path)[0] - get_inode_mock.side_effect = self.mock_get_inode(path, inode) + get_inode_mock.side_effect = self.mock_get_inode(inode) with state: state.save(PathInfo(path), md5) diff --git a/tests/func/test_system.py b/tests/func/test_system.py index 432493fb3..867825282 100644 --- a/tests/func/test_system.py +++ b/tests/func/test_system.py @@ -11,6 +11,6 @@ def test_getdirinfo(tmp_path): fd = os.open(locked, os.O_WRONLY | os.O_CREAT | os.O_EXCL) try: # should not raise WinError - System.getdirinfo(locked) + System._getdirinfo(locked) finally: os.close(fd)