From ff63376d490399f6fa9bf86ed8c88b114e4c46d9 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 23 Sep 2018 12:19:18 +0300 Subject: [PATCH] dvc: use ctypes instead of cffi Signed-off-by: Ruslan Kuprieiev --- dvc/reflink_ffi.py | 88 --------------------------------------------- dvc/remote/local.py | 2 +- dvc/system.py | 67 ++++++++++++++++++++++++++++++---- requirements.txt | 1 - setup.py | 3 -- 5 files changed, 61 insertions(+), 100 deletions(-) delete mode 100644 dvc/reflink_ffi.py diff --git a/dvc/reflink_ffi.py b/dvc/reflink_ffi.py deleted file mode 100644 index 6056f47ad..000000000 --- a/dvc/reflink_ffi.py +++ /dev/null @@ -1,88 +0,0 @@ -import platform -from cffi import FFI - - -WINDOWS_SRC = """ -int reflink(char *src, char *dst) -{ - return -1; -} -""" - -DARWIN_SRC = """ -#include -/* NOTE: sys/clonefile.h is available since OS X 10.12 */ -#if MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 -#include /* for clonefile(2) */ -#include -int reflink(char *src, char *dst) -{ - return clonefile(src, dst, 0); -} -#else -int reflink(char *src, char *dst) -{ - return -1; -} -#endif -""" - -LINUX_SRC = """ -#include -#include -#include -#include -#include -#include -#include -int reflink(char *src, char *dst) -{ - int ret = -1; -#ifdef FICLONE - int src_fd = -1; - int dst_fd = -1; - - src_fd = open(src, O_RDONLY); - if (src_fd < 0) - return ret; - - dst_fd = open(dst, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR); - if (src_fd < 0) { - close(src_fd); - return ret; - } - - ret = ioctl(dst_fd, FICLONE, src_fd); - if (ret != 0) - unlink(dst); - -out: - if (src_fd >= 0) - close(src_fd); - if (dst_fd >= 0) - close(dst_fd); -#endif - return ret; -} -""" - -ffibuilder = FFI() - -ffibuilder.cdef(""" -int reflink(char *src, char *dst); -""") - -system = platform.system() -if system == 'Windows': - src = WINDOWS_SRC -elif system == 'Linux': - src = LINUX_SRC -elif system == 'Darwin': - src = DARWIN_SRC -else: - raise NotImplementedError - -ffibuilder.set_source("reflink", src) - -if __file__ == '__main__': - ffibuilder.compile(verbose=True) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 84e328e24..85bf8db2f 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -108,7 +108,7 @@ class RemoteLOCAL(RemoteBase): os.path.relpath(cache), os.path.relpath(path))) return - except Exception as exc: + except DvcException as exc: msg = 'Cache type \'{}\' is not supported: {}' Logger.debug(msg.format(self.cache_types[0], str(exc))) del self.cache_types[0] diff --git a/dvc/system.py b/dvc/system.py index 09bf6836b..1a3aac2ee 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -1,5 +1,4 @@ import os -import ctypes if os.name == 'nt': import ntfsutils.hardlink as winlink @@ -19,6 +18,7 @@ class System(object): @staticmethod def symlink(source, link_name): + import ctypes if System.is_unix(): return os.symlink(source, link_name) @@ -33,17 +33,70 @@ class System(object): if func(link_name, source, flags) == 0: raise ctypes.WinError() + @staticmethod + def _reflink_darwin(src, dst): + import ctypes + + clonefile = ctypes.CDLL('libc.dylib').clonefile + clonefile.argtypes = [ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_int] + clonefile.restype = ctypes.c_int + + return clonefile(ctypes.c_char_p(src.encode('utf-8')), + ctypes.c_char_p(src.encode('utf-8')), + ctypes.c_int(0)) + + @staticmethod + def _reflink_windows(src, dst): + return -1 + + @staticmethod + def _reflink_linux(src, dst): + import os + import fcntl + + FICLONE = 0x40049409 + + s = open(src, 'r') + d = open(dst, 'w+') + + try: + ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno()) + except IOError: + s.close() + d.close() + os.unlink(dst) + raise + + s.close() + d.close() + + if ret != 0: + os.unlink(dst) + + return ret + @staticmethod def reflink(source, link_name): - from dvc.reflink import reflink + import platform from dvc.exceptions import DvcException - ret = reflink(source, link_name) + system = platform.system() + try: + if system == 'Windows': + ret = System._reflink_windows(source, link_name) + elif system == 'Darwin': + ret = System._reflink_darwin(source, link_name) + elif system == 'Linux': + ret = System._reflink_linux(source, link_name) + else: + ret = -1 + except IOError: + ret = -1 + if ret != 0: - raise DvcException("Failed to reflink '{}' -> '{}' returned " - "non-zero code '{}'".format(source, - link_name, - ret)) + raise DvcException('Reflink is not supported') # FIXME # Temporary fix while waiting for the PR to be merged and released: diff --git a/requirements.txt b/requirements.txt index 53a59ef2a..638b4f363 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,4 +23,3 @@ wheel>=0.31.1 futures>=3.2.0; python_version == "2.7" grandalf==0.6 asciicanvas==0.0.3 -cffi>=1.0.0 diff --git a/setup.py b/setup.py index e173e232d..c5807d915 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,6 @@ install_requires = [ "requests>=2.18.4", "grandalf==0.6", "asciicanvas==0.0.3", - "cffi>=1.0.0", ] # Extra dependencies for remote integrations @@ -61,7 +60,6 @@ setup( # NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1 ':python_version=="2.7"': ['futures'], }, - setup_requires=['cffi>=1.0.0'], keywords='data science, data version control, machine learning', classifiers=[ 'Development Status :: 4 - Beta', @@ -75,5 +73,4 @@ setup( 'console_scripts': ['dvc = dvc.main:main'] }, zip_safe=False, - cffi_modules=["dvc/reflink_ffi.py:ffibuilder"] )