Merge pull request #1156 from efiop/master

dvc: use ctypes instead of cffi

dvc/reflink_ffi.py

@@ -1,88 +0,0 @@
-import platform
-from cffi import FFI
-
-
-WINDOWS_SRC = """
-int reflink(char *src, char *dst)
-{
-    return -1;
-}
-"""
-
-DARWIN_SRC = """
-#include <AvailabilityMacros.h>
-/* NOTE: sys/clonefile.h is available since OS X 10.12 */
-#if MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12
-#include <sys/clonefile.h> /* for clonefile(2) */
-#include <errno.h>
-int reflink(char *src, char *dst)
-{
-    return clonefile(src, dst, 0);
-}
-#else
-int reflink(char *src, char *dst)
-{
-    return -1;
-}
-#endif
-"""
-
-LINUX_SRC = """
-#include <sys/stat.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <linux/fs.h>
-#include <sys/ioctl.h>
-#include <assert.h>
-#include <errno.h>
-int reflink(char *src, char *dst)
-{
-    int ret = -1;
-#ifdef FICLONE
-    int src_fd = -1;
-    int dst_fd = -1;
-
-    src_fd = open(src, O_RDONLY);
-    if (src_fd < 0)
-        return ret;
-
-    dst_fd = open(dst, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR);
-    if (src_fd < 0) {
-        close(src_fd);
-        return ret;
-    }
-
-    ret = ioctl(dst_fd, FICLONE, src_fd);
-    if (ret != 0)
-        unlink(dst);
-
-out:
-    if (src_fd >= 0)
-        close(src_fd);
-    if (dst_fd >= 0)
-        close(dst_fd);
-#endif
-    return ret;
-}
-"""
-
-ffibuilder = FFI()
-
-ffibuilder.cdef("""
-int reflink(char *src, char *dst);
-""")
-
-system = platform.system()
-if system == 'Windows':
-    src = WINDOWS_SRC
-elif system == 'Linux':
-    src = LINUX_SRC
-elif system == 'Darwin':
-    src = DARWIN_SRC
-else:
-    raise NotImplementedError
-
-ffibuilder.set_source("reflink", src)
-
-if __file__ == '__main__':
-    ffibuilder.compile(verbose=True)

dvc/remote/local.py

@@ -108,7 +108,7 @@ class RemoteLOCAL(RemoteBase):
                                        os.path.relpath(cache),
                                        os.path.relpath(path)))
                 return
-            except Exception as exc:
+            except DvcException as exc:
                 msg = 'Cache type \'{}\' is not supported: {}'
                 Logger.debug(msg.format(self.cache_types[0], str(exc)))
                 del self.cache_types[0]

dvc/system.py

@@ -1,5 +1,4 @@
 import os
-import ctypes
 
 if os.name == 'nt':
     import ntfsutils.hardlink as winlink
@@ -19,6 +18,7 @@ class System(object):
 
     @staticmethod
     def symlink(source, link_name):
+        import ctypes
         if System.is_unix():
             return os.symlink(source, link_name)
 
@@ -33,17 +33,70 @@ class System(object):
         if func(link_name, source, flags) == 0:
             raise ctypes.WinError()
 
+    @staticmethod
+    def _reflink_darwin(src, dst):
+        import ctypes
+
+        clonefile = ctypes.CDLL('libc.dylib').clonefile
+        clonefile.argtypes = [ctypes.c_char_p,
+                              ctypes.c_char_p,
+                              ctypes.c_int]
+        clonefile.restype = ctypes.c_int
+
+        return clonefile(ctypes.c_char_p(src.encode('utf-8')),
+                         ctypes.c_char_p(src.encode('utf-8')),
+                         ctypes.c_int(0))
+
+    @staticmethod
+    def _reflink_windows(src, dst):
+        return -1
+
+    @staticmethod
+    def _reflink_linux(src, dst):
+        import os
+        import fcntl
+
+        FICLONE = 0x40049409
+
+        s = open(src, 'r')
+        d = open(dst, 'w+')
+
+        try:
+            ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
+        except IOError:
+            s.close()
+            d.close()
+            os.unlink(dst)
+            raise
+
+        s.close()
+        d.close()
+
+        if ret != 0:
+            os.unlink(dst)
+
+        return ret
+
     @staticmethod
     def reflink(source, link_name):
-        from dvc.reflink import reflink
+        import platform
         from dvc.exceptions import DvcException
 
-        ret = reflink(source, link_name)
+        system = platform.system()
+        try:
+            if system == 'Windows':
+                ret = System._reflink_windows(source, link_name)
+            elif system == 'Darwin':
+                ret = System._reflink_darwin(source, link_name)
+            elif system == 'Linux':
+                ret = System._reflink_linux(source, link_name)
+            else:
+                ret = -1
+        except IOError:
+            ret = -1
+
         if ret != 0:
-            raise DvcException("Failed to reflink '{}' -> '{}' returned "
-                               "non-zero code '{}'".format(source,
-                                                           link_name,
-                                                           ret))
+            raise DvcException('Reflink is not supported')
 
     # FIXME
     # Temporary fix while waiting for the PR to be merged and released:

requirements.txt

@@ -23,4 +23,3 @@ wheel>=0.31.1
 futures>=3.2.0; python_version == "2.7"
 grandalf==0.6
 asciicanvas==0.0.3
-cffi>=1.0.0

setup.py

@@ -24,7 +24,6 @@ install_requires = [
     "requests>=2.18.4",
     "grandalf==0.6",
     "asciicanvas==0.0.3",
-    "cffi>=1.0.0",
 ]
 
 # Extra dependencies for remote integrations
@@ -61,7 +60,6 @@ setup(
         # NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1
         ':python_version=="2.7"': ['futures'],
     },
-    setup_requires=['cffi>=1.0.0'],
     keywords='data science, data version control, machine learning',
     classifiers=[
         'Development Status :: 4 - Beta',
@@ -75,5 +73,4 @@ setup(
         'console_scripts': ['dvc = dvc.main:main']
     },
     zip_safe=False,
-    cffi_modules=["dvc/reflink_ffi.py:ffibuilder"]
 )