dvc: use heuristics to detect binary files

Fixes #859

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/binary_file_extensions.py

@@ -1,51 +0,0 @@
-# source: https://github.com/sindresorhus/binary-extensions
-# 
-# MIT License
-# 
-# Copyright (c) Sindre Sorhus <[email protected]> (sindresorhus.com)
-# 
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-# 
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-# 
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-BINARY_FILE_EXTENSIONS = [
-    "3ds", "3g2", "3gp", "7z", "a", "aac", "adp", "ai", "aif", "aiff", "alz",
-    "ape", "apk", "ar", "arj", "asf", "au", "avi", "bak", "baml", "bh", "bin",
-    "bk", "bmp", "btif", "bz2", "bzip2", "cab", "caf", "cgm", "class", "cmx",
-    "cpio", "cr2", "csv", "cur", "dat", "dcm", "deb", "dex", "djvu", "dll",
-    "dmg", "dng", "doc", "docm", "docx", "dot", "dotm", "dra", "DS_Store",
-    "dsk", "dts", "dtshd", "dvb", "dwg", "dxf", "ecelp4800", "ecelp7470",
-    "ecelp9600", "egg", "eol", "eot", "epub", "exe", "f4v", "fbs", "fh", "fla",
-    "flac", "fli", "flv", "fpx", "fst", "fvt", "g3", "gif", "graffle", "gz",
-    "gzip", "h261", "h263", "h264", "icns", "ico", "ief", "img", "ipa", "iso",
-    "jar", "jpeg", "jpg", "jpgv", "jpm", "jxr", "key", "ktx", "lha", "lib",
-    "lvp", "lz", "lzh", "lzma", "lzo", "m3u", "m4a", "m4v", "mar", "mdi", "mht",
-    "mid", "midi", "mj2", "mka", "mkv", "mmr", "mng", "mobi", "mov", "movie",
-    "mp3", "mp4", "mp4a", "mpeg", "mpg", "mpga", "mxu", "nef", "npx", "numbers",
-    "o", "oga", "ogg", "ogv", "otf", "pages", "pbm", "pcx", "pdb", "pdf", "pea",
-    "pgm", "pic", "png", "pnm", "pot", "potm", "potx", "ppa", "ppam", "ppm",
-    "pps", "ppsm", "ppsx", "ppt", "pptm", "pptx", "psd", "pya", "pyc", "pyo",
-    "pyv", "qt", "rar", "ras", "raw", "resources", "rgb", "rip", "rlc", "rmf",
-    "rmvb", "rtf", "rz", "s3m", "s7z", "scpt", "sgi", "shar", "sil", "sketch",
-    "slk", "smv", "so", "sub", "swf", "tar", "tbz", "tbz2", "tga", "tgz",
-    "thmx", "tif", "tiff", "tlz", "ttc", "ttf", "txz", "udf", "uvh", "uvi",
-    "uvm", "uvp", "uvs", "uvu", "viv", "vob", "war", "wav", "wax", "wbmp",
-    "wdp", "weba", "webm", "webp", "whl", "wim", "wm", "wma", "wmv", "wmx",
-    "woff", "woff2", "wvx", "xbm", "xif", "xla", "xlam", "xls", "xlsb", "xlsm",
-    "xlsx", "xlt", "xltm", "xltx", "xm", "xmind", "xpi", "xpm", "xwd", "xz",
-    "z", "zip", "zipx",
-]

dvc/istextfile.py

@@ -0,0 +1,36 @@
+# Based on https://eli.thegreenplace.net/2011/10/19/perls-guess-if-file-is-text-or-binary-implemented-in-python
+import sys
+PY3 = sys.version_info[0] == 3
+
+# A function that takes an integer in the 8-bit range and returns
+# a single-character byte object in py3 / a single-character string
+# in py2.
+#
+int2byte = (lambda x: bytes((x,))) if PY3 else chr
+
+
+_text_characters = (
+        b''.join(int2byte(i) for i in range(32, 127)) +
+        b'\n\r\t\f\b')
+
+
+def istextfile(fname, blocksize=512):
+    """ Uses heuristics to guess whether the given file is text or binary,
+        by reading a single block of bytes from the file.
+        If more than 30% of the chars in the block are non-text, or there
+        are NUL ('\x00') bytes in the block, assume this is a binary file.
+    """
+    with open(fname, 'rb') as fd:
+        block = fd.read(blocksize)
+
+    if b'\x00' in block:
+        # Files with null bytes are binary
+        return False
+    elif not block:
+        # An empty file is considered a valid text file
+        return True
+
+    # Use translate's 'deletechars' argument to efficiently remove all
+    # occurrences of _text_characters from the block
+    nontext = block.translate(None, _text_characters)
+    return float(len(nontext)) / len(block) <= 0.30

dvc/utils.py

@@ -9,12 +9,11 @@ import json
 import stat
 import shutil
 import hashlib
-from binaryornot.check import is_binary
 from multiprocessing.pool import ThreadPool
 
+from dvc.istextfile import istextfile
 from dvc.progress import progress
 from dvc.logger import Logger
-from dvc.binary_file_extensions import BINARY_FILE_EXTENSIONS
 
 
 LOCAL_CHUNK_SIZE = 1024*1024
@@ -28,13 +27,7 @@ def file_md5(fname):
     """ get the (md5 hexdigest, md5 digest) of a file """
     if os.path.exists(fname):
         hash_md5 = hashlib.md5()
-        # when dealing with large collections of binary files, the is_binary
-        # call becomes a bottleneck, here we first check the extension to
-        # avoid this bottleneck when possible:
-        if fname.split('.')[-1] in BINARY_FILE_EXTENSIONS:
-            binary = True
-        else:
-            binary = is_binary(fname)
+        binary = not istextfile(fname)
 
         with open(fname, 'rb') as fobj:
             while True:

requirements.txt

@@ -17,7 +17,6 @@ nanotime>=0.5.2
 pyasn1>=0.4.1
 schema>=0.6.7
 paramiko>=2.4.1
-binaryornot==0.4.4
 jsonpath-rw==1.4.0
 reflink==0.2.0
 requests>=2.18.4

setup.py

@@ -20,7 +20,6 @@ install_requires = [
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
     "schema>=0.6.7",
-    "binaryornot==0.4.4",
     "jsonpath-rw==1.4.0",
     "reflink==0.2.0",
     "requests>=2.18.4",