diff --git a/dvc/binary_file_extensions.py b/dvc/binary_file_extensions.py deleted file mode 100644 index cecc8bbcf..000000000 --- a/dvc/binary_file_extensions.py +++ /dev/null @@ -1,51 +0,0 @@ -# source: https://github.com/sindresorhus/binary-extensions -# -# MIT License -# -# Copyright (c) Sindre Sorhus (sindresorhus.com) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -BINARY_FILE_EXTENSIONS = [ - "3ds", "3g2", "3gp", "7z", "a", "aac", "adp", "ai", "aif", "aiff", "alz", - "ape", "apk", "ar", "arj", "asf", "au", "avi", "bak", "baml", "bh", "bin", - "bk", "bmp", "btif", "bz2", "bzip2", "cab", "caf", "cgm", "class", "cmx", - "cpio", "cr2", "csv", "cur", "dat", "dcm", "deb", "dex", "djvu", "dll", - "dmg", "dng", "doc", "docm", "docx", "dot", "dotm", "dra", "DS_Store", - "dsk", "dts", "dtshd", "dvb", "dwg", "dxf", "ecelp4800", "ecelp7470", - "ecelp9600", "egg", "eol", "eot", "epub", "exe", "f4v", "fbs", "fh", "fla", - "flac", "fli", "flv", "fpx", "fst", "fvt", "g3", "gif", "graffle", "gz", - "gzip", "h261", "h263", "h264", "icns", "ico", "ief", "img", "ipa", "iso", - "jar", "jpeg", "jpg", "jpgv", "jpm", "jxr", "key", "ktx", "lha", "lib", - "lvp", "lz", "lzh", "lzma", "lzo", "m3u", "m4a", "m4v", "mar", "mdi", "mht", - "mid", "midi", "mj2", "mka", "mkv", "mmr", "mng", "mobi", "mov", "movie", - "mp3", "mp4", "mp4a", "mpeg", "mpg", "mpga", "mxu", "nef", "npx", "numbers", - "o", "oga", "ogg", "ogv", "otf", "pages", "pbm", "pcx", "pdb", "pdf", "pea", - "pgm", "pic", "png", "pnm", "pot", "potm", "potx", "ppa", "ppam", "ppm", - "pps", "ppsm", "ppsx", "ppt", "pptm", "pptx", "psd", "pya", "pyc", "pyo", - "pyv", "qt", "rar", "ras", "raw", "resources", "rgb", "rip", "rlc", "rmf", - "rmvb", "rtf", "rz", "s3m", "s7z", "scpt", "sgi", "shar", "sil", "sketch", - "slk", "smv", "so", "sub", "swf", "tar", "tbz", "tbz2", "tga", "tgz", - "thmx", "tif", "tiff", "tlz", "ttc", "ttf", "txz", "udf", "uvh", "uvi", - "uvm", "uvp", "uvs", "uvu", "viv", "vob", "war", "wav", "wax", "wbmp", - "wdp", "weba", "webm", "webp", "whl", "wim", "wm", "wma", "wmv", "wmx", - "woff", "woff2", "wvx", "xbm", "xif", "xla", "xlam", "xls", "xlsb", "xlsm", - "xlsx", "xlt", "xltm", "xltx", "xm", "xmind", "xpi", "xpm", "xwd", "xz", - "z", "zip", "zipx", -] diff --git a/dvc/istextfile.py b/dvc/istextfile.py new file mode 100644 index 000000000..1c624acba --- /dev/null +++ b/dvc/istextfile.py @@ -0,0 +1,36 @@ +# Based on https://eli.thegreenplace.net/2011/10/19/perls-guess-if-file-is-text-or-binary-implemented-in-python +import sys +PY3 = sys.version_info[0] == 3 + +# A function that takes an integer in the 8-bit range and returns +# a single-character byte object in py3 / a single-character string +# in py2. +# +int2byte = (lambda x: bytes((x,))) if PY3 else chr + + +_text_characters = ( + b''.join(int2byte(i) for i in range(32, 127)) + + b'\n\r\t\f\b') + + +def istextfile(fname, blocksize=512): + """ Uses heuristics to guess whether the given file is text or binary, + by reading a single block of bytes from the file. + If more than 30% of the chars in the block are non-text, or there + are NUL ('\x00') bytes in the block, assume this is a binary file. + """ + with open(fname, 'rb') as fd: + block = fd.read(blocksize) + + if b'\x00' in block: + # Files with null bytes are binary + return False + elif not block: + # An empty file is considered a valid text file + return True + + # Use translate's 'deletechars' argument to efficiently remove all + # occurrences of _text_characters from the block + nontext = block.translate(None, _text_characters) + return float(len(nontext)) / len(block) <= 0.30 diff --git a/dvc/utils.py b/dvc/utils.py index 25c98198d..f4d23f9a0 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -9,12 +9,11 @@ import json import stat import shutil import hashlib -from binaryornot.check import is_binary from multiprocessing.pool import ThreadPool +from dvc.istextfile import istextfile from dvc.progress import progress from dvc.logger import Logger -from dvc.binary_file_extensions import BINARY_FILE_EXTENSIONS LOCAL_CHUNK_SIZE = 1024*1024 @@ -28,13 +27,7 @@ def file_md5(fname): """ get the (md5 hexdigest, md5 digest) of a file """ if os.path.exists(fname): hash_md5 = hashlib.md5() - # when dealing with large collections of binary files, the is_binary - # call becomes a bottleneck, here we first check the extension to - # avoid this bottleneck when possible: - if fname.split('.')[-1] in BINARY_FILE_EXTENSIONS: - binary = True - else: - binary = is_binary(fname) + binary = not istextfile(fname) with open(fname, 'rb') as fobj: while True: diff --git a/requirements.txt b/requirements.txt index e2a1d95d1..d6e2c66da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,6 @@ nanotime>=0.5.2 pyasn1>=0.4.1 schema>=0.6.7 paramiko>=2.4.1 -binaryornot==0.4.4 jsonpath-rw==1.4.0 reflink==0.2.0 requests>=2.18.4 diff --git a/setup.py b/setup.py index 5bb4fb875..c4ae9963b 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,6 @@ install_requires = [ "nanotime>=0.5.2", "pyasn1>=0.4.1", "schema>=0.6.7", - "binaryornot==0.4.4", "jsonpath-rw==1.4.0", "reflink==0.2.0", "requests>=2.18.4",