mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-07 15:28:28 +01:00
Peter Rowlands (변기호)
b7cd785eba
* erepo: load external repos directly from git tree * erepo: add fetch_external * remote/RepoTree: add callback for keeping track of download counts when saving tree to cache * repo: use erepo.fetch_external() in repo.fetch() * erepo: return cache save_infos in fetch_external * DependencyRepo: use erepo.fetch_external and cache.checkout instead of pull_to * erepo: remove pull_to * DvcTree: support granularity for isdir() * DvcTree: fix indentation bug * RepoTree: add copytree() method * LocalRemote: use working tree for local cache walk_files * repo: state should be noop for GitTree based erepos * erepo: add get_external() * test_get: remove unneeded caplog check * erepo: add get_rev() * tests: fix erepo test bug * ls: use RepoTree * remote: support trees in _collect_dir * remote: support trees in get_dir_checksum * RepoTree: fix dir_cache relpaths in windows * remote: fix duplicated fetch bug * utils: don't skip md5 dos2unix check for git blobs * otherwise md5 checksums will not match when comparing git version to cached filesystem version on windows
35 lines
1.1 KiB
Python
35 lines
1.1 KiB
Python
"""Use heuristics to guess if it is a text file or a binary file."""
|
|
|
|
# Based on https://eli.thegreenplace.net/2011/10/19/
|
|
# perls-guess-if-file-is-text-or-binary-implemented-in-python
|
|
|
|
|
|
TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
|
|
|
|
|
|
def istextfile(fname, blocksize=512, tree=None):
|
|
""" Uses heuristics to guess whether the given file is text or binary,
|
|
by reading a single block of bytes from the file.
|
|
If more than 30% of the chars in the block are non-text, or there
|
|
are NUL ('\x00') bytes in the block, assume this is a binary file.
|
|
"""
|
|
if tree:
|
|
open_func = tree.open
|
|
else:
|
|
open_func = open
|
|
with open_func(fname, "rb") as fobj:
|
|
block = fobj.read(blocksize)
|
|
|
|
if not block:
|
|
# An empty file is considered a valid text file
|
|
return True
|
|
|
|
if b"\x00" in block:
|
|
# Files with null bytes are binary
|
|
return False
|
|
|
|
# Use translate's 'deletechars' argument to efficiently remove all
|
|
# occurrences of TEXT_CHARS from the block
|
|
nontext = block.translate(None, TEXT_CHARS)
|
|
return float(len(nontext)) / len(block) <= 0.30
|