diff --git a/dvc/istextfile.py b/dvc/istextfile.py index d7250fb6a..81144927c 100644 --- a/dvc/istextfile.py +++ b/dvc/istextfile.py @@ -5,6 +5,7 @@ TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b" +DEFAULT_CHUNK_SIZE = 512 def istextblock(block): @@ -22,7 +23,7 @@ def istextblock(block): return float(len(nontext)) / len(block) <= 0.30 -def istextfile(fname, fs, blocksize=512): +def istextfile(fname, fs, blocksize=DEFAULT_CHUNK_SIZE): """Uses heuristics to guess whether the given file is text or binary, by reading a single block of bytes from the file. If more than 30% of the chars in the block are non-text, or there diff --git a/dvc/utils/stream.py b/dvc/utils/stream.py index fc71de920..8db69a29c 100644 --- a/dvc/utils/stream.py +++ b/dvc/utils/stream.py @@ -4,7 +4,7 @@ import io from funcy import cached_property from dvc.hash_info import HashInfo -from dvc.istextfile import istextblock +from dvc.istextfile import DEFAULT_CHUNK_SIZE, istextblock from dvc.utils import dos2unix @@ -89,7 +89,7 @@ class HashedStreamReader(io.IOBase): def read(self, n=-1): chunk = self._reader(n) if self.is_text_file is None: - self.is_text_file = istextblock(chunk) + self.is_text_file = istextblock(chunk[:DEFAULT_CHUNK_SIZE]) if self.is_text_file: data = dos2unix(chunk) diff --git a/tests/unit/utils/test_stream.py b/tests/unit/utils/test_stream.py index 439dcd7ff..2a80c3f0d 100644 --- a/tests/unit/utils/test_stream.py +++ b/tests/unit/utils/test_stream.py @@ -1,4 +1,7 @@ +import pytest + from dvc.fs.local import LocalFileSystem +from dvc.istextfile import DEFAULT_CHUNK_SIZE, istextfile from dvc.utils import file_md5 from dvc.utils.stream import HashedStreamReader @@ -46,3 +49,26 @@ def test_hashed_stream_reader_as_chunks(tmp_dir): hex_digest = file_md5(foo, LocalFileSystem()) assert not stream_reader.is_text_file assert hex_digest == stream_reader.hash_info.value + + +@pytest.mark.parametrize( + "contents", + [b"x" * DEFAULT_CHUNK_SIZE + b"\x00", b"clean", b"not clean \x00"], +) +def test_hashed_stream_reader_compatibility(tmp_dir, contents): + # Always read more than the DEFAULT_CHUNK_SIZE (512 bytes). + # This imitates the read actions performed by upload_fobj. + chunk_size = DEFAULT_CHUNK_SIZE * 2 + + tmp_dir.gen("data", contents) + data = tmp_dir / "data" + + with open(data, "rb") as fobj: + stream_reader = HashedStreamReader(fobj) + stream_reader.read(chunk_size) + + local_fs = LocalFileSystem() + hex_digest = file_md5(data, local_fs) + + assert stream_reader.is_text_file is istextfile(data, local_fs) + assert stream_reader.hash_info.value == hex_digest