import-url: use the same chunk size for istextblock() calls (#6303)

dvc/istextfile.py

@@ -5,6 +5,7 @@
 
 
 TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
+DEFAULT_CHUNK_SIZE = 512
 
 
 def istextblock(block):
@@ -22,7 +23,7 @@ def istextblock(block):
     return float(len(nontext)) / len(block) <= 0.30
 
 
-def istextfile(fname, fs, blocksize=512):
+def istextfile(fname, fs, blocksize=DEFAULT_CHUNK_SIZE):
     """Uses heuristics to guess whether the given file is text or binary,
     by reading a single block of bytes from the file.
     If more than 30% of the chars in the block are non-text, or there

dvc/utils/stream.py

@@ -4,7 +4,7 @@ import io
 from funcy import cached_property
 
 from dvc.hash_info import HashInfo
-from dvc.istextfile import istextblock
+from dvc.istextfile import DEFAULT_CHUNK_SIZE, istextblock
 from dvc.utils import dos2unix
 
 
@@ -89,7 +89,7 @@ class HashedStreamReader(io.IOBase):
     def read(self, n=-1):
         chunk = self._reader(n)
         if self.is_text_file is None:
-            self.is_text_file = istextblock(chunk)
+            self.is_text_file = istextblock(chunk[:DEFAULT_CHUNK_SIZE])
 
         if self.is_text_file:
             data = dos2unix(chunk)

tests/unit/utils/test_stream.py

@@ -1,4 +1,7 @@
+import pytest
+
 from dvc.fs.local import LocalFileSystem
+from dvc.istextfile import DEFAULT_CHUNK_SIZE, istextfile
 from dvc.utils import file_md5
 from dvc.utils.stream import HashedStreamReader
 
@@ -46,3 +49,26 @@ def test_hashed_stream_reader_as_chunks(tmp_dir):
     hex_digest = file_md5(foo, LocalFileSystem())
     assert not stream_reader.is_text_file
     assert hex_digest == stream_reader.hash_info.value
+
+
+@pytest.mark.parametrize(
+    "contents",
+    [b"x" * DEFAULT_CHUNK_SIZE + b"\x00", b"clean", b"not clean \x00"],
+)
+def test_hashed_stream_reader_compatibility(tmp_dir, contents):
+    # Always read more than the DEFAULT_CHUNK_SIZE (512 bytes).
+    # This imitates the read actions performed by upload_fobj.
+    chunk_size = DEFAULT_CHUNK_SIZE * 2
+
+    tmp_dir.gen("data", contents)
+    data = tmp_dir / "data"
+
+    with open(data, "rb") as fobj:
+        stream_reader = HashedStreamReader(fobj)
+        stream_reader.read(chunk_size)
+
+    local_fs = LocalFileSystem()
+    hex_digest = file_md5(data, local_fs)
+
+    assert stream_reader.is_text_file is istextfile(data, local_fs)
+    assert stream_reader.hash_info.value == hex_digest