add, import-url: preserve size for to-remote/to-cache (#6162)

This commit is contained in:
Batuhan Taskaya
2021-06-16 18:42:51 +03:00
committed by GitHub
parent 78299ad345
commit ea89f4a102
3 changed files with 9 additions and 1 deletions
+5 -1
View File
@@ -70,6 +70,7 @@ class HashedStreamReader(io.IOBase):
def __init__(self, fobj):
self.fobj = fobj
self.md5 = hashlib.md5()
self.total_read = 0
self.is_text_file = None
super().__init__()
@@ -95,9 +96,12 @@ class HashedStreamReader(io.IOBase):
else:
data = chunk
self.md5.update(data)
self.total_read += len(data)
return chunk
@property
def hash_info(self):
return HashInfo(self.PARAM_CHECKSUM, self.md5.hexdigest(), nfiles=1)
return HashInfo(
self.PARAM_CHECKSUM, self.md5.hexdigest(), size=self.total_read
)
+3
View File
@@ -1024,6 +1024,7 @@ def test_add_to_remote(tmp_dir, dvc, local_cloud, local_remote):
hash_info = stage.outs[0].hash_info
assert local_remote.hash_to_path_info(hash_info.value).read_text() == "foo"
assert hash_info.size == len("foo")
def test_add_to_remote_absolute(tmp_dir, make_tmp_dir, dvc, local_remote):
@@ -1067,6 +1068,8 @@ def test_add_to_cache_dir(tmp_dir, dvc, local_cloud):
(stage,) = dvc.add(str(local_cloud / "data"), out="data")
assert len(stage.deps) == 0
assert len(stage.outs) == 1
assert stage.outs[0].hash_info.size == len("foo") + len("bar")
assert stage.outs[0].hash_info.nfiles == 2
data = tmp_dir / "data"
assert data.read_text() == {"foo": "foo", "bar": "bar"}
+1
View File
@@ -282,6 +282,7 @@ def test_import_url_to_remote_single_file(
hash_info = stage.outs[0].hash_info
assert local_remote.hash_to_path_info(hash_info.value).read_text() == "foo"
assert hash_info.size == len("foo")
@pytest.mark.parametrize(