mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-06 15:28:28 +01:00
Batuhan Taskaya
51ada6d272
* Implement export-to-remote functionality * Redistrubition of functionality from tree to remote * Pass chunk size into the open_url * Move functionality away from remote to cache * Basic progress bars for both transfer types * Proper job calculation * Redesign .open() usage in order to work with different trees * Align the signatures of other open() functions to accept **kwargs * Implement chunked uploading to the ssh * Use local cache to compute directory hash (md5) * S3: multipart upload test * Wrap local cache's hash_to_path with PathInfo * Use .isdir() instead of a try/except statement for calculating the from_infos * New CLI layout (add/import-url) * Adjust CLI tests to accept new arguments * tests: import_url/add func/unit tests * Introduce cache.transfer() * Invalid CLI combination tests * Store dir_info key in the transfer_directory * LocalTree.upload_fobj * Tests for utils * Lazy imports on some places * --to-remote migration (and repo._transfer) * Use proper temporary location, more efficient pbar calc * Directory transfer logic rewrite * Use shutil.copyfileobj() * Rebase... * Better tests * Inline transfer operation to .add() / .transfer() * Open in the 'rb' mode for the tests * More tests, better invalid option handling at repo level * .read() certain amount of bytes to validate the content * Better invalid argument handling * Better progress bars by using .getsize() API * For fallback upload, use the from_info.name instead of temporary naem * introduce --jobs for add/import-url
32 lines
886 B
Python
32 lines
886 B
Python
from dvc.utils import file_md5
|
|
from dvc.utils.stream import HashedStreamReader
|
|
|
|
|
|
def test_hashed_stream_reader(tmp_dir):
|
|
tmp_dir.gen({"foo": "foo"})
|
|
|
|
foo = tmp_dir / "foo"
|
|
with open(foo, "rb") as fobj:
|
|
stream_reader = HashedStreamReader(fobj)
|
|
assert stream_reader.read(3) == b"foo"
|
|
|
|
hex_digest, _ = file_md5(foo)
|
|
assert stream_reader.is_text_file
|
|
assert hex_digest == stream_reader.hash_info.value
|
|
|
|
|
|
def test_hashed_stream_reader_as_chunks(tmp_dir):
|
|
tmp_dir.gen({"foo": b"foo \x00" * 16})
|
|
|
|
foo = tmp_dir / "foo"
|
|
with open(foo, "rb") as fobj:
|
|
stream_reader = HashedStreamReader(fobj)
|
|
while True:
|
|
chunk = stream_reader.read(16)
|
|
if not chunk:
|
|
break
|
|
|
|
hex_digest, _ = file_md5(foo)
|
|
assert not stream_reader.is_text_file
|
|
assert hex_digest == stream_reader.hash_info.value
|