mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-11 23:10:10 +01:00
Batuhan Taskaya
51ada6d272
* Implement export-to-remote functionality * Redistrubition of functionality from tree to remote * Pass chunk size into the open_url * Move functionality away from remote to cache * Basic progress bars for both transfer types * Proper job calculation * Redesign .open() usage in order to work with different trees * Align the signatures of other open() functions to accept **kwargs * Implement chunked uploading to the ssh * Use local cache to compute directory hash (md5) * S3: multipart upload test * Wrap local cache's hash_to_path with PathInfo * Use .isdir() instead of a try/except statement for calculating the from_infos * New CLI layout (add/import-url) * Adjust CLI tests to accept new arguments * tests: import_url/add func/unit tests * Introduce cache.transfer() * Invalid CLI combination tests * Store dir_info key in the transfer_directory * LocalTree.upload_fobj * Tests for utils * Lazy imports on some places * --to-remote migration (and repo._transfer) * Use proper temporary location, more efficient pbar calc * Directory transfer logic rewrite * Use shutil.copyfileobj() * Rebase... * Better tests * Inline transfer operation to .add() / .transfer() * Open in the 'rb' mode for the tests * More tests, better invalid option handling at repo level * .read() certain amount of bytes to validate the content * Better invalid argument handling * Better progress bars by using .getsize() API * For fallback upload, use the from_info.name instead of temporary naem * introduce --jobs for add/import-url
89 lines
2.3 KiB
Python
89 lines
2.3 KiB
Python
import hashlib
|
|
import io
|
|
|
|
from dvc.hash_info import HashInfo
|
|
from dvc.istextfile import istextblock
|
|
from dvc.utils import dos2unix
|
|
|
|
|
|
class IterStream(io.RawIOBase):
|
|
"""Wraps an iterator yielding bytes as a file object"""
|
|
|
|
def __init__(self, iterator): # pylint: disable=super-init-not-called
|
|
self.iterator = iterator
|
|
self.leftover = b""
|
|
|
|
def readable(self):
|
|
return True
|
|
|
|
def writable(self) -> bool:
|
|
return False
|
|
|
|
# Python 3 requires only .readinto() method, it still uses other ones
|
|
# under some circumstances and falls back if those are absent. Since
|
|
# iterator already constructs byte strings for us, .readinto() is not the
|
|
# most optimal, so we provide .read1() too.
|
|
|
|
def readinto(self, b):
|
|
try:
|
|
n = len(b) # We're supposed to return at most this much
|
|
chunk = self.leftover or next(self.iterator)
|
|
output, self.leftover = chunk[:n], chunk[n:]
|
|
|
|
n_out = len(output)
|
|
b[:n_out] = output
|
|
return n_out
|
|
except StopIteration:
|
|
return 0 # indicate EOF
|
|
|
|
readinto1 = readinto
|
|
|
|
def read1(self, n=-1):
|
|
try:
|
|
chunk = self.leftover or next(self.iterator)
|
|
except StopIteration:
|
|
return b""
|
|
|
|
# Return an arbitrary number or bytes
|
|
if n <= 0:
|
|
self.leftover = b""
|
|
return chunk
|
|
|
|
output, self.leftover = chunk[:n], chunk[n:]
|
|
return output
|
|
|
|
def peek(self, n):
|
|
while len(self.leftover) < n:
|
|
try:
|
|
self.leftover += next(self.iterator)
|
|
except StopIteration:
|
|
break
|
|
return self.leftover[:n]
|
|
|
|
|
|
class HashedStreamReader:
|
|
|
|
PARAM_CHECKSUM = "md5"
|
|
|
|
def __init__(self, fobj):
|
|
self.md5 = hashlib.md5()
|
|
self.is_text_file = None
|
|
self.reader = fobj.read1 if hasattr(fobj, "read1") else fobj.read
|
|
|
|
def read(self, n=-1):
|
|
chunk = self.reader(n)
|
|
if self.is_text_file is None:
|
|
self.is_text_file = istextblock(chunk)
|
|
|
|
if self.is_text_file:
|
|
data = dos2unix(chunk)
|
|
else:
|
|
data = chunk
|
|
self.md5.update(data)
|
|
|
|
return chunk
|
|
|
|
@property
|
|
def hash_info(self):
|
|
return HashInfo(self.PARAM_CHECKSUM, self.md5.hexdigest(), nfiles=1)
|