mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-08 15:28:26 +01:00
Batuhan Taskaya
51ada6d272
* Implement export-to-remote functionality * Redistrubition of functionality from tree to remote * Pass chunk size into the open_url * Move functionality away from remote to cache * Basic progress bars for both transfer types * Proper job calculation * Redesign .open() usage in order to work with different trees * Align the signatures of other open() functions to accept **kwargs * Implement chunked uploading to the ssh * Use local cache to compute directory hash (md5) * S3: multipart upload test * Wrap local cache's hash_to_path with PathInfo * Use .isdir() instead of a try/except statement for calculating the from_infos * New CLI layout (add/import-url) * Adjust CLI tests to accept new arguments * tests: import_url/add func/unit tests * Introduce cache.transfer() * Invalid CLI combination tests * Store dir_info key in the transfer_directory * LocalTree.upload_fobj * Tests for utils * Lazy imports on some places * --to-remote migration (and repo._transfer) * Use proper temporary location, more efficient pbar calc * Directory transfer logic rewrite * Use shutil.copyfileobj() * Rebase... * Better tests * Inline transfer operation to .add() / .transfer() * Open in the 'rb' mode for the tests * More tests, better invalid option handling at repo level * .read() certain amount of bytes to validate the content * Better invalid argument handling * Better progress bars by using .getsize() API * For fallback upload, use the from_info.name instead of temporary naem * introduce --jobs for add/import-url
66 lines
1.9 KiB
Python
66 lines
1.9 KiB
Python
import io
|
|
from contextlib import contextmanager
|
|
|
|
from dvc.utils.stream import IterStream
|
|
|
|
|
|
@contextmanager
|
|
def open_url(url, mode="r", encoding=None, **iter_opts):
|
|
"""Opens an url as a readable stream.
|
|
|
|
Resumes on connection error.
|
|
Url could be a string or a callable returning a string.
|
|
"""
|
|
assert mode in {"r", "rt", "rb"}
|
|
|
|
with iter_url(url, **iter_opts) as (response, it):
|
|
bytes_stream = IterStream(it)
|
|
|
|
if mode == "rb":
|
|
yield bytes_stream
|
|
else:
|
|
encoding = encoding or response.encoding
|
|
yield io.TextIOWrapper(bytes_stream, encoding=encoding)
|
|
|
|
|
|
@contextmanager
|
|
def iter_url(url, chunk_size=io.DEFAULT_BUFFER_SIZE):
|
|
"""Iterate over chunks requested from url."""
|
|
import requests
|
|
|
|
def request(headers=None):
|
|
the_url = url() if callable(url) else url
|
|
response = requests.get(the_url, stream=True, headers=headers)
|
|
if response.status_code == 404:
|
|
raise FileNotFoundError(f"Can't open {the_url}")
|
|
response.raise_for_status()
|
|
return response
|
|
|
|
def gen(response):
|
|
try:
|
|
pos = 0
|
|
while True:
|
|
try:
|
|
for chunk in response.iter_content(chunk_size):
|
|
pos += len(chunk)
|
|
yield chunk
|
|
break
|
|
except requests.ConnectionError:
|
|
response.close()
|
|
if response.headers.get("Accept-Ranges") != "bytes":
|
|
raise
|
|
|
|
# Reopen request from where we stopped
|
|
headers = {"Range": f"bytes={pos}-"}
|
|
response = request(headers)
|
|
finally:
|
|
response.close()
|
|
|
|
response = request()
|
|
it = gen(response)
|
|
try:
|
|
yield response, it
|
|
finally:
|
|
# Ensure connection is closed
|
|
it.close()
|