mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-04 15:28:27 +01:00
Batuhan Taskaya
51ada6d272
* Implement export-to-remote functionality * Redistrubition of functionality from tree to remote * Pass chunk size into the open_url * Move functionality away from remote to cache * Basic progress bars for both transfer types * Proper job calculation * Redesign .open() usage in order to work with different trees * Align the signatures of other open() functions to accept **kwargs * Implement chunked uploading to the ssh * Use local cache to compute directory hash (md5) * S3: multipart upload test * Wrap local cache's hash_to_path with PathInfo * Use .isdir() instead of a try/except statement for calculating the from_infos * New CLI layout (add/import-url) * Adjust CLI tests to accept new arguments * tests: import_url/add func/unit tests * Introduce cache.transfer() * Invalid CLI combination tests * Store dir_info key in the transfer_directory * LocalTree.upload_fobj * Tests for utils * Lazy imports on some places * --to-remote migration (and repo._transfer) * Use proper temporary location, more efficient pbar calc * Directory transfer logic rewrite * Use shutil.copyfileobj() * Rebase... * Better tests * Inline transfer operation to .add() / .transfer() * Open in the 'rb' mode for the tests * More tests, better invalid option handling at repo level * .read() certain amount of bytes to validate the content * Better invalid argument handling * Better progress bars by using .getsize() API * For fallback upload, use the from_info.name instead of temporary naem * introduce --jobs for add/import-url
149 lines
3.5 KiB
Python
149 lines
3.5 KiB
Python
import logging
|
|
|
|
from dvc.cli import parse_args
|
|
from dvc.command.imp_url import CmdImportUrl
|
|
from dvc.exceptions import DvcException
|
|
|
|
|
|
def test_import_url(mocker):
|
|
cli_args = parse_args(
|
|
[
|
|
"import-url",
|
|
"src",
|
|
"out",
|
|
"--file",
|
|
"file",
|
|
"--jobs",
|
|
"4",
|
|
"--desc",
|
|
"description",
|
|
]
|
|
)
|
|
assert cli_args.func == CmdImportUrl
|
|
|
|
cmd = cli_args.func(cli_args)
|
|
m = mocker.patch.object(cmd.repo, "imp_url", autospec=True)
|
|
|
|
assert cmd.run() == 0
|
|
|
|
m.assert_called_once_with(
|
|
"src",
|
|
out="out",
|
|
fname="file",
|
|
no_exec=False,
|
|
remote=None,
|
|
to_remote=False,
|
|
desc="description",
|
|
jobs=4,
|
|
)
|
|
|
|
|
|
def test_failed_import_url(mocker, caplog):
|
|
cli_args = parse_args(["import-url", "http://somesite.com/file_name"])
|
|
assert cli_args.func == CmdImportUrl
|
|
|
|
cmd = cli_args.func(cli_args)
|
|
mocker.patch.object(cmd.repo, "imp_url", side_effect=DvcException("error"))
|
|
with caplog.at_level(logging.ERROR, logger="dvc"):
|
|
assert cmd.run() == 1
|
|
expected_error = (
|
|
"failed to import http://somesite.com/file_name. "
|
|
"You could also try downloading it manually, and "
|
|
"adding it with `dvc add`."
|
|
)
|
|
assert expected_error in caplog.text
|
|
|
|
|
|
def test_import_url_no_exec(mocker):
|
|
cli_args = parse_args(
|
|
[
|
|
"import-url",
|
|
"--no-exec",
|
|
"src",
|
|
"out",
|
|
"--file",
|
|
"file",
|
|
"--desc",
|
|
"description",
|
|
]
|
|
)
|
|
|
|
cmd = cli_args.func(cli_args)
|
|
m = mocker.patch.object(cmd.repo, "imp_url", autospec=True)
|
|
|
|
assert cmd.run() == 0
|
|
|
|
m.assert_called_once_with(
|
|
"src",
|
|
out="out",
|
|
fname="file",
|
|
no_exec=True,
|
|
remote=None,
|
|
to_remote=False,
|
|
desc="description",
|
|
jobs=None,
|
|
)
|
|
|
|
|
|
def test_import_url_to_remote(mocker):
|
|
cli_args = parse_args(
|
|
[
|
|
"import-url",
|
|
"s3://bucket/foo",
|
|
"bar",
|
|
"--to-remote",
|
|
"--remote",
|
|
"remote",
|
|
"--desc",
|
|
"description",
|
|
]
|
|
)
|
|
assert cli_args.func == CmdImportUrl
|
|
|
|
cmd = cli_args.func(cli_args)
|
|
m = mocker.patch.object(cmd.repo, "imp_url", autospec=True)
|
|
|
|
assert cmd.run() == 0
|
|
|
|
m.assert_called_once_with(
|
|
"s3://bucket/foo",
|
|
out="bar",
|
|
fname=None,
|
|
no_exec=False,
|
|
remote="remote",
|
|
to_remote=True,
|
|
desc="description",
|
|
jobs=None,
|
|
)
|
|
|
|
|
|
def test_import_url_to_remote_invalid_combination(mocker, caplog):
|
|
cli_args = parse_args(
|
|
[
|
|
"import-url",
|
|
"s3://bucket/foo",
|
|
"bar",
|
|
"--to-remote",
|
|
"--remote",
|
|
"remote",
|
|
"--no-exec",
|
|
]
|
|
)
|
|
assert cli_args.func == CmdImportUrl
|
|
|
|
cmd = cli_args.func(cli_args)
|
|
with caplog.at_level(logging.ERROR, logger="dvc"):
|
|
assert cmd.run() == 1
|
|
expected_msg = "--no-exec can't be combined with --to-remote"
|
|
assert expected_msg in caplog.text
|
|
|
|
cli_args = parse_args(
|
|
["import-url", "s3://bucket/foo", "bar", "--remote", "remote"]
|
|
)
|
|
|
|
cmd = cli_args.func(cli_args)
|
|
with caplog.at_level(logging.ERROR, logger="dvc"):
|
|
assert cmd.run() == 1
|
|
expected_msg = "--remote can't be used without --to-remote"
|
|
assert expected_msg in caplog.text
|