From d5f5e14183e53d7c9611ecd513037e37ae9a2d52 Mon Sep 17 00:00:00 2001 From: Ronan Lamy Date: Thu, 6 Oct 2022 21:41:28 +0100 Subject: [PATCH] make get-url testable with the dvc plugins --- dvc/repo/get_url.py | 4 ++-- dvc/testing/workspace_tests.py | 34 ++++++++++++++++++++++++++++++++++ tests/func/test_get_url.py | 7 ++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/dvc/repo/get_url.py b/dvc/repo/get_url.py index e7195961e..91fedf0f6 100644 --- a/dvc/repo/get_url.py +++ b/dvc/repo/get_url.py @@ -6,12 +6,12 @@ from dvc.fs import download, parse_external_url from dvc.utils import resolve_output -def get_url(url, out=None, jobs=None): +def get_url(url, out=None, *, config=None, jobs=None): out = resolve_output(url, out) out = os.path.abspath(out) (out,) = output.loads_from(None, [out], use_cache=False) - src_fs, src_path = parse_external_url(url) + src_fs, src_path = parse_external_url(url, config) if not src_fs.exists(src_path): raise URLMissingError(url) download(src_fs, src_path, out, jobs=jobs) diff --git a/dvc/testing/workspace_tests.py b/dvc/testing/workspace_tests.py index 42477c7fd..55218ae2e 100644 --- a/dvc/testing/workspace_tests.py +++ b/dvc/testing/workspace_tests.py @@ -3,6 +3,7 @@ import os import pytest from dvc.exceptions import URLMissingError +from dvc.repo import Repo from dvc.repo.ls_url import ls_url, parse_external_url @@ -196,3 +197,36 @@ class TestLsUrl: def test_nonexistent(self, cloud): with pytest.raises(URLMissingError): ls_url(str(cloud / "dir"), config=cloud.config) + + +class TestGetUrl: + def test_get_file(self, cloud, tmp_dir): + cloud.gen({"foo": "foo contents"}) + + Repo.get_url(str(cloud / "foo"), "foo_imported", config=cloud.config) + + assert (tmp_dir / "foo_imported").is_file() + assert (tmp_dir / "foo_imported").read_text() == "foo contents" + + def test_get_dir(self, cloud, tmp_dir): + cloud.gen({"foo": {"foo": "foo contents"}}) + + Repo.get_url(str(cloud / "foo"), "foo_imported", config=cloud.config) + + assert (tmp_dir / "foo_imported").is_dir() + assert (tmp_dir / "foo_imported" / "foo").is_file() + assert (tmp_dir / "foo_imported" / "foo").read_text() == "foo contents" + + @pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"]) + def test_get_url_to_dir(self, cloud, tmp_dir, dname): + cloud.gen({"src": {"foo": "foo contents"}}) + tmp_dir.gen({"dir": {"subdir": {}}}) + + Repo.get_url(str(cloud / "src" / "foo"), dname, config=cloud.config) + + assert (tmp_dir / dname).is_dir() + assert (tmp_dir / dname / "foo").read_text() == "foo contents" + + def test_get_url_nonexistent(self, cloud): + with pytest.raises(URLMissingError): + Repo.get_url(str(cloud / "nonexistent"), config=cloud.config) diff --git a/tests/func/test_get_url.py b/tests/func/test_get_url.py index 7b2f3bb16..ff7e5bf93 100644 --- a/tests/func/test_get_url.py +++ b/tests/func/test_get_url.py @@ -4,6 +4,7 @@ import pytest from dvc.exceptions import URLMissingError from dvc.repo import Repo +from dvc.testing.workspace_tests import TestGetUrl as _TestGetUrl def test_get_file(tmp_dir): @@ -26,7 +27,7 @@ def test_get_dir(tmp_dir): @pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"]) -def test_get_url_to_dir(tmp_dir, tmp_path_factory, dname): +def test_get_url_to_dir(tmp_dir, dname): tmp_dir.gen({"src": {"foo": "foo contents"}, "dir": {"subdir": {}}}) Repo.get_url(os.path.join("src", "foo"), dname) @@ -38,3 +39,7 @@ def test_get_url_to_dir(tmp_dir, tmp_path_factory, dname): def test_get_url_nonexistent(tmp_dir): with pytest.raises(URLMissingError): Repo.get_url("nonexistent") + + +class TestGetUrl(_TestGetUrl): + pass