diff --git a/dvc/repo/ls_url.py b/dvc/repo/ls_url.py index d2248baab..7f211678f 100644 --- a/dvc/repo/ls_url.py +++ b/dvc/repo/ls_url.py @@ -10,10 +10,16 @@ def parse_external_url(url): def ls_url(url, recursive=False): fs, fs_path = parse_external_url(url) + return _ls_url(fs, fs_path, orig_url=url, recursive=recursive) + + +def _ls_url(fs, fs_path, orig_url=None, recursive=False): + if orig_url is None: + orig_url = fs_path try: info = fs.info(fs_path) except FileNotFoundError as exc: - raise URLMissingError(url) from exc + raise URLMissingError(orig_url) from exc if info["type"] != "directory": return [{"path": info["name"], "isdir": False}] diff --git a/dvc/testing/tmp_dir.py b/dvc/testing/tmp_dir.py index ea4cd0d77..f90066167 100644 --- a/dvc/testing/tmp_dir.py +++ b/dvc/testing/tmp_dir.py @@ -51,6 +51,7 @@ import sys from contextlib import contextmanager from functools import partialmethod +from dvc.fs import LocalFileSystem from dvc.utils import serialize from dvc.utils.fs import makedirs @@ -70,6 +71,10 @@ class TmpDir(pathlib.Path): def config(self): return {"url": self.url} + @property + def fs(self): + return LocalFileSystem(**self.config) + def __new__(cls, *args, **kwargs): if cls is TmpDir: cls = ( # pylint: disable=self-cls-assignment diff --git a/dvc/testing/workspace_tests.py b/dvc/testing/workspace_tests.py index 0b883a7ba..f9dde1911 100644 --- a/dvc/testing/workspace_tests.py +++ b/dvc/testing/workspace_tests.py @@ -2,6 +2,9 @@ import os import pytest +from dvc.exceptions import URLMissingError +from dvc.repo.ls_url import _ls_url + class TestImport: def test_import(self, tmp_dir, dvc, workspace): @@ -134,3 +137,66 @@ class TestAdd: assert ( workspace / "cache" / dir_hash_value[:2] / dir_hash_value[2:] ).is_file() + + +def match_files(fs, entries, expected): + entries_content = { + (fs.path.normpath(d["path"]), d["isdir"]) for d in entries + } + expected_content = { + (fs.path.normpath(d["path"]), d["isdir"]) for d in expected + } + assert entries_content == expected_content + + +class TestLsUrl: + @pytest.mark.parametrize("fname", ["foo", "foo.dvc", "dir/foo"]) + def test_file(self, cloud, fname): + cloud.gen({fname: "foo contents"}) + fs = cloud.fs + fs_path = fs.path.join(cloud.fs_path, fname) + + result = _ls_url(fs, fs_path) + + match_files(fs, result, [{"path": fs_path, "isdir": False}]) + + def test_dir(self, cloud): + cloud.gen( + {"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"} + ) + fs = cloud.fs + fs_path = fs.path.join(cloud.fs_path, "dir") + + result = _ls_url(fs, fs_path) + match_files( + fs, + result, + [ + {"path": "foo", "isdir": False}, + {"path": "subdir", "isdir": True}, + ], + ) + + def test_recursive(self, cloud): + cloud.gen( + {"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"} + ) + fs = cloud.fs + fs_path = fs.path.join(cloud.fs_path, "dir") + + result = _ls_url(fs, fs_path, recursive=True) + + match_files( + fs, + result, + [ + {"path": "foo", "isdir": False}, + {"path": "subdir/bar", "isdir": False}, + ], + ) + + def test_nonexistent(self, cloud): + fs = cloud.fs + fs_path = fs.path.join(cloud.fs_path, "nonexistent") + with pytest.raises(URLMissingError): + _ls_url(fs, fs_path) diff --git a/tests/func/test_ls_url.py b/tests/func/test_ls_url.py index 797646675..cd4dbbdb2 100644 --- a/tests/func/test_ls_url.py +++ b/tests/func/test_ls_url.py @@ -4,6 +4,7 @@ import pytest from dvc.exceptions import URLMissingError from dvc.repo import Repo +from dvc.testing.workspace_tests import TestLsUrl as _TestLsUrl def match_files(entries, expected): @@ -49,3 +50,7 @@ def test_recursive(tmp_dir): def test_nonexistent(tmp_dir): with pytest.raises(URLMissingError): Repo.ls_url("nonexistent") + + +class TestLsUrl(_TestLsUrl): + pass