From 72eb4a57370910c6e0ffa3f1b035968d5357b890 Mon Sep 17 00:00:00 2001 From: Saugat Pachhai Date: Thu, 12 Nov 2020 13:52:46 +0545 Subject: [PATCH] Add test server and tests for webdav (#4827) * Add test server and tests for webdav * Fix webdav unittests --- dvc/tree/webdav.py | 6 ---- setup.py | 1 + tests/func/test_api.py | 14 +++++++- tests/func/test_data_cloud.py | 11 +++++- tests/remotes/__init__.py | 1 + tests/remotes/webdav.py | 60 ++++++++++++++++++++++++++++++++ tests/unit/remote/test_webdav.py | 9 +++-- tests/utils/httpd.py | 34 +++++++++++------- 8 files changed, 113 insertions(+), 23 deletions(-) create mode 100644 tests/remotes/webdav.py diff --git a/dvc/tree/webdav.py b/dvc/tree/webdav.py index ca5294cf4..901a2883d 100644 --- a/dvc/tree/webdav.py +++ b/dvc/tree/webdav.py @@ -81,12 +81,6 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method if self.user is None and self.path_info.user is not None: self.user = self.path_info.user - # If username specified add to path_info - if self.user is not None: - self.path_info.user = self.user - else: - self.path_info = None - # Webdav client @wrap_prop(threading.Lock()) @cached_property diff --git a/setup.py b/setup.py index d698315ac..0df9c255c 100644 --- a/setup.py +++ b/setup.py @@ -150,6 +150,7 @@ tests_requirements = [ "wget", "filelock", "black==19.10b0", + "wsgidav", ] setup( diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 15fae724a..92955007f 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -10,7 +10,17 @@ from dvc.path_info import URLInfo from dvc.utils.fs import remove from tests.unit.tree.test_repo import make_subrepo -cloud_names = ["s3", "gs", "azure", "gdrive", "oss", "ssh", "http", "hdfs"] +cloud_names = [ + "s3", + "gs", + "azure", + "gdrive", + "oss", + "ssh", + "http", + "hdfs", + "webdav", +] clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names] all_clouds = [pytest.lazy_fixture("local_cloud")] + clouds @@ -72,6 +82,7 @@ def test_open(tmp_dir, dvc, remote): "http", "hdfs", "ssh", + "webdav", ] ], ) @@ -124,6 +135,7 @@ def test_open_granular(tmp_dir, dvc, remote): "ssh", "http", "hdfs", + "webdav", ] ], indirect=True, diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index eb687a692..9ef36eb0c 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -21,7 +21,16 @@ from dvc.utils.serialize import dump_yaml, load_yaml all_clouds = [ pytest.lazy_fixture(cloud) - for cloud in ["s3", "gs", "azure", "gdrive", "ssh", "http", "hdfs"] + for cloud in [ + "s3", + "gs", + "azure", + "gdrive", + "ssh", + "http", + "hdfs", + "webdav", + ] ] + [ pytest.param( pytest.lazy_fixture("oss"), diff --git a/tests/remotes/__init__.py b/tests/remotes/__init__.py index 417bd6899..1750dcef1 100644 --- a/tests/remotes/__init__.py +++ b/tests/remotes/__init__.py @@ -14,6 +14,7 @@ from .oss import ( # noqa: F401 real_oss, ) from .s3 import S3, TEST_AWS_REPO_BUCKET, real_s3, s3 # noqa: F401 +from .webdav import Webdav, webdav, webdav_server # noqa: F401 from .gdrive import ( # noqa: F401; noqa: F401 TEST_GDRIVE_REPO_BUCKET, diff --git a/tests/remotes/webdav.py b/tests/remotes/webdav.py new file mode 100644 index 000000000..696db8e09 --- /dev/null +++ b/tests/remotes/webdav.py @@ -0,0 +1,60 @@ +# pylint:disable=abstract-method +import os +from pathlib import Path +from wsgiref.simple_server import make_server + +import pytest +from funcy import first +from wsgidav.wsgidav_app import WsgiDAVApp + +from dvc.path_info import WebDAVURLInfo +from dvc.utils.fs import makedirs +from tests.utils.httpd import run_server_on_thread + +from .base import Base + +AUTH = {"user1": {"password": "password1"}} + + +class Webdav(Base, WebDAVURLInfo): + @staticmethod + def get_url(port): # pylint: disable=arguments-differ + return f"webdav://localhost:{port}" + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + assert parents + makedirs(self.path, exist_ok=exist_ok) + + def write_bytes(self, contents): + Path(self.path).write_bytes(contents) + + def write_text(self, contents, encoding=None, errors=None): + Path(self.path).write_text(contents, encoding=encoding, errors=errors) + + +@pytest.fixture +def webdav_server(tmp_path_factory): + host, port = "localhost", 0 + directory = os.fspath(tmp_path_factory.mktemp("http")) + dirmap = {"/": directory} + + app = WsgiDAVApp( + { + "host": host, + "port": port, + "provider_mapping": dirmap, + "simple_dc": {"user_mapping": {"*": AUTH}}, + } + ) + server = make_server(host, port, app) + with run_server_on_thread(server) as httpd: + yield httpd + + +@pytest.fixture +def webdav(webdav_server): + url = Webdav.get_url(webdav_server.server_port) + ret = Webdav(url) + user, secrets = first(AUTH.items()) + ret.config = {"url": url, "user": user, **secrets} + yield ret diff --git a/tests/unit/remote/test_webdav.py b/tests/unit/remote/test_webdav.py index 7691f6b55..907e812b4 100644 --- a/tests/unit/remote/test_webdav.py +++ b/tests/unit/remote/test_webdav.py @@ -1,3 +1,5 @@ +import pytest + from dvc.tree.webdav import WebDAVTree # Test configuration @@ -16,12 +18,13 @@ def test_init(dvc): # Test username from configuration -def test_user(dvc): - config = {"url": url, "user": user} +@pytest.mark.parametrize( + "config", [{"url": url, "user": user}, {"url": userurl}] +) +def test_user(dvc, config): tree = WebDAVTree(dvc, config) assert tree.user == user - assert tree.path_info.user == user # Test username extraction from url diff --git a/tests/utils/httpd.py b/tests/utils/httpd.py index 74ddbc2b7..44c0d7bfe 100644 --- a/tests/utils/httpd.py +++ b/tests/utils/httpd.py @@ -2,6 +2,7 @@ import hashlib import os import sys import threading +from contextlib import contextmanager from http import HTTPStatus from http.server import HTTPServer @@ -89,26 +90,35 @@ class TestRequestHandler(RangeRequestHandler): self.end_headers() +@contextmanager +def run_server_on_thread(server): + thread = threading.Thread(target=server.serve_forever) + thread.daemon = True + thread.start() + + yield server + + server.socket.close() + server.shutdown() + server.server_close() + + class StaticFileServer: _lock = threading.Lock() def __init__(self, directory): from functools import partial - self._lock.acquire() - self._httpd = HTTPServer( - ("localhost", 0), partial(TestRequestHandler, directory=directory), - ) - self._thread = None + addr = ("localhost", 0) + req = partial(TestRequestHandler, directory=directory) + server = HTTPServer(addr, req) + self.runner = run_server_on_thread(server) + # pylint: disable=no-member def __enter__(self): - self._thread = threading.Thread(target=self._httpd.serve_forever) - self._thread.daemon = True - self._thread.start() - return self._httpd + self._lock.acquire() + return self.runner.__enter__() def __exit__(self, *args): - self._httpd.socket.close() - self._httpd.shutdown() - self._httpd.server_close() + self.runner.__exit__(*args) self._lock.release()