Add test server and tests for webdav (#4827)

* Add test server and tests for webdav

* Fix webdav unittests
This commit is contained in:
Saugat Pachhai
2020-11-12 13:52:46 +05:45
committed by GitHub
parent 72ec2a8541
commit 72eb4a5737
8 changed files with 113 additions and 23 deletions
-6
View File
@@ -81,12 +81,6 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
if self.user is None and self.path_info.user is not None:
self.user = self.path_info.user
# If username specified add to path_info
if self.user is not None:
self.path_info.user = self.user
else:
self.path_info = None
# Webdav client
@wrap_prop(threading.Lock())
@cached_property
+1
View File
@@ -150,6 +150,7 @@ tests_requirements = [
"wget",
"filelock",
"black==19.10b0",
"wsgidav",
]
setup(
+13 -1
View File
@@ -10,7 +10,17 @@ from dvc.path_info import URLInfo
from dvc.utils.fs import remove
from tests.unit.tree.test_repo import make_subrepo
cloud_names = ["s3", "gs", "azure", "gdrive", "oss", "ssh", "http", "hdfs"]
cloud_names = [
"s3",
"gs",
"azure",
"gdrive",
"oss",
"ssh",
"http",
"hdfs",
"webdav",
]
clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names]
all_clouds = [pytest.lazy_fixture("local_cloud")] + clouds
@@ -72,6 +82,7 @@ def test_open(tmp_dir, dvc, remote):
"http",
"hdfs",
"ssh",
"webdav",
]
],
)
@@ -124,6 +135,7 @@ def test_open_granular(tmp_dir, dvc, remote):
"ssh",
"http",
"hdfs",
"webdav",
]
],
indirect=True,
+10 -1
View File
@@ -21,7 +21,16 @@ from dvc.utils.serialize import dump_yaml, load_yaml
all_clouds = [
pytest.lazy_fixture(cloud)
for cloud in ["s3", "gs", "azure", "gdrive", "ssh", "http", "hdfs"]
for cloud in [
"s3",
"gs",
"azure",
"gdrive",
"ssh",
"http",
"hdfs",
"webdav",
]
] + [
pytest.param(
pytest.lazy_fixture("oss"),
+1
View File
@@ -14,6 +14,7 @@ from .oss import ( # noqa: F401
real_oss,
)
from .s3 import S3, TEST_AWS_REPO_BUCKET, real_s3, s3 # noqa: F401
from .webdav import Webdav, webdav, webdav_server # noqa: F401
from .gdrive import ( # noqa: F401; noqa: F401
TEST_GDRIVE_REPO_BUCKET,
+60
View File
@@ -0,0 +1,60 @@
# pylint:disable=abstract-method
import os
from pathlib import Path
from wsgiref.simple_server import make_server
import pytest
from funcy import first
from wsgidav.wsgidav_app import WsgiDAVApp
from dvc.path_info import WebDAVURLInfo
from dvc.utils.fs import makedirs
from tests.utils.httpd import run_server_on_thread
from .base import Base
AUTH = {"user1": {"password": "password1"}}
class Webdav(Base, WebDAVURLInfo):
@staticmethod
def get_url(port): # pylint: disable=arguments-differ
return f"webdav://localhost:{port}"
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
assert parents
makedirs(self.path, exist_ok=exist_ok)
def write_bytes(self, contents):
Path(self.path).write_bytes(contents)
def write_text(self, contents, encoding=None, errors=None):
Path(self.path).write_text(contents, encoding=encoding, errors=errors)
@pytest.fixture
def webdav_server(tmp_path_factory):
host, port = "localhost", 0
directory = os.fspath(tmp_path_factory.mktemp("http"))
dirmap = {"/": directory}
app = WsgiDAVApp(
{
"host": host,
"port": port,
"provider_mapping": dirmap,
"simple_dc": {"user_mapping": {"*": AUTH}},
}
)
server = make_server(host, port, app)
with run_server_on_thread(server) as httpd:
yield httpd
@pytest.fixture
def webdav(webdav_server):
url = Webdav.get_url(webdav_server.server_port)
ret = Webdav(url)
user, secrets = first(AUTH.items())
ret.config = {"url": url, "user": user, **secrets}
yield ret
+6 -3
View File
@@ -1,3 +1,5 @@
import pytest
from dvc.tree.webdav import WebDAVTree
# Test configuration
@@ -16,12 +18,13 @@ def test_init(dvc):
# Test username from configuration
def test_user(dvc):
config = {"url": url, "user": user}
@pytest.mark.parametrize(
"config", [{"url": url, "user": user}, {"url": userurl}]
)
def test_user(dvc, config):
tree = WebDAVTree(dvc, config)
assert tree.user == user
assert tree.path_info.user == user
# Test username extraction from url
+22 -12
View File
@@ -2,6 +2,7 @@ import hashlib
import os
import sys
import threading
from contextlib import contextmanager
from http import HTTPStatus
from http.server import HTTPServer
@@ -89,26 +90,35 @@ class TestRequestHandler(RangeRequestHandler):
self.end_headers()
@contextmanager
def run_server_on_thread(server):
thread = threading.Thread(target=server.serve_forever)
thread.daemon = True
thread.start()
yield server
server.socket.close()
server.shutdown()
server.server_close()
class StaticFileServer:
_lock = threading.Lock()
def __init__(self, directory):
from functools import partial
self._lock.acquire()
self._httpd = HTTPServer(
("localhost", 0), partial(TestRequestHandler, directory=directory),
)
self._thread = None
addr = ("localhost", 0)
req = partial(TestRequestHandler, directory=directory)
server = HTTPServer(addr, req)
self.runner = run_server_on_thread(server)
# pylint: disable=no-member
def __enter__(self):
self._thread = threading.Thread(target=self._httpd.serve_forever)
self._thread.daemon = True
self._thread.start()
return self._httpd
self._lock.acquire()
return self.runner.__enter__()
def __exit__(self, *args):
self._httpd.socket.close()
self._httpd.shutdown()
self._httpd.server_close()
self.runner.__exit__(*args)
self._lock.release()