mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 23:11:44 +01:00
Add test server and tests for webdav (#4827)
* Add test server and tests for webdav * Fix webdav unittests
This commit is contained in:
@@ -81,12 +81,6 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
|
||||
if self.user is None and self.path_info.user is not None:
|
||||
self.user = self.path_info.user
|
||||
|
||||
# If username specified add to path_info
|
||||
if self.user is not None:
|
||||
self.path_info.user = self.user
|
||||
else:
|
||||
self.path_info = None
|
||||
|
||||
# Webdav client
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
|
||||
@@ -150,6 +150,7 @@ tests_requirements = [
|
||||
"wget",
|
||||
"filelock",
|
||||
"black==19.10b0",
|
||||
"wsgidav",
|
||||
]
|
||||
|
||||
setup(
|
||||
|
||||
+13
-1
@@ -10,7 +10,17 @@ from dvc.path_info import URLInfo
|
||||
from dvc.utils.fs import remove
|
||||
from tests.unit.tree.test_repo import make_subrepo
|
||||
|
||||
cloud_names = ["s3", "gs", "azure", "gdrive", "oss", "ssh", "http", "hdfs"]
|
||||
cloud_names = [
|
||||
"s3",
|
||||
"gs",
|
||||
"azure",
|
||||
"gdrive",
|
||||
"oss",
|
||||
"ssh",
|
||||
"http",
|
||||
"hdfs",
|
||||
"webdav",
|
||||
]
|
||||
clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names]
|
||||
all_clouds = [pytest.lazy_fixture("local_cloud")] + clouds
|
||||
|
||||
@@ -72,6 +82,7 @@ def test_open(tmp_dir, dvc, remote):
|
||||
"http",
|
||||
"hdfs",
|
||||
"ssh",
|
||||
"webdav",
|
||||
]
|
||||
],
|
||||
)
|
||||
@@ -124,6 +135,7 @@ def test_open_granular(tmp_dir, dvc, remote):
|
||||
"ssh",
|
||||
"http",
|
||||
"hdfs",
|
||||
"webdav",
|
||||
]
|
||||
],
|
||||
indirect=True,
|
||||
|
||||
@@ -21,7 +21,16 @@ from dvc.utils.serialize import dump_yaml, load_yaml
|
||||
|
||||
all_clouds = [
|
||||
pytest.lazy_fixture(cloud)
|
||||
for cloud in ["s3", "gs", "azure", "gdrive", "ssh", "http", "hdfs"]
|
||||
for cloud in [
|
||||
"s3",
|
||||
"gs",
|
||||
"azure",
|
||||
"gdrive",
|
||||
"ssh",
|
||||
"http",
|
||||
"hdfs",
|
||||
"webdav",
|
||||
]
|
||||
] + [
|
||||
pytest.param(
|
||||
pytest.lazy_fixture("oss"),
|
||||
|
||||
@@ -14,6 +14,7 @@ from .oss import ( # noqa: F401
|
||||
real_oss,
|
||||
)
|
||||
from .s3 import S3, TEST_AWS_REPO_BUCKET, real_s3, s3 # noqa: F401
|
||||
from .webdav import Webdav, webdav, webdav_server # noqa: F401
|
||||
|
||||
from .gdrive import ( # noqa: F401; noqa: F401
|
||||
TEST_GDRIVE_REPO_BUCKET,
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# pylint:disable=abstract-method
|
||||
import os
|
||||
from pathlib import Path
|
||||
from wsgiref.simple_server import make_server
|
||||
|
||||
import pytest
|
||||
from funcy import first
|
||||
from wsgidav.wsgidav_app import WsgiDAVApp
|
||||
|
||||
from dvc.path_info import WebDAVURLInfo
|
||||
from dvc.utils.fs import makedirs
|
||||
from tests.utils.httpd import run_server_on_thread
|
||||
|
||||
from .base import Base
|
||||
|
||||
AUTH = {"user1": {"password": "password1"}}
|
||||
|
||||
|
||||
class Webdav(Base, WebDAVURLInfo):
|
||||
@staticmethod
|
||||
def get_url(port): # pylint: disable=arguments-differ
|
||||
return f"webdav://localhost:{port}"
|
||||
|
||||
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
|
||||
assert parents
|
||||
makedirs(self.path, exist_ok=exist_ok)
|
||||
|
||||
def write_bytes(self, contents):
|
||||
Path(self.path).write_bytes(contents)
|
||||
|
||||
def write_text(self, contents, encoding=None, errors=None):
|
||||
Path(self.path).write_text(contents, encoding=encoding, errors=errors)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def webdav_server(tmp_path_factory):
|
||||
host, port = "localhost", 0
|
||||
directory = os.fspath(tmp_path_factory.mktemp("http"))
|
||||
dirmap = {"/": directory}
|
||||
|
||||
app = WsgiDAVApp(
|
||||
{
|
||||
"host": host,
|
||||
"port": port,
|
||||
"provider_mapping": dirmap,
|
||||
"simple_dc": {"user_mapping": {"*": AUTH}},
|
||||
}
|
||||
)
|
||||
server = make_server(host, port, app)
|
||||
with run_server_on_thread(server) as httpd:
|
||||
yield httpd
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def webdav(webdav_server):
|
||||
url = Webdav.get_url(webdav_server.server_port)
|
||||
ret = Webdav(url)
|
||||
user, secrets = first(AUTH.items())
|
||||
ret.config = {"url": url, "user": user, **secrets}
|
||||
yield ret
|
||||
@@ -1,3 +1,5 @@
|
||||
import pytest
|
||||
|
||||
from dvc.tree.webdav import WebDAVTree
|
||||
|
||||
# Test configuration
|
||||
@@ -16,12 +18,13 @@ def test_init(dvc):
|
||||
|
||||
|
||||
# Test username from configuration
|
||||
def test_user(dvc):
|
||||
config = {"url": url, "user": user}
|
||||
@pytest.mark.parametrize(
|
||||
"config", [{"url": url, "user": user}, {"url": userurl}]
|
||||
)
|
||||
def test_user(dvc, config):
|
||||
tree = WebDAVTree(dvc, config)
|
||||
|
||||
assert tree.user == user
|
||||
assert tree.path_info.user == user
|
||||
|
||||
|
||||
# Test username extraction from url
|
||||
|
||||
+22
-12
@@ -2,6 +2,7 @@ import hashlib
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from http import HTTPStatus
|
||||
from http.server import HTTPServer
|
||||
|
||||
@@ -89,26 +90,35 @@ class TestRequestHandler(RangeRequestHandler):
|
||||
self.end_headers()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def run_server_on_thread(server):
|
||||
thread = threading.Thread(target=server.serve_forever)
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
||||
yield server
|
||||
|
||||
server.socket.close()
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
class StaticFileServer:
|
||||
_lock = threading.Lock()
|
||||
|
||||
def __init__(self, directory):
|
||||
from functools import partial
|
||||
|
||||
self._lock.acquire()
|
||||
self._httpd = HTTPServer(
|
||||
("localhost", 0), partial(TestRequestHandler, directory=directory),
|
||||
)
|
||||
self._thread = None
|
||||
addr = ("localhost", 0)
|
||||
req = partial(TestRequestHandler, directory=directory)
|
||||
server = HTTPServer(addr, req)
|
||||
self.runner = run_server_on_thread(server)
|
||||
|
||||
# pylint: disable=no-member
|
||||
def __enter__(self):
|
||||
self._thread = threading.Thread(target=self._httpd.serve_forever)
|
||||
self._thread.daemon = True
|
||||
self._thread.start()
|
||||
return self._httpd
|
||||
self._lock.acquire()
|
||||
return self.runner.__enter__()
|
||||
|
||||
def __exit__(self, *args):
|
||||
self._httpd.socket.close()
|
||||
self._httpd.shutdown()
|
||||
self._httpd.server_close()
|
||||
self.runner.__exit__(*args)
|
||||
self._lock.release()
|
||||
|
||||
Reference in New Issue
Block a user