From d1881fd1b11f0df5ca6cdd0fa5a481c14a0f8598 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 12 May 2019 16:39:40 +0300 Subject: [PATCH] remote: separate http and https Fixes #1988 Signed-off-by: Ruslan Kuprieiev --- dvc/dependency/__init__.py | 4 +++- dvc/dependency/http.py | 4 ++-- dvc/dependency/https.py | 8 ++++++++ dvc/path/__init__.py | 1 + dvc/path/http.py | 9 ++------- dvc/path/https.py | 6 ++++++ dvc/path/utils.py | 2 ++ dvc/remote/__init__.py | 2 ++ dvc/remote/http.py | 10 +++++----- dvc/remote/https.py | 10 ++++++++++ 10 files changed, 41 insertions(+), 15 deletions(-) create mode 100644 dvc/dependency/https.py create mode 100644 dvc/path/https.py create mode 100644 dvc/remote/https.py diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 86a41b3c5..085a46be6 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -10,6 +10,7 @@ from dvc.dependency.local import DependencyLOCAL from dvc.dependency.hdfs import DependencyHDFS from dvc.dependency.ssh import DependencySSH from dvc.dependency.http import DependencyHTTP +from dvc.dependency.https import DependencyHTTPS from dvc.remote import Remote @@ -17,6 +18,7 @@ DEPS = [ DependencyGS, DependencyHDFS, DependencyHTTP, + DependencyHTTPS, DependencyS3, DependencySSH, # NOTE: DependencyLOCAL is the default choice @@ -29,7 +31,7 @@ DEP_MAP = { "gs": DependencyGS, "hdfs": DependencyHDFS, "http": DependencyHTTP, - "https": DependencyHTTP, + "https": DependencyHTTPS, } diff --git a/dvc/dependency/http.py b/dvc/dependency/http.py index 646ca9817..c90a760f3 100644 --- a/dvc/dependency/http.py +++ b/dvc/dependency/http.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -from dvc.path.http import HTTPPathInfo +from dvc.path.utils import PathInfo from dvc.utils.compat import urlparse, urljoin from dvc.output.base import OutputBase from dvc.remote.http import RemoteHTTP @@ -17,4 +17,4 @@ class DependencyHTTP(DependencyBase, OutputBase): if path.startswith("remote"): path = urljoin(self.remote.cache_dir, urlparse(path).path) - self.path_info = HTTPPathInfo(url=self.url, path=path) + self.path_info = PathInfo(self.scheme, url=self.url, path=path) diff --git a/dvc/dependency/https.py b/dvc/dependency/https.py new file mode 100644 index 000000000..91233bede --- /dev/null +++ b/dvc/dependency/https.py @@ -0,0 +1,8 @@ +from __future__ import unicode_literals + +from dvc.remote.https import RemoteHTTPS +from .http import DependencyHTTP + + +class DependencyHTTPS(DependencyHTTP): + REMOTE = RemoteHTTPS diff --git a/dvc/path/__init__.py b/dvc/path/__init__.py index df7c6d3ef..aa0433a85 100644 --- a/dvc/path/__init__.py +++ b/dvc/path/__init__.py @@ -7,6 +7,7 @@ class Schemes: S3 = "s3" AZURE = "azure" HTTP = "http" + HTTPS = "https" GS = "gs" LOCAL = "local" OSS = "oss" diff --git a/dvc/path/http.py b/dvc/path/http.py index c9b9bc4a5..ed71c609f 100644 --- a/dvc/path/http.py +++ b/dvc/path/http.py @@ -1,14 +1,9 @@ from dvc.path import BasePathInfo, Schemes -from dvc.utils.compat import urlparse, urlunsplit +from dvc.utils.compat import urlunsplit class HTTPPathInfo(BasePathInfo): - @property - def scheme(self): - if self.path: - return urlparse(self.path).scheme - else: - return Schemes.HTTP + scheme = Schemes.HTTP def __str__(self): if not self.url: diff --git a/dvc/path/https.py b/dvc/path/https.py new file mode 100644 index 000000000..957f02123 --- /dev/null +++ b/dvc/path/https.py @@ -0,0 +1,6 @@ +from . import Schemes +from .http import HTTPPathInfo + + +class HTTPSPathInfo(HTTPPathInfo): + scheme = Schemes.HTTPS diff --git a/dvc/path/utils.py b/dvc/path/utils.py index cf421ede7..0c7f084f1 100644 --- a/dvc/path/utils.py +++ b/dvc/path/utils.py @@ -3,6 +3,7 @@ from dvc.path.azure import AzurePathInfo from dvc.path.gs import GSPathInfo from dvc.path.hdfs import HDFSPathInfo from dvc.path.http import HTTPPathInfo +from dvc.path.https import HTTPSPathInfo from dvc.path.local import LocalPathInfo from dvc.path.oss import OSSPathInfo from dvc.path.s3 import S3PathInfo @@ -14,6 +15,7 @@ PATH_MAP = { Schemes.S3: S3PathInfo, Schemes.AZURE: AzurePathInfo, Schemes.HTTP: HTTPPathInfo, + Schemes.HTTPS: HTTPSPathInfo, Schemes.GS: GSPathInfo, Schemes.LOCAL: LocalPathInfo, Schemes.OSS: OSSPathInfo, diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index 5c35c038d..5dadbc03d 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -7,6 +7,7 @@ from dvc.remote.local import RemoteLOCAL from dvc.remote.s3 import RemoteS3 from dvc.remote.ssh import RemoteSSH from dvc.remote.http import RemoteHTTP +from dvc.remote.https import RemoteHTTPS from dvc.remote.oss import RemoteOSS @@ -15,6 +16,7 @@ REMOTES = [ RemoteGS, RemoteHDFS, RemoteHTTP, + RemoteHTTPS, RemoteS3, RemoteSSH, RemoteOSS, diff --git a/dvc/remote/http.py b/dvc/remote/http.py index 64d0670c3..6ddd73c7c 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals from dvc.path import Schemes -from dvc.path.http import HTTPPathInfo +from dvc.path.utils import PathInfo from dvc.utils.compat import open, makedirs import os @@ -34,7 +34,7 @@ class ProgressBarCallback(object): class RemoteHTTP(RemoteBase): scheme = Schemes.HTTP - REGEX = r"^https?://.*$" + REGEX = r"^http://.*$" REQUEST_TIMEOUT = 10 CHUNK_SIZE = 2 ** 16 PARAM_CHECKSUM = "etag" @@ -44,7 +44,7 @@ class RemoteHTTP(RemoteBase): self.cache_dir = config.get(Config.SECTION_REMOTE_URL) self.url = self.cache_dir - self.path_info = HTTPPathInfo() + self.path_info = PathInfo(self.scheme) @property def prefix(self): @@ -61,7 +61,7 @@ class RemoteHTTP(RemoteBase): names = self._verify_path_args(to_infos, from_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): - if from_info.scheme not in ["http", "https"]: + if from_info.scheme != self.scheme: raise NotImplementedError if to_info.scheme != "local": @@ -99,7 +99,7 @@ class RemoteHTTP(RemoteBase): def exists(self, path_info): assert not isinstance(path_info, list) - assert path_info.scheme in ["http", "https"] + assert path_info.scheme == self.scheme return bool(self._request("HEAD", path_info.path)) def cache_exists(self, md5s): diff --git a/dvc/remote/https.py b/dvc/remote/https.py new file mode 100644 index 000000000..0653b0a1e --- /dev/null +++ b/dvc/remote/https.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals + +from dvc.path import Schemes + +from .http import RemoteHTTP + + +class RemoteHTTPS(RemoteHTTP): + scheme = Schemes.HTTPS + REGEX = r"^https://.*$"