remote: separate http and https

Fixes #1988

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/dependency/__init__.py

@@ -10,6 +10,7 @@ from dvc.dependency.local import DependencyLOCAL
 from dvc.dependency.hdfs import DependencyHDFS
 from dvc.dependency.ssh import DependencySSH
 from dvc.dependency.http import DependencyHTTP
+from dvc.dependency.https import DependencyHTTPS
 
 from dvc.remote import Remote
 
@@ -17,6 +18,7 @@ DEPS = [
     DependencyGS,
     DependencyHDFS,
     DependencyHTTP,
+    DependencyHTTPS,
     DependencyS3,
     DependencySSH,
     # NOTE: DependencyLOCAL is the default choice
@@ -29,7 +31,7 @@ DEP_MAP = {
     "gs": DependencyGS,
     "hdfs": DependencyHDFS,
     "http": DependencyHTTP,
-    "https": DependencyHTTP,
+    "https": DependencyHTTPS,
 }
 
 

dvc/dependency/http.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from dvc.path.http import HTTPPathInfo
+from dvc.path.utils import PathInfo
 from dvc.utils.compat import urlparse, urljoin
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP
@@ -17,4 +17,4 @@ class DependencyHTTP(DependencyBase, OutputBase):
         if path.startswith("remote"):
             path = urljoin(self.remote.cache_dir, urlparse(path).path)
 
-        self.path_info = HTTPPathInfo(url=self.url, path=path)
+        self.path_info = PathInfo(self.scheme, url=self.url, path=path)

dvc/dependency/https.py

@@ -0,0 +1,8 @@
+from __future__ import unicode_literals
+
+from dvc.remote.https import RemoteHTTPS
+from .http import DependencyHTTP
+
+
+class DependencyHTTPS(DependencyHTTP):
+    REMOTE = RemoteHTTPS

dvc/path/__init__.py

@@ -7,6 +7,7 @@ class Schemes:
     S3 = "s3"
     AZURE = "azure"
     HTTP = "http"
+    HTTPS = "https"
     GS = "gs"
     LOCAL = "local"
     OSS = "oss"

dvc/path/http.py

@@ -1,14 +1,9 @@
 from dvc.path import BasePathInfo, Schemes
-from dvc.utils.compat import urlparse, urlunsplit
+from dvc.utils.compat import urlunsplit
 
 
 class HTTPPathInfo(BasePathInfo):
-    @property
-    def scheme(self):
-        if self.path:
-            return urlparse(self.path).scheme
-        else:
-            return Schemes.HTTP
+    scheme = Schemes.HTTP
 
     def __str__(self):
         if not self.url:

dvc/path/https.py

@@ -0,0 +1,6 @@
+from . import Schemes
+from .http import HTTPPathInfo
+
+
+class HTTPSPathInfo(HTTPPathInfo):
+    scheme = Schemes.HTTPS

dvc/path/utils.py

@@ -3,6 +3,7 @@ from dvc.path.azure import AzurePathInfo
 from dvc.path.gs import GSPathInfo
 from dvc.path.hdfs import HDFSPathInfo
 from dvc.path.http import HTTPPathInfo
+from dvc.path.https import HTTPSPathInfo
 from dvc.path.local import LocalPathInfo
 from dvc.path.oss import OSSPathInfo
 from dvc.path.s3 import S3PathInfo
@@ -14,6 +15,7 @@ PATH_MAP = {
     Schemes.S3: S3PathInfo,
     Schemes.AZURE: AzurePathInfo,
     Schemes.HTTP: HTTPPathInfo,
+    Schemes.HTTPS: HTTPSPathInfo,
     Schemes.GS: GSPathInfo,
     Schemes.LOCAL: LocalPathInfo,
     Schemes.OSS: OSSPathInfo,

dvc/remote/__init__.py

@@ -7,6 +7,7 @@ from dvc.remote.local import RemoteLOCAL
 from dvc.remote.s3 import RemoteS3
 from dvc.remote.ssh import RemoteSSH
 from dvc.remote.http import RemoteHTTP
+from dvc.remote.https import RemoteHTTPS
 from dvc.remote.oss import RemoteOSS
 
 
@@ -15,6 +16,7 @@ REMOTES = [
     RemoteGS,
     RemoteHDFS,
     RemoteHTTP,
+    RemoteHTTPS,
     RemoteS3,
     RemoteSSH,
     RemoteOSS,

dvc/remote/http.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from dvc.path import Schemes
-from dvc.path.http import HTTPPathInfo
+from dvc.path.utils import PathInfo
 from dvc.utils.compat import open, makedirs
 
 import os
@@ -34,7 +34,7 @@ class ProgressBarCallback(object):
 
 class RemoteHTTP(RemoteBase):
     scheme = Schemes.HTTP
-    REGEX = r"^https?://.*$"
+    REGEX = r"^http://.*$"
     REQUEST_TIMEOUT = 10
     CHUNK_SIZE = 2 ** 16
     PARAM_CHECKSUM = "etag"
@@ -44,7 +44,7 @@ class RemoteHTTP(RemoteBase):
         self.cache_dir = config.get(Config.SECTION_REMOTE_URL)
         self.url = self.cache_dir
 
-        self.path_info = HTTPPathInfo()
+        self.path_info = PathInfo(self.scheme)
 
     @property
     def prefix(self):
@@ -61,7 +61,7 @@ class RemoteHTTP(RemoteBase):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme not in ["http", "https"]:
+            if from_info.scheme != self.scheme:
                 raise NotImplementedError
 
             if to_info.scheme != "local":
@@ -99,7 +99,7 @@ class RemoteHTTP(RemoteBase):
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info.scheme in ["http", "https"]
+        assert path_info.scheme == self.scheme
         return bool(self._request("HEAD", path_info.path))
 
     def cache_exists(self, md5s):

dvc/remote/https.py

@@ -0,0 +1,10 @@
+from __future__ import unicode_literals
+
+from dvc.path import Schemes
+
+from .http import RemoteHTTP
+
+
+class RemoteHTTPS(RemoteHTTP):
+    scheme = Schemes.HTTPS
+    REGEX = r"^https://.*$"