diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 73649f06b..490a60bfa 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -8,7 +8,7 @@ from dvc.config import Config, ConfigError from dvc.remote import Remote from dvc.remote.s3 import RemoteS3 from dvc.remote.gs import RemoteGS -from dvc.remote.azure import RemoteAzure +from dvc.remote.azure import RemoteAZURE from dvc.remote.oss import RemoteOSS from dvc.remote.ssh import RemoteSSH from dvc.remote.hdfs import RemoteHDFS @@ -34,7 +34,7 @@ class DataCloud(object): CLOUD_MAP = { "aws": RemoteS3, "gcp": RemoteGS, - "azure": RemoteAzure, + "azure": RemoteAZURE, "oss": RemoteOSS, "ssh": RemoteSSH, "hdfs": RemoteHDFS, @@ -124,7 +124,7 @@ class DataCloud(object): Args: targets (list): list of targets to push to the cloud. jobs (int): number of jobs that can be running simultaneously. - remote (dvc.remote.base.RemoteBase): optional remote to push to. + remote (dvc.remote.base.RemoteBASE): optional remote to push to. By default remote from core.remote config option is used. show_checksums (bool): show checksums instead of file names in information messages. @@ -142,7 +142,7 @@ class DataCloud(object): Args: targets (list): list of targets to pull from the cloud. jobs (int): number of jobs that can be running simultaneously. - remote (dvc.remote.base.RemoteBase): optional remote to pull from. + remote (dvc.remote.base.RemoteBASE): optional remote to pull from. By default remote from core.remote config option is used. show_checksums (bool): show checksums instead of file names in information messages. @@ -160,7 +160,7 @@ class DataCloud(object): Args: targets (list): list of targets to check status for. jobs (int): number of jobs that can be running simultaneously. - remote (dvc.remote.base.RemoteBase): optional remote to compare + remote (dvc.remote.base.RemoteBASE): optional remote to compare targets to. By default remote from core.remote config option is used. show_checksums (bool): show checksums instead of file names in diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 86a41b3c5..5c5ba67d4 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals import schema +from dvc.scheme import Schemes + import dvc.output as output from dvc.output.base import OutputBase from dvc.dependency.s3 import DependencyS3 @@ -10,6 +12,7 @@ from dvc.dependency.local import DependencyLOCAL from dvc.dependency.hdfs import DependencyHDFS from dvc.dependency.ssh import DependencySSH from dvc.dependency.http import DependencyHTTP +from dvc.dependency.https import DependencyHTTPS from dvc.remote import Remote @@ -17,19 +20,20 @@ DEPS = [ DependencyGS, DependencyHDFS, DependencyHTTP, + DependencyHTTPS, DependencyS3, DependencySSH, # NOTE: DependencyLOCAL is the default choice ] DEP_MAP = { - "local": DependencyLOCAL, - "ssh": DependencySSH, - "s3": DependencyS3, - "gs": DependencyGS, - "hdfs": DependencyHDFS, - "http": DependencyHTTP, - "https": DependencyHTTP, + Schemes.LOCAL: DependencyLOCAL, + Schemes.SSH: DependencySSH, + Schemes.S3: DependencyS3, + Schemes.GS: DependencyGS, + Schemes.HDFS: DependencyHDFS, + Schemes.HTTP: DependencyHTTP, + Schemes.HTTPS: DependencyHTTPS, } diff --git a/dvc/dependency/http.py b/dvc/dependency/http.py index 646ca9817..24ddfe497 100644 --- a/dvc/dependency/http.py +++ b/dvc/dependency/http.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -from dvc.path.http import HTTPPathInfo +from dvc.path import Path from dvc.utils.compat import urlparse, urljoin from dvc.output.base import OutputBase from dvc.remote.http import RemoteHTTP @@ -17,4 +17,4 @@ class DependencyHTTP(DependencyBase, OutputBase): if path.startswith("remote"): path = urljoin(self.remote.cache_dir, urlparse(path).path) - self.path_info = HTTPPathInfo(url=self.url, path=path) + self.path_info = Path(self.scheme, url=self.url, path=path) diff --git a/dvc/dependency/https.py b/dvc/dependency/https.py new file mode 100644 index 000000000..91233bede --- /dev/null +++ b/dvc/dependency/https.py @@ -0,0 +1,8 @@ +from __future__ import unicode_literals + +from dvc.remote.https import RemoteHTTPS +from .http import DependencyHTTP + + +class DependencyHTTPS(DependencyHTTP): + REMOTE = RemoteHTTPS diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index 54e76c52d..60db6a50c 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals import schema -from dvc.path import Schemes +from dvc.scheme import Schemes from dvc.utils.compat import urlparse, str from dvc.output.base import OutputBase diff --git a/dvc/output/base.py b/dvc/output/base.py index 9e9be29c6..784c12e99 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -8,7 +8,7 @@ from schema import Or, Optional from dvc.exceptions import DvcException from dvc.utils.compat import str -from dvc.remote.base import RemoteBase +from dvc.remote.base import RemoteBASE logger = logging.getLogger(__name__) @@ -35,7 +35,7 @@ class OutputAlreadyTrackedError(DvcException): class OutputBase(object): IS_DEPENDENCY = False - REMOTE = RemoteBase + REMOTE = RemoteBASE PARAM_PATH = "path" PARAM_CACHE = "cache" diff --git a/dvc/output/hdfs.py b/dvc/output/hdfs.py index dd01d7488..94225c90b 100644 --- a/dvc/output/hdfs.py +++ b/dvc/output/hdfs.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals import posixpath -from dvc.path.hdfs import HDFSPathInfo +from dvc.path.hdfs import PathHDFS from dvc.utils.compat import urlparse from dvc.output.base import OutputBase from dvc.remote.hdfs import RemoteHDFS @@ -35,4 +35,4 @@ class OutputHDFS(OutputBase): if remote: path = posixpath.join(remote.url, urlparse(path).path.lstrip("/")) user = remote.user if remote else self.group("user") - self.path_info = HDFSPathInfo(user=user, url=self.url, path=path) + self.path_info = PathHDFS(user=user, url=self.url, path=path) diff --git a/dvc/output/local.py b/dvc/output/local.py index 848e21fd2..1064009ca 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import os import logging -from dvc.path.local import LocalPathInfo +from dvc.path.local import PathLOCAL from dvc.utils.compat import urlparse from dvc.istextfile import istextfile from dvc.exceptions import DvcException @@ -50,7 +50,7 @@ class OutputLOCAL(OutputBase): p = os.path.join(stage.wdir, p) p = os.path.abspath(os.path.normpath(p)) - self.path_info = LocalPathInfo(url=self.url, path=p) + self.path_info = PathLOCAL(url=self.url, path=p) def __str__(self): return self.rel_path diff --git a/dvc/output/s3.py b/dvc/output/s3.py index caa039970..4c5cc4724 100644 --- a/dvc/output/s3.py +++ b/dvc/output/s3.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals import posixpath -from dvc.path.utils import PathInfo +from dvc.path import Path from dvc.remote.s3 import RemoteS3 from dvc.utils.compat import urlparse from dvc.output.base import OutputBase @@ -37,6 +37,6 @@ class OutputS3(OutputBase): if remote: path = posixpath.join(remote.prefix, path) - self.path_info = PathInfo( + self.path_info = Path( self.scheme, bucket=bucket, path=path, url=self.url ) diff --git a/dvc/output/ssh.py b/dvc/output/ssh.py index 184b41218..f461175fd 100644 --- a/dvc/output/ssh.py +++ b/dvc/output/ssh.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import getpass import posixpath -from dvc.path.ssh import SSHPathInfo +from dvc.path.ssh import PathSSH from dvc.utils.compat import urlparse from dvc.output.base import OutputBase from dvc.remote.ssh import RemoteSSH @@ -49,6 +49,6 @@ class OutputSSH(OutputBase): else: path = parsed.path - self.path_info = SSHPathInfo( + self.path_info = PathSSH( host=host, user=user, port=port, url=self.url, path=path ) diff --git a/dvc/path/__init__.py b/dvc/path/__init__.py index df7c6d3ef..0af623dee 100644 --- a/dvc/path/__init__.py +++ b/dvc/path/__init__.py @@ -1,34 +1,29 @@ -from dvc.utils.compat import urlunsplit +from dvc.scheme import Schemes + +from dvc.path.azure import PathAZURE +from dvc.path.gs import PathGS +from dvc.path.hdfs import PathHDFS +from dvc.path.http import PathHTTP +from dvc.path.https import PathHTTPS +from dvc.path.local import PathLOCAL +from dvc.path.oss import PathOSS +from dvc.path.s3 import PathS3 +from dvc.path.ssh import PathSSH -class Schemes: - SSH = "ssh" - HDFS = "hdfs" - S3 = "s3" - AZURE = "azure" - HTTP = "http" - GS = "gs" - LOCAL = "local" - OSS = "oss" +PATH_MAP = { + Schemes.SSH: PathSSH, + Schemes.HDFS: PathHDFS, + Schemes.S3: PathS3, + Schemes.AZURE: PathAZURE, + Schemes.HTTP: PathHTTP, + Schemes.HTTPS: PathHTTPS, + Schemes.GS: PathGS, + Schemes.LOCAL: PathLOCAL, + Schemes.OSS: PathOSS, +} -class BasePathInfo(object): - scheme = None - - def __init__(self, url=None, path=None): - self.url = url - self.path = path - - def __str__(self): - return self.url - - -class DefaultCloudPathInfo(BasePathInfo): - def __init__(self, bucket, url=None, path=None): - super(DefaultCloudPathInfo, self).__init__(url, path) - self.bucket = bucket - - def __str__(self): - if not self.url: - return urlunsplit((self.scheme, self.bucket, self.path, "", "")) - return self.url +def Path(scheme, *args, **kwargs): + cls = PATH_MAP[scheme] + return cls(*args, **kwargs) diff --git a/dvc/path/azure.py b/dvc/path/azure.py index 956d91581..019cf0af4 100644 --- a/dvc/path/azure.py +++ b/dvc/path/azure.py @@ -1,5 +1,6 @@ -from dvc.path import Schemes, DefaultCloudPathInfo +from dvc.scheme import Schemes +from .base import PathCloudBASE -class AzurePathInfo(DefaultCloudPathInfo): +class PathAZURE(PathCloudBASE): scheme = Schemes.AZURE diff --git a/dvc/path/base.py b/dvc/path/base.py new file mode 100644 index 000000000..616df4041 --- /dev/null +++ b/dvc/path/base.py @@ -0,0 +1,23 @@ +from dvc.utils.compat import urlunsplit + + +class PathBASE(object): + scheme = None + + def __init__(self, url=None, path=None): + self.url = url + self.path = path + + def __str__(self): + return self.url + + +class PathCloudBASE(PathBASE): + def __init__(self, bucket, url=None, path=None): + super(PathCloudBASE, self).__init__(url, path) + self.bucket = bucket + + def __str__(self): + if not self.url: + return urlunsplit((self.scheme, self.bucket, self.path, "", "")) + return self.url diff --git a/dvc/path/gs.py b/dvc/path/gs.py index 087c584be..f2befca6c 100644 --- a/dvc/path/gs.py +++ b/dvc/path/gs.py @@ -1,5 +1,6 @@ -from dvc.path import Schemes, DefaultCloudPathInfo +from dvc.scheme import Schemes +from .base import PathCloudBASE -class GSPathInfo(DefaultCloudPathInfo): +class PathGS(PathCloudBASE): scheme = Schemes.GS diff --git a/dvc/path/hdfs.py b/dvc/path/hdfs.py index d5ce13314..628bd2c29 100644 --- a/dvc/path/hdfs.py +++ b/dvc/path/hdfs.py @@ -1,12 +1,13 @@ from dvc.utils.compat import urlunsplit -from dvc.path import BasePathInfo, Schemes +from dvc.scheme import Schemes +from .base import PathBASE -class HDFSPathInfo(BasePathInfo): +class PathHDFS(PathBASE): scheme = Schemes.HDFS def __init__(self, user, url=None, path=None): - super(HDFSPathInfo, self).__init__(url, path) + super(PathHDFS, self).__init__(url, path) self.user = user def __str__(self): diff --git a/dvc/path/http.py b/dvc/path/http.py index c9b9bc4a5..579db8198 100644 --- a/dvc/path/http.py +++ b/dvc/path/http.py @@ -1,14 +1,11 @@ -from dvc.path import BasePathInfo, Schemes -from dvc.utils.compat import urlparse, urlunsplit +from dvc.scheme import Schemes +from dvc.utils.compat import urlunsplit + +from .base import PathBASE -class HTTPPathInfo(BasePathInfo): - @property - def scheme(self): - if self.path: - return urlparse(self.path).scheme - else: - return Schemes.HTTP +class PathHTTP(PathBASE): + scheme = Schemes.HTTP def __str__(self): if not self.url: diff --git a/dvc/path/https.py b/dvc/path/https.py new file mode 100644 index 000000000..b631045eb --- /dev/null +++ b/dvc/path/https.py @@ -0,0 +1,6 @@ +from dvc.scheme import Schemes +from .http import PathHTTP + + +class PathHTTPS(PathHTTP): + scheme = Schemes.HTTPS diff --git a/dvc/path/local.py b/dvc/path/local.py index cbde5c2a4..4d90364c5 100644 --- a/dvc/path/local.py +++ b/dvc/path/local.py @@ -1,9 +1,10 @@ import os -from dvc.path import BasePathInfo, Schemes +from dvc.scheme import Schemes +from .base import PathBASE -class LocalPathInfo(BasePathInfo): +class PathLOCAL(PathBASE): scheme = Schemes.LOCAL def __str__(self): diff --git a/dvc/path/oss.py b/dvc/path/oss.py index 606007b26..67cb3dd19 100644 --- a/dvc/path/oss.py +++ b/dvc/path/oss.py @@ -1,5 +1,6 @@ -from dvc.path import Schemes, DefaultCloudPathInfo +from dvc.scheme import Schemes +from .base import PathCloudBASE -class OSSPathInfo(DefaultCloudPathInfo): +class PathOSS(PathCloudBASE): scheme = Schemes.OSS diff --git a/dvc/path/s3.py b/dvc/path/s3.py index a1c04fa75..df46e87e3 100644 --- a/dvc/path/s3.py +++ b/dvc/path/s3.py @@ -1,5 +1,6 @@ -from dvc.path import Schemes, DefaultCloudPathInfo +from dvc.scheme import Schemes +from .base import PathCloudBASE -class S3PathInfo(DefaultCloudPathInfo): +class PathS3(PathCloudBASE): scheme = Schemes.S3 diff --git a/dvc/path/ssh.py b/dvc/path/ssh.py index 3c9c2e185..13f0b8508 100644 --- a/dvc/path/ssh.py +++ b/dvc/path/ssh.py @@ -1,12 +1,13 @@ from dvc.utils.compat import urlunsplit -from dvc.path import BasePathInfo, Schemes +from dvc.scheme import Schemes +from .base import PathBASE -class SSHPathInfo(BasePathInfo): +class PathSSH(PathBASE): scheme = Schemes.SSH def __init__(self, host, user, port, url=None, path=None): - super(SSHPathInfo, self).__init__(url, path) + super(PathSSH, self).__init__(url, path) self.host = host self.user = user self.port = port diff --git a/dvc/path/utils.py b/dvc/path/utils.py deleted file mode 100644 index cf421ede7..000000000 --- a/dvc/path/utils.py +++ /dev/null @@ -1,25 +0,0 @@ -from dvc.path import Schemes -from dvc.path.azure import AzurePathInfo -from dvc.path.gs import GSPathInfo -from dvc.path.hdfs import HDFSPathInfo -from dvc.path.http import HTTPPathInfo -from dvc.path.local import LocalPathInfo -from dvc.path.oss import OSSPathInfo -from dvc.path.s3 import S3PathInfo -from dvc.path.ssh import SSHPathInfo - -PATH_MAP = { - Schemes.SSH: SSHPathInfo, - Schemes.HDFS: HDFSPathInfo, - Schemes.S3: S3PathInfo, - Schemes.AZURE: AzurePathInfo, - Schemes.HTTP: HTTPPathInfo, - Schemes.GS: GSPathInfo, - Schemes.LOCAL: LocalPathInfo, - Schemes.OSS: OSSPathInfo, -} - - -def PathInfo(scheme, *args, **kwargs): - cls = PATH_MAP[scheme] - return cls(*args, **kwargs) diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index 5c35c038d..4eb124199 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -1,20 +1,22 @@ from __future__ import unicode_literals -from dvc.remote.azure import RemoteAzure +from dvc.remote.azure import RemoteAZURE from dvc.remote.gs import RemoteGS from dvc.remote.hdfs import RemoteHDFS from dvc.remote.local import RemoteLOCAL from dvc.remote.s3 import RemoteS3 from dvc.remote.ssh import RemoteSSH from dvc.remote.http import RemoteHTTP +from dvc.remote.https import RemoteHTTPS from dvc.remote.oss import RemoteOSS REMOTES = [ - RemoteAzure, + RemoteAZURE, RemoteGS, RemoteHDFS, RemoteHTTP, + RemoteHTTPS, RemoteS3, RemoteSSH, RemoteOSS, diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 263c2f724..df86d7540 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -5,8 +5,8 @@ import os import re import logging -from dvc.path import Schemes -from dvc.path.azure import AzurePathInfo +from dvc.scheme import Schemes +from dvc.path.azure import PathAZURE try: from azure.storage.blob import BlockBlobService @@ -18,7 +18,7 @@ from dvc.utils import tmp_fname, move from dvc.utils.compat import urlparse, makedirs from dvc.progress import progress from dvc.config import Config -from dvc.remote.base import RemoteBase +from dvc.remote.base import RemoteBASE logger = logging.getLogger(__name__) @@ -32,7 +32,7 @@ class Callback(object): progress.update_target(self.name, current, total) -class RemoteAzure(RemoteBase): +class RemoteAZURE(RemoteBASE): scheme = Schemes.AZURE REGEX = ( r"azure://((?P[^=;]*)?|(" @@ -45,7 +45,7 @@ class RemoteAzure(RemoteBase): COPY_POLL_SECONDS = 5 def __init__(self, repo, config): - super(RemoteAzure, self).__init__(repo, config) + super(RemoteAZURE, self).__init__(repo, config) self.url = config.get(Config.SECTION_REMOTE_URL, "azure://") match = re.match(self.REGEX, self.url) # backward compatibility @@ -72,7 +72,7 @@ class RemoteAzure(RemoteBase): raise ValueError("azure storage connection string missing") self.__blob_service = None - self.path_info = AzurePathInfo(bucket=self.bucket) + self.path_info = PathAZURE(bucket=self.bucket) @property def blob_service(self): diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 766933686..64f8bae6c 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from copy import copy -from dvc.path.local import LocalPathInfo +from dvc.path.local import PathLOCAL from dvc.utils.compat import str import os @@ -66,7 +66,7 @@ class RemoteMissingDepsError(DvcException): pass -class RemoteBase(object): +class RemoteBASE(object): scheme = None REGEX = None REQUIRES = {} @@ -211,7 +211,7 @@ class RemoteBase(object): with open(tmp, "w+") as fobj: json.dump(dir_info, fobj, sort_keys=True) - from_info = LocalPathInfo(path=tmp) + from_info = PathLOCAL(path=tmp) self.cache.upload([from_info], [to_info], no_progress_bar=True) checksum = self.get_file_checksum(to_info) + self.CHECKSUM_DIR_SUFFIX @@ -235,7 +235,7 @@ class RemoteBase(object): fobj = tempfile.NamedTemporaryFile(delete=False) path = fobj.name - to_info = LocalPathInfo(path=path) + to_info = PathLOCAL(path=path) self.cache.download([path_info], [to_info], no_progress_bar=True) try: diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index b41aff3e6..2e747f1dd 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -3,8 +3,8 @@ from __future__ import unicode_literals import os import logging -from dvc.path import Schemes -from dvc.path.gs import GSPathInfo +from dvc.scheme import Schemes +from dvc.path.gs import PathGS try: from google.cloud import storage @@ -13,7 +13,7 @@ except ImportError: from dvc.utils import tmp_fname, move from dvc.utils.compat import urlparse, makedirs -from dvc.remote.base import RemoteBase +from dvc.remote.base import RemoteBASE from dvc.config import Config from dvc.progress import progress from dvc.exceptions import DvcException @@ -22,7 +22,7 @@ from dvc.exceptions import DvcException logger = logging.getLogger(__name__) -class RemoteGS(RemoteBase): +class RemoteGS(RemoteBASE): scheme = Schemes.GS REGEX = r"^gs://(?P.*)$" REQUIRES = {"google.cloud.storage": storage} @@ -41,7 +41,7 @@ class RemoteGS(RemoteBase): self.bucket = parsed.netloc self.prefix = parsed.path.lstrip("/") - self.path_info = GSPathInfo(bucket=self.bucket) + self.path_info = PathGS(bucket=self.bucket) @staticmethod def compat_config(config): diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index b6922c1db..b8a3b22cc 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -8,16 +8,16 @@ import logging from subprocess import Popen, PIPE from dvc.config import Config -from dvc.path import Schemes -from dvc.path.hdfs import HDFSPathInfo -from dvc.remote.base import RemoteBase, RemoteCmdError +from dvc.scheme import Schemes +from dvc.path.hdfs import PathHDFS +from dvc.remote.base import RemoteBASE, RemoteCmdError from dvc.utils import fix_env, tmp_fname logger = logging.getLogger(__name__) -class RemoteHDFS(RemoteBase): +class RemoteHDFS(RemoteBASE): scheme = Schemes.HDFS REGEX = r"^hdfs://((?P.*)@)?.*$" PARAM_CHECKSUM = "checksum" @@ -32,7 +32,7 @@ class RemoteHDFS(RemoteBase): Config.SECTION_REMOTE_USER, getpass.getuser() ) - self.path_info = HDFSPathInfo(user=self.user) + self.path_info = PathHDFS(user=self.user) def hadoop_fs(self, cmd, user=None): cmd = "hadoop fs -" + cmd diff --git a/dvc/remote/http.py b/dvc/remote/http.py index 64d0670c3..6af541104 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals -from dvc.path import Schemes -from dvc.path.http import HTTPPathInfo +from dvc.scheme import Schemes +from dvc.path import Path from dvc.utils.compat import open, makedirs import os @@ -12,7 +12,7 @@ import logging from dvc.progress import progress from dvc.exceptions import DvcException from dvc.config import Config -from dvc.remote.base import RemoteBase +from dvc.remote.base import RemoteBASE from dvc.utils import move @@ -32,9 +32,9 @@ class ProgressBarCallback(object): progress.update_target(self.name, self.current, self.total) -class RemoteHTTP(RemoteBase): +class RemoteHTTP(RemoteBASE): scheme = Schemes.HTTP - REGEX = r"^https?://.*$" + REGEX = r"^http://.*$" REQUEST_TIMEOUT = 10 CHUNK_SIZE = 2 ** 16 PARAM_CHECKSUM = "etag" @@ -44,7 +44,7 @@ class RemoteHTTP(RemoteBase): self.cache_dir = config.get(Config.SECTION_REMOTE_URL) self.url = self.cache_dir - self.path_info = HTTPPathInfo() + self.path_info = Path(self.scheme) @property def prefix(self): @@ -61,7 +61,7 @@ class RemoteHTTP(RemoteBase): names = self._verify_path_args(to_infos, from_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): - if from_info.scheme not in ["http", "https"]: + if from_info.scheme != self.scheme: raise NotImplementedError if to_info.scheme != "local": @@ -99,7 +99,7 @@ class RemoteHTTP(RemoteBase): def exists(self, path_info): assert not isinstance(path_info, list) - assert path_info.scheme in ["http", "https"] + assert path_info.scheme == self.scheme return bool(self._request("HEAD", path_info.path)) def cache_exists(self, md5s): diff --git a/dvc/remote/https.py b/dvc/remote/https.py new file mode 100644 index 000000000..e775a5161 --- /dev/null +++ b/dvc/remote/https.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals + +from dvc.scheme import Schemes + +from .http import RemoteHTTP + + +class RemoteHTTPS(RemoteHTTP): + scheme = Schemes.HTTPS + REGEX = r"^https://.*$" diff --git a/dvc/remote/local/__init__.py b/dvc/remote/local/__init__.py index 7d853b702..4638f723a 100644 --- a/dvc/remote/local/__init__.py +++ b/dvc/remote/local/__init__.py @@ -2,8 +2,9 @@ from __future__ import unicode_literals from copy import copy -from dvc.path import BasePathInfo, Schemes -from dvc.path.local import LocalPathInfo +from dvc.scheme import Schemes +from dvc.path.base import PathBASE +from dvc.path.local import PathLOCAL from dvc.remote.local.slow_link_detection import slow_link_guard from dvc.utils.compat import str, makedirs @@ -17,7 +18,7 @@ import logging from dvc.system import System from dvc.remote.base import ( - RemoteBase, + RemoteBASE, STATUS_MAP, STATUS_NEW, STATUS_DELETED, @@ -43,7 +44,7 @@ from dvc.utils.fs import get_mtime_and_size, get_inode logger = logging.getLogger(__name__) -class RemoteLOCAL(RemoteBase): +class RemoteLOCAL(RemoteBASE): scheme = Schemes.LOCAL REGEX = r"^(?P.*)$" PARAM_CHECKSUM = "md5" @@ -80,7 +81,7 @@ class RemoteLOCAL(RemoteBase): os.mkdir(self.cache_dir) self._dir_info = {} - self.path_info = LocalPathInfo() + self.path_info = PathLOCAL() @staticmethod def compat_config(config): @@ -116,7 +117,7 @@ class RemoteLOCAL(RemoteBase): return self.checksum_to_path(md5) def exists(self, path_info): - assert isinstance(path_info, BasePathInfo) + assert isinstance(path_info, PathBASE) assert path_info.scheme == "local" return os.path.lexists(path_info.path) diff --git a/dvc/remote/oss.py b/dvc/remote/oss.py index b07d87999..56879501d 100644 --- a/dvc/remote/oss.py +++ b/dvc/remote/oss.py @@ -4,8 +4,8 @@ from __future__ import unicode_literals import os import logging -from dvc.path import Schemes -from dvc.path.oss import OSSPathInfo +from dvc.scheme import Schemes +from dvc.path.oss import PathOSS try: import oss2 @@ -16,14 +16,14 @@ from dvc.utils import tmp_fname, move from dvc.utils.compat import urlparse, makedirs from dvc.progress import progress from dvc.config import Config -from dvc.remote.base import RemoteBase +from dvc.remote.base import RemoteBASE from dvc.remote.azure import Callback logger = logging.getLogger(__name__) -class RemoteOSS(RemoteBase): +class RemoteOSS(RemoteBASE): """ oss2 document: https://www.alibabacloud.com/help/doc-detail/32026.htm @@ -73,7 +73,7 @@ class RemoteOSS(RemoteBase): ) self._bucket = None - self.path_info = OSSPathInfo(bucket=self.bucket) + self.path_info = PathOSS(bucket=self.bucket) @property def oss_service(self): diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 0964090a1..99ff523d2 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -4,8 +4,8 @@ import os import threading import logging -from dvc.path import Schemes -from dvc.path.s3 import S3PathInfo +from dvc.scheme import Schemes +from dvc.path.s3 import PathS3 try: import boto3 @@ -16,7 +16,7 @@ from dvc.utils import tmp_fname, move from dvc.utils.compat import urlparse, makedirs from dvc.progress import progress from dvc.config import Config -from dvc.remote.base import RemoteBase +from dvc.remote.base import RemoteBASE from dvc.exceptions import DvcException, ETagMismatchError logger = logging.getLogger(__name__) @@ -35,7 +35,7 @@ class Callback(object): progress.update_target(self.name, self.current, self.total) -class RemoteS3(RemoteBase): +class RemoteS3(RemoteBASE): scheme = Schemes.S3 REGEX = r"^s3://(?P.*)$" REQUIRES = {"boto3": boto3} @@ -72,7 +72,7 @@ class RemoteS3(RemoteBase): self.bucket = parsed.netloc self.prefix = parsed.path.lstrip("/") - self.path_info = S3PathInfo(bucket=self.bucket) + self.path_info = PathS3(bucket=self.bucket) @staticmethod def compat_config(config): diff --git a/dvc/remote/ssh/__init__.py b/dvc/remote/ssh/__init__.py index b9b924570..836d5644e 100644 --- a/dvc/remote/ssh/__init__.py +++ b/dvc/remote/ssh/__init__.py @@ -5,7 +5,7 @@ import getpass import logging from dvc.path import Schemes -from dvc.path.ssh import SSHPathInfo +from dvc.path.ssh import PathSSH try: import paramiko @@ -16,13 +16,13 @@ import dvc.prompt as prompt from dvc.remote.ssh.connection import SSHConnection from dvc.config import Config from dvc.utils.compat import urlparse -from dvc.remote.base import RemoteBase +from dvc.remote.base import RemoteBASE logger = logging.getLogger(__name__) -class RemoteSSH(RemoteBase): +class RemoteSSH(RemoteBASE): scheme = Schemes.SSH # NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and @@ -68,7 +68,7 @@ class RemoteSSH(RemoteBase): Config.SECTION_REMOTE_ASK_PASSWORD, False ) - self.path_info = SSHPathInfo( + self.path_info = PathSSH( host=self.host, user=self.user, port=self.port ) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 4e6a477f4..e50b3eb1d 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -12,7 +12,7 @@ from dvc.exceptions import ( TargetNotDirectoryError, ) from dvc.ignore import DvcIgnoreFileHandler -from dvc.path.local import LocalPathInfo +from dvc.path.local import PathLOCAL logger = logging.getLogger(__name__) @@ -117,7 +117,7 @@ class Repo(object): return Repo(root_dir) def unprotect(self, target): - path_info = LocalPathInfo(path=target) + path_info = PathLOCAL(path=target) return self.cache.local.unprotect(path_info) def _ignore(self): diff --git a/dvc/scheme.py b/dvc/scheme.py new file mode 100644 index 000000000..e12b768f5 --- /dev/null +++ b/dvc/scheme.py @@ -0,0 +1,13 @@ +from __future__ import unicode_literals + + +class Schemes: + SSH = "ssh" + HDFS = "hdfs" + S3 = "s3" + AZURE = "azure" + HTTP = "http" + HTTPS = "https" + GS = "gs" + LOCAL = "local" + OSS = "oss" diff --git a/dvc/stage.py b/dvc/stage.py index 3e0e81e00..a5b018230 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -619,14 +619,14 @@ class Stage(object): return stage def dumpd(self): - from dvc.remote.base import RemoteBase + from dvc.remote.base import RemoteBASE return { key: value for key, value in { Stage.PARAM_MD5: self.md5, Stage.PARAM_CMD: self.cmd, - Stage.PARAM_WDIR: RemoteBase.to_posixpath( + Stage.PARAM_WDIR: RemoteBASE.to_posixpath( os.path.relpath(self.wdir, os.path.dirname(self.path)) ), Stage.PARAM_LOCKED: self.locked, diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 66a07ad44..45067b0c1 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -19,7 +19,7 @@ from dvc.data_cloud import ( DataCloud, RemoteS3, RemoteGS, - RemoteAzure, + RemoteAZURE, RemoteOSS, RemoteLOCAL, RemoteSSH, @@ -244,7 +244,7 @@ class TestDataCloud(TestDvc): ("gs://mybucket/", RemoteGS), ("ssh://user@localhost:/", RemoteSSH), ("http://localhost:8000/", RemoteHTTP), - ("azure://ContainerName=mybucket;conn_string;", RemoteAzure), + ("azure://ContainerName=mybucket;conn_string;", RemoteAZURE), ("oss://mybucket/", RemoteOSS), (TestDvc.mkdtemp(), RemoteLOCAL), ] @@ -403,7 +403,7 @@ class TestRemoteGS(TestDataCloudBase): return RemoteGS -class TestRemoteAzureCompat(TestDataCloudBase): +class TestRemoteAZURECompat(TestDataCloudBase): def _should_test(self): return _should_test_azure() @@ -411,10 +411,10 @@ class TestRemoteAzureCompat(TestDataCloudBase): return get_azure_url_compat() def _get_cloud_class(self): - return RemoteAzure + return RemoteAZURE -class TestRemoteAzure(TestRemoteAzureCompat): +class TestRemoteAZURE(TestRemoteAZURECompat): def _get_url(self): return get_azure_url() @@ -673,7 +673,7 @@ class TestRemoteGSCLI(TestDataCloudCLIBase): self._test_cloud(TEST_REMOTE) -class TestRemoteAzureCLI(TestDataCloudCLIBase): +class TestRemoteAZURECLI(TestDataCloudCLIBase): def _should_test(self): return _should_test_azure() diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index a0c527a39..9c2ca42ad 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -1,7 +1,7 @@ import os import configobj -from dvc.path.local import LocalPathInfo -from dvc.remote.base import RemoteBase +from dvc.path.local import PathLOCAL +from dvc.remote.base import RemoteBASE from mock import patch from dvc.main import main @@ -199,10 +199,10 @@ def test_dir_checksum_should_be_key_order_agnostic(dvc): with open(file2, "w") as fobj: fobj.write("2") - path_info = LocalPathInfo(path=data_dir) + path_info = PathLOCAL(path=data_dir) with dvc.state: with patch.object( - RemoteBase, + RemoteBASE, "_collect_dir", return_value=[ {"relpath": "1", "md5": "1"}, @@ -212,7 +212,7 @@ def test_dir_checksum_should_be_key_order_agnostic(dvc): checksum1 = dvc.cache.local.get_dir_checksum(path_info) with patch.object( - RemoteBase, + RemoteBASE, "_collect_dir", return_value=[ {"md5": "1", "relpath": "1"}, diff --git a/tests/func/test_s3.py b/tests/func/test_s3.py index 718a8a41c..c6768b93e 100644 --- a/tests/func/test_s3.py +++ b/tests/func/test_s3.py @@ -4,7 +4,7 @@ from copy import copy import boto3 import pytest -from dvc.path.s3 import S3PathInfo +from dvc.path.s3 import PathS3 from dvc.remote.s3 import RemoteS3 from tests.func.test_data_cloud import TEST_AWS_REPO_BUCKET, _should_test_aws @@ -12,7 +12,7 @@ from tests.func.test_data_cloud import TEST_AWS_REPO_BUCKET, _should_test_aws def _get_src_dst(): prefix = str(uuid.uuid4()) - from_info = S3PathInfo( + from_info = PathS3( bucket=TEST_AWS_REPO_BUCKET, path=posixpath.join(prefix, "from") ) to_info = copy(from_info) diff --git a/tests/func/test_state.py b/tests/func/test_state.py index 5c0fbc96c..70fb0db0b 100644 --- a/tests/func/test_state.py +++ b/tests/func/test_state.py @@ -1,6 +1,6 @@ import os import mock -from dvc.path.local import LocalPathInfo +from dvc.path.local import PathLOCAL from dvc.utils.compat import str @@ -15,7 +15,7 @@ from tests.basic_env import TestDvc class TestState(TestDvc): def test_update(self): path = os.path.join(self.dvc.root_dir, self.FOO) - path_info = LocalPathInfo(path=path) + path_info = PathLOCAL(path=path) md5 = file_md5(path)[0] state = State(self.dvc, self.dvc.config.config) @@ -78,6 +78,6 @@ class TestGetStateRecordForInode(TestDvc): get_inode_mock.side_effect = self.mock_get_inode(path, inode) with state: - state.save(LocalPathInfo(path=path), md5) + state.save(PathLOCAL(path=path), md5) ret = state.get_state_record_for_inode(inode) self.assertIsNotNone(ret) diff --git a/tests/unit/remote/test_azure.py b/tests/unit/remote/test_azure.py index 676651f61..4ddb57ae9 100644 --- a/tests/unit/remote/test_azure.py +++ b/tests/unit/remote/test_azure.py @@ -1,9 +1,9 @@ from unittest import TestCase -from dvc.remote.azure import RemoteAzure +from dvc.remote.azure import RemoteAZURE -class TestRemoteAzure(TestCase): +class TestRemoteAZURE(TestCase): container_name = "container-name" connection_string = ( "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" @@ -20,7 +20,7 @@ class TestRemoteAzure(TestCase): connection_string=self.connection_string, ) config = {"url": url} - remote = RemoteAzure(None, config) + remote = RemoteAZURE(None, config) self.assertEqual(remote.url, url) self.assertEqual(remote.prefix, "") self.assertEqual(remote.bucket, self.container_name) @@ -30,7 +30,7 @@ class TestRemoteAzure(TestCase): prefix = "some/prefix" url = "azure://{}/{}".format(self.container_name, prefix) config = {"url": url, "connection_string": self.connection_string} - remote = RemoteAzure(None, config) + remote = RemoteAZURE(None, config) self.assertEqual(remote.url, url) self.assertEqual(remote.prefix, prefix) self.assertEqual(remote.bucket, self.container_name) diff --git a/tests/unit/remote/test_base.py b/tests/unit/remote/test_base.py index a63bfa55b..39ed3e5ee 100644 --- a/tests/unit/remote/test_base.py +++ b/tests/unit/remote/test_base.py @@ -1,15 +1,15 @@ import mock from unittest import TestCase -from dvc.path import BasePathInfo -from dvc.remote.base import RemoteBase, RemoteCmdError, RemoteMissingDepsError +from dvc.path.base import PathBASE +from dvc.remote.base import RemoteBASE, RemoteCmdError, RemoteMissingDepsError -class TestRemoteBase(object): - REMOTE_CLS = RemoteBase +class TestRemoteBASE(object): + REMOTE_CLS = RemoteBASE -class TestMissingDeps(TestCase, TestRemoteBase): +class TestMissingDeps(TestCase, TestRemoteBASE): def test(self): REQUIRES = {"foo": None, "bar": None, "mock": mock} with mock.patch.object(self.REMOTE_CLS, "REQUIRES", REQUIRES): @@ -17,7 +17,7 @@ class TestMissingDeps(TestCase, TestRemoteBase): self.REMOTE_CLS(None, {}) -class TestCmdError(TestCase, TestRemoteBase): +class TestCmdError(TestCase, TestRemoteBASE): def test(self): repo = None config = {} @@ -41,13 +41,13 @@ class TestCacheExists(TestCase): "url": "base://example/prefix", "connection_string": "1234567", } - remote = RemoteBase(None, config) + remote = RemoteBASE(None, config) remote.PARAM_CHECKSUM = "checksum" - remote.path_info = BasePathInfo(None, None) + remote.path_info = PathBASE(None, None) remote.url = "" remote.prefix = "" - path_info = BasePathInfo("example", None) + path_info = PathBASE("example", None) checksum_info = {remote.PARAM_CHECKSUM: "1234567890"} with mock.patch.object(remote, "_checkout") as mock_checkout: