dvc: refactor PathInfos

Following same conventions from remotes/outsputs/dependencies.

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/dependency/__init__.py

@@ -2,6 +2,8 @@ from __future__ import unicode_literals
 
 import schema
 
+from dvc.scheme import Schemes
+
 import dvc.output as output
 from dvc.output.base import OutputBase
 from dvc.dependency.s3 import DependencyS3
@@ -25,13 +27,13 @@ DEPS = [
 ]
 
 DEP_MAP = {
-    "local": DependencyLOCAL,
-    "ssh": DependencySSH,
-    "s3": DependencyS3,
-    "gs": DependencyGS,
-    "hdfs": DependencyHDFS,
-    "http": DependencyHTTP,
-    "https": DependencyHTTPS,
+    Schemes.LOCAL: DependencyLOCAL,
+    Schemes.SSH: DependencySSH,
+    Schemes.S3: DependencyS3,
+    Schemes.GS: DependencyGS,
+    Schemes.HDFS: DependencyHDFS,
+    Schemes.HTTP: DependencyHTTP,
+    Schemes.HTTPS: DependencyHTTPS,
 }
 
 

dvc/dependency/http.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from dvc.path.utils import PathInfo
+from dvc.path import Path
 from dvc.utils.compat import urlparse, urljoin
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP
@@ -17,4 +17,4 @@ class DependencyHTTP(DependencyBase, OutputBase):
         if path.startswith("remote"):
             path = urljoin(self.remote.cache_dir, urlparse(path).path)
 
-        self.path_info = PathInfo(self.scheme, url=self.url, path=path)
+        self.path_info = Path(self.scheme, url=self.url, path=path)

dvc/output/__init__.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import schema
 
-from dvc.path import Schemes
+from dvc.scheme import Schemes
 from dvc.utils.compat import urlparse, str
 
 from dvc.output.base import OutputBase

dvc/output/hdfs.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import posixpath
 
-from dvc.path.hdfs import HDFSPathInfo
+from dvc.path.hdfs import PathHDFS
 from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
 from dvc.remote.hdfs import RemoteHDFS
@@ -35,4 +35,4 @@ class OutputHDFS(OutputBase):
         if remote:
             path = posixpath.join(remote.url, urlparse(path).path.lstrip("/"))
         user = remote.user if remote else self.group("user")
-        self.path_info = HDFSPathInfo(user=user, url=self.url, path=path)
+        self.path_info = PathHDFS(user=user, url=self.url, path=path)

dvc/output/local.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import os
 import logging
 
-from dvc.path.local import LocalPathInfo
+from dvc.path.local import PathLOCAL
 from dvc.utils.compat import urlparse
 from dvc.istextfile import istextfile
 from dvc.exceptions import DvcException
@@ -50,7 +50,7 @@ class OutputLOCAL(OutputBase):
             p = os.path.join(stage.wdir, p)
         p = os.path.abspath(os.path.normpath(p))
 
-        self.path_info = LocalPathInfo(url=self.url, path=p)
+        self.path_info = PathLOCAL(url=self.url, path=p)
 
     def __str__(self):
         return self.rel_path

dvc/output/s3.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import posixpath
 
-from dvc.path.utils import PathInfo
+from dvc.path import Path
 from dvc.remote.s3 import RemoteS3
 from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
@@ -37,6 +37,6 @@ class OutputS3(OutputBase):
         if remote:
             path = posixpath.join(remote.prefix, path)
 
-        self.path_info = PathInfo(
+        self.path_info = Path(
             self.scheme, bucket=bucket, path=path, url=self.url
         )

dvc/output/ssh.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import getpass
 import posixpath
 
-from dvc.path.ssh import SSHPathInfo
+from dvc.path.ssh import PathSSH
 from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
 from dvc.remote.ssh import RemoteSSH
@@ -49,6 +49,6 @@ class OutputSSH(OutputBase):
         else:
             path = parsed.path
 
-        self.path_info = SSHPathInfo(
+        self.path_info = PathSSH(
             host=host, user=user, port=port, url=self.url, path=path
         )

dvc/path/__init__.py

@@ -1,35 +1,29 @@
-from dvc.utils.compat import urlunsplit
+from dvc.scheme import Schemes
+
+from dvc.path.azure import PathAZURE
+from dvc.path.gs import PathGS
+from dvc.path.hdfs import PathHDFS
+from dvc.path.http import PathHTTP
+from dvc.path.https import PathHTTPS
+from dvc.path.local import PathLOCAL
+from dvc.path.oss import PathOSS
+from dvc.path.s3 import PathS3
+from dvc.path.ssh import PathSSH
 
 
-class Schemes:
-    SSH = "ssh"
-    HDFS = "hdfs"
-    S3 = "s3"
-    AZURE = "azure"
-    HTTP = "http"
-    HTTPS = "https"
-    GS = "gs"
-    LOCAL = "local"
-    OSS = "oss"
+PATH_MAP = {
+    Schemes.SSH: PathSSH,
+    Schemes.HDFS: PathHDFS,
+    Schemes.S3: PathS3,
+    Schemes.AZURE: PathAZURE,
+    Schemes.HTTP: PathHTTP,
+    Schemes.HTTPS: PathHTTPS,
+    Schemes.GS: PathGS,
+    Schemes.LOCAL: PathLOCAL,
+    Schemes.OSS: PathOSS,
+}
 
 
-class BasePathInfo(object):
-    scheme = None
-
-    def __init__(self, url=None, path=None):
-        self.url = url
-        self.path = path
-
-    def __str__(self):
-        return self.url
-
-
-class DefaultCloudPathInfo(BasePathInfo):
-    def __init__(self, bucket, url=None, path=None):
-        super(DefaultCloudPathInfo, self).__init__(url, path)
-        self.bucket = bucket
-
-    def __str__(self):
-        if not self.url:
-            return urlunsplit((self.scheme, self.bucket, self.path, "", ""))
-        return self.url
+def Path(scheme, *args, **kwargs):
+    cls = PATH_MAP[scheme]
+    return cls(*args, **kwargs)

dvc/path/azure.py

@@ -1,5 +1,6 @@
-from dvc.path import Schemes, DefaultCloudPathInfo
+from dvc.scheme import Schemes
+from .base import PathCloudBASE
 
 
-class AzurePathInfo(DefaultCloudPathInfo):
+class PathAZURE(PathCloudBASE):
     scheme = Schemes.AZURE

dvc/path/base.py

@@ -0,0 +1,23 @@
+from dvc.utils.compat import urlunsplit
+
+
+class PathBASE(object):
+    scheme = None
+
+    def __init__(self, url=None, path=None):
+        self.url = url
+        self.path = path
+
+    def __str__(self):
+        return self.url
+
+
+class PathCloudBASE(PathBASE):
+    def __init__(self, bucket, url=None, path=None):
+        super(PathCloudBASE, self).__init__(url, path)
+        self.bucket = bucket
+
+    def __str__(self):
+        if not self.url:
+            return urlunsplit((self.scheme, self.bucket, self.path, "", ""))
+        return self.url

dvc/path/gs.py

@@ -1,5 +1,6 @@
-from dvc.path import Schemes, DefaultCloudPathInfo
+from dvc.scheme import Schemes
+from .base import PathCloudBASE
 
 
-class GSPathInfo(DefaultCloudPathInfo):
+class PathGS(PathCloudBASE):
     scheme = Schemes.GS

dvc/path/hdfs.py

@@ -1,12 +1,13 @@
 from dvc.utils.compat import urlunsplit
-from dvc.path import BasePathInfo, Schemes
+from dvc.scheme import Schemes
+from .base import PathBASE
 
 
-class HDFSPathInfo(BasePathInfo):
+class PathHDFS(PathBASE):
     scheme = Schemes.HDFS
 
     def __init__(self, user, url=None, path=None):
-        super(HDFSPathInfo, self).__init__(url, path)
+        super(PathHDFS, self).__init__(url, path)
         self.user = user
 
     def __str__(self):

dvc/path/http.py

@@ -1,8 +1,10 @@
-from dvc.path import BasePathInfo, Schemes
+from dvc.scheme import Schemes
 from dvc.utils.compat import urlunsplit
 
+from .base import PathBASE
 
-class HTTPPathInfo(BasePathInfo):
+
+class PathHTTP(PathBASE):
     scheme = Schemes.HTTP
 
     def __str__(self):

dvc/path/https.py

@@ -1,6 +1,6 @@
-from . import Schemes
-from .http import HTTPPathInfo
+from dvc.scheme import Schemes
+from .http import PathHTTP
 
 
-class HTTPSPathInfo(HTTPPathInfo):
+class PathHTTPS(PathHTTP):
     scheme = Schemes.HTTPS

dvc/path/local.py

@@ -1,9 +1,10 @@
 import os
 
-from dvc.path import BasePathInfo, Schemes
+from dvc.scheme import Schemes
+from .base import PathBASE
 
 
-class LocalPathInfo(BasePathInfo):
+class PathLOCAL(PathBASE):
     scheme = Schemes.LOCAL
 
     def __str__(self):

dvc/path/oss.py

@@ -1,5 +1,6 @@
-from dvc.path import Schemes, DefaultCloudPathInfo
+from dvc.scheme import Schemes
+from .base import PathCloudBASE
 
 
-class OSSPathInfo(DefaultCloudPathInfo):
+class PathOSS(PathCloudBASE):
     scheme = Schemes.OSS

dvc/path/s3.py

@@ -1,5 +1,6 @@
-from dvc.path import Schemes, DefaultCloudPathInfo
+from dvc.scheme import Schemes
+from .base import PathCloudBASE
 
 
-class S3PathInfo(DefaultCloudPathInfo):
+class PathS3(PathCloudBASE):
     scheme = Schemes.S3

dvc/path/ssh.py

@@ -1,12 +1,13 @@
 from dvc.utils.compat import urlunsplit
-from dvc.path import BasePathInfo, Schemes
+from dvc.scheme import Schemes
+from .base import PathBASE
 
 
-class SSHPathInfo(BasePathInfo):
+class PathSSH(PathBASE):
     scheme = Schemes.SSH
 
     def __init__(self, host, user, port, url=None, path=None):
-        super(SSHPathInfo, self).__init__(url, path)
+        super(PathSSH, self).__init__(url, path)
         self.host = host
         self.user = user
         self.port = port

dvc/path/utils.py

@@ -1,27 +0,0 @@
-from dvc.path import Schemes
-from dvc.path.azure import AzurePathInfo
-from dvc.path.gs import GSPathInfo
-from dvc.path.hdfs import HDFSPathInfo
-from dvc.path.http import HTTPPathInfo
-from dvc.path.https import HTTPSPathInfo
-from dvc.path.local import LocalPathInfo
-from dvc.path.oss import OSSPathInfo
-from dvc.path.s3 import S3PathInfo
-from dvc.path.ssh import SSHPathInfo
-
-PATH_MAP = {
-    Schemes.SSH: SSHPathInfo,
-    Schemes.HDFS: HDFSPathInfo,
-    Schemes.S3: S3PathInfo,
-    Schemes.AZURE: AzurePathInfo,
-    Schemes.HTTP: HTTPPathInfo,
-    Schemes.HTTPS: HTTPSPathInfo,
-    Schemes.GS: GSPathInfo,
-    Schemes.LOCAL: LocalPathInfo,
-    Schemes.OSS: OSSPathInfo,
-}
-
-
-def PathInfo(scheme, *args, **kwargs):
-    cls = PATH_MAP[scheme]
-    return cls(*args, **kwargs)

dvc/remote/azure.py

@@ -5,8 +5,8 @@ import os
 import re
 import logging
 
-from dvc.path import Schemes
-from dvc.path.azure import AzurePathInfo
+from dvc.scheme import Schemes
+from dvc.path.azure import PathAZURE
 
 try:
     from azure.storage.blob import BlockBlobService
@@ -72,7 +72,7 @@ class RemoteAzure(RemoteBase):
             raise ValueError("azure storage connection string missing")
 
         self.__blob_service = None
-        self.path_info = AzurePathInfo(bucket=self.bucket)
+        self.path_info = PathAZURE(bucket=self.bucket)
 
     @property
     def blob_service(self):

dvc/remote/base.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from copy import copy
 
-from dvc.path.local import LocalPathInfo
+from dvc.path.local import PathLOCAL
 from dvc.utils.compat import str
 
 import os
@@ -211,7 +211,7 @@ class RemoteBase(object):
         with open(tmp, "w+") as fobj:
             json.dump(dir_info, fobj, sort_keys=True)
 
-        from_info = LocalPathInfo(path=tmp)
+        from_info = PathLOCAL(path=tmp)
         self.cache.upload([from_info], [to_info], no_progress_bar=True)
 
         checksum = self.get_file_checksum(to_info) + self.CHECKSUM_DIR_SUFFIX
@@ -235,7 +235,7 @@ class RemoteBase(object):
 
         fobj = tempfile.NamedTemporaryFile(delete=False)
         path = fobj.name
-        to_info = LocalPathInfo(path=path)
+        to_info = PathLOCAL(path=path)
         self.cache.download([path_info], [to_info], no_progress_bar=True)
 
         try:

dvc/remote/gs.py

@@ -3,8 +3,8 @@ from __future__ import unicode_literals
 import os
 import logging
 
-from dvc.path import Schemes
-from dvc.path.gs import GSPathInfo
+from dvc.scheme import Schemes
+from dvc.path.gs import PathGS
 
 try:
     from google.cloud import storage
@@ -41,7 +41,7 @@ class RemoteGS(RemoteBase):
         self.bucket = parsed.netloc
         self.prefix = parsed.path.lstrip("/")
 
-        self.path_info = GSPathInfo(bucket=self.bucket)
+        self.path_info = PathGS(bucket=self.bucket)
 
     @staticmethod
     def compat_config(config):

dvc/remote/hdfs.py

@@ -8,8 +8,8 @@ import logging
 from subprocess import Popen, PIPE
 
 from dvc.config import Config
-from dvc.path import Schemes
-from dvc.path.hdfs import HDFSPathInfo
+from dvc.scheme import Schemes
+from dvc.path.hdfs import PathHDFS
 from dvc.remote.base import RemoteBase, RemoteCmdError
 from dvc.utils import fix_env, tmp_fname
 
@@ -32,7 +32,7 @@ class RemoteHDFS(RemoteBase):
                 Config.SECTION_REMOTE_USER, getpass.getuser()
             )
 
-        self.path_info = HDFSPathInfo(user=self.user)
+        self.path_info = PathHDFS(user=self.user)
 
     def hadoop_fs(self, cmd, user=None):
         cmd = "hadoop fs -" + cmd

dvc/remote/http.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.path import Schemes
-from dvc.path.utils import PathInfo
+from dvc.scheme import Schemes
+from dvc.path import Path
 from dvc.utils.compat import open, makedirs
 
 import os
@@ -44,7 +44,7 @@ class RemoteHTTP(RemoteBase):
         self.cache_dir = config.get(Config.SECTION_REMOTE_URL)
         self.url = self.cache_dir
 
-        self.path_info = PathInfo(self.scheme)
+        self.path_info = Path(self.scheme)
 
     @property
     def prefix(self):

dvc/remote/https.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from dvc.path import Schemes
+from dvc.scheme import Schemes
 
 from .http import RemoteHTTP
 

dvc/remote/local/__init__.py

@@ -2,8 +2,9 @@ from __future__ import unicode_literals
 
 from copy import copy
 
-from dvc.path import BasePathInfo, Schemes
-from dvc.path.local import LocalPathInfo
+from dvc.scheme import Schemes
+from dvc.path.base import PathBASE
+from dvc.path.local import PathLOCAL
 from dvc.remote.local.slow_link_detection import slow_link_guard
 from dvc.utils.compat import str, makedirs
 
@@ -80,7 +81,7 @@ class RemoteLOCAL(RemoteBase):
             os.mkdir(self.cache_dir)
 
         self._dir_info = {}
-        self.path_info = LocalPathInfo()
+        self.path_info = PathLOCAL()
 
     @staticmethod
     def compat_config(config):
@@ -116,7 +117,7 @@ class RemoteLOCAL(RemoteBase):
         return self.checksum_to_path(md5)
 
     def exists(self, path_info):
-        assert isinstance(path_info, BasePathInfo)
+        assert isinstance(path_info, PathBASE)
         assert path_info.scheme == "local"
         return os.path.lexists(path_info.path)
 

dvc/remote/oss.py

@@ -4,8 +4,8 @@ from __future__ import unicode_literals
 import os
 import logging
 
-from dvc.path import Schemes
-from dvc.path.oss import OSSPathInfo
+from dvc.scheme import Schemes
+from dvc.path.oss import PathOSS
 
 try:
     import oss2
@@ -73,7 +73,7 @@ class RemoteOSS(RemoteBase):
         )
 
         self._bucket = None
-        self.path_info = OSSPathInfo(bucket=self.bucket)
+        self.path_info = PathOSS(bucket=self.bucket)
 
     @property
     def oss_service(self):

dvc/remote/s3.py

@@ -4,8 +4,8 @@ import os
 import threading
 import logging
 
-from dvc.path import Schemes
-from dvc.path.s3 import S3PathInfo
+from dvc.scheme import Schemes
+from dvc.path.s3 import PathS3
 
 try:
     import boto3
@@ -72,7 +72,7 @@ class RemoteS3(RemoteBase):
         self.bucket = parsed.netloc
         self.prefix = parsed.path.lstrip("/")
 
-        self.path_info = S3PathInfo(bucket=self.bucket)
+        self.path_info = PathS3(bucket=self.bucket)
 
     @staticmethod
     def compat_config(config):

dvc/remote/ssh/__init__.py

@@ -5,7 +5,7 @@ import getpass
 import logging
 
 from dvc.path import Schemes
-from dvc.path.ssh import SSHPathInfo
+from dvc.path.ssh import PathSSH
 
 try:
     import paramiko
@@ -68,7 +68,7 @@ class RemoteSSH(RemoteBase):
             Config.SECTION_REMOTE_ASK_PASSWORD, False
         )
 
-        self.path_info = SSHPathInfo(
+        self.path_info = PathSSH(
             host=self.host, user=self.user, port=self.port
         )
 

dvc/repo/__init__.py

@@ -12,7 +12,7 @@ from dvc.exceptions import (
     TargetNotDirectoryError,
 )
 from dvc.ignore import DvcIgnoreFileHandler
-from dvc.path.local import LocalPathInfo
+from dvc.path.local import PathLOCAL
 
 logger = logging.getLogger(__name__)
 
@@ -117,7 +117,7 @@ class Repo(object):
         return Repo(root_dir)
 
     def unprotect(self, target):
-        path_info = LocalPathInfo(path=target)
+        path_info = PathLOCAL(path=target)
         return self.cache.local.unprotect(path_info)
 
     def _ignore(self):

dvc/scheme.py

@@ -0,0 +1,13 @@
+from __future__ import unicode_literals
+
+
+class Schemes:
+    SSH = "ssh"
+    HDFS = "hdfs"
+    S3 = "s3"
+    AZURE = "azure"
+    HTTP = "http"
+    HTTPS = "https"
+    GS = "gs"
+    LOCAL = "local"
+    OSS = "oss"

tests/func/test_remote.py

@@ -1,6 +1,6 @@
 import os
 import configobj
-from dvc.path.local import LocalPathInfo
+from dvc.path.local import PathLOCAL
 from dvc.remote.base import RemoteBase
 from mock import patch
 
@@ -199,7 +199,7 @@ def test_dir_checksum_should_be_key_order_agnostic(dvc):
     with open(file2, "w") as fobj:
         fobj.write("2")
 
-    path_info = LocalPathInfo(path=data_dir)
+    path_info = PathLOCAL(path=data_dir)
     with dvc.state:
         with patch.object(
             RemoteBase,

tests/func/test_s3.py

@@ -4,7 +4,7 @@ from copy import copy
 
 import boto3
 import pytest
-from dvc.path.s3 import S3PathInfo
+from dvc.path.s3 import PathS3
 
 from dvc.remote.s3 import RemoteS3
 from tests.func.test_data_cloud import TEST_AWS_REPO_BUCKET, _should_test_aws
@@ -12,7 +12,7 @@ from tests.func.test_data_cloud import TEST_AWS_REPO_BUCKET, _should_test_aws
 
 def _get_src_dst():
     prefix = str(uuid.uuid4())
-    from_info = S3PathInfo(
+    from_info = PathS3(
         bucket=TEST_AWS_REPO_BUCKET, path=posixpath.join(prefix, "from")
     )
     to_info = copy(from_info)

tests/func/test_state.py

@@ -1,6 +1,6 @@
 import os
 import mock
-from dvc.path.local import LocalPathInfo
+from dvc.path.local import PathLOCAL
 
 from dvc.utils.compat import str
 
@@ -15,7 +15,7 @@ from tests.basic_env import TestDvc
 class TestState(TestDvc):
     def test_update(self):
         path = os.path.join(self.dvc.root_dir, self.FOO)
-        path_info = LocalPathInfo(path=path)
+        path_info = PathLOCAL(path=path)
         md5 = file_md5(path)[0]
 
         state = State(self.dvc, self.dvc.config.config)
@@ -78,6 +78,6 @@ class TestGetStateRecordForInode(TestDvc):
         get_inode_mock.side_effect = self.mock_get_inode(path, inode)
 
         with state:
-            state.save(LocalPathInfo(path=path), md5)
+            state.save(PathLOCAL(path=path), md5)
             ret = state.get_state_record_for_inode(inode)
             self.assertIsNotNone(ret)

tests/unit/remote/test_base.py

@@ -1,7 +1,7 @@
 import mock
 from unittest import TestCase
 
-from dvc.path import BasePathInfo
+from dvc.path.base import PathBASE
 from dvc.remote.base import RemoteBase, RemoteCmdError, RemoteMissingDepsError
 
 
@@ -44,10 +44,10 @@ class TestCacheExists(TestCase):
         remote = RemoteBase(None, config)
 
         remote.PARAM_CHECKSUM = "checksum"
-        remote.path_info = BasePathInfo(None, None)
+        remote.path_info = PathBASE(None, None)
         remote.url = ""
         remote.prefix = ""
-        path_info = BasePathInfo("example", None)
+        path_info = PathBASE("example", None)
         checksum_info = {remote.PARAM_CHECKSUM: "1234567890"}
 
         with mock.patch.object(remote, "_checkout") as mock_checkout: