Merge pull request #1990 from efiop/small-fixes

dvc: refactoring: unify path/remote logic and naming
This commit is contained in:
Ruslan Kuprieiev
2019-05-12 17:55:05 +03:00
committed by GitHub
42 changed files with 229 additions and 188 deletions
+5 -5
View File
@@ -8,7 +8,7 @@ from dvc.config import Config, ConfigError
from dvc.remote import Remote
from dvc.remote.s3 import RemoteS3
from dvc.remote.gs import RemoteGS
from dvc.remote.azure import RemoteAzure
from dvc.remote.azure import RemoteAZURE
from dvc.remote.oss import RemoteOSS
from dvc.remote.ssh import RemoteSSH
from dvc.remote.hdfs import RemoteHDFS
@@ -34,7 +34,7 @@ class DataCloud(object):
CLOUD_MAP = {
"aws": RemoteS3,
"gcp": RemoteGS,
"azure": RemoteAzure,
"azure": RemoteAZURE,
"oss": RemoteOSS,
"ssh": RemoteSSH,
"hdfs": RemoteHDFS,
@@ -124,7 +124,7 @@ class DataCloud(object):
Args:
targets (list): list of targets to push to the cloud.
jobs (int): number of jobs that can be running simultaneously.
remote (dvc.remote.base.RemoteBase): optional remote to push to.
remote (dvc.remote.base.RemoteBASE): optional remote to push to.
By default remote from core.remote config option is used.
show_checksums (bool): show checksums instead of file names in
information messages.
@@ -142,7 +142,7 @@ class DataCloud(object):
Args:
targets (list): list of targets to pull from the cloud.
jobs (int): number of jobs that can be running simultaneously.
remote (dvc.remote.base.RemoteBase): optional remote to pull from.
remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
By default remote from core.remote config option is used.
show_checksums (bool): show checksums instead of file names in
information messages.
@@ -160,7 +160,7 @@ class DataCloud(object):
Args:
targets (list): list of targets to check status for.
jobs (int): number of jobs that can be running simultaneously.
remote (dvc.remote.base.RemoteBase): optional remote to compare
remote (dvc.remote.base.RemoteBASE): optional remote to compare
targets to. By default remote from core.remote config option
is used.
show_checksums (bool): show checksums instead of file names in
+11 -7
View File
@@ -2,6 +2,8 @@ from __future__ import unicode_literals
import schema
from dvc.scheme import Schemes
import dvc.output as output
from dvc.output.base import OutputBase
from dvc.dependency.s3 import DependencyS3
@@ -10,6 +12,7 @@ from dvc.dependency.local import DependencyLOCAL
from dvc.dependency.hdfs import DependencyHDFS
from dvc.dependency.ssh import DependencySSH
from dvc.dependency.http import DependencyHTTP
from dvc.dependency.https import DependencyHTTPS
from dvc.remote import Remote
@@ -17,19 +20,20 @@ DEPS = [
DependencyGS,
DependencyHDFS,
DependencyHTTP,
DependencyHTTPS,
DependencyS3,
DependencySSH,
# NOTE: DependencyLOCAL is the default choice
]
DEP_MAP = {
"local": DependencyLOCAL,
"ssh": DependencySSH,
"s3": DependencyS3,
"gs": DependencyGS,
"hdfs": DependencyHDFS,
"http": DependencyHTTP,
"https": DependencyHTTP,
Schemes.LOCAL: DependencyLOCAL,
Schemes.SSH: DependencySSH,
Schemes.S3: DependencyS3,
Schemes.GS: DependencyGS,
Schemes.HDFS: DependencyHDFS,
Schemes.HTTP: DependencyHTTP,
Schemes.HTTPS: DependencyHTTPS,
}
+2 -2
View File
@@ -1,6 +1,6 @@
from __future__ import unicode_literals
from dvc.path.http import HTTPPathInfo
from dvc.path import Path
from dvc.utils.compat import urlparse, urljoin
from dvc.output.base import OutputBase
from dvc.remote.http import RemoteHTTP
@@ -17,4 +17,4 @@ class DependencyHTTP(DependencyBase, OutputBase):
if path.startswith("remote"):
path = urljoin(self.remote.cache_dir, urlparse(path).path)
self.path_info = HTTPPathInfo(url=self.url, path=path)
self.path_info = Path(self.scheme, url=self.url, path=path)
+8
View File
@@ -0,0 +1,8 @@
from __future__ import unicode_literals
from dvc.remote.https import RemoteHTTPS
from .http import DependencyHTTP
class DependencyHTTPS(DependencyHTTP):
REMOTE = RemoteHTTPS
+1 -1
View File
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
import schema
from dvc.path import Schemes
from dvc.scheme import Schemes
from dvc.utils.compat import urlparse, str
from dvc.output.base import OutputBase
+2 -2
View File
@@ -8,7 +8,7 @@ from schema import Or, Optional
from dvc.exceptions import DvcException
from dvc.utils.compat import str
from dvc.remote.base import RemoteBase
from dvc.remote.base import RemoteBASE
logger = logging.getLogger(__name__)
@@ -35,7 +35,7 @@ class OutputAlreadyTrackedError(DvcException):
class OutputBase(object):
IS_DEPENDENCY = False
REMOTE = RemoteBase
REMOTE = RemoteBASE
PARAM_PATH = "path"
PARAM_CACHE = "cache"
+2 -2
View File
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
import posixpath
from dvc.path.hdfs import HDFSPathInfo
from dvc.path.hdfs import PathHDFS
from dvc.utils.compat import urlparse
from dvc.output.base import OutputBase
from dvc.remote.hdfs import RemoteHDFS
@@ -35,4 +35,4 @@ class OutputHDFS(OutputBase):
if remote:
path = posixpath.join(remote.url, urlparse(path).path.lstrip("/"))
user = remote.user if remote else self.group("user")
self.path_info = HDFSPathInfo(user=user, url=self.url, path=path)
self.path_info = PathHDFS(user=user, url=self.url, path=path)
+2 -2
View File
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
import os
import logging
from dvc.path.local import LocalPathInfo
from dvc.path.local import PathLOCAL
from dvc.utils.compat import urlparse
from dvc.istextfile import istextfile
from dvc.exceptions import DvcException
@@ -50,7 +50,7 @@ class OutputLOCAL(OutputBase):
p = os.path.join(stage.wdir, p)
p = os.path.abspath(os.path.normpath(p))
self.path_info = LocalPathInfo(url=self.url, path=p)
self.path_info = PathLOCAL(url=self.url, path=p)
def __str__(self):
return self.rel_path
+2 -2
View File
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
import posixpath
from dvc.path.utils import PathInfo
from dvc.path import Path
from dvc.remote.s3 import RemoteS3
from dvc.utils.compat import urlparse
from dvc.output.base import OutputBase
@@ -37,6 +37,6 @@ class OutputS3(OutputBase):
if remote:
path = posixpath.join(remote.prefix, path)
self.path_info = PathInfo(
self.path_info = Path(
self.scheme, bucket=bucket, path=path, url=self.url
)
+2 -2
View File
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
import getpass
import posixpath
from dvc.path.ssh import SSHPathInfo
from dvc.path.ssh import PathSSH
from dvc.utils.compat import urlparse
from dvc.output.base import OutputBase
from dvc.remote.ssh import RemoteSSH
@@ -49,6 +49,6 @@ class OutputSSH(OutputBase):
else:
path = parsed.path
self.path_info = SSHPathInfo(
self.path_info = PathSSH(
host=host, user=user, port=port, url=self.url, path=path
)
+25 -30
View File
@@ -1,34 +1,29 @@
from dvc.utils.compat import urlunsplit
from dvc.scheme import Schemes
from dvc.path.azure import PathAZURE
from dvc.path.gs import PathGS
from dvc.path.hdfs import PathHDFS
from dvc.path.http import PathHTTP
from dvc.path.https import PathHTTPS
from dvc.path.local import PathLOCAL
from dvc.path.oss import PathOSS
from dvc.path.s3 import PathS3
from dvc.path.ssh import PathSSH
class Schemes:
SSH = "ssh"
HDFS = "hdfs"
S3 = "s3"
AZURE = "azure"
HTTP = "http"
GS = "gs"
LOCAL = "local"
OSS = "oss"
PATH_MAP = {
Schemes.SSH: PathSSH,
Schemes.HDFS: PathHDFS,
Schemes.S3: PathS3,
Schemes.AZURE: PathAZURE,
Schemes.HTTP: PathHTTP,
Schemes.HTTPS: PathHTTPS,
Schemes.GS: PathGS,
Schemes.LOCAL: PathLOCAL,
Schemes.OSS: PathOSS,
}
class BasePathInfo(object):
scheme = None
def __init__(self, url=None, path=None):
self.url = url
self.path = path
def __str__(self):
return self.url
class DefaultCloudPathInfo(BasePathInfo):
def __init__(self, bucket, url=None, path=None):
super(DefaultCloudPathInfo, self).__init__(url, path)
self.bucket = bucket
def __str__(self):
if not self.url:
return urlunsplit((self.scheme, self.bucket, self.path, "", ""))
return self.url
def Path(scheme, *args, **kwargs):
cls = PATH_MAP[scheme]
return cls(*args, **kwargs)
+3 -2
View File
@@ -1,5 +1,6 @@
from dvc.path import Schemes, DefaultCloudPathInfo
from dvc.scheme import Schemes
from .base import PathCloudBASE
class AzurePathInfo(DefaultCloudPathInfo):
class PathAZURE(PathCloudBASE):
scheme = Schemes.AZURE
+23
View File
@@ -0,0 +1,23 @@
from dvc.utils.compat import urlunsplit
class PathBASE(object):
scheme = None
def __init__(self, url=None, path=None):
self.url = url
self.path = path
def __str__(self):
return self.url
class PathCloudBASE(PathBASE):
def __init__(self, bucket, url=None, path=None):
super(PathCloudBASE, self).__init__(url, path)
self.bucket = bucket
def __str__(self):
if not self.url:
return urlunsplit((self.scheme, self.bucket, self.path, "", ""))
return self.url
+3 -2
View File
@@ -1,5 +1,6 @@
from dvc.path import Schemes, DefaultCloudPathInfo
from dvc.scheme import Schemes
from .base import PathCloudBASE
class GSPathInfo(DefaultCloudPathInfo):
class PathGS(PathCloudBASE):
scheme = Schemes.GS
+4 -3
View File
@@ -1,12 +1,13 @@
from dvc.utils.compat import urlunsplit
from dvc.path import BasePathInfo, Schemes
from dvc.scheme import Schemes
from .base import PathBASE
class HDFSPathInfo(BasePathInfo):
class PathHDFS(PathBASE):
scheme = Schemes.HDFS
def __init__(self, user, url=None, path=None):
super(HDFSPathInfo, self).__init__(url, path)
super(PathHDFS, self).__init__(url, path)
self.user = user
def __str__(self):
+6 -9
View File
@@ -1,14 +1,11 @@
from dvc.path import BasePathInfo, Schemes
from dvc.utils.compat import urlparse, urlunsplit
from dvc.scheme import Schemes
from dvc.utils.compat import urlunsplit
from .base import PathBASE
class HTTPPathInfo(BasePathInfo):
@property
def scheme(self):
if self.path:
return urlparse(self.path).scheme
else:
return Schemes.HTTP
class PathHTTP(PathBASE):
scheme = Schemes.HTTP
def __str__(self):
if not self.url:
+6
View File
@@ -0,0 +1,6 @@
from dvc.scheme import Schemes
from .http import PathHTTP
class PathHTTPS(PathHTTP):
scheme = Schemes.HTTPS
+3 -2
View File
@@ -1,9 +1,10 @@
import os
from dvc.path import BasePathInfo, Schemes
from dvc.scheme import Schemes
from .base import PathBASE
class LocalPathInfo(BasePathInfo):
class PathLOCAL(PathBASE):
scheme = Schemes.LOCAL
def __str__(self):
+3 -2
View File
@@ -1,5 +1,6 @@
from dvc.path import Schemes, DefaultCloudPathInfo
from dvc.scheme import Schemes
from .base import PathCloudBASE
class OSSPathInfo(DefaultCloudPathInfo):
class PathOSS(PathCloudBASE):
scheme = Schemes.OSS
+3 -2
View File
@@ -1,5 +1,6 @@
from dvc.path import Schemes, DefaultCloudPathInfo
from dvc.scheme import Schemes
from .base import PathCloudBASE
class S3PathInfo(DefaultCloudPathInfo):
class PathS3(PathCloudBASE):
scheme = Schemes.S3
+4 -3
View File
@@ -1,12 +1,13 @@
from dvc.utils.compat import urlunsplit
from dvc.path import BasePathInfo, Schemes
from dvc.scheme import Schemes
from .base import PathBASE
class SSHPathInfo(BasePathInfo):
class PathSSH(PathBASE):
scheme = Schemes.SSH
def __init__(self, host, user, port, url=None, path=None):
super(SSHPathInfo, self).__init__(url, path)
super(PathSSH, self).__init__(url, path)
self.host = host
self.user = user
self.port = port
-25
View File
@@ -1,25 +0,0 @@
from dvc.path import Schemes
from dvc.path.azure import AzurePathInfo
from dvc.path.gs import GSPathInfo
from dvc.path.hdfs import HDFSPathInfo
from dvc.path.http import HTTPPathInfo
from dvc.path.local import LocalPathInfo
from dvc.path.oss import OSSPathInfo
from dvc.path.s3 import S3PathInfo
from dvc.path.ssh import SSHPathInfo
PATH_MAP = {
Schemes.SSH: SSHPathInfo,
Schemes.HDFS: HDFSPathInfo,
Schemes.S3: S3PathInfo,
Schemes.AZURE: AzurePathInfo,
Schemes.HTTP: HTTPPathInfo,
Schemes.GS: GSPathInfo,
Schemes.LOCAL: LocalPathInfo,
Schemes.OSS: OSSPathInfo,
}
def PathInfo(scheme, *args, **kwargs):
cls = PATH_MAP[scheme]
return cls(*args, **kwargs)
+4 -2
View File
@@ -1,20 +1,22 @@
from __future__ import unicode_literals
from dvc.remote.azure import RemoteAzure
from dvc.remote.azure import RemoteAZURE
from dvc.remote.gs import RemoteGS
from dvc.remote.hdfs import RemoteHDFS
from dvc.remote.local import RemoteLOCAL
from dvc.remote.s3 import RemoteS3
from dvc.remote.ssh import RemoteSSH
from dvc.remote.http import RemoteHTTP
from dvc.remote.https import RemoteHTTPS
from dvc.remote.oss import RemoteOSS
REMOTES = [
RemoteAzure,
RemoteAZURE,
RemoteGS,
RemoteHDFS,
RemoteHTTP,
RemoteHTTPS,
RemoteS3,
RemoteSSH,
RemoteOSS,
+6 -6
View File
@@ -5,8 +5,8 @@ import os
import re
import logging
from dvc.path import Schemes
from dvc.path.azure import AzurePathInfo
from dvc.scheme import Schemes
from dvc.path.azure import PathAZURE
try:
from azure.storage.blob import BlockBlobService
@@ -18,7 +18,7 @@ from dvc.utils import tmp_fname, move
from dvc.utils.compat import urlparse, makedirs
from dvc.progress import progress
from dvc.config import Config
from dvc.remote.base import RemoteBase
from dvc.remote.base import RemoteBASE
logger = logging.getLogger(__name__)
@@ -32,7 +32,7 @@ class Callback(object):
progress.update_target(self.name, current, total)
class RemoteAzure(RemoteBase):
class RemoteAZURE(RemoteBASE):
scheme = Schemes.AZURE
REGEX = (
r"azure://((?P<path>[^=;]*)?|("
@@ -45,7 +45,7 @@ class RemoteAzure(RemoteBase):
COPY_POLL_SECONDS = 5
def __init__(self, repo, config):
super(RemoteAzure, self).__init__(repo, config)
super(RemoteAZURE, self).__init__(repo, config)
self.url = config.get(Config.SECTION_REMOTE_URL, "azure://")
match = re.match(self.REGEX, self.url) # backward compatibility
@@ -72,7 +72,7 @@ class RemoteAzure(RemoteBase):
raise ValueError("azure storage connection string missing")
self.__blob_service = None
self.path_info = AzurePathInfo(bucket=self.bucket)
self.path_info = PathAZURE(bucket=self.bucket)
@property
def blob_service(self):
+4 -4
View File
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
from copy import copy
from dvc.path.local import LocalPathInfo
from dvc.path.local import PathLOCAL
from dvc.utils.compat import str
import os
@@ -66,7 +66,7 @@ class RemoteMissingDepsError(DvcException):
pass
class RemoteBase(object):
class RemoteBASE(object):
scheme = None
REGEX = None
REQUIRES = {}
@@ -211,7 +211,7 @@ class RemoteBase(object):
with open(tmp, "w+") as fobj:
json.dump(dir_info, fobj, sort_keys=True)
from_info = LocalPathInfo(path=tmp)
from_info = PathLOCAL(path=tmp)
self.cache.upload([from_info], [to_info], no_progress_bar=True)
checksum = self.get_file_checksum(to_info) + self.CHECKSUM_DIR_SUFFIX
@@ -235,7 +235,7 @@ class RemoteBase(object):
fobj = tempfile.NamedTemporaryFile(delete=False)
path = fobj.name
to_info = LocalPathInfo(path=path)
to_info = PathLOCAL(path=path)
self.cache.download([path_info], [to_info], no_progress_bar=True)
try:
+5 -5
View File
@@ -3,8 +3,8 @@ from __future__ import unicode_literals
import os
import logging
from dvc.path import Schemes
from dvc.path.gs import GSPathInfo
from dvc.scheme import Schemes
from dvc.path.gs import PathGS
try:
from google.cloud import storage
@@ -13,7 +13,7 @@ except ImportError:
from dvc.utils import tmp_fname, move
from dvc.utils.compat import urlparse, makedirs
from dvc.remote.base import RemoteBase
from dvc.remote.base import RemoteBASE
from dvc.config import Config
from dvc.progress import progress
from dvc.exceptions import DvcException
@@ -22,7 +22,7 @@ from dvc.exceptions import DvcException
logger = logging.getLogger(__name__)
class RemoteGS(RemoteBase):
class RemoteGS(RemoteBASE):
scheme = Schemes.GS
REGEX = r"^gs://(?P<path>.*)$"
REQUIRES = {"google.cloud.storage": storage}
@@ -41,7 +41,7 @@ class RemoteGS(RemoteBase):
self.bucket = parsed.netloc
self.prefix = parsed.path.lstrip("/")
self.path_info = GSPathInfo(bucket=self.bucket)
self.path_info = PathGS(bucket=self.bucket)
@staticmethod
def compat_config(config):
+5 -5
View File
@@ -8,16 +8,16 @@ import logging
from subprocess import Popen, PIPE
from dvc.config import Config
from dvc.path import Schemes
from dvc.path.hdfs import HDFSPathInfo
from dvc.remote.base import RemoteBase, RemoteCmdError
from dvc.scheme import Schemes
from dvc.path.hdfs import PathHDFS
from dvc.remote.base import RemoteBASE, RemoteCmdError
from dvc.utils import fix_env, tmp_fname
logger = logging.getLogger(__name__)
class RemoteHDFS(RemoteBase):
class RemoteHDFS(RemoteBASE):
scheme = Schemes.HDFS
REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
PARAM_CHECKSUM = "checksum"
@@ -32,7 +32,7 @@ class RemoteHDFS(RemoteBase):
Config.SECTION_REMOTE_USER, getpass.getuser()
)
self.path_info = HDFSPathInfo(user=self.user)
self.path_info = PathHDFS(user=self.user)
def hadoop_fs(self, cmd, user=None):
cmd = "hadoop fs -" + cmd
+8 -8
View File
@@ -1,7 +1,7 @@
from __future__ import unicode_literals
from dvc.path import Schemes
from dvc.path.http import HTTPPathInfo
from dvc.scheme import Schemes
from dvc.path import Path
from dvc.utils.compat import open, makedirs
import os
@@ -12,7 +12,7 @@ import logging
from dvc.progress import progress
from dvc.exceptions import DvcException
from dvc.config import Config
from dvc.remote.base import RemoteBase
from dvc.remote.base import RemoteBASE
from dvc.utils import move
@@ -32,9 +32,9 @@ class ProgressBarCallback(object):
progress.update_target(self.name, self.current, self.total)
class RemoteHTTP(RemoteBase):
class RemoteHTTP(RemoteBASE):
scheme = Schemes.HTTP
REGEX = r"^https?://.*$"
REGEX = r"^http://.*$"
REQUEST_TIMEOUT = 10
CHUNK_SIZE = 2 ** 16
PARAM_CHECKSUM = "etag"
@@ -44,7 +44,7 @@ class RemoteHTTP(RemoteBase):
self.cache_dir = config.get(Config.SECTION_REMOTE_URL)
self.url = self.cache_dir
self.path_info = HTTPPathInfo()
self.path_info = Path(self.scheme)
@property
def prefix(self):
@@ -61,7 +61,7 @@ class RemoteHTTP(RemoteBase):
names = self._verify_path_args(to_infos, from_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info.scheme not in ["http", "https"]:
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme != "local":
@@ -99,7 +99,7 @@ class RemoteHTTP(RemoteBase):
def exists(self, path_info):
assert not isinstance(path_info, list)
assert path_info.scheme in ["http", "https"]
assert path_info.scheme == self.scheme
return bool(self._request("HEAD", path_info.path))
def cache_exists(self, md5s):
+10
View File
@@ -0,0 +1,10 @@
from __future__ import unicode_literals
from dvc.scheme import Schemes
from .http import RemoteHTTP
class RemoteHTTPS(RemoteHTTP):
scheme = Schemes.HTTPS
REGEX = r"^https://.*$"
+7 -6
View File
@@ -2,8 +2,9 @@ from __future__ import unicode_literals
from copy import copy
from dvc.path import BasePathInfo, Schemes
from dvc.path.local import LocalPathInfo
from dvc.scheme import Schemes
from dvc.path.base import PathBASE
from dvc.path.local import PathLOCAL
from dvc.remote.local.slow_link_detection import slow_link_guard
from dvc.utils.compat import str, makedirs
@@ -17,7 +18,7 @@ import logging
from dvc.system import System
from dvc.remote.base import (
RemoteBase,
RemoteBASE,
STATUS_MAP,
STATUS_NEW,
STATUS_DELETED,
@@ -43,7 +44,7 @@ from dvc.utils.fs import get_mtime_and_size, get_inode
logger = logging.getLogger(__name__)
class RemoteLOCAL(RemoteBase):
class RemoteLOCAL(RemoteBASE):
scheme = Schemes.LOCAL
REGEX = r"^(?P<path>.*)$"
PARAM_CHECKSUM = "md5"
@@ -80,7 +81,7 @@ class RemoteLOCAL(RemoteBase):
os.mkdir(self.cache_dir)
self._dir_info = {}
self.path_info = LocalPathInfo()
self.path_info = PathLOCAL()
@staticmethod
def compat_config(config):
@@ -116,7 +117,7 @@ class RemoteLOCAL(RemoteBase):
return self.checksum_to_path(md5)
def exists(self, path_info):
assert isinstance(path_info, BasePathInfo)
assert isinstance(path_info, PathBASE)
assert path_info.scheme == "local"
return os.path.lexists(path_info.path)
+5 -5
View File
@@ -4,8 +4,8 @@ from __future__ import unicode_literals
import os
import logging
from dvc.path import Schemes
from dvc.path.oss import OSSPathInfo
from dvc.scheme import Schemes
from dvc.path.oss import PathOSS
try:
import oss2
@@ -16,14 +16,14 @@ from dvc.utils import tmp_fname, move
from dvc.utils.compat import urlparse, makedirs
from dvc.progress import progress
from dvc.config import Config
from dvc.remote.base import RemoteBase
from dvc.remote.base import RemoteBASE
from dvc.remote.azure import Callback
logger = logging.getLogger(__name__)
class RemoteOSS(RemoteBase):
class RemoteOSS(RemoteBASE):
"""
oss2 document:
https://www.alibabacloud.com/help/doc-detail/32026.htm
@@ -73,7 +73,7 @@ class RemoteOSS(RemoteBase):
)
self._bucket = None
self.path_info = OSSPathInfo(bucket=self.bucket)
self.path_info = PathOSS(bucket=self.bucket)
@property
def oss_service(self):
+5 -5
View File
@@ -4,8 +4,8 @@ import os
import threading
import logging
from dvc.path import Schemes
from dvc.path.s3 import S3PathInfo
from dvc.scheme import Schemes
from dvc.path.s3 import PathS3
try:
import boto3
@@ -16,7 +16,7 @@ from dvc.utils import tmp_fname, move
from dvc.utils.compat import urlparse, makedirs
from dvc.progress import progress
from dvc.config import Config
from dvc.remote.base import RemoteBase
from dvc.remote.base import RemoteBASE
from dvc.exceptions import DvcException, ETagMismatchError
logger = logging.getLogger(__name__)
@@ -35,7 +35,7 @@ class Callback(object):
progress.update_target(self.name, self.current, self.total)
class RemoteS3(RemoteBase):
class RemoteS3(RemoteBASE):
scheme = Schemes.S3
REGEX = r"^s3://(?P<path>.*)$"
REQUIRES = {"boto3": boto3}
@@ -72,7 +72,7 @@ class RemoteS3(RemoteBase):
self.bucket = parsed.netloc
self.prefix = parsed.path.lstrip("/")
self.path_info = S3PathInfo(bucket=self.bucket)
self.path_info = PathS3(bucket=self.bucket)
@staticmethod
def compat_config(config):
+4 -4
View File
@@ -5,7 +5,7 @@ import getpass
import logging
from dvc.path import Schemes
from dvc.path.ssh import SSHPathInfo
from dvc.path.ssh import PathSSH
try:
import paramiko
@@ -16,13 +16,13 @@ import dvc.prompt as prompt
from dvc.remote.ssh.connection import SSHConnection
from dvc.config import Config
from dvc.utils.compat import urlparse
from dvc.remote.base import RemoteBase
from dvc.remote.base import RemoteBASE
logger = logging.getLogger(__name__)
class RemoteSSH(RemoteBase):
class RemoteSSH(RemoteBASE):
scheme = Schemes.SSH
# NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and
@@ -68,7 +68,7 @@ class RemoteSSH(RemoteBase):
Config.SECTION_REMOTE_ASK_PASSWORD, False
)
self.path_info = SSHPathInfo(
self.path_info = PathSSH(
host=self.host, user=self.user, port=self.port
)
+2 -2
View File
@@ -12,7 +12,7 @@ from dvc.exceptions import (
TargetNotDirectoryError,
)
from dvc.ignore import DvcIgnoreFileHandler
from dvc.path.local import LocalPathInfo
from dvc.path.local import PathLOCAL
logger = logging.getLogger(__name__)
@@ -117,7 +117,7 @@ class Repo(object):
return Repo(root_dir)
def unprotect(self, target):
path_info = LocalPathInfo(path=target)
path_info = PathLOCAL(path=target)
return self.cache.local.unprotect(path_info)
def _ignore(self):
+13
View File
@@ -0,0 +1,13 @@
from __future__ import unicode_literals
class Schemes:
SSH = "ssh"
HDFS = "hdfs"
S3 = "s3"
AZURE = "azure"
HTTP = "http"
HTTPS = "https"
GS = "gs"
LOCAL = "local"
OSS = "oss"
+2 -2
View File
@@ -619,14 +619,14 @@ class Stage(object):
return stage
def dumpd(self):
from dvc.remote.base import RemoteBase
from dvc.remote.base import RemoteBASE
return {
key: value
for key, value in {
Stage.PARAM_MD5: self.md5,
Stage.PARAM_CMD: self.cmd,
Stage.PARAM_WDIR: RemoteBase.to_posixpath(
Stage.PARAM_WDIR: RemoteBASE.to_posixpath(
os.path.relpath(self.wdir, os.path.dirname(self.path))
),
Stage.PARAM_LOCKED: self.locked,
+6 -6
View File
@@ -19,7 +19,7 @@ from dvc.data_cloud import (
DataCloud,
RemoteS3,
RemoteGS,
RemoteAzure,
RemoteAZURE,
RemoteOSS,
RemoteLOCAL,
RemoteSSH,
@@ -244,7 +244,7 @@ class TestDataCloud(TestDvc):
("gs://mybucket/", RemoteGS),
("ssh://user@localhost:/", RemoteSSH),
("http://localhost:8000/", RemoteHTTP),
("azure://ContainerName=mybucket;conn_string;", RemoteAzure),
("azure://ContainerName=mybucket;conn_string;", RemoteAZURE),
("oss://mybucket/", RemoteOSS),
(TestDvc.mkdtemp(), RemoteLOCAL),
]
@@ -403,7 +403,7 @@ class TestRemoteGS(TestDataCloudBase):
return RemoteGS
class TestRemoteAzureCompat(TestDataCloudBase):
class TestRemoteAZURECompat(TestDataCloudBase):
def _should_test(self):
return _should_test_azure()
@@ -411,10 +411,10 @@ class TestRemoteAzureCompat(TestDataCloudBase):
return get_azure_url_compat()
def _get_cloud_class(self):
return RemoteAzure
return RemoteAZURE
class TestRemoteAzure(TestRemoteAzureCompat):
class TestRemoteAZURE(TestRemoteAZURECompat):
def _get_url(self):
return get_azure_url()
@@ -673,7 +673,7 @@ class TestRemoteGSCLI(TestDataCloudCLIBase):
self._test_cloud(TEST_REMOTE)
class TestRemoteAzureCLI(TestDataCloudCLIBase):
class TestRemoteAZURECLI(TestDataCloudCLIBase):
def _should_test(self):
return _should_test_azure()
+5 -5
View File
@@ -1,7 +1,7 @@
import os
import configobj
from dvc.path.local import LocalPathInfo
from dvc.remote.base import RemoteBase
from dvc.path.local import PathLOCAL
from dvc.remote.base import RemoteBASE
from mock import patch
from dvc.main import main
@@ -199,10 +199,10 @@ def test_dir_checksum_should_be_key_order_agnostic(dvc):
with open(file2, "w") as fobj:
fobj.write("2")
path_info = LocalPathInfo(path=data_dir)
path_info = PathLOCAL(path=data_dir)
with dvc.state:
with patch.object(
RemoteBase,
RemoteBASE,
"_collect_dir",
return_value=[
{"relpath": "1", "md5": "1"},
@@ -212,7 +212,7 @@ def test_dir_checksum_should_be_key_order_agnostic(dvc):
checksum1 = dvc.cache.local.get_dir_checksum(path_info)
with patch.object(
RemoteBase,
RemoteBASE,
"_collect_dir",
return_value=[
{"md5": "1", "relpath": "1"},
+2 -2
View File
@@ -4,7 +4,7 @@ from copy import copy
import boto3
import pytest
from dvc.path.s3 import S3PathInfo
from dvc.path.s3 import PathS3
from dvc.remote.s3 import RemoteS3
from tests.func.test_data_cloud import TEST_AWS_REPO_BUCKET, _should_test_aws
@@ -12,7 +12,7 @@ from tests.func.test_data_cloud import TEST_AWS_REPO_BUCKET, _should_test_aws
def _get_src_dst():
prefix = str(uuid.uuid4())
from_info = S3PathInfo(
from_info = PathS3(
bucket=TEST_AWS_REPO_BUCKET, path=posixpath.join(prefix, "from")
)
to_info = copy(from_info)
+3 -3
View File
@@ -1,6 +1,6 @@
import os
import mock
from dvc.path.local import LocalPathInfo
from dvc.path.local import PathLOCAL
from dvc.utils.compat import str
@@ -15,7 +15,7 @@ from tests.basic_env import TestDvc
class TestState(TestDvc):
def test_update(self):
path = os.path.join(self.dvc.root_dir, self.FOO)
path_info = LocalPathInfo(path=path)
path_info = PathLOCAL(path=path)
md5 = file_md5(path)[0]
state = State(self.dvc, self.dvc.config.config)
@@ -78,6 +78,6 @@ class TestGetStateRecordForInode(TestDvc):
get_inode_mock.side_effect = self.mock_get_inode(path, inode)
with state:
state.save(LocalPathInfo(path=path), md5)
state.save(PathLOCAL(path=path), md5)
ret = state.get_state_record_for_inode(inode)
self.assertIsNotNone(ret)
+4 -4
View File
@@ -1,9 +1,9 @@
from unittest import TestCase
from dvc.remote.azure import RemoteAzure
from dvc.remote.azure import RemoteAZURE
class TestRemoteAzure(TestCase):
class TestRemoteAZURE(TestCase):
container_name = "container-name"
connection_string = (
"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;"
@@ -20,7 +20,7 @@ class TestRemoteAzure(TestCase):
connection_string=self.connection_string,
)
config = {"url": url}
remote = RemoteAzure(None, config)
remote = RemoteAZURE(None, config)
self.assertEqual(remote.url, url)
self.assertEqual(remote.prefix, "")
self.assertEqual(remote.bucket, self.container_name)
@@ -30,7 +30,7 @@ class TestRemoteAzure(TestCase):
prefix = "some/prefix"
url = "azure://{}/{}".format(self.container_name, prefix)
config = {"url": url, "connection_string": self.connection_string}
remote = RemoteAzure(None, config)
remote = RemoteAZURE(None, config)
self.assertEqual(remote.url, url)
self.assertEqual(remote.prefix, prefix)
self.assertEqual(remote.bucket, self.container_name)
+9 -9
View File
@@ -1,15 +1,15 @@
import mock
from unittest import TestCase
from dvc.path import BasePathInfo
from dvc.remote.base import RemoteBase, RemoteCmdError, RemoteMissingDepsError
from dvc.path.base import PathBASE
from dvc.remote.base import RemoteBASE, RemoteCmdError, RemoteMissingDepsError
class TestRemoteBase(object):
REMOTE_CLS = RemoteBase
class TestRemoteBASE(object):
REMOTE_CLS = RemoteBASE
class TestMissingDeps(TestCase, TestRemoteBase):
class TestMissingDeps(TestCase, TestRemoteBASE):
def test(self):
REQUIRES = {"foo": None, "bar": None, "mock": mock}
with mock.patch.object(self.REMOTE_CLS, "REQUIRES", REQUIRES):
@@ -17,7 +17,7 @@ class TestMissingDeps(TestCase, TestRemoteBase):
self.REMOTE_CLS(None, {})
class TestCmdError(TestCase, TestRemoteBase):
class TestCmdError(TestCase, TestRemoteBASE):
def test(self):
repo = None
config = {}
@@ -41,13 +41,13 @@ class TestCacheExists(TestCase):
"url": "base://example/prefix",
"connection_string": "1234567",
}
remote = RemoteBase(None, config)
remote = RemoteBASE(None, config)
remote.PARAM_CHECKSUM = "checksum"
remote.path_info = BasePathInfo(None, None)
remote.path_info = PathBASE(None, None)
remote.url = ""
remote.prefix = ""
path_info = BasePathInfo("example", None)
path_info = PathBASE("example", None)
checksum_info = {remote.PARAM_CHECKSUM: "1234567890"}
with mock.patch.object(remote, "_checkout") as mock_checkout: