mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
Merge pull request #1990 from efiop/small-fixes
dvc: refactoring: unify path/remote logic and naming
This commit is contained in:
+5
-5
@@ -8,7 +8,7 @@ from dvc.config import Config, ConfigError
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
from dvc.remote.gs import RemoteGS
|
||||
from dvc.remote.azure import RemoteAzure
|
||||
from dvc.remote.azure import RemoteAZURE
|
||||
from dvc.remote.oss import RemoteOSS
|
||||
from dvc.remote.ssh import RemoteSSH
|
||||
from dvc.remote.hdfs import RemoteHDFS
|
||||
@@ -34,7 +34,7 @@ class DataCloud(object):
|
||||
CLOUD_MAP = {
|
||||
"aws": RemoteS3,
|
||||
"gcp": RemoteGS,
|
||||
"azure": RemoteAzure,
|
||||
"azure": RemoteAZURE,
|
||||
"oss": RemoteOSS,
|
||||
"ssh": RemoteSSH,
|
||||
"hdfs": RemoteHDFS,
|
||||
@@ -124,7 +124,7 @@ class DataCloud(object):
|
||||
Args:
|
||||
targets (list): list of targets to push to the cloud.
|
||||
jobs (int): number of jobs that can be running simultaneously.
|
||||
remote (dvc.remote.base.RemoteBase): optional remote to push to.
|
||||
remote (dvc.remote.base.RemoteBASE): optional remote to push to.
|
||||
By default remote from core.remote config option is used.
|
||||
show_checksums (bool): show checksums instead of file names in
|
||||
information messages.
|
||||
@@ -142,7 +142,7 @@ class DataCloud(object):
|
||||
Args:
|
||||
targets (list): list of targets to pull from the cloud.
|
||||
jobs (int): number of jobs that can be running simultaneously.
|
||||
remote (dvc.remote.base.RemoteBase): optional remote to pull from.
|
||||
remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
|
||||
By default remote from core.remote config option is used.
|
||||
show_checksums (bool): show checksums instead of file names in
|
||||
information messages.
|
||||
@@ -160,7 +160,7 @@ class DataCloud(object):
|
||||
Args:
|
||||
targets (list): list of targets to check status for.
|
||||
jobs (int): number of jobs that can be running simultaneously.
|
||||
remote (dvc.remote.base.RemoteBase): optional remote to compare
|
||||
remote (dvc.remote.base.RemoteBASE): optional remote to compare
|
||||
targets to. By default remote from core.remote config option
|
||||
is used.
|
||||
show_checksums (bool): show checksums instead of file names in
|
||||
|
||||
@@ -2,6 +2,8 @@ from __future__ import unicode_literals
|
||||
|
||||
import schema
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
import dvc.output as output
|
||||
from dvc.output.base import OutputBase
|
||||
from dvc.dependency.s3 import DependencyS3
|
||||
@@ -10,6 +12,7 @@ from dvc.dependency.local import DependencyLOCAL
|
||||
from dvc.dependency.hdfs import DependencyHDFS
|
||||
from dvc.dependency.ssh import DependencySSH
|
||||
from dvc.dependency.http import DependencyHTTP
|
||||
from dvc.dependency.https import DependencyHTTPS
|
||||
|
||||
from dvc.remote import Remote
|
||||
|
||||
@@ -17,19 +20,20 @@ DEPS = [
|
||||
DependencyGS,
|
||||
DependencyHDFS,
|
||||
DependencyHTTP,
|
||||
DependencyHTTPS,
|
||||
DependencyS3,
|
||||
DependencySSH,
|
||||
# NOTE: DependencyLOCAL is the default choice
|
||||
]
|
||||
|
||||
DEP_MAP = {
|
||||
"local": DependencyLOCAL,
|
||||
"ssh": DependencySSH,
|
||||
"s3": DependencyS3,
|
||||
"gs": DependencyGS,
|
||||
"hdfs": DependencyHDFS,
|
||||
"http": DependencyHTTP,
|
||||
"https": DependencyHTTP,
|
||||
Schemes.LOCAL: DependencyLOCAL,
|
||||
Schemes.SSH: DependencySSH,
|
||||
Schemes.S3: DependencyS3,
|
||||
Schemes.GS: DependencyGS,
|
||||
Schemes.HDFS: DependencyHDFS,
|
||||
Schemes.HTTP: DependencyHTTP,
|
||||
Schemes.HTTPS: DependencyHTTPS,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from dvc.path.http import HTTPPathInfo
|
||||
from dvc.path import Path
|
||||
from dvc.utils.compat import urlparse, urljoin
|
||||
from dvc.output.base import OutputBase
|
||||
from dvc.remote.http import RemoteHTTP
|
||||
@@ -17,4 +17,4 @@ class DependencyHTTP(DependencyBase, OutputBase):
|
||||
if path.startswith("remote"):
|
||||
path = urljoin(self.remote.cache_dir, urlparse(path).path)
|
||||
|
||||
self.path_info = HTTPPathInfo(url=self.url, path=path)
|
||||
self.path_info = Path(self.scheme, url=self.url, path=path)
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from dvc.remote.https import RemoteHTTPS
|
||||
from .http import DependencyHTTP
|
||||
|
||||
|
||||
class DependencyHTTPS(DependencyHTTP):
|
||||
REMOTE = RemoteHTTPS
|
||||
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
|
||||
|
||||
import schema
|
||||
|
||||
from dvc.path import Schemes
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils.compat import urlparse, str
|
||||
|
||||
from dvc.output.base import OutputBase
|
||||
|
||||
+2
-2
@@ -8,7 +8,7 @@ from schema import Or, Optional
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils.compat import str
|
||||
from dvc.remote.base import RemoteBase
|
||||
from dvc.remote.base import RemoteBASE
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -35,7 +35,7 @@ class OutputAlreadyTrackedError(DvcException):
|
||||
class OutputBase(object):
|
||||
IS_DEPENDENCY = False
|
||||
|
||||
REMOTE = RemoteBase
|
||||
REMOTE = RemoteBASE
|
||||
|
||||
PARAM_PATH = "path"
|
||||
PARAM_CACHE = "cache"
|
||||
|
||||
+2
-2
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
|
||||
|
||||
import posixpath
|
||||
|
||||
from dvc.path.hdfs import HDFSPathInfo
|
||||
from dvc.path.hdfs import PathHDFS
|
||||
from dvc.utils.compat import urlparse
|
||||
from dvc.output.base import OutputBase
|
||||
from dvc.remote.hdfs import RemoteHDFS
|
||||
@@ -35,4 +35,4 @@ class OutputHDFS(OutputBase):
|
||||
if remote:
|
||||
path = posixpath.join(remote.url, urlparse(path).path.lstrip("/"))
|
||||
user = remote.user if remote else self.group("user")
|
||||
self.path_info = HDFSPathInfo(user=user, url=self.url, path=path)
|
||||
self.path_info = PathHDFS(user=user, url=self.url, path=path)
|
||||
|
||||
+2
-2
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
|
||||
import os
|
||||
import logging
|
||||
|
||||
from dvc.path.local import LocalPathInfo
|
||||
from dvc.path.local import PathLOCAL
|
||||
from dvc.utils.compat import urlparse
|
||||
from dvc.istextfile import istextfile
|
||||
from dvc.exceptions import DvcException
|
||||
@@ -50,7 +50,7 @@ class OutputLOCAL(OutputBase):
|
||||
p = os.path.join(stage.wdir, p)
|
||||
p = os.path.abspath(os.path.normpath(p))
|
||||
|
||||
self.path_info = LocalPathInfo(url=self.url, path=p)
|
||||
self.path_info = PathLOCAL(url=self.url, path=p)
|
||||
|
||||
def __str__(self):
|
||||
return self.rel_path
|
||||
|
||||
+2
-2
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
|
||||
|
||||
import posixpath
|
||||
|
||||
from dvc.path.utils import PathInfo
|
||||
from dvc.path import Path
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
from dvc.utils.compat import urlparse
|
||||
from dvc.output.base import OutputBase
|
||||
@@ -37,6 +37,6 @@ class OutputS3(OutputBase):
|
||||
if remote:
|
||||
path = posixpath.join(remote.prefix, path)
|
||||
|
||||
self.path_info = PathInfo(
|
||||
self.path_info = Path(
|
||||
self.scheme, bucket=bucket, path=path, url=self.url
|
||||
)
|
||||
|
||||
+2
-2
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
|
||||
import getpass
|
||||
import posixpath
|
||||
|
||||
from dvc.path.ssh import SSHPathInfo
|
||||
from dvc.path.ssh import PathSSH
|
||||
from dvc.utils.compat import urlparse
|
||||
from dvc.output.base import OutputBase
|
||||
from dvc.remote.ssh import RemoteSSH
|
||||
@@ -49,6 +49,6 @@ class OutputSSH(OutputBase):
|
||||
else:
|
||||
path = parsed.path
|
||||
|
||||
self.path_info = SSHPathInfo(
|
||||
self.path_info = PathSSH(
|
||||
host=host, user=user, port=port, url=self.url, path=path
|
||||
)
|
||||
|
||||
+25
-30
@@ -1,34 +1,29 @@
|
||||
from dvc.utils.compat import urlunsplit
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from dvc.path.azure import PathAZURE
|
||||
from dvc.path.gs import PathGS
|
||||
from dvc.path.hdfs import PathHDFS
|
||||
from dvc.path.http import PathHTTP
|
||||
from dvc.path.https import PathHTTPS
|
||||
from dvc.path.local import PathLOCAL
|
||||
from dvc.path.oss import PathOSS
|
||||
from dvc.path.s3 import PathS3
|
||||
from dvc.path.ssh import PathSSH
|
||||
|
||||
|
||||
class Schemes:
|
||||
SSH = "ssh"
|
||||
HDFS = "hdfs"
|
||||
S3 = "s3"
|
||||
AZURE = "azure"
|
||||
HTTP = "http"
|
||||
GS = "gs"
|
||||
LOCAL = "local"
|
||||
OSS = "oss"
|
||||
PATH_MAP = {
|
||||
Schemes.SSH: PathSSH,
|
||||
Schemes.HDFS: PathHDFS,
|
||||
Schemes.S3: PathS3,
|
||||
Schemes.AZURE: PathAZURE,
|
||||
Schemes.HTTP: PathHTTP,
|
||||
Schemes.HTTPS: PathHTTPS,
|
||||
Schemes.GS: PathGS,
|
||||
Schemes.LOCAL: PathLOCAL,
|
||||
Schemes.OSS: PathOSS,
|
||||
}
|
||||
|
||||
|
||||
class BasePathInfo(object):
|
||||
scheme = None
|
||||
|
||||
def __init__(self, url=None, path=None):
|
||||
self.url = url
|
||||
self.path = path
|
||||
|
||||
def __str__(self):
|
||||
return self.url
|
||||
|
||||
|
||||
class DefaultCloudPathInfo(BasePathInfo):
|
||||
def __init__(self, bucket, url=None, path=None):
|
||||
super(DefaultCloudPathInfo, self).__init__(url, path)
|
||||
self.bucket = bucket
|
||||
|
||||
def __str__(self):
|
||||
if not self.url:
|
||||
return urlunsplit((self.scheme, self.bucket, self.path, "", ""))
|
||||
return self.url
|
||||
def Path(scheme, *args, **kwargs):
|
||||
cls = PATH_MAP[scheme]
|
||||
return cls(*args, **kwargs)
|
||||
|
||||
+3
-2
@@ -1,5 +1,6 @@
|
||||
from dvc.path import Schemes, DefaultCloudPathInfo
|
||||
from dvc.scheme import Schemes
|
||||
from .base import PathCloudBASE
|
||||
|
||||
|
||||
class AzurePathInfo(DefaultCloudPathInfo):
|
||||
class PathAZURE(PathCloudBASE):
|
||||
scheme = Schemes.AZURE
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
from dvc.utils.compat import urlunsplit
|
||||
|
||||
|
||||
class PathBASE(object):
|
||||
scheme = None
|
||||
|
||||
def __init__(self, url=None, path=None):
|
||||
self.url = url
|
||||
self.path = path
|
||||
|
||||
def __str__(self):
|
||||
return self.url
|
||||
|
||||
|
||||
class PathCloudBASE(PathBASE):
|
||||
def __init__(self, bucket, url=None, path=None):
|
||||
super(PathCloudBASE, self).__init__(url, path)
|
||||
self.bucket = bucket
|
||||
|
||||
def __str__(self):
|
||||
if not self.url:
|
||||
return urlunsplit((self.scheme, self.bucket, self.path, "", ""))
|
||||
return self.url
|
||||
+3
-2
@@ -1,5 +1,6 @@
|
||||
from dvc.path import Schemes, DefaultCloudPathInfo
|
||||
from dvc.scheme import Schemes
|
||||
from .base import PathCloudBASE
|
||||
|
||||
|
||||
class GSPathInfo(DefaultCloudPathInfo):
|
||||
class PathGS(PathCloudBASE):
|
||||
scheme = Schemes.GS
|
||||
|
||||
+4
-3
@@ -1,12 +1,13 @@
|
||||
from dvc.utils.compat import urlunsplit
|
||||
from dvc.path import BasePathInfo, Schemes
|
||||
from dvc.scheme import Schemes
|
||||
from .base import PathBASE
|
||||
|
||||
|
||||
class HDFSPathInfo(BasePathInfo):
|
||||
class PathHDFS(PathBASE):
|
||||
scheme = Schemes.HDFS
|
||||
|
||||
def __init__(self, user, url=None, path=None):
|
||||
super(HDFSPathInfo, self).__init__(url, path)
|
||||
super(PathHDFS, self).__init__(url, path)
|
||||
self.user = user
|
||||
|
||||
def __str__(self):
|
||||
|
||||
+6
-9
@@ -1,14 +1,11 @@
|
||||
from dvc.path import BasePathInfo, Schemes
|
||||
from dvc.utils.compat import urlparse, urlunsplit
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils.compat import urlunsplit
|
||||
|
||||
from .base import PathBASE
|
||||
|
||||
|
||||
class HTTPPathInfo(BasePathInfo):
|
||||
@property
|
||||
def scheme(self):
|
||||
if self.path:
|
||||
return urlparse(self.path).scheme
|
||||
else:
|
||||
return Schemes.HTTP
|
||||
class PathHTTP(PathBASE):
|
||||
scheme = Schemes.HTTP
|
||||
|
||||
def __str__(self):
|
||||
if not self.url:
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
from dvc.scheme import Schemes
|
||||
from .http import PathHTTP
|
||||
|
||||
|
||||
class PathHTTPS(PathHTTP):
|
||||
scheme = Schemes.HTTPS
|
||||
+3
-2
@@ -1,9 +1,10 @@
|
||||
import os
|
||||
|
||||
from dvc.path import BasePathInfo, Schemes
|
||||
from dvc.scheme import Schemes
|
||||
from .base import PathBASE
|
||||
|
||||
|
||||
class LocalPathInfo(BasePathInfo):
|
||||
class PathLOCAL(PathBASE):
|
||||
scheme = Schemes.LOCAL
|
||||
|
||||
def __str__(self):
|
||||
|
||||
+3
-2
@@ -1,5 +1,6 @@
|
||||
from dvc.path import Schemes, DefaultCloudPathInfo
|
||||
from dvc.scheme import Schemes
|
||||
from .base import PathCloudBASE
|
||||
|
||||
|
||||
class OSSPathInfo(DefaultCloudPathInfo):
|
||||
class PathOSS(PathCloudBASE):
|
||||
scheme = Schemes.OSS
|
||||
|
||||
+3
-2
@@ -1,5 +1,6 @@
|
||||
from dvc.path import Schemes, DefaultCloudPathInfo
|
||||
from dvc.scheme import Schemes
|
||||
from .base import PathCloudBASE
|
||||
|
||||
|
||||
class S3PathInfo(DefaultCloudPathInfo):
|
||||
class PathS3(PathCloudBASE):
|
||||
scheme = Schemes.S3
|
||||
|
||||
+4
-3
@@ -1,12 +1,13 @@
|
||||
from dvc.utils.compat import urlunsplit
|
||||
from dvc.path import BasePathInfo, Schemes
|
||||
from dvc.scheme import Schemes
|
||||
from .base import PathBASE
|
||||
|
||||
|
||||
class SSHPathInfo(BasePathInfo):
|
||||
class PathSSH(PathBASE):
|
||||
scheme = Schemes.SSH
|
||||
|
||||
def __init__(self, host, user, port, url=None, path=None):
|
||||
super(SSHPathInfo, self).__init__(url, path)
|
||||
super(PathSSH, self).__init__(url, path)
|
||||
self.host = host
|
||||
self.user = user
|
||||
self.port = port
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
from dvc.path import Schemes
|
||||
from dvc.path.azure import AzurePathInfo
|
||||
from dvc.path.gs import GSPathInfo
|
||||
from dvc.path.hdfs import HDFSPathInfo
|
||||
from dvc.path.http import HTTPPathInfo
|
||||
from dvc.path.local import LocalPathInfo
|
||||
from dvc.path.oss import OSSPathInfo
|
||||
from dvc.path.s3 import S3PathInfo
|
||||
from dvc.path.ssh import SSHPathInfo
|
||||
|
||||
PATH_MAP = {
|
||||
Schemes.SSH: SSHPathInfo,
|
||||
Schemes.HDFS: HDFSPathInfo,
|
||||
Schemes.S3: S3PathInfo,
|
||||
Schemes.AZURE: AzurePathInfo,
|
||||
Schemes.HTTP: HTTPPathInfo,
|
||||
Schemes.GS: GSPathInfo,
|
||||
Schemes.LOCAL: LocalPathInfo,
|
||||
Schemes.OSS: OSSPathInfo,
|
||||
}
|
||||
|
||||
|
||||
def PathInfo(scheme, *args, **kwargs):
|
||||
cls = PATH_MAP[scheme]
|
||||
return cls(*args, **kwargs)
|
||||
@@ -1,20 +1,22 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from dvc.remote.azure import RemoteAzure
|
||||
from dvc.remote.azure import RemoteAZURE
|
||||
from dvc.remote.gs import RemoteGS
|
||||
from dvc.remote.hdfs import RemoteHDFS
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
from dvc.remote.ssh import RemoteSSH
|
||||
from dvc.remote.http import RemoteHTTP
|
||||
from dvc.remote.https import RemoteHTTPS
|
||||
from dvc.remote.oss import RemoteOSS
|
||||
|
||||
|
||||
REMOTES = [
|
||||
RemoteAzure,
|
||||
RemoteAZURE,
|
||||
RemoteGS,
|
||||
RemoteHDFS,
|
||||
RemoteHTTP,
|
||||
RemoteHTTPS,
|
||||
RemoteS3,
|
||||
RemoteSSH,
|
||||
RemoteOSS,
|
||||
|
||||
+6
-6
@@ -5,8 +5,8 @@ import os
|
||||
import re
|
||||
import logging
|
||||
|
||||
from dvc.path import Schemes
|
||||
from dvc.path.azure import AzurePathInfo
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.path.azure import PathAZURE
|
||||
|
||||
try:
|
||||
from azure.storage.blob import BlockBlobService
|
||||
@@ -18,7 +18,7 @@ from dvc.utils import tmp_fname, move
|
||||
from dvc.utils.compat import urlparse, makedirs
|
||||
from dvc.progress import progress
|
||||
from dvc.config import Config
|
||||
from dvc.remote.base import RemoteBase
|
||||
from dvc.remote.base import RemoteBASE
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -32,7 +32,7 @@ class Callback(object):
|
||||
progress.update_target(self.name, current, total)
|
||||
|
||||
|
||||
class RemoteAzure(RemoteBase):
|
||||
class RemoteAZURE(RemoteBASE):
|
||||
scheme = Schemes.AZURE
|
||||
REGEX = (
|
||||
r"azure://((?P<path>[^=;]*)?|("
|
||||
@@ -45,7 +45,7 @@ class RemoteAzure(RemoteBase):
|
||||
COPY_POLL_SECONDS = 5
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super(RemoteAzure, self).__init__(repo, config)
|
||||
super(RemoteAZURE, self).__init__(repo, config)
|
||||
|
||||
self.url = config.get(Config.SECTION_REMOTE_URL, "azure://")
|
||||
match = re.match(self.REGEX, self.url) # backward compatibility
|
||||
@@ -72,7 +72,7 @@ class RemoteAzure(RemoteBase):
|
||||
raise ValueError("azure storage connection string missing")
|
||||
|
||||
self.__blob_service = None
|
||||
self.path_info = AzurePathInfo(bucket=self.bucket)
|
||||
self.path_info = PathAZURE(bucket=self.bucket)
|
||||
|
||||
@property
|
||||
def blob_service(self):
|
||||
|
||||
+4
-4
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
|
||||
|
||||
from copy import copy
|
||||
|
||||
from dvc.path.local import LocalPathInfo
|
||||
from dvc.path.local import PathLOCAL
|
||||
from dvc.utils.compat import str
|
||||
|
||||
import os
|
||||
@@ -66,7 +66,7 @@ class RemoteMissingDepsError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class RemoteBase(object):
|
||||
class RemoteBASE(object):
|
||||
scheme = None
|
||||
REGEX = None
|
||||
REQUIRES = {}
|
||||
@@ -211,7 +211,7 @@ class RemoteBase(object):
|
||||
with open(tmp, "w+") as fobj:
|
||||
json.dump(dir_info, fobj, sort_keys=True)
|
||||
|
||||
from_info = LocalPathInfo(path=tmp)
|
||||
from_info = PathLOCAL(path=tmp)
|
||||
self.cache.upload([from_info], [to_info], no_progress_bar=True)
|
||||
|
||||
checksum = self.get_file_checksum(to_info) + self.CHECKSUM_DIR_SUFFIX
|
||||
@@ -235,7 +235,7 @@ class RemoteBase(object):
|
||||
|
||||
fobj = tempfile.NamedTemporaryFile(delete=False)
|
||||
path = fobj.name
|
||||
to_info = LocalPathInfo(path=path)
|
||||
to_info = PathLOCAL(path=path)
|
||||
self.cache.download([path_info], [to_info], no_progress_bar=True)
|
||||
|
||||
try:
|
||||
|
||||
+5
-5
@@ -3,8 +3,8 @@ from __future__ import unicode_literals
|
||||
import os
|
||||
import logging
|
||||
|
||||
from dvc.path import Schemes
|
||||
from dvc.path.gs import GSPathInfo
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.path.gs import PathGS
|
||||
|
||||
try:
|
||||
from google.cloud import storage
|
||||
@@ -13,7 +13,7 @@ except ImportError:
|
||||
|
||||
from dvc.utils import tmp_fname, move
|
||||
from dvc.utils.compat import urlparse, makedirs
|
||||
from dvc.remote.base import RemoteBase
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from dvc.config import Config
|
||||
from dvc.progress import progress
|
||||
from dvc.exceptions import DvcException
|
||||
@@ -22,7 +22,7 @@ from dvc.exceptions import DvcException
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RemoteGS(RemoteBase):
|
||||
class RemoteGS(RemoteBASE):
|
||||
scheme = Schemes.GS
|
||||
REGEX = r"^gs://(?P<path>.*)$"
|
||||
REQUIRES = {"google.cloud.storage": storage}
|
||||
@@ -41,7 +41,7 @@ class RemoteGS(RemoteBase):
|
||||
self.bucket = parsed.netloc
|
||||
self.prefix = parsed.path.lstrip("/")
|
||||
|
||||
self.path_info = GSPathInfo(bucket=self.bucket)
|
||||
self.path_info = PathGS(bucket=self.bucket)
|
||||
|
||||
@staticmethod
|
||||
def compat_config(config):
|
||||
|
||||
+5
-5
@@ -8,16 +8,16 @@ import logging
|
||||
from subprocess import Popen, PIPE
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.path import Schemes
|
||||
from dvc.path.hdfs import HDFSPathInfo
|
||||
from dvc.remote.base import RemoteBase, RemoteCmdError
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.path.hdfs import PathHDFS
|
||||
from dvc.remote.base import RemoteBASE, RemoteCmdError
|
||||
from dvc.utils import fix_env, tmp_fname
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RemoteHDFS(RemoteBase):
|
||||
class RemoteHDFS(RemoteBASE):
|
||||
scheme = Schemes.HDFS
|
||||
REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
@@ -32,7 +32,7 @@ class RemoteHDFS(RemoteBase):
|
||||
Config.SECTION_REMOTE_USER, getpass.getuser()
|
||||
)
|
||||
|
||||
self.path_info = HDFSPathInfo(user=self.user)
|
||||
self.path_info = PathHDFS(user=self.user)
|
||||
|
||||
def hadoop_fs(self, cmd, user=None):
|
||||
cmd = "hadoop fs -" + cmd
|
||||
|
||||
+8
-8
@@ -1,7 +1,7 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from dvc.path import Schemes
|
||||
from dvc.path.http import HTTPPathInfo
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.path import Path
|
||||
from dvc.utils.compat import open, makedirs
|
||||
|
||||
import os
|
||||
@@ -12,7 +12,7 @@ import logging
|
||||
from dvc.progress import progress
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.config import Config
|
||||
from dvc.remote.base import RemoteBase
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from dvc.utils import move
|
||||
|
||||
|
||||
@@ -32,9 +32,9 @@ class ProgressBarCallback(object):
|
||||
progress.update_target(self.name, self.current, self.total)
|
||||
|
||||
|
||||
class RemoteHTTP(RemoteBase):
|
||||
class RemoteHTTP(RemoteBASE):
|
||||
scheme = Schemes.HTTP
|
||||
REGEX = r"^https?://.*$"
|
||||
REGEX = r"^http://.*$"
|
||||
REQUEST_TIMEOUT = 10
|
||||
CHUNK_SIZE = 2 ** 16
|
||||
PARAM_CHECKSUM = "etag"
|
||||
@@ -44,7 +44,7 @@ class RemoteHTTP(RemoteBase):
|
||||
self.cache_dir = config.get(Config.SECTION_REMOTE_URL)
|
||||
self.url = self.cache_dir
|
||||
|
||||
self.path_info = HTTPPathInfo()
|
||||
self.path_info = Path(self.scheme)
|
||||
|
||||
@property
|
||||
def prefix(self):
|
||||
@@ -61,7 +61,7 @@ class RemoteHTTP(RemoteBase):
|
||||
names = self._verify_path_args(to_infos, from_infos, names)
|
||||
|
||||
for to_info, from_info, name in zip(to_infos, from_infos, names):
|
||||
if from_info.scheme not in ["http", "https"]:
|
||||
if from_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
if to_info.scheme != "local":
|
||||
@@ -99,7 +99,7 @@ class RemoteHTTP(RemoteBase):
|
||||
|
||||
def exists(self, path_info):
|
||||
assert not isinstance(path_info, list)
|
||||
assert path_info.scheme in ["http", "https"]
|
||||
assert path_info.scheme == self.scheme
|
||||
return bool(self._request("HEAD", path_info.path))
|
||||
|
||||
def cache_exists(self, md5s):
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .http import RemoteHTTP
|
||||
|
||||
|
||||
class RemoteHTTPS(RemoteHTTP):
|
||||
scheme = Schemes.HTTPS
|
||||
REGEX = r"^https://.*$"
|
||||
@@ -2,8 +2,9 @@ from __future__ import unicode_literals
|
||||
|
||||
from copy import copy
|
||||
|
||||
from dvc.path import BasePathInfo, Schemes
|
||||
from dvc.path.local import LocalPathInfo
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.path.base import PathBASE
|
||||
from dvc.path.local import PathLOCAL
|
||||
from dvc.remote.local.slow_link_detection import slow_link_guard
|
||||
from dvc.utils.compat import str, makedirs
|
||||
|
||||
@@ -17,7 +18,7 @@ import logging
|
||||
|
||||
from dvc.system import System
|
||||
from dvc.remote.base import (
|
||||
RemoteBase,
|
||||
RemoteBASE,
|
||||
STATUS_MAP,
|
||||
STATUS_NEW,
|
||||
STATUS_DELETED,
|
||||
@@ -43,7 +44,7 @@ from dvc.utils.fs import get_mtime_and_size, get_inode
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RemoteLOCAL(RemoteBase):
|
||||
class RemoteLOCAL(RemoteBASE):
|
||||
scheme = Schemes.LOCAL
|
||||
REGEX = r"^(?P<path>.*)$"
|
||||
PARAM_CHECKSUM = "md5"
|
||||
@@ -80,7 +81,7 @@ class RemoteLOCAL(RemoteBase):
|
||||
os.mkdir(self.cache_dir)
|
||||
|
||||
self._dir_info = {}
|
||||
self.path_info = LocalPathInfo()
|
||||
self.path_info = PathLOCAL()
|
||||
|
||||
@staticmethod
|
||||
def compat_config(config):
|
||||
@@ -116,7 +117,7 @@ class RemoteLOCAL(RemoteBase):
|
||||
return self.checksum_to_path(md5)
|
||||
|
||||
def exists(self, path_info):
|
||||
assert isinstance(path_info, BasePathInfo)
|
||||
assert isinstance(path_info, PathBASE)
|
||||
assert path_info.scheme == "local"
|
||||
return os.path.lexists(path_info.path)
|
||||
|
||||
|
||||
+5
-5
@@ -4,8 +4,8 @@ from __future__ import unicode_literals
|
||||
import os
|
||||
import logging
|
||||
|
||||
from dvc.path import Schemes
|
||||
from dvc.path.oss import OSSPathInfo
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.path.oss import PathOSS
|
||||
|
||||
try:
|
||||
import oss2
|
||||
@@ -16,14 +16,14 @@ from dvc.utils import tmp_fname, move
|
||||
from dvc.utils.compat import urlparse, makedirs
|
||||
from dvc.progress import progress
|
||||
from dvc.config import Config
|
||||
from dvc.remote.base import RemoteBase
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from dvc.remote.azure import Callback
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RemoteOSS(RemoteBase):
|
||||
class RemoteOSS(RemoteBASE):
|
||||
"""
|
||||
oss2 document:
|
||||
https://www.alibabacloud.com/help/doc-detail/32026.htm
|
||||
@@ -73,7 +73,7 @@ class RemoteOSS(RemoteBase):
|
||||
)
|
||||
|
||||
self._bucket = None
|
||||
self.path_info = OSSPathInfo(bucket=self.bucket)
|
||||
self.path_info = PathOSS(bucket=self.bucket)
|
||||
|
||||
@property
|
||||
def oss_service(self):
|
||||
|
||||
+5
-5
@@ -4,8 +4,8 @@ import os
|
||||
import threading
|
||||
import logging
|
||||
|
||||
from dvc.path import Schemes
|
||||
from dvc.path.s3 import S3PathInfo
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.path.s3 import PathS3
|
||||
|
||||
try:
|
||||
import boto3
|
||||
@@ -16,7 +16,7 @@ from dvc.utils import tmp_fname, move
|
||||
from dvc.utils.compat import urlparse, makedirs
|
||||
from dvc.progress import progress
|
||||
from dvc.config import Config
|
||||
from dvc.remote.base import RemoteBase
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from dvc.exceptions import DvcException, ETagMismatchError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -35,7 +35,7 @@ class Callback(object):
|
||||
progress.update_target(self.name, self.current, self.total)
|
||||
|
||||
|
||||
class RemoteS3(RemoteBase):
|
||||
class RemoteS3(RemoteBASE):
|
||||
scheme = Schemes.S3
|
||||
REGEX = r"^s3://(?P<path>.*)$"
|
||||
REQUIRES = {"boto3": boto3}
|
||||
@@ -72,7 +72,7 @@ class RemoteS3(RemoteBase):
|
||||
self.bucket = parsed.netloc
|
||||
self.prefix = parsed.path.lstrip("/")
|
||||
|
||||
self.path_info = S3PathInfo(bucket=self.bucket)
|
||||
self.path_info = PathS3(bucket=self.bucket)
|
||||
|
||||
@staticmethod
|
||||
def compat_config(config):
|
||||
|
||||
@@ -5,7 +5,7 @@ import getpass
|
||||
import logging
|
||||
|
||||
from dvc.path import Schemes
|
||||
from dvc.path.ssh import SSHPathInfo
|
||||
from dvc.path.ssh import PathSSH
|
||||
|
||||
try:
|
||||
import paramiko
|
||||
@@ -16,13 +16,13 @@ import dvc.prompt as prompt
|
||||
from dvc.remote.ssh.connection import SSHConnection
|
||||
from dvc.config import Config
|
||||
from dvc.utils.compat import urlparse
|
||||
from dvc.remote.base import RemoteBase
|
||||
from dvc.remote.base import RemoteBASE
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RemoteSSH(RemoteBase):
|
||||
class RemoteSSH(RemoteBASE):
|
||||
scheme = Schemes.SSH
|
||||
|
||||
# NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and
|
||||
@@ -68,7 +68,7 @@ class RemoteSSH(RemoteBase):
|
||||
Config.SECTION_REMOTE_ASK_PASSWORD, False
|
||||
)
|
||||
|
||||
self.path_info = SSHPathInfo(
|
||||
self.path_info = PathSSH(
|
||||
host=self.host, user=self.user, port=self.port
|
||||
)
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ from dvc.exceptions import (
|
||||
TargetNotDirectoryError,
|
||||
)
|
||||
from dvc.ignore import DvcIgnoreFileHandler
|
||||
from dvc.path.local import LocalPathInfo
|
||||
from dvc.path.local import PathLOCAL
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -117,7 +117,7 @@ class Repo(object):
|
||||
return Repo(root_dir)
|
||||
|
||||
def unprotect(self, target):
|
||||
path_info = LocalPathInfo(path=target)
|
||||
path_info = PathLOCAL(path=target)
|
||||
return self.cache.local.unprotect(path_info)
|
||||
|
||||
def _ignore(self):
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
|
||||
class Schemes:
|
||||
SSH = "ssh"
|
||||
HDFS = "hdfs"
|
||||
S3 = "s3"
|
||||
AZURE = "azure"
|
||||
HTTP = "http"
|
||||
HTTPS = "https"
|
||||
GS = "gs"
|
||||
LOCAL = "local"
|
||||
OSS = "oss"
|
||||
+2
-2
@@ -619,14 +619,14 @@ class Stage(object):
|
||||
return stage
|
||||
|
||||
def dumpd(self):
|
||||
from dvc.remote.base import RemoteBase
|
||||
from dvc.remote.base import RemoteBASE
|
||||
|
||||
return {
|
||||
key: value
|
||||
for key, value in {
|
||||
Stage.PARAM_MD5: self.md5,
|
||||
Stage.PARAM_CMD: self.cmd,
|
||||
Stage.PARAM_WDIR: RemoteBase.to_posixpath(
|
||||
Stage.PARAM_WDIR: RemoteBASE.to_posixpath(
|
||||
os.path.relpath(self.wdir, os.path.dirname(self.path))
|
||||
),
|
||||
Stage.PARAM_LOCKED: self.locked,
|
||||
|
||||
@@ -19,7 +19,7 @@ from dvc.data_cloud import (
|
||||
DataCloud,
|
||||
RemoteS3,
|
||||
RemoteGS,
|
||||
RemoteAzure,
|
||||
RemoteAZURE,
|
||||
RemoteOSS,
|
||||
RemoteLOCAL,
|
||||
RemoteSSH,
|
||||
@@ -244,7 +244,7 @@ class TestDataCloud(TestDvc):
|
||||
("gs://mybucket/", RemoteGS),
|
||||
("ssh://user@localhost:/", RemoteSSH),
|
||||
("http://localhost:8000/", RemoteHTTP),
|
||||
("azure://ContainerName=mybucket;conn_string;", RemoteAzure),
|
||||
("azure://ContainerName=mybucket;conn_string;", RemoteAZURE),
|
||||
("oss://mybucket/", RemoteOSS),
|
||||
(TestDvc.mkdtemp(), RemoteLOCAL),
|
||||
]
|
||||
@@ -403,7 +403,7 @@ class TestRemoteGS(TestDataCloudBase):
|
||||
return RemoteGS
|
||||
|
||||
|
||||
class TestRemoteAzureCompat(TestDataCloudBase):
|
||||
class TestRemoteAZURECompat(TestDataCloudBase):
|
||||
def _should_test(self):
|
||||
return _should_test_azure()
|
||||
|
||||
@@ -411,10 +411,10 @@ class TestRemoteAzureCompat(TestDataCloudBase):
|
||||
return get_azure_url_compat()
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return RemoteAzure
|
||||
return RemoteAZURE
|
||||
|
||||
|
||||
class TestRemoteAzure(TestRemoteAzureCompat):
|
||||
class TestRemoteAZURE(TestRemoteAZURECompat):
|
||||
def _get_url(self):
|
||||
return get_azure_url()
|
||||
|
||||
@@ -673,7 +673,7 @@ class TestRemoteGSCLI(TestDataCloudCLIBase):
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
|
||||
|
||||
class TestRemoteAzureCLI(TestDataCloudCLIBase):
|
||||
class TestRemoteAZURECLI(TestDataCloudCLIBase):
|
||||
def _should_test(self):
|
||||
return _should_test_azure()
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
import configobj
|
||||
from dvc.path.local import LocalPathInfo
|
||||
from dvc.remote.base import RemoteBase
|
||||
from dvc.path.local import PathLOCAL
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from mock import patch
|
||||
|
||||
from dvc.main import main
|
||||
@@ -199,10 +199,10 @@ def test_dir_checksum_should_be_key_order_agnostic(dvc):
|
||||
with open(file2, "w") as fobj:
|
||||
fobj.write("2")
|
||||
|
||||
path_info = LocalPathInfo(path=data_dir)
|
||||
path_info = PathLOCAL(path=data_dir)
|
||||
with dvc.state:
|
||||
with patch.object(
|
||||
RemoteBase,
|
||||
RemoteBASE,
|
||||
"_collect_dir",
|
||||
return_value=[
|
||||
{"relpath": "1", "md5": "1"},
|
||||
@@ -212,7 +212,7 @@ def test_dir_checksum_should_be_key_order_agnostic(dvc):
|
||||
checksum1 = dvc.cache.local.get_dir_checksum(path_info)
|
||||
|
||||
with patch.object(
|
||||
RemoteBase,
|
||||
RemoteBASE,
|
||||
"_collect_dir",
|
||||
return_value=[
|
||||
{"md5": "1", "relpath": "1"},
|
||||
|
||||
@@ -4,7 +4,7 @@ from copy import copy
|
||||
|
||||
import boto3
|
||||
import pytest
|
||||
from dvc.path.s3 import S3PathInfo
|
||||
from dvc.path.s3 import PathS3
|
||||
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
from tests.func.test_data_cloud import TEST_AWS_REPO_BUCKET, _should_test_aws
|
||||
@@ -12,7 +12,7 @@ from tests.func.test_data_cloud import TEST_AWS_REPO_BUCKET, _should_test_aws
|
||||
|
||||
def _get_src_dst():
|
||||
prefix = str(uuid.uuid4())
|
||||
from_info = S3PathInfo(
|
||||
from_info = PathS3(
|
||||
bucket=TEST_AWS_REPO_BUCKET, path=posixpath.join(prefix, "from")
|
||||
)
|
||||
to_info = copy(from_info)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import mock
|
||||
from dvc.path.local import LocalPathInfo
|
||||
from dvc.path.local import PathLOCAL
|
||||
|
||||
from dvc.utils.compat import str
|
||||
|
||||
@@ -15,7 +15,7 @@ from tests.basic_env import TestDvc
|
||||
class TestState(TestDvc):
|
||||
def test_update(self):
|
||||
path = os.path.join(self.dvc.root_dir, self.FOO)
|
||||
path_info = LocalPathInfo(path=path)
|
||||
path_info = PathLOCAL(path=path)
|
||||
md5 = file_md5(path)[0]
|
||||
|
||||
state = State(self.dvc, self.dvc.config.config)
|
||||
@@ -78,6 +78,6 @@ class TestGetStateRecordForInode(TestDvc):
|
||||
get_inode_mock.side_effect = self.mock_get_inode(path, inode)
|
||||
|
||||
with state:
|
||||
state.save(LocalPathInfo(path=path), md5)
|
||||
state.save(PathLOCAL(path=path), md5)
|
||||
ret = state.get_state_record_for_inode(inode)
|
||||
self.assertIsNotNone(ret)
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
from unittest import TestCase
|
||||
|
||||
from dvc.remote.azure import RemoteAzure
|
||||
from dvc.remote.azure import RemoteAZURE
|
||||
|
||||
|
||||
class TestRemoteAzure(TestCase):
|
||||
class TestRemoteAZURE(TestCase):
|
||||
container_name = "container-name"
|
||||
connection_string = (
|
||||
"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;"
|
||||
@@ -20,7 +20,7 @@ class TestRemoteAzure(TestCase):
|
||||
connection_string=self.connection_string,
|
||||
)
|
||||
config = {"url": url}
|
||||
remote = RemoteAzure(None, config)
|
||||
remote = RemoteAZURE(None, config)
|
||||
self.assertEqual(remote.url, url)
|
||||
self.assertEqual(remote.prefix, "")
|
||||
self.assertEqual(remote.bucket, self.container_name)
|
||||
@@ -30,7 +30,7 @@ class TestRemoteAzure(TestCase):
|
||||
prefix = "some/prefix"
|
||||
url = "azure://{}/{}".format(self.container_name, prefix)
|
||||
config = {"url": url, "connection_string": self.connection_string}
|
||||
remote = RemoteAzure(None, config)
|
||||
remote = RemoteAZURE(None, config)
|
||||
self.assertEqual(remote.url, url)
|
||||
self.assertEqual(remote.prefix, prefix)
|
||||
self.assertEqual(remote.bucket, self.container_name)
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
import mock
|
||||
from unittest import TestCase
|
||||
|
||||
from dvc.path import BasePathInfo
|
||||
from dvc.remote.base import RemoteBase, RemoteCmdError, RemoteMissingDepsError
|
||||
from dvc.path.base import PathBASE
|
||||
from dvc.remote.base import RemoteBASE, RemoteCmdError, RemoteMissingDepsError
|
||||
|
||||
|
||||
class TestRemoteBase(object):
|
||||
REMOTE_CLS = RemoteBase
|
||||
class TestRemoteBASE(object):
|
||||
REMOTE_CLS = RemoteBASE
|
||||
|
||||
|
||||
class TestMissingDeps(TestCase, TestRemoteBase):
|
||||
class TestMissingDeps(TestCase, TestRemoteBASE):
|
||||
def test(self):
|
||||
REQUIRES = {"foo": None, "bar": None, "mock": mock}
|
||||
with mock.patch.object(self.REMOTE_CLS, "REQUIRES", REQUIRES):
|
||||
@@ -17,7 +17,7 @@ class TestMissingDeps(TestCase, TestRemoteBase):
|
||||
self.REMOTE_CLS(None, {})
|
||||
|
||||
|
||||
class TestCmdError(TestCase, TestRemoteBase):
|
||||
class TestCmdError(TestCase, TestRemoteBASE):
|
||||
def test(self):
|
||||
repo = None
|
||||
config = {}
|
||||
@@ -41,13 +41,13 @@ class TestCacheExists(TestCase):
|
||||
"url": "base://example/prefix",
|
||||
"connection_string": "1234567",
|
||||
}
|
||||
remote = RemoteBase(None, config)
|
||||
remote = RemoteBASE(None, config)
|
||||
|
||||
remote.PARAM_CHECKSUM = "checksum"
|
||||
remote.path_info = BasePathInfo(None, None)
|
||||
remote.path_info = PathBASE(None, None)
|
||||
remote.url = ""
|
||||
remote.prefix = ""
|
||||
path_info = BasePathInfo("example", None)
|
||||
path_info = PathBASE("example", None)
|
||||
checksum_info = {remote.PARAM_CHECKSUM: "1234567890"}
|
||||
|
||||
with mock.patch.object(remote, "_checkout") as mock_checkout:
|
||||
|
||||
Reference in New Issue
Block a user