Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-09 23:10:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: dvc/remotes name unification (#3684)

Browse Source 
Partially fixes #2089
This commit is contained in:
nik123
2020-04-26 20:55:00 +03:00
committed by GitHub
parent b312895e77
commit fefbcc0b4c
49 changed files with 251 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/data_cloud.py
+3 -3
View File
@@ -54,7 +54,7 @@ class DataCloud(object):
Args:
cache (NamedCache): named checksums to push to the cloud.
jobs (int): number of jobs that can be running simultaneously.
remote (dvc.remote.base.RemoteBASE): optional remote to push to.
remote (dvc.remote.base.BaseRemote): optional remote to push to.
By default remote from core.remote config option is used.
show_checksums (bool): show checksums instead of file names in
information messages.
@@ -72,7 +72,7 @@ class DataCloud(object):
Args:
cache (NamedCache): named checksums to pull from the cloud.
jobs (int): number of jobs that can be running simultaneously.
remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
remote (dvc.remote.base.BaseRemote): optional remote to pull from.
By default remote from core.remote config option is used.
show_checksums (bool): show checksums instead of file names in
information messages.
@@ -103,7 +103,7 @@ class DataCloud(object):
Args:
cache (NamedCache): named checksums to check status for.
jobs (int): number of jobs that can be running simultaneously.
remote (dvc.remote.base.RemoteBASE): optional remote to compare
remote (dvc.remote.base.BaseRemote): optional remote to compare
cache to. By default remote from core.remote config option
is used.
show_checksums (bool): show checksums instead of file names in
dvc/dependency/http.py
+2 -2
View File
@@ -1,7 +1,7 @@
from dvc.dependency.base import DependencyBase
from dvc.output.base import OutputBase
from dvc.remote.http import RemoteHTTP
from dvc.remote.http import HTTPRemote
class DependencyHTTP(DependencyBase, OutputBase):
REMOTE = RemoteHTTP
REMOTE = HTTPRemote
dvc/dependency/https.py
+2 -2
View File
@@ -1,6 +1,6 @@
from .http import DependencyHTTP
from dvc.remote.https import RemoteHTTPS
from dvc.remote.https import HTTPSRemote
class DependencyHTTPS(DependencyHTTP):
REMOTE = RemoteHTTPS
REMOTE = HTTPSRemote
dvc/output/__init__.py
+6 -6
View File
@@ -8,9 +8,9 @@ from dvc.output.local import OutputLOCAL
from dvc.output.s3 import OutputS3
from dvc.output.ssh import OutputSSH
from dvc.remote import Remote
from dvc.remote.hdfs import RemoteHDFS
from dvc.remote.local import RemoteLOCAL
from dvc.remote.s3 import RemoteS3
from dvc.remote.hdfs import HDFSRemote
from dvc.remote.local import LocalRemote
from dvc.remote.s3 import S3Remote
from dvc.scheme import Schemes
OUTS = [
@@ -44,9 +44,9 @@ CHECKSUM_SCHEMA = Any(
# so when a few types of outputs share the same name, we only need
# specify it once.
CHECKSUMS_SCHEMA = {
RemoteLOCAL.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
RemoteS3.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
RemoteHDFS.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
LocalRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
S3Remote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
HDFSRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
}
TAGS_SCHEMA = {str: CHECKSUMS_SCHEMA}
dvc/output/base.py
+2 -2
View File
@@ -9,7 +9,7 @@ import dvc.prompt as prompt
from dvc.cache import NamedCache
from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError
from dvc.exceptions import DvcException
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
logger = logging.getLogger(__name__)
@@ -41,7 +41,7 @@ class OutputIsStageFileError(DvcException):
class OutputBase(object):
IS_DEPENDENCY = False
REMOTE = RemoteBASE
REMOTE = BaseRemote
PARAM_PATH = "path"
PARAM_CACHE = "cache"
dvc/output/gs.py
+2 -2
View File
@@ -1,6 +1,6 @@
from dvc.output.s3 import OutputS3
from dvc.remote.gs import RemoteGS
from dvc.remote.gs import GSRemote
class OutputGS(OutputS3):
REMOTE = RemoteGS
REMOTE = GSRemote
dvc/output/hdfs.py
+2 -2
View File
@@ -1,6 +1,6 @@
from dvc.output.base import OutputBase
from dvc.remote.hdfs import RemoteHDFS
from dvc.remote.hdfs import HDFSRemote
class OutputHDFS(OutputBase):
REMOTE = RemoteHDFS
REMOTE = HDFSRemote
dvc/output/local.py
+2 -2
View File
@@ -5,7 +5,7 @@ from urllib.parse import urlparse
from dvc.exceptions import DvcException
from dvc.istextfile import istextfile
from dvc.output.base import OutputBase
from dvc.remote.local import RemoteLOCAL
from dvc.remote.local import LocalRemote
from dvc.utils import relpath
from dvc.compat import fspath_py35
from dvc.utils.fs import path_isin
@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
class OutputLOCAL(OutputBase):
REMOTE = RemoteLOCAL
REMOTE = LocalRemote
sep = os.sep
def __init__(self, stage, path, *args, **kwargs):
dvc/output/s3.py
+2 -2
View File
@@ -1,6 +1,6 @@
from dvc.output.base import OutputBase
from dvc.remote.s3 import RemoteS3
from dvc.remote.s3 import S3Remote
class OutputS3(OutputBase):
REMOTE = RemoteS3
REMOTE = S3Remote
dvc/output/ssh.py
+2 -2
View File
@@ -1,6 +1,6 @@
from dvc.output.base import OutputBase
from dvc.remote.ssh import RemoteSSH
from dvc.remote.ssh import SSHRemote
class OutputSSH(OutputBase):
REMOTE = RemoteSSH
REMOTE = SSHRemote
dvc/remote/__init__.py
+21 -21
View File
@@ -1,29 +1,29 @@
import posixpath
from urllib.parse import urlparse
from dvc.remote.azure import RemoteAZURE
from dvc.remote.gdrive import RemoteGDrive
from dvc.remote.gs import RemoteGS
from dvc.remote.hdfs import RemoteHDFS
from dvc.remote.http import RemoteHTTP
from dvc.remote.https import RemoteHTTPS
from dvc.remote.local import RemoteLOCAL
from dvc.remote.oss import RemoteOSS
from dvc.remote.s3 import RemoteS3
from dvc.remote.ssh import RemoteSSH
from dvc.remote.azure import AzureRemote
from dvc.remote.gdrive import GDriveRemote
from dvc.remote.gs import GSRemote
from dvc.remote.hdfs import HDFSRemote
from dvc.remote.http import HTTPRemote
from dvc.remote.https import HTTPSRemote
from dvc.remote.local import LocalRemote
from dvc.remote.oss import OSSRemote
from dvc.remote.s3 import S3Remote
from dvc.remote.ssh import SSHRemote
REMOTES = [
RemoteAZURE,
RemoteGDrive,
RemoteGS,
RemoteHDFS,
RemoteHTTP,
RemoteHTTPS,
RemoteS3,
RemoteSSH,
RemoteOSS,
# NOTE: RemoteLOCAL is the default
AzureRemote,
GDriveRemote,
GSRemote,
HDFSRemote,
HTTPRemote,
HTTPSRemote,
S3Remote,
SSHRemote,
OSSRemote,
# NOTE: LocalRemote is the default
]
@@ -31,7 +31,7 @@ def _get(remote_conf):
for remote in REMOTES:
if remote.supported(remote_conf):
return remote
return RemoteLOCAL
return LocalRemote
def Remote(repo, **kwargs):
dvc/remote/azure.py
+2 -2
View File
@@ -10,14 +10,14 @@ from funcy import cached_property, wrap_prop
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.scheme import Schemes
logger = logging.getLogger(__name__)
class RemoteAZURE(RemoteBASE):
class AzureRemote(BaseRemote):
scheme = Schemes.AZURE
path_cls = CloudURLInfo
REGEX = (
dvc/remote/base.py
+1 -1
View File
@@ -82,7 +82,7 @@ def index_locked(f):
return wrapper
class RemoteBASE(object):
class BaseRemote(object):
scheme = "base"
path_cls = URLInfo
REQUIRES = {}
dvc/remote/gdrive.py
+6 -6
View File
@@ -12,7 +12,7 @@ from funcy.py3 import cat
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from dvc.path_info import CloudURLInfo
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.exceptions import DvcException
from dvc.utils import tmp_fname, format_link
@@ -94,7 +94,7 @@ class GDriveURLInfo(CloudURLInfo):
self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
class RemoteGDrive(RemoteBASE):
class GDriveRemote(BaseRemote):
scheme = Schemes.GDRIVE
path_cls = GDriveURLInfo
REQUIRES = {"pydrive2": "pydrive2"}
@@ -139,7 +139,7 @@ class RemoteGDrive(RemoteBASE):
self._validate_config()
self._gdrive_user_credentials_path = (
tmp_fname(os.path.join(self.repo.tmp_dir, ""))
if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA)
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA)
else config.get(
"gdrive_user_credentials_file",
os.path.join(
@@ -183,12 +183,12 @@ class RemoteGDrive(RemoteBASE):
from pydrive2.auth import GoogleAuth
from pydrive2.drive import GoogleDrive
if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA):
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
with open(
self._gdrive_user_credentials_path, "w"
) as credentials_file:
credentials_file.write(
os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA)
os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA)
)
GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
@@ -237,7 +237,7 @@ class RemoteGDrive(RemoteBASE):
except Exception as exc:
raise DvcException("Google Drive authentication failed") from exc
finally:
if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA):
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
os.remove(self._gdrive_user_credentials_path)
return GoogleDrive(gauth)
dvc/remote/gs.py
+2 -2
View File
@@ -11,7 +11,7 @@ from funcy import cached_property, wrap_prop
from dvc.exceptions import DvcException
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.scheme import Schemes
logger = logging.getLogger(__name__)
@@ -66,7 +66,7 @@ def _upload_to_bucket(
blob.upload_from_file(wrapped)
class RemoteGS(RemoteBASE):
class GSRemote(BaseRemote):
scheme = Schemes.GS
path_cls = CloudURLInfo
REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
dvc/remote/hdfs.py
+2 -2
View File
@@ -8,7 +8,7 @@ from collections import deque
from contextlib import closing, contextmanager
from urllib.parse import urlparse
from .base import RemoteBASE, RemoteCmdError
from .base import BaseRemote, RemoteCmdError
from .pool import get_connection
from dvc.scheme import Schemes
from dvc.utils import fix_env, tmp_fname
@@ -16,7 +16,7 @@ from dvc.utils import fix_env, tmp_fname
logger = logging.getLogger(__name__)
class RemoteHDFS(RemoteBASE):
class HDFSRemote(BaseRemote):
scheme = Schemes.HDFS
REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
PARAM_CHECKSUM = "checksum"
dvc/remote/http.py
+2 -2
View File
@@ -8,7 +8,7 @@ from dvc.path_info import HTTPURLInfo
import dvc.prompt as prompt
from dvc.exceptions import DvcException, HTTPError
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.scheme import Schemes
logger = logging.getLogger(__name__)
@@ -23,7 +23,7 @@ def ask_password(host, user):
)
class RemoteHTTP(RemoteBASE):
class HTTPRemote(BaseRemote):
scheme = Schemes.HTTP
path_cls = HTTPURLInfo
SESSION_RETRIES = 5
dvc/remote/https.py
+2 -2
View File
@@ -1,6 +1,6 @@
from .http import RemoteHTTP
from .http import HTTPRemote
from dvc.scheme import Schemes
class RemoteHTTPS(RemoteHTTP):
class HTTPSRemote(HTTPRemote):
scheme = Schemes.HTTPS
dvc/remote/local.py
+2 -2
View File
@@ -15,7 +15,7 @@ from dvc.path_info import PathInfo
from dvc.progress import Tqdm
from dvc.remote.base import (
index_locked,
RemoteBASE,
BaseRemote,
STATUS_MAP,
STATUS_DELETED,
STATUS_MISSING,
@@ -31,7 +31,7 @@ from dvc.utils.fs import copyfile, move, makedirs, remove, walk_files
logger = logging.getLogger(__name__)
class RemoteLOCAL(RemoteBASE):
class LocalRemote(BaseRemote):
scheme = Schemes.LOCAL
path_cls = PathInfo
PARAM_CHECKSUM = "md5"
dvc/remote/oss.py
+2 -2
View File
@@ -7,14 +7,14 @@ from funcy import cached_property, wrap_prop
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.scheme import Schemes
logger = logging.getLogger(__name__)
class RemoteOSS(RemoteBASE):
class OSSRemote(BaseRemote):
"""
oss2 document:
https://www.alibabacloud.com/help/doc-detail/32026.htm
dvc/remote/s3.py
+2 -2
View File
@@ -12,13 +12,13 @@ from dvc.exceptions import DvcException
from dvc.exceptions import ETagMismatchError
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.scheme import Schemes
logger = logging.getLogger(__name__)
class RemoteS3(RemoteBASE):
class S3Remote(BaseRemote):
scheme = Schemes.S3
path_cls = CloudURLInfo
REQUIRES = {"boto3": "boto3"}
dvc/remote/ssh/__init__.py
+3 -3
View File
@@ -14,7 +14,7 @@ from funcy import memoize, wrap_with, silent, first
import dvc.prompt as prompt
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.remote.pool import get_connection
from dvc.scheme import Schemes
from dvc.utils import to_chunks
@@ -33,7 +33,7 @@ def ask_password(host, user, port):
)
class RemoteSSH(RemoteBASE):
class SSHRemote(BaseRemote):
scheme = Schemes.SSH
REQUIRES = {"paramiko": "paramiko"}
@@ -103,7 +103,7 @@ class RemoteSSH(RemoteBASE):
def _load_user_ssh_config(hostname):
import paramiko
user_config_file = RemoteSSH.ssh_config_filename()
user_config_file = SSHRemote.ssh_config_filename()
user_ssh_config = {}
if hostname and os.path.exists(user_config_file):
ssh_config = paramiko.SSHConfig()
dvc/stage/__init__.py
+4 -4
View File
@@ -355,8 +355,8 @@ class Stage(params.StageParams):
"""
Checks if this stage has been already ran and stored
"""
from dvc.remote.local import RemoteLOCAL
from dvc.remote.s3 import RemoteS3
from dvc.remote.local import LocalRemote
from dvc.remote.s3 import S3Remote
old = self.reload()
if old._changed_outs():
@@ -376,8 +376,8 @@ class Stage(params.StageParams):
new_d.pop(self.PARAM_MD5, None)
outs = old_d.get(self.PARAM_OUTS, [])
for out in outs:
out.pop(RemoteLOCAL.PARAM_CHECKSUM, None)
out.pop(RemoteS3.PARAM_CHECKSUM, None)
out.pop(LocalRemote.PARAM_CHECKSUM, None)
out.pop(S3Remote.PARAM_CHECKSUM, None)
if old_d != new_d:
return False
tests/func/remote/test_index.py
+8 -8
View File
@@ -2,9 +2,9 @@ import pytest
from dvc.compat import fspath
from dvc.exceptions import DownloadError, UploadError
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.remote.index import RemoteIndex
from dvc.remote.local import RemoteLOCAL
from dvc.remote.local import LocalRemote
from dvc.utils.fs import remove
@@ -17,12 +17,12 @@ def remote(tmp_dir, dvc, tmp_path_factory, mocker):
# patch cache_exists since the RemoteLOCAL normally overrides
# RemoteBASE.cache_exists.
def cache_exists(self, *args, **kwargs):
return RemoteBASE.cache_exists(self, *args, **kwargs)
return BaseRemote.cache_exists(self, *args, **kwargs)
mocker.patch.object(RemoteLOCAL, "cache_exists", cache_exists)
mocker.patch.object(LocalRemote, "cache_exists", cache_exists)
# patch index class since RemoteLOCAL normally overrides index class
mocker.patch.object(RemoteLOCAL, "INDEX_CLS", RemoteIndex)
mocker.patch.object(LocalRemote, "INDEX_CLS", RemoteIndex)
return dvc.cloud.get_remote("upstream")
@@ -79,7 +79,7 @@ def test_clear_on_download_err(tmp_dir, dvc, tmp_path_factory, remote, mocker):
remove(dvc.cache.local.cache_dir)
mocked_clear = mocker.patch.object(remote.INDEX_CLS, "clear")
mocker.patch.object(RemoteLOCAL, "_download", side_effect=Exception)
mocker.patch.object(LocalRemote, "_download", side_effect=Exception)
with pytest.raises(DownloadError):
dvc.pull()
mocked_clear.assert_called_once_with()
@@ -89,14 +89,14 @@ def test_partial_upload(tmp_dir, dvc, tmp_path_factory, remote, mocker):
tmp_dir.dvc_gen({"foo": "foo content"})
tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})
original = RemoteLOCAL._upload
original = LocalRemote._upload
def unreliable_upload(self, from_file, to_info, name=None, **kwargs):
if "baz" in name:
raise Exception("stop baz")
return original(self, from_file, to_info, name, **kwargs)
mocker.patch.object(RemoteLOCAL, "_upload", unreliable_upload)
mocker.patch.object(LocalRemote, "_upload", unreliable_upload)
with pytest.raises(UploadError):
dvc.push()
with remote.index:
tests/func/remote/test_local.py
+6 -6
View File
@@ -3,7 +3,7 @@ import os
import mock
from dvc.exceptions import DvcException
from dvc.remote.local import RemoteLOCAL
from dvc.remote.local import LocalRemote
from tests.utils import trees_equal
@@ -11,7 +11,7 @@ def test_dont_fail_on_unpacked_create_fail(tmp_dir, dvc):
(stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
with mock.patch.object(
RemoteLOCAL, "_create_unpacked_dir", side_effect=DvcException("msg")
LocalRemote, "_create_unpacked_dir", side_effect=DvcException("msg")
) as unpacked_create_spy, dvc.state:
assert not dvc.cache.local.changed_cache(stage.outs[0].checksum)
assert unpacked_create_spy.call_count == 1
@@ -19,14 +19,14 @@ def test_dont_fail_on_unpacked_create_fail(tmp_dir, dvc):
def test_remove_unpacked_on_create_fail(tmp_dir, dvc):
(stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX
unpacked_dir = stage.outs[0].cache_path + LocalRemote.UNPACKED_DIR_SUFFIX
# artificial unpacked dir for test purpose
os.makedirs(unpacked_dir)
assert os.path.exists(unpacked_dir)
with mock.patch.object(
RemoteLOCAL, "_create_unpacked_dir", side_effect=DvcException("msg")
LocalRemote, "_create_unpacked_dir", side_effect=DvcException("msg")
), dvc.state:
assert not dvc.cache.local.changed_cache(stage.outs[0].checksum)
@@ -35,7 +35,7 @@ def test_remove_unpacked_on_create_fail(tmp_dir, dvc):
def test_create_unpacked_on_status(tmp_dir, dvc):
(stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX
unpacked_dir = stage.outs[0].cache_path + LocalRemote.UNPACKED_DIR_SUFFIX
assert not os.path.exists(unpacked_dir)
with dvc.state:
@@ -48,7 +48,7 @@ def test_dir_cache_changed_on_single_cache_file_modification(tmp_dir, dvc):
(stage,) = tmp_dir.dvc_gen(
{"dir": {"file1": "file1 content", "file2": "file2 content"}}
)
unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX
unpacked_dir = stage.outs[0].cache_path + LocalRemote.UNPACKED_DIR_SUFFIX
assert not os.path.exists(unpacked_dir)
file_md5 = stage.outs[0].dir_cache[0]["md5"]
tests/func/test_add.py
+4 -4
View File
@@ -17,7 +17,7 @@ from dvc.exceptions import RecursiveAddingWhileUsingFilename
from dvc.exceptions import StageFileCorruptedError
from dvc.main import main
from dvc.output.base import OutputAlreadyTrackedError
from dvc.remote import RemoteLOCAL
from dvc.remote import LocalRemote
from dvc.repo import Repo as DvcRepo
from dvc.stage import Stage
from dvc.system import System
@@ -322,7 +322,7 @@ class TestAddCommit(TestDvc):
def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
tmp_dir.gen({"data/data": "foo"})
get_dir_checksum_counter = mocker.spy(RemoteLOCAL, "get_dir_checksum")
get_dir_checksum_counter = mocker.spy(LocalRemote, "get_dir_checksum")
ret = main(["add", "data"])
assert ret == 0
@@ -566,7 +566,7 @@ def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
dvc.add("dir")
tmp_dir.gen("dir/new_file", "new_file_content")
unprotect_spy = mocker.spy(RemoteLOCAL, "unprotect")
unprotect_spy = mocker.spy(LocalRemote, "unprotect")
dvc.add("dir")
assert not unprotect_spy.mock.called
@@ -686,7 +686,7 @@ def test_add_empty_files(tmp_dir, dvc, link):
def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker):
dvc.cache.local.cache_types = ["hardlink"]
tmp_dir.gen({"foo": "", "bar": "", "lorem": "lorem", "ipsum": "ipsum"})
m = mocker.spy(RemoteLOCAL, "is_hardlink")
m = mocker.spy(LocalRemote, "is_hardlink")
stages = dvc.add(["foo", "bar", "lorem", "ipsum"])
assert m.call_count == 1
tests/func/test_checkout.py
+6 -6
View File
@@ -12,7 +12,7 @@ from dvc.exceptions import CheckoutErrorSuggestGit
from dvc.exceptions import ConfirmRemoveError
from dvc.exceptions import DvcException
from dvc.main import main
from dvc.remote.local import RemoteLOCAL
from dvc.remote.local import LocalRemote
from dvc.repo import Repo as DvcRepo
from dvc.stage import Stage
from dvc.dvcfile import Dvcfile, DVC_FILE_SUFFIX
@@ -25,7 +25,7 @@ from dvc.utils import relpath
from dvc.utils.fs import walk_files
from dvc.utils.stage import dump_stage_file
from dvc.utils.stage import load_stage_file
from dvc.remote import RemoteS3
from dvc.remote import S3Remote
from tests.remotes import S3
from tests.basic_env import TestDvc
from tests.basic_env import TestDvcGit
@@ -306,8 +306,8 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
class TestCheckoutMissingMd5InStageFile(TestRepro):
def test(self):
d = load_stage_file(self.file1_stage)
del d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM]
del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM]
dump_stage_file(self.file1_stage, d)
with pytest.raises(CheckoutError):
@@ -738,9 +738,9 @@ def test_checkout_recursive(tmp_dir, dvc):
not S3.should_test(), reason="Only run with S3 credentials"
)
def test_checkout_for_external_outputs(tmp_dir, dvc):
dvc.cache.s3 = RemoteS3(dvc, {"url": S3.get_url()})
dvc.cache.s3 = S3Remote(dvc, {"url": S3.get_url()})
remote = RemoteS3(dvc, {"url": S3.get_url()})
remote = S3Remote(dvc, {"url": S3.get_url()})
file_path = remote.path_info / "foo"
remote.s3.put_object(
Bucket=remote.path_info.bucket, Key=file_path.path, Body="foo"
tests/func/test_data_cloud.py
+42 -42
View File
@@ -11,15 +11,15 @@ from dvc.compat import fspath
from dvc.cache import NamedCache
from dvc.data_cloud import DataCloud
from dvc.main import main
from dvc.remote import RemoteAZURE
from dvc.remote import RemoteGDrive
from dvc.remote import RemoteGS
from dvc.remote import RemoteHDFS
from dvc.remote import RemoteHTTP
from dvc.remote import RemoteLOCAL
from dvc.remote import RemoteOSS
from dvc.remote import RemoteS3
from dvc.remote import RemoteSSH
from dvc.remote import AzureRemote
from dvc.remote import GDriveRemote
from dvc.remote import GSRemote
from dvc.remote import HDFSRemote
from dvc.remote import HTTPRemote
from dvc.remote import LocalRemote
from dvc.remote import OSSRemote
from dvc.remote import S3Remote
from dvc.remote import SSHRemote
from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
from dvc.utils import file_md5
from dvc.utils.fs import remove
@@ -53,13 +53,13 @@ class TestDataCloud(TestDvc):
config = copy.deepcopy(TEST_CONFIG)
clist = [
("s3://mybucket/", RemoteS3),
("gs://mybucket/", RemoteGS),
("ssh://user@localhost:/", RemoteSSH),
("http://localhost:8000/", RemoteHTTP),
("azure://ContainerName=mybucket;conn_string;", RemoteAZURE),
("oss://mybucket/", RemoteOSS),
(TestDvc.mkdtemp(), RemoteLOCAL),
("s3://mybucket/", S3Remote),
("gs://mybucket/", GSRemote),
("ssh://user@localhost:/", SSHRemote),
("http://localhost:8000/", HTTPRemote),
("azure://ContainerName=mybucket;conn_string;", AzureRemote),
("oss://mybucket/", OSSRemote),
(TestDvc.mkdtemp(), LocalRemote),
]
for scheme, cl in clist:
@@ -184,9 +184,9 @@ class TestDataCloudBase(TestDvc):
self._test_cloud()
class TestRemoteS3(S3, TestDataCloudBase):
class TestS3Remote(S3, TestDataCloudBase):
def _get_cloud_class(self):
return RemoteS3
return S3Remote
def setup_gdrive_cloud(remote_url, dvc):
@@ -203,7 +203,7 @@ def setup_gdrive_cloud(remote_url, dvc):
remote._gdrive_create_dir("root", remote.path_info.path)
class TestRemoteGDrive(GDrive, TestDataCloudBase):
class TestGDriveRemote(GDrive, TestDataCloudBase):
def _setup_cloud(self):
self._ensure_should_run()
@@ -214,10 +214,10 @@ class TestRemoteGDrive(GDrive, TestDataCloudBase):
self.assertIsInstance(remote, self._get_cloud_class())
def _get_cloud_class(self):
return RemoteGDrive
return GDriveRemote
class TestRemoteGS(GCP, TestDataCloudBase):
class TestGSRemote(GCP, TestDataCloudBase):
def _setup_cloud(self):
self._ensure_should_run()
@@ -234,26 +234,26 @@ class TestRemoteGS(GCP, TestDataCloudBase):
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
def _get_cloud_class(self):
return RemoteGS
return GSRemote
class TestRemoteAZURE(Azure, TestDataCloudBase):
class TestAzureRemote(Azure, TestDataCloudBase):
def _get_cloud_class(self):
return RemoteAZURE
return AzureRemote
class TestRemoteOSS(OSS, TestDataCloudBase):
class TestOSSRemote(OSS, TestDataCloudBase):
def _get_cloud_class(self):
return RemoteOSS
return OSSRemote
class TestRemoteLOCAL(Local, TestDataCloudBase):
class TestLocalRemote(Local, TestDataCloudBase):
def _get_cloud_class(self):
return RemoteLOCAL
return LocalRemote
@pytest.mark.usefixtures("ssh_server")
class TestRemoteSSHMocked(SSHMocked, TestDataCloudBase):
class TestSSHRemoteMocked(SSHMocked, TestDataCloudBase):
@pytest.fixture(autouse=True)
def setup_method_fixture(self, request, ssh_server):
self.ssh_server = ssh_server
@@ -284,16 +284,16 @@ class TestRemoteSSHMocked(SSHMocked, TestDataCloudBase):
return self.ssh_server.test_creds["key_filename"]
def _get_cloud_class(self):
return RemoteSSH
return SSHRemote
class TestRemoteHDFS(HDFS, TestDataCloudBase):
class TestHDFSRemote(HDFS, TestDataCloudBase):
def _get_cloud_class(self):
return RemoteHDFS
return HDFSRemote
@pytest.mark.usefixtures("http_server")
class TestRemoteHTTP(HTTP, TestDataCloudBase):
class TestHTTPRemote(HTTP, TestDataCloudBase):
@pytest.fixture(autouse=True)
def setup_method_fixture(self, request, http_server):
self.http_server = http_server
@@ -303,7 +303,7 @@ class TestRemoteHTTP(HTTP, TestDataCloudBase):
return super().get_url(self.http_server.server_port)
def _get_cloud_class(self):
return RemoteHTTP
return HTTPRemote
class TestDataCloudCLIBase(TestDvc):
@@ -395,7 +395,7 @@ class TestDataCloudCLIBase(TestDvc):
self._test()
class TestRemoteLOCALCLI(Local, TestDataCloudCLIBase):
class TestLocalRemoteCLI(Local, TestDataCloudCLIBase):
def _test(self):
url = self.get_url()
@@ -413,7 +413,7 @@ class TestRemoteHDFSCLI(HDFS, TestDataCloudCLIBase):
self._test_cloud(TEST_REMOTE)
class TestRemoteS3CLI(S3, TestDataCloudCLIBase):
class TestS3RemoteCLI(S3, TestDataCloudCLIBase):
def _test(self):
url = self.get_url()
@@ -422,7 +422,7 @@ class TestRemoteS3CLI(S3, TestDataCloudCLIBase):
self._test_cloud(TEST_REMOTE)
class TestRemoteGDriveCLI(GDrive, TestDataCloudCLIBase):
class TestGDriveRemoteCLI(GDrive, TestDataCloudCLIBase):
def _setup_cloud(self):
setup_gdrive_cloud(self.get_url(), self.dvc)
@@ -461,7 +461,7 @@ class TestRemoteGDriveCLI(GDrive, TestDataCloudCLIBase):
self._test_cloud(TEST_REMOTE)
class TestRemoteGSCLI(GCP, TestDataCloudCLIBase):
class TestGSRemoteCLI(GCP, TestDataCloudCLIBase):
def _test(self):
url = self.get_url()
@@ -479,7 +479,7 @@ class TestRemoteGSCLI(GCP, TestDataCloudCLIBase):
self._test_cloud(TEST_REMOTE)
class TestRemoteAZURECLI(Azure, TestDataCloudCLIBase):
class TestAzureRemoteCLI(Azure, TestDataCloudCLIBase):
def _test(self):
url = self.get_url()
@@ -488,7 +488,7 @@ class TestRemoteAZURECLI(Azure, TestDataCloudCLIBase):
self._test_cloud(TEST_REMOTE)
class TestRemoteOSSCLI(OSS, TestDataCloudCLIBase):
class TestOSSRemoteCLI(OSS, TestDataCloudCLIBase):
def _test(self):
url = self.get_url()
@@ -548,7 +548,7 @@ class TestRecursiveSyncOperations(Local, TestDataCloudBase):
self.assertEqual(ret, 0)
def _get_cloud_class(self):
return RemoteLOCAL
return LocalRemote
def _prepare_repo(self):
remote = self.cloud.get_remote()
@@ -619,7 +619,7 @@ class TestRecursiveSyncOperations(Local, TestDataCloudBase):
def test_checksum_recalculation(mocker, dvc, tmp_dir):
tmp_dir.gen({"foo": "foo"})
test_get_file_checksum = mocker.spy(RemoteLOCAL, "get_file_checksum")
test_get_file_checksum = mocker.spy(LocalRemote, "get_file_checksum")
url = Local.get_url()
ret = main(["remote", "add", "-d", TEST_REMOTE, url])
assert ret == 0
tests/func/test_external_repo.py
+2 -2
View File
@@ -5,7 +5,7 @@ from dvc.compat import fspath
from dvc.external_repo import external_repo
from dvc.scm.git import Git
from dvc.remote import RemoteLOCAL
from dvc.remote import LocalRemote
from dvc.utils import relpath
from dvc.utils.fs import remove
@@ -47,7 +47,7 @@ def test_cache_reused(erepo_dir, mocker):
with erepo_dir.chdir():
erepo_dir.dvc_gen("file", "text", commit="add file")
download_spy = mocker.spy(RemoteLOCAL, "download")
download_spy = mocker.spy(LocalRemote, "download")
# Use URL to prevent any fishy optimizations
url = "file://{}".format(erepo_dir)
tests/func/test_gc.py
+3 -3
View File
@@ -9,7 +9,7 @@ from dvc.compat import fspath
from dvc.exceptions import CollectCacheError
from dvc.main import main
from dvc.repo import Repo as DvcRepo
from dvc.remote.local import RemoteLOCAL
from dvc.remote.local import LocalRemote
from dvc.utils.fs import remove
from tests.basic_env import TestDir, TestDvcGit
@@ -216,7 +216,7 @@ def test_gc_no_unpacked_dir(tmp_dir, dvc):
os.remove("dir.dvc")
unpackeddir = (
dir_stages[0].outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX
dir_stages[0].outs[0].cache_path + LocalRemote.UNPACKED_DIR_SUFFIX
)
assert os.path.exists(unpackeddir)
@@ -324,7 +324,7 @@ def test_gc_cloud_remove_order(tmp_dir, scm, dvc, tmp_path_factory, mocker):
dvc.remove(dir2.relpath)
dvc.gc(workspace=True)
mocked_remove = mocker.patch.object(RemoteLOCAL, "remove", autospec=True)
mocked_remove = mocker.patch.object(LocalRemote, "remove", autospec=True)
dvc.gc(workspace=True, cloud=True)
assert len(mocked_remove.mock_calls) == 8
# dir (and unpacked dir) should be first 4 checksums removed from
tests/func/test_ignore.py
+3 -3
View File
@@ -14,7 +14,7 @@ from dvc.scm.tree import WorkingTree
from dvc.utils import relpath
from dvc.compat import fspath_py35, fspath
from dvc.utils.fs import get_mtime_and_size
from dvc.remote import RemoteLOCAL
from dvc.remote import LocalRemote
from tests.dir_helpers import TmpDir
from tests.utils import to_posixpath
@@ -138,7 +138,7 @@ def test_match_nested(tmp_dir, dvc):
}
)
remote = RemoteLOCAL(dvc, {})
remote = LocalRemote(dvc, {})
result = {fspath(f) for f in remote.walk_files(".")}
assert result == {".dvcignore", "foo"}
@@ -148,6 +148,6 @@ def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
ext_dir = TmpDir(fspath_py35(tmp_path_factory.mktemp("external_dir")))
ext_dir.gen({"y.backup": "y", "tmp": "ext tmp"})
remote = RemoteLOCAL(dvc, {})
remote = LocalRemote(dvc, {})
result = {relpath(f, ext_dir) for f in remote.walk_files(ext_dir)}
assert result == {"y.backup", "tmp"}
tests/func/test_import.py
+1 -1
View File
@@ -239,7 +239,7 @@ def test_download_error_pulling_imported_stage(tmp_dir, dvc, erepo_dir):
remove(dst_cache)
with patch(
"dvc.remote.RemoteLOCAL._download", side_effect=Exception
"dvc.remote.LocalRemote._download", side_effect=Exception
), pytest.raises(DownloadError):
dvc.pull(["foo_imported.dvc"])
tests/func/test_remote.py
+9 -9
View File
@@ -9,8 +9,8 @@ from dvc.config import Config
from dvc.exceptions import DownloadError, UploadError
from dvc.main import main
from dvc.path_info import PathInfo
from dvc.remote import RemoteLOCAL
from dvc.remote.base import RemoteBASE, RemoteCacheRequiredError
from dvc.remote import LocalRemote
from dvc.remote.base import BaseRemote, RemoteCacheRequiredError
from dvc.compat import fspath
from dvc.utils.fs import remove
from tests.basic_env import TestDvc
@@ -150,7 +150,7 @@ def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc):
path_info = PathInfo("data")
with dvc.state:
with patch.object(
RemoteBASE,
BaseRemote,
"_collect_dir",
return_value=[
{"relpath": "1", "md5": "1"},
@@ -160,7 +160,7 @@ def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc):
checksum1 = dvc.cache.local.get_dir_checksum(path_info)
with patch.object(
RemoteBASE,
BaseRemote,
"_collect_dir",
return_value=[
{"md5": "1", "relpath": "1"},
@@ -182,14 +182,14 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory):
baz = tmp_dir.dvc_gen({"baz": {"foo": "baz content"}})[0].outs[0]
# Faulty upload version, failing on foo
original = RemoteLOCAL._upload
original = LocalRemote._upload
def unreliable_upload(self, from_file, to_info, name=None, **kwargs):
if "foo" in name:
raise Exception("stop foo")
return original(self, from_file, to_info, name, **kwargs)
with patch.object(RemoteLOCAL, "_upload", unreliable_upload):
with patch.object(LocalRemote, "_upload", unreliable_upload):
with pytest.raises(UploadError) as upload_error_info:
dvc.push()
assert upload_error_info.value.amount == 3
@@ -203,7 +203,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory):
dvc.push()
remove(dvc.cache.local.cache_dir)
with patch.object(RemoteLOCAL, "_download", side_effect=Exception):
with patch.object(LocalRemote, "_download", side_effect=Exception):
with pytest.raises(DownloadError) as download_error_info:
dvc.pull()
# error count should be len(.dir + standalone file checksums)
@@ -219,7 +219,7 @@ def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker):
tmp_dir.dvc_gen({"file": "file content"})
mocker.patch.object(
RemoteLOCAL,
LocalRemote,
"_upload",
side_effect=OSError(errno.EMFILE, "Too many open files"),
)
@@ -252,7 +252,7 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker):
tmp_dir.dvc_gen({"foo": {"bar": "bar content"}})
tmp_dir.dvc_gen({"baz": "baz content"})
mocked_upload = mocker.patch.object(RemoteLOCAL, "_upload", return_value=0)
mocked_upload = mocker.patch.object(LocalRemote, "_upload", return_value=0)
dvc.push()
# last uploaded file should be dir checksum
assert mocked_upload.call_args[0][0].endswith(".dir")
tests/func/test_repro.py
+5 -5
View File
@@ -25,7 +25,7 @@ from dvc.exceptions import StagePathAsOutputError
from dvc.main import main
from dvc.output.base import OutputBase
from dvc.path_info import URLInfo
from dvc.remote.local import RemoteLOCAL
from dvc.remote.local import LocalRemote
from dvc.repo import Repo as DvcRepo
from dvc.stage import Stage
from dvc.dvcfile import Dvcfile
@@ -748,8 +748,8 @@ class TestReproChangedDirData(SingleStageRun, TestDvc):
class TestReproMissingMd5InStageFile(TestRepro):
def test(self):
d = load_stage_file(self.file1_stage)
del d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM]
del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM]
dump_stage_file(self.file1_stage, d)
stages = self.dvc.reproduce(self.file1_stage)
@@ -1329,9 +1329,9 @@ class TestReproAlreadyCached(TestRepro):
self.assertEqual(ret, 0)
patch_download = patch.object(
RemoteLOCAL,
LocalRemote,
"download",
side_effect=RemoteLOCAL.download,
side_effect=LocalRemote.download,
autospec=True,
)
tests/func/test_s3.py
+8 -8
View File
@@ -5,7 +5,7 @@ import moto.s3.models as s3model
import pytest
from moto import mock_s3
from dvc.remote.s3 import RemoteS3
from dvc.remote.s3 import S3Remote
from tests.remotes import S3
@@ -31,7 +31,7 @@ def reduced_min_part_size(f):
def _get_src_dst():
base_info = RemoteS3.path_cls(S3.get_url())
base_info = S3Remote.path_cls(S3.get_url())
return base_info / "from", base_info / "to"
@@ -43,16 +43,16 @@ def test_copy_singlepart_preserve_etag():
s3.create_bucket(Bucket=from_info.bucket)
s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data")
RemoteS3._copy(s3, from_info, to_info, {})
S3Remote._copy(s3, from_info, to_info, {})
@mock_s3
@pytest.mark.parametrize(
"base_info",
[RemoteS3.path_cls("s3://bucket/"), RemoteS3.path_cls("s3://bucket/ns/")],
[S3Remote.path_cls("s3://bucket/"), S3Remote.path_cls("s3://bucket/ns/")],
)
def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm):
remote = RemoteS3(dvc, {"url": str(base_info.parent)})
remote = S3Remote(dvc, {"url": str(base_info.parent)})
remote.s3.create_bucket(Bucket=base_info.bucket)
remote.s3.put_object(
Bucket=base_info.bucket, Key=(base_info / "from").path, Body="data"
@@ -65,8 +65,8 @@ def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm):
@mock_s3
def test_makedirs_doesnot_try_on_top_level_paths(tmp_dir, dvc, scm):
base_info = RemoteS3.path_cls("s3://bucket/")
remote = RemoteS3(dvc, {"url": str(base_info)})
base_info = S3Remote.path_cls("s3://bucket/")
remote = S3Remote(dvc, {"url": str(base_info)})
remote.makedirs(base_info)
@@ -108,4 +108,4 @@ def test_copy_multipart_preserve_etag():
s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
_upload_multipart(s3, from_info.bucket, from_info.path)
RemoteS3._copy(s3, from_info, to_info, {})
S3Remote._copy(s3, from_info, to_info, {})
tests/func/test_stage.py
+3 -3
View File
@@ -4,7 +4,7 @@ import pytest
from dvc.main import main
from dvc.output.local import OutputLOCAL
from dvc.remote.local import RemoteLOCAL
from dvc.remote.local import LocalRemote
from dvc.repo import Repo
from dvc.stage import Stage
from dvc.dvcfile import Dvcfile
@@ -54,8 +54,8 @@ def test_empty_list():
def test_list():
lst = [
{OutputLOCAL.PARAM_PATH: "foo", RemoteLOCAL.PARAM_CHECKSUM: "123"},
{OutputLOCAL.PARAM_PATH: "bar", RemoteLOCAL.PARAM_CHECKSUM: None},
{OutputLOCAL.PARAM_PATH: "foo", LocalRemote.PARAM_CHECKSUM: "123"},
{OutputLOCAL.PARAM_PATH: "bar", LocalRemote.PARAM_CHECKSUM: None},
{OutputLOCAL.PARAM_PATH: "baz"},
]
d = {Stage.PARAM_DEPS: lst}
tests/remotes.py
+6 -6
View File
@@ -7,9 +7,9 @@ from contextlib import contextmanager
from subprocess import CalledProcessError, check_output, Popen
from dvc.utils import env2bool
from dvc.remote import RemoteGDrive
from dvc.remote.gs import RemoteGS
from dvc.remote.s3 import RemoteS3
from dvc.remote import GDriveRemote
from dvc.remote.gs import GSRemote
from dvc.remote.s3 import S3Remote
from tests.basic_env import TestDvc
from moto.s3 import mock_s3
@@ -80,7 +80,7 @@ class S3Mocked(S3):
@contextmanager
def remote(cls, repo):
with mock_s3():
yield RemoteS3(repo, {"url": cls.get_url()})
yield S3Remote(repo, {"url": cls.get_url()})
@staticmethod
def put_objects(remote, objects):
@@ -128,7 +128,7 @@ class GCP:
@classmethod
@contextmanager
def remote(cls, repo):
yield RemoteGS(repo, {"url": cls.get_url()})
yield GSRemote(repo, {"url": cls.get_url()})
@staticmethod
def put_objects(remote, objects):
@@ -141,7 +141,7 @@ class GCP:
class GDrive:
@staticmethod
def should_test():
return os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA) is not None
return os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA) is not None
def get_url(self):
if not getattr(self, "_remote_url", None):
tests/unit/output/test_local.py
+2 -2
View File
@@ -2,7 +2,7 @@ import os
from mock import patch
from dvc.output import OutputLOCAL
from dvc.remote.local import RemoteLOCAL
from dvc.remote.local import LocalRemote
from dvc.stage import Stage
from dvc.utils import relpath
from tests.basic_env import TestDvc
@@ -78,7 +78,7 @@ class TestGetFilesNumber(TestDvc):
@patch.object(OutputLOCAL, "checksum", "12345678.dir")
@patch.object(
RemoteLOCAL,
LocalRemote,
"get_dir_cache",
return_value=[{"md5": "asdf"}, {"md5": "qwe"}],
)
tests/unit/remote/ssh/test_ssh.py
+21 -21
View File
@@ -6,7 +6,7 @@ import pytest
from mock import mock_open
from mock import patch
from dvc.remote.ssh import RemoteSSH
from dvc.remote.ssh import SSHRemote
from dvc.system import System
from tests.remotes import SSHMocked
@@ -21,20 +21,20 @@ def test_url(dvc):
url = "ssh://{}@{}:{}{}".format(user, host, port, path)
config = {"url": url}
remote = RemoteSSH(dvc, config)
remote = SSHRemote(dvc, config)
assert remote.path_info == url
# SCP-like URL ssh://[user@]host.xz:/absolute/path
url = "ssh://{}@{}:{}".format(user, host, path)
config = {"url": url}
remote = RemoteSSH(dvc, config)
remote = SSHRemote(dvc, config)
assert remote.path_info == url
def test_no_path(dvc):
config = {"url": "ssh://127.0.0.1"}
remote = RemoteSSH(dvc, config)
remote = SSHRemote(dvc, config)
assert remote.path_info.path == ""
@@ -68,10 +68,10 @@ else:
def test_ssh_host_override_from_config(
mock_file, mock_exists, dvc, config, expected_host
):
remote = RemoteSSH(dvc, config)
remote = SSHRemote(dvc, config)
mock_exists.assert_called_with(RemoteSSH.ssh_config_filename())
mock_file.assert_called_with(RemoteSSH.ssh_config_filename())
mock_exists.assert_called_with(SSHRemote.ssh_config_filename())
mock_file.assert_called_with(SSHRemote.ssh_config_filename())
assert remote.path_info.host == expected_host
@@ -96,10 +96,10 @@ def test_ssh_host_override_from_config(
read_data=mock_ssh_config,
)
def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user):
remote = RemoteSSH(dvc, config)
remote = SSHRemote(dvc, config)
mock_exists.assert_called_with(RemoteSSH.ssh_config_filename())
mock_file.assert_called_with(RemoteSSH.ssh_config_filename())
mock_exists.assert_called_with(SSHRemote.ssh_config_filename())
mock_file.assert_called_with(SSHRemote.ssh_config_filename())
assert remote.path_info.user == expected_user
@@ -109,7 +109,7 @@ def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user):
({"url": "ssh://example.com:2222"}, 2222),
({"url": "ssh://example.com"}, 1234),
({"url": "ssh://example.com", "port": 4321}, 4321),
({"url": "ssh://not_in_ssh_config.com"}, RemoteSSH.DEFAULT_PORT),
({"url": "ssh://not_in_ssh_config.com"}, SSHRemote.DEFAULT_PORT),
({"url": "ssh://not_in_ssh_config.com:2222"}, 2222),
({"url": "ssh://not_in_ssh_config.com:2222", "port": 4321}, 4321),
],
@@ -121,10 +121,10 @@ def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user):
read_data=mock_ssh_config,
)
def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port):
remote = RemoteSSH(dvc, config)
remote = SSHRemote(dvc, config)
mock_exists.assert_called_with(RemoteSSH.ssh_config_filename())
mock_file.assert_called_with(RemoteSSH.ssh_config_filename())
mock_exists.assert_called_with(SSHRemote.ssh_config_filename())
mock_file.assert_called_with(SSHRemote.ssh_config_filename())
assert remote.path_info.port == expected_port
@@ -156,10 +156,10 @@ def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port):
read_data=mock_ssh_config,
)
def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile):
remote = RemoteSSH(dvc, config)
remote = SSHRemote(dvc, config)
mock_exists.assert_called_with(RemoteSSH.ssh_config_filename())
mock_file.assert_called_with(RemoteSSH.ssh_config_filename())
mock_exists.assert_called_with(SSHRemote.ssh_config_filename())
mock_file.assert_called_with(SSHRemote.ssh_config_filename())
assert remote.keyfile == expected_keyfile
@@ -178,10 +178,10 @@ def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile):
read_data=mock_ssh_config,
)
def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth):
remote = RemoteSSH(dvc, config)
remote = SSHRemote(dvc, config)
mock_exists.assert_called_with(RemoteSSH.ssh_config_filename())
mock_file.assert_called_with(RemoteSSH.ssh_config_filename())
mock_exists.assert_called_with(SSHRemote.ssh_config_filename())
mock_file.assert_called_with(SSHRemote.ssh_config_filename())
assert remote.gss_auth == expected_gss_auth
@@ -195,7 +195,7 @@ def test_hardlink_optimization(dvc, tmp_dir, ssh_server):
"user": user,
"keyfile": ssh_server.test_creds["key_filename"],
}
remote = RemoteSSH(dvc, config)
remote = SSHRemote(dvc, config)
from_info = remote.path_info / "empty"
to_info = remote.path_info / "link"
tests/unit/remote/test_azure.py
+3 -3
View File
@@ -1,4 +1,4 @@
from dvc.remote.azure import RemoteAZURE
from dvc.remote.azure import AzureRemote
container_name = "container-name"
@@ -17,7 +17,7 @@ def test_init_compat(dvc):
container_name=container_name, connection_string=connection_string,
)
config = {"url": url}
remote = RemoteAZURE(dvc, config)
remote = AzureRemote(dvc, config)
assert remote.path_info == "azure://" + container_name
assert remote.connection_string == connection_string
@@ -26,6 +26,6 @@ def test_init(dvc):
prefix = "some/prefix"
url = "azure://{}/{}".format(container_name, prefix)
config = {"url": url, "connection_string": connection_string}
remote = RemoteAZURE(dvc, config)
remote = AzureRemote(dvc, config)
assert remote.path_info == url
assert remote.connection_string == connection_string
tests/unit/remote/test_base.py
+9 -9
View File
@@ -3,7 +3,7 @@ import mock
import pytest
from dvc.path_info import PathInfo
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.remote.base import RemoteCmdError
from dvc.remote.base import RemoteMissingDepsError
@@ -16,7 +16,7 @@ class _CallableOrNone(object):
CallableOrNone = _CallableOrNone()
REMOTE_CLS = RemoteBASE
REMOTE_CLS = BaseRemote
def test_missing_deps(dvc):
@@ -42,10 +42,10 @@ def test_cmd_error(dvc):
REMOTE_CLS(dvc, config).remove("file")
@mock.patch.object(RemoteBASE, "_cache_checksums_traverse")
@mock.patch.object(RemoteBASE, "_cache_object_exists")
@mock.patch.object(BaseRemote, "_cache_checksums_traverse")
@mock.patch.object(BaseRemote, "_cache_object_exists")
def test_cache_exists(object_exists, traverse, dvc):
remote = RemoteBASE(dvc, {})
remote = BaseRemote(dvc, {})
# remote does not support traverse
remote.CAN_TRAVERSE = False
@@ -98,13 +98,13 @@ def test_cache_exists(object_exists, traverse, dvc):
@mock.patch.object(
RemoteBASE, "cache_checksums", return_value=[],
BaseRemote, "cache_checksums", return_value=[],
)
@mock.patch.object(
RemoteBASE, "path_to_checksum", side_effect=lambda x: x,
BaseRemote, "path_to_checksum", side_effect=lambda x: x,
)
def test_cache_checksums_traverse(path_to_checksum, cache_checksums, dvc):
remote = RemoteBASE(dvc, {})
remote = BaseRemote(dvc, {})
remote.path_info = PathInfo("foo")
# parallel traverse
@@ -129,7 +129,7 @@ def test_cache_checksums_traverse(path_to_checksum, cache_checksums, dvc):
def test_cache_checksums(dvc):
remote = RemoteBASE(dvc, {})
remote = BaseRemote(dvc, {})
remote.path_info = PathInfo("foo")
with mock.patch.object(
tests/unit/remote/test_gdrive.py
+7 -7
View File
@@ -2,7 +2,7 @@ import pytest
import os
from dvc.remote.gdrive import (
RemoteGDrive,
GDriveRemote,
GDriveAccessTokenRefreshError,
GDriveMissedCredentialKeyError,
)
@@ -20,21 +20,21 @@ class TestRemoteGDrive(object):
}
def test_init(self, dvc):
remote = RemoteGDrive(dvc, self.CONFIG)
remote = GDriveRemote(dvc, self.CONFIG)
assert str(remote.path_info) == self.CONFIG["url"]
def test_drive(self, dvc):
remote = RemoteGDrive(dvc, self.CONFIG)
remote = GDriveRemote(dvc, self.CONFIG)
os.environ[
RemoteGDrive.GDRIVE_CREDENTIALS_DATA
GDriveRemote.GDRIVE_CREDENTIALS_DATA
] = USER_CREDS_TOKEN_REFRESH_ERROR
with pytest.raises(GDriveAccessTokenRefreshError):
remote._drive
os.environ[RemoteGDrive.GDRIVE_CREDENTIALS_DATA] = ""
remote = RemoteGDrive(dvc, self.CONFIG)
os.environ[GDriveRemote.GDRIVE_CREDENTIALS_DATA] = ""
remote = GDriveRemote(dvc, self.CONFIG)
os.environ[
RemoteGDrive.GDRIVE_CREDENTIALS_DATA
GDriveRemote.GDRIVE_CREDENTIALS_DATA
] = USER_CREDS_MISSED_KEY_ERROR
with pytest.raises(GDriveMissedCredentialKeyError):
remote._drive
tests/unit/remote/test_gs.py
+4 -4
View File
@@ -3,7 +3,7 @@ import pytest
import requests
from dvc.remote.gs import dynamic_chunk_size
from dvc.remote.gs import RemoteGS
from dvc.remote.gs import GSRemote
BUCKET = "bucket"
@@ -19,7 +19,7 @@ CONFIG = {
def test_init(dvc):
remote = RemoteGS(dvc, CONFIG)
remote = GSRemote(dvc, CONFIG)
assert remote.path_info == URL
assert remote.projectname == PROJECT
assert remote.credentialpath == CREDENTIALPATH
@@ -27,7 +27,7 @@ def test_init(dvc):
@mock.patch("google.cloud.storage.Client.from_service_account_json")
def test_gs(mock_client, dvc):
remote = RemoteGS(dvc, CONFIG)
remote = GSRemote(dvc, CONFIG)
assert remote.credentialpath
remote.gs()
mock_client.assert_called_once_with(CREDENTIALPATH)
@@ -37,7 +37,7 @@ def test_gs(mock_client, dvc):
def test_gs_no_credspath(mock_client, dvc):
config = CONFIG.copy()
del config["credentialpath"]
remote = RemoteGS(dvc, config)
remote = GSRemote(dvc, config)
remote.gs()
mock_client.assert_called_with(PROJECT)
tests/unit/remote/test_http.py
+6 -6
View File
@@ -2,7 +2,7 @@ import pytest
from dvc.exceptions import HTTPError
from dvc.path_info import URLInfo
from dvc.remote.http import RemoteHTTP
from dvc.remote.http import HTTPRemote
from tests.utils.httpd import StaticFileServer
@@ -11,7 +11,7 @@ def test_download_fails_on_error_code(dvc):
url = "http://localhost:{}/".format(httpd.server_port)
config = {"url": url}
remote = RemoteHTTP(dvc, config)
remote = HTTPRemote(dvc, config)
with pytest.raises(HTTPError):
remote._download(URLInfo(url) / "missing.txt", "missing.txt")
@@ -25,7 +25,7 @@ def test_public_auth_method(dvc):
"password": "",
}
remote = RemoteHTTP(dvc, config)
remote = HTTPRemote(dvc, config)
assert remote.auth_method() is None
@@ -44,7 +44,7 @@ def test_basic_auth_method(dvc):
"password": password,
}
remote = RemoteHTTP(dvc, config)
remote = HTTPRemote(dvc, config)
assert remote.auth_method() == auth
assert isinstance(remote.auth_method(), HTTPBasicAuth)
@@ -64,7 +64,7 @@ def test_digest_auth_method(dvc):
"password": password,
}
remote = RemoteHTTP(dvc, config)
remote = HTTPRemote(dvc, config)
assert remote.auth_method() == auth
assert isinstance(remote.auth_method(), HTTPDigestAuth)
@@ -81,7 +81,7 @@ def test_custom_auth_method(dvc):
"password": password,
}
remote = RemoteHTTP(dvc, config)
remote = HTTPRemote(dvc, config)
assert remote.auth_method() is None
assert header in remote.headers
tests/unit/remote/test_local.py
+5 -5
View File
@@ -5,7 +5,7 @@ import pytest
from dvc.cache import NamedCache
from dvc.path_info import PathInfo
from dvc.remote.local import RemoteLOCAL
from dvc.remote.local import LocalRemote
def test_status_download_optimization(mocker, dvc):
@@ -13,7 +13,7 @@ def test_status_download_optimization(mocker, dvc):
And the desired files to fetch are already on the local cache,
Don't check the existence of the desired files on the remote cache
"""
remote = RemoteLOCAL(dvc, {})
remote = LocalRemote(dvc, {})
infos = NamedCache()
infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo")
@@ -33,7 +33,7 @@ def test_status_download_optimization(mocker, dvc):
@pytest.mark.parametrize("link_name", ["hardlink", "symlink"])
def test_is_protected(tmp_dir, dvc, link_name):
remote = RemoteLOCAL(dvc, {})
remote = LocalRemote(dvc, {})
link_method = getattr(remote, link_name)
(tmp_dir / "foo").write_text("foo")
@@ -66,7 +66,7 @@ def test_is_protected(tmp_dir, dvc, link_name):
def test_protect_ignore_errors(tmp_dir, mocker, err):
tmp_dir.gen("foo", "foo")
foo = PathInfo("foo")
remote = RemoteLOCAL(None, {})
remote = LocalRemote(None, {})
remote.protect(foo)
@@ -80,7 +80,7 @@ def test_protect_ignore_errors(tmp_dir, mocker, err):
def test_protect_ignore_erofs(tmp_dir, mocker):
tmp_dir.gen("foo", "foo")
foo = PathInfo("foo")
remote = RemoteLOCAL(None, {})
remote = LocalRemote(None, {})
mock_chmod = mocker.patch(
"os.chmod", side_effect=OSError(errno.EROFS, "read-only fs")
tests/unit/remote/test_oss.py
+2 -2
View File
@@ -1,4 +1,4 @@
from dvc.remote.oss import RemoteOSS
from dvc.remote.oss import OSSRemote
bucket_name = "bucket-name"
@@ -16,7 +16,7 @@ def test_init(dvc):
"oss_key_secret": key_secret,
"oss_endpoint": endpoint,
}
remote = RemoteOSS(dvc, config)
remote = OSSRemote(dvc, config)
assert remote.path_info == url
assert remote.endpoint == endpoint
assert remote.key_id == key_id
tests/unit/remote/test_remote.py
+2 -2
View File
@@ -1,6 +1,6 @@
import pytest
from dvc.remote import Remote, RemoteS3, RemoteGS
from dvc.remote import Remote, S3Remote, GSRemote
def test_remote_with_checksum_jobs(dvc):
@@ -29,7 +29,7 @@ def test_remote_without_checksum_jobs_default(dvc):
assert remote.checksum_jobs == remote.CHECKSUM_JOBS
@pytest.mark.parametrize("remote_cls", [RemoteGS, RemoteS3])
@pytest.mark.parametrize("remote_cls", [GSRemote, S3Remote])
def test_makedirs_not_create_for_top_level_path(remote_cls, dvc, mocker):
url = "{.scheme}://bucket/".format(remote_cls)
remote = remote_cls(dvc, {"url": url})
tests/unit/remote/test_remote_dir.py
+4 -4
View File
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
import pytest
import os
from dvc.remote.s3 import RemoteS3
from dvc.remote.s3 import S3Remote
from dvc.utils.fs import walk_files
from dvc.path_info import PathInfo
from tests.remotes import GCP, S3Mocked
@@ -90,15 +90,15 @@ def test_copy_preserve_etag_across_buckets(remote, dvc):
s3 = remote.s3
s3.create_bucket(Bucket="another")
another = RemoteS3(dvc, {"url": "s3://another", "region": "us-east-1"})
another = S3Remote(dvc, {"url": "s3://another", "region": "us-east-1"})
from_info = remote.path_info / "foo"
to_info = another.path_info / "foo"
remote.copy(from_info, to_info)
from_etag = RemoteS3.get_etag(s3, from_info.bucket, from_info.path)
to_etag = RemoteS3.get_etag(s3, "another", "foo")
from_etag = S3Remote.get_etag(s3, from_info.bucket, from_info.path)
to_etag = S3Remote.get_etag(s3, "another", "foo")
assert from_etag == to_etag
tests/unit/remote/test_s3.py
+4 -4
View File
@@ -1,7 +1,7 @@
import pytest
from dvc.config import ConfigError
from dvc.remote.s3 import RemoteS3
from dvc.remote.s3 import S3Remote
bucket_name = "bucket-name"
@@ -21,7 +21,7 @@ def grants():
def test_init(dvc):
config = {"url": url}
remote = RemoteS3(dvc, config)
remote = S3Remote(dvc, config)
assert remote.path_info == url
@@ -34,7 +34,7 @@ def test_grants(dvc):
"grant_write_acp": "id=write-acp-permission-id",
"grant_full_control": "id=full-control-permission-id",
}
remote = RemoteS3(dvc, config)
remote = S3Remote(dvc, config)
assert (
remote.extra_args["GrantRead"]
@@ -53,4 +53,4 @@ def test_grants_mutually_exclusive_acl_error(dvc, grants):
config = {"url": url, "acl": "public-read", grant_option: grant_value}
with pytest.raises(ConfigError):
RemoteS3(dvc, config)
S3Remote(dvc, config)