Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-11 07:18:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

Consolidate trees (#4243)

Browse Source 
* dvc: get rid of is_walking_tree

No longer needed. Some applications of `isinstance()` will be refactored
in the future, as trees will just provide the needed functionality.

* dvc: move GitTree to dvc/tree/

* dvc: rename *RemoteTree to *Tree
This commit is contained in:
Ruslan Kuprieiev
2020-07-19 18:43:16 +03:00
committed by GitHub
parent 7b1823e324
commit a077135d76
62 changed files with 254 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/config.py
+2 -2
View File
@@ -237,7 +237,7 @@ class Config(dict):
def __init__(
self, dvc_dir=None, validate=True, tree=None,
): # pylint: disable=super-init-not-called
from dvc.tree.local import LocalRemoteTree
from dvc.tree.local import LocalTree
self.dvc_dir = dvc_dir
@@ -251,7 +251,7 @@ class Config(dict):
else:
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self.wtree = LocalRemoteTree(None, {"url": self.dvc_dir})
self.wtree = LocalTree(None, {"url": self.dvc_dir})
self.tree = tree or self.wtree
self.load(validate=validate)
dvc/dependency/azure.py
+2 -2
View File
@@ -1,8 +1,8 @@
from dvc.dependency.base import BaseDependency
from dvc.output.base import BaseOutput
from ..tree.azure import AzureRemoteTree
from ..tree.azure import AzureTree
class AzureDependency(BaseDependency, BaseOutput):
TREE_CLS = AzureRemoteTree
TREE_CLS = AzureTree
dvc/dependency/http.py
+2 -2
View File
@@ -1,8 +1,8 @@
from dvc.dependency.base import BaseDependency
from dvc.output.base import BaseOutput
from ..tree.http import HTTPRemoteTree
from ..tree.http import HTTPTree
class HTTPDependency(BaseDependency, BaseOutput):
TREE_CLS = HTTPRemoteTree
TREE_CLS = HTTPTree
dvc/dependency/https.py
+2 -2
View File
@@ -1,6 +1,6 @@
from ..tree.https import HTTPSRemoteTree
from ..tree.https import HTTPSTree
from .http import HTTPDependency
class HTTPSDependency(HTTPDependency):
TREE_CLS = HTTPSRemoteTree
TREE_CLS = HTTPSTree
dvc/external_repo.py
+2 -4
View File
@@ -20,7 +20,7 @@ from dvc.path_info import PathInfo
from dvc.repo import Repo
from dvc.repo.tree import RepoTree
from dvc.scm.git import Git
from dvc.scm.tree import is_working_tree
from dvc.tree.local import LocalTree
from dvc.utils.fs import remove
logger = logging.getLogger(__name__)
@@ -102,10 +102,8 @@ class BaseExternalRepo:
return RepoTree(self, fetch=True)
def get_rev(self):
if is_working_tree(self.tree):
if isinstance(self.tree, LocalTree):
return self.scm.get_rev()
if hasattr(self.tree, "tree"):
return self.tree.tree.rev
return self.tree.rev
def fetch_external(self, paths: Iterable, **kwargs):
dvc/output/__init__.py
+6 -6
View File
@@ -13,9 +13,9 @@ from dvc.output.ssh import SSHOutput
from dvc.scheme import Schemes
from ..tree import get_cloud_tree
from ..tree.hdfs import HDFSRemoteTree
from ..tree.local import LocalRemoteTree
from ..tree.s3 import S3RemoteTree
from ..tree.hdfs import HDFSTree
from ..tree.local import LocalTree
from ..tree.s3 import S3Tree
OUTS = [
HDFSOutput,
@@ -48,9 +48,9 @@ CHECKSUM_SCHEMA = Any(
# so when a few types of outputs share the same name, we only need
# specify it once.
CHECKSUMS_SCHEMA = {
LocalRemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
S3RemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
HDFSRemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
LocalTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
S3Tree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
HDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
}
SCHEMA = CHECKSUMS_SCHEMA.copy()
dvc/output/base.py
+2 -2
View File
@@ -13,7 +13,7 @@ from dvc.exceptions import (
RemoteCacheRequiredError,
)
from ..tree.base import BaseRemoteTree
from ..tree.base import BaseTree
logger = logging.getLogger(__name__)
@@ -48,7 +48,7 @@ class OutputIsStageFileError(DvcException):
class BaseOutput:
IS_DEPENDENCY = False
TREE_CLS = BaseRemoteTree
TREE_CLS = BaseTree
PARAM_PATH = "path"
PARAM_CACHE = "cache"
dvc/output/gs.py
+2 -2
View File
@@ -1,7 +1,7 @@
from dvc.output.s3 import S3Output
from ..tree.gs import GSRemoteTree
from ..tree.gs import GSTree
class GSOutput(S3Output):
TREE_CLS = GSRemoteTree
TREE_CLS = GSTree
dvc/output/hdfs.py
+2 -2
View File
@@ -1,7 +1,7 @@
from dvc.output.base import BaseOutput
from ..tree.hdfs import HDFSRemoteTree
from ..tree.hdfs import HDFSTree
class HDFSOutput(BaseOutput):
TREE_CLS = HDFSRemoteTree
TREE_CLS = HDFSTree
dvc/output/local.py
+7 -4
View File
@@ -5,17 +5,16 @@ from urllib.parse import urlparse
from dvc.exceptions import DvcException
from dvc.istextfile import istextfile
from dvc.output.base import BaseOutput
from dvc.scm.tree import is_working_tree
from dvc.utils import relpath
from dvc.utils.fs import path_isin
from ..tree.local import LocalRemoteTree
from ..tree.local import LocalTree
logger = logging.getLogger(__name__)
class LocalOutput(BaseOutput):
TREE_CLS = LocalRemoteTree
TREE_CLS = LocalTree
sep = os.sep
def __init__(self, stage, path, *args, **kwargs):
@@ -23,7 +22,11 @@ class LocalOutput(BaseOutput):
path = relpath(path, stage.wdir)
super().__init__(stage, path, *args, **kwargs)
if self.is_in_repo and self.repo and is_working_tree(self.repo.tree):
if (
self.is_in_repo
and self.repo
and isinstance(self.repo.tree, LocalTree)
):
self.tree = self.repo.tree
def _parse_path(self, tree, path):
dvc/output/s3.py
+2 -2
View File
@@ -1,7 +1,7 @@
from dvc.output.base import BaseOutput
from ..tree.s3 import S3RemoteTree
from ..tree.s3 import S3Tree
class S3Output(BaseOutput):
TREE_CLS = S3RemoteTree
TREE_CLS = S3Tree
dvc/output/ssh.py
+2 -2
View File
@@ -1,7 +1,7 @@
from dvc.output.base import BaseOutput
from ..tree.ssh import SSHRemoteTree
from ..tree.ssh import SSHTree
class SSHOutput(BaseOutput):
TREE_CLS = SSHRemoteTree
TREE_CLS = SSHTree
dvc/remote/local.py
+2 -2
View File
@@ -20,7 +20,7 @@ from dvc.remote.base import (
)
from dvc.remote.index import RemoteIndexNoop
from ..tree.local import LocalRemoteTree
from ..tree.local import LocalTree
logger = logging.getLogger(__name__)
@@ -52,7 +52,7 @@ class LocalRemote(Remote):
class LocalCache(CloudCache):
DEFAULT_CACHE_TYPES = ["reflink", "copy"]
CACHE_MODE = LocalRemoteTree.CACHE_MODE
CACHE_MODE = LocalTree.CACHE_MODE
def __init__(self, tree):
super().__init__(tree)
dvc/repo/__init__.py
+2 -2
View File
@@ -76,7 +76,7 @@ class Repo:
from dvc.repo.metrics import Metrics
from dvc.repo.plots import Plots
from dvc.repo.params import Params
from dvc.tree.local import LocalRemoteTree
from dvc.tree.local import LocalTree
from dvc.utils.fs import makedirs
from dvc.stage.cache import StageCache
@@ -91,7 +91,7 @@ class Repo:
else:
root_dir = self.find_root(root_dir)
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
self.tree = LocalRemoteTree(
self.tree = LocalTree(
self,
{"url": self.root_dir},
use_dvcignore=True,
dvc/repo/brancher.py
+2 -4
View File
@@ -1,6 +1,6 @@
from funcy import group_by
from dvc.tree.local import LocalRemoteTree
from dvc.tree.local import LocalTree
def brancher( # noqa: E302
@@ -29,9 +29,7 @@ def brancher( # noqa: E302
scm = self.scm
self.tree = LocalRemoteTree(
self, {"url": self.root_dir}, use_dvcignore=True
)
self.tree = LocalTree(self, {"url": self.root_dir}, use_dvcignore=True)
yield "workspace"
if revs and "workspace" in revs:
dvc/repo/diff.py
+2 -2
View File
@@ -1,7 +1,7 @@
import os
from dvc.repo import locked
from dvc.scm.tree import is_working_tree
from dvc.tree.local import LocalTree
@locked
@@ -33,7 +33,7 @@ def diff(self, a_rev="HEAD", b_rev=None):
else os.path.join(str(output), "")
)
on_working_tree = is_working_tree(self.tree)
on_working_tree = isinstance(self.tree, LocalTree)
def _to_checksum(output):
if on_working_tree:
dvc/repo/tree.py
+3 -3
View File
@@ -5,14 +5,14 @@ from dvc.dvcfile import is_valid_filename
from dvc.exceptions import OutputNotFoundError
from dvc.path_info import PathInfo
from dvc.remote.base import RemoteActionNotImplemented
from dvc.tree.base import BaseRemoteTree
from dvc.tree.base import BaseTree
from dvc.utils import file_md5
from dvc.utils.fs import copy_fobj_to_file, makedirs
logger = logging.getLogger(__name__)
class DvcTree(BaseRemoteTree): # pylint:disable=abstract-method
class DvcTree(BaseTree): # pylint:disable=abstract-method
"""DVC repo tree.
Args:
@@ -237,7 +237,7 @@ class DvcTree(BaseRemoteTree): # pylint:disable=abstract-method
return out.checksum
class RepoTree(BaseRemoteTree): # pylint:disable=abstract-method
class RepoTree(BaseTree): # pylint:disable=abstract-method
"""DVC + git-tracked files tree.
Args:
dvc/scm/git/__init__.py → dvc/scm/git.py
+2 -1
View File
@@ -17,7 +17,6 @@ from dvc.scm.base import (
RevError,
SCMError,
)
from dvc.scm.git.tree import GitTree
from dvc.utils import fix_env, is_binary, relpath
from dvc.utils.fs import path_isin
@@ -376,6 +375,8 @@ class Git(Base):
return basename == self.ignore_file or Git.GIT_DIR in path_parts
def get_tree(self, rev, **kwargs):
from dvc.tree.git import GitTree
return GitTree(self.repo, self.resolve_rev(rev), **kwargs)
def get_rev(self):
dvc/scm/tree.py
-6
View File
@@ -1,6 +0,0 @@
def is_working_tree(tree):
from dvc.tree.local import LocalRemoteTree
return isinstance(tree, LocalRemoteTree) or isinstance(
getattr(tree, "tree", None), LocalRemoteTree
)
dvc/stage/utils.py
+4 -4
View File
@@ -8,8 +8,8 @@ from dvc import dependency, output
from dvc.utils.fs import path_isin
from ..dependency import ParamsDependency
from ..tree.local import LocalRemoteTree
from ..tree.s3 import S3RemoteTree
from ..tree.local import LocalTree
from ..tree.s3 import S3Tree
from ..utils import dict_md5, format_link, relpath
from .exceptions import (
MissingDataSource,
@@ -133,8 +133,8 @@ def stage_dump_eq(stage_cls, old_d, new_d):
new_d.pop(stage_cls.PARAM_MD5, None)
outs = old_d.get(stage_cls.PARAM_OUTS, [])
for out in outs:
out.pop(LocalRemoteTree.PARAM_CHECKSUM, None)
out.pop(S3RemoteTree.PARAM_CHECKSUM, None)
out.pop(LocalTree.PARAM_CHECKSUM, None)
out.pop(S3Tree.PARAM_CHECKSUM, None)
# outs and deps are lists of dicts. To check equality, we need to make
# them independent of the order, so, we convert them to dicts.
dvc/state.py
+3 -3
View File
@@ -89,12 +89,12 @@ class State: # pylint: disable=too-many-instance-attributes
MAX_UINT = 2 ** 64 - 2
def __init__(self, local_cache):
from dvc.tree.local import LocalRemoteTree
from dvc.tree.local import LocalTree
repo = local_cache.repo
self.repo = repo
self.root_dir = repo.root_dir
self.tree = LocalRemoteTree(None, {"url": self.root_dir})
self.tree = LocalTree(None, {"url": self.root_dir})
state_config = repo.config.get("state", {})
self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
@@ -396,7 +396,7 @@ class State: # pylint: disable=too-many-instance-attributes
assert isinstance(path_info, str) or path_info.scheme == "local"
path = os.fspath(path_info)
# NOTE: use os.path.exists instead of LocalRemoteTree.exists
# NOTE: use os.path.exists instead of LocalTree.exists
# because it uses lexists() and will return True for broken
# symlinks that we cannot stat() in get_mtime_and_size
if not os.path.exists(path):
dvc/tree/__init__.py
+21 -21
View File
@@ -1,28 +1,28 @@
import posixpath
from urllib.parse import urlparse
from .azure import AzureRemoteTree
from .gdrive import GDriveRemoteTree
from .gs import GSRemoteTree
from .hdfs import HDFSRemoteTree
from .http import HTTPRemoteTree
from .https import HTTPSRemoteTree
from .local import LocalRemoteTree
from .oss import OSSRemoteTree
from .s3 import S3RemoteTree
from .ssh import SSHRemoteTree
from .azure import AzureTree
from .gdrive import GDriveTree
from .gs import GSTree
from .hdfs import HDFSTree
from .http import HTTPTree
from .https import HTTPSTree
from .local import LocalTree
from .oss import OSSTree
from .s3 import S3Tree
from .ssh import SSHTree
TREES = [
AzureRemoteTree,
GDriveRemoteTree,
GSRemoteTree,
HDFSRemoteTree,
HTTPRemoteTree,
HTTPSRemoteTree,
S3RemoteTree,
SSHRemoteTree,
OSSRemoteTree,
# NOTE: LocalRemoteTree is the default
AzureTree,
GDriveTree,
GSTree,
HDFSTree,
HTTPTree,
HTTPSTree,
S3Tree,
SSHTree,
OSSTree,
# NOTE: LocalTree is the default
]
@@ -30,7 +30,7 @@ def _get_tree(remote_conf):
for tree_cls in TREES:
if tree_cls.supported(remote_conf):
return tree_cls
return LocalRemoteTree
return LocalTree
def _get_conf(repo, **kwargs):
dvc/tree/azure.py
+2 -2
View File
@@ -9,12 +9,12 @@ from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from .base import BaseRemoteTree
from .base import BaseTree
logger = logging.getLogger(__name__)
class AzureRemoteTree(BaseRemoteTree):
class AzureTree(BaseTree):
scheme = Schemes.AZURE
PATH_CLS = CloudURLInfo
REQUIRES = {
dvc/tree/base.py
+1 -1
View File
@@ -57,7 +57,7 @@ class RemoteMissingDepsError(DvcException):
pass
class BaseRemoteTree:
class BaseTree:
scheme = "base"
REQUIRES = {}
PATH_CLS = URLInfo
dvc/tree/gdrive.py
+10 -12
View File
@@ -18,7 +18,7 @@ from dvc.scheme import Schemes
from dvc.utils import format_link, tmp_fname
from dvc.utils.stream import IterStream
from .base import BaseRemoteTree
from .base import BaseTree
logger = logging.getLogger(__name__)
FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
@@ -85,7 +85,7 @@ class GDriveURLInfo(CloudURLInfo):
self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
class GDriveRemoteTree(BaseRemoteTree):
class GDriveTree(BaseTree):
scheme = Schemes.GDRIVE
PATH_CLS = GDriveURLInfo
REQUIRES = {"pydrive2": "pydrive2"}
@@ -130,7 +130,7 @@ class GDriveRemoteTree(BaseRemoteTree):
self._validate_config()
self._gdrive_user_credentials_path = (
tmp_fname(os.path.join(self.repo.tmp_dir, ""))
if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA)
if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA)
else config.get(
"gdrive_user_credentials_file",
os.path.join(
@@ -172,8 +172,8 @@ class GDriveRemoteTree(BaseRemoteTree):
Useful for tests, exception messages, etc. Returns either env variable
name if it's set or actual path to the credentials file.
"""
if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA):
return GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA
if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
return GDriveTree.GDRIVE_CREDENTIALS_DATA
if os.path.exists(self._gdrive_user_credentials_path):
return self._gdrive_user_credentials_path
return None
@@ -187,7 +187,7 @@ class GDriveRemoteTree(BaseRemoteTree):
DVC config client id or secret but forgets to remove the cached
credentials file.
"""
if not os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA):
if not os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
if (
settings["client_config"]["client_id"]
!= auth.credentials.client_id
@@ -210,11 +210,9 @@ class GDriveRemoteTree(BaseRemoteTree):
from pydrive2.auth import GoogleAuth
from pydrive2.drive import GoogleDrive
if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA):
if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
with open(self._gdrive_user_credentials_path, "w") as cred_file:
cred_file.write(
os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA)
)
cred_file.write(os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA))
auth_settings = {
"client_config_backend": "settings",
@@ -260,7 +258,7 @@ class GDriveRemoteTree(BaseRemoteTree):
gauth.ServiceAuth()
else:
gauth.CommandLineAuth()
GDriveRemoteTree._validate_credentials(gauth, auth_settings)
GDriveTree._validate_credentials(gauth, auth_settings)
# Handle AuthenticationError, RefreshError and other auth failures
# It's hard to come up with a narrow exception, since PyDrive throws
@@ -269,7 +267,7 @@ class GDriveRemoteTree(BaseRemoteTree):
except Exception as exc:
raise GDriveAuthError(self.credentials_location) from exc
finally:
if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA):
if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
os.remove(self._gdrive_user_credentials_path)
return GoogleDrive(gauth)
dvc/scm/git/tree.py → dvc/tree/git.py
+2 -2
View File
@@ -6,7 +6,7 @@ import stat
from funcy import cached_property
from dvc.exceptions import DvcException
from dvc.tree.base import BaseRemoteTree
from dvc.tree.base import BaseTree
from dvc.utils import relpath
# see git-fast-import(1)
@@ -21,7 +21,7 @@ def _item_basename(item):
return os.path.basename(item.path)
class GitTree(BaseRemoteTree): # pylint:disable=abstract-method
class GitTree(BaseTree): # pylint:disable=abstract-method
"""Proxies the repo file access methods to Git objects"""
def __init__(self, git, rev, use_dvcignore=False, dvcignore_root=None):
dvc/tree/gs.py
+2 -2
View File
@@ -11,7 +11,7 @@ from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from .base import BaseRemoteTree
from .base import BaseTree
logger = logging.getLogger(__name__)
@@ -66,7 +66,7 @@ def _upload_to_bucket(
blob.upload_from_file(wrapped)
class GSRemoteTree(BaseRemoteTree):
class GSTree(BaseTree):
scheme = Schemes.GS
PATH_CLS = CloudURLInfo
REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
dvc/tree/hdfs.py
+2 -2
View File
@@ -10,13 +10,13 @@ from urllib.parse import urlparse
from dvc.scheme import Schemes
from dvc.utils import fix_env, tmp_fname
from .base import BaseRemoteTree, RemoteCmdError
from .base import BaseTree, RemoteCmdError
from .pool import get_connection
logger = logging.getLogger(__name__)
class HDFSRemoteTree(BaseRemoteTree):
class HDFSTree(BaseTree):
scheme = Schemes.HDFS
REQUIRES = {"pyarrow": "pyarrow"}
REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
dvc/tree/http.py
+2 -2
View File
@@ -10,7 +10,7 @@ from dvc.path_info import HTTPURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from .base import BaseRemoteTree
from .base import BaseTree
logger = logging.getLogger(__name__)
@@ -24,7 +24,7 @@ def ask_password(host, user):
)
class HTTPRemoteTree(BaseRemoteTree): # pylint:disable=abstract-method
class HTTPTree(BaseTree): # pylint:disable=abstract-method
scheme = Schemes.HTTP
PATH_CLS = HTTPURLInfo
PARAM_CHECKSUM = "etag"
dvc/tree/https.py
+2 -2
View File
@@ -1,7 +1,7 @@
from dvc.scheme import Schemes
from .http import HTTPRemoteTree
from .http import HTTPTree
class HTTPSRemoteTree(HTTPRemoteTree): # pylint:disable=abstract-method
class HTTPSTree(HTTPTree): # pylint:disable=abstract-method
scheme = Schemes.HTTPS
dvc/tree/local.py
+2 -2
View File
@@ -20,12 +20,12 @@ from dvc.utils.fs import (
walk_files,
)
from .base import BaseRemoteTree
from .base import BaseTree
logger = logging.getLogger(__name__)
class LocalRemoteTree(BaseRemoteTree):
class LocalTree(BaseTree):
scheme = Schemes.LOCAL
PATH_CLS = PathInfo
PARAM_CHECKSUM = "md5"
dvc/tree/oss.py
+2 -2
View File
@@ -8,12 +8,12 @@ from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from .base import BaseRemoteTree
from .base import BaseTree
logger = logging.getLogger(__name__)
class OSSRemoteTree(BaseRemoteTree): # pylint:disable=abstract-method
class OSSTree(BaseTree): # pylint:disable=abstract-method
"""
oss2 document:
https://www.alibabacloud.com/help/doc-detail/32026.htm
dvc/tree/s3.py
+5 -7
View File
@@ -10,12 +10,12 @@ from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from .base import BaseRemoteTree
from .base import BaseTree
logger = logging.getLogger(__name__)
class S3RemoteTree(BaseRemoteTree):
class S3Tree(BaseTree):
scheme = Schemes.S3
PATH_CLS = CloudURLInfo
REQUIRES = {"boto3": "boto3"}
@@ -238,7 +238,7 @@ class S3RemoteTree(BaseRemoteTree):
parts = []
byte_position = 0
for i in range(1, n_parts + 1):
obj = S3RemoteTree.get_head_object(
obj = S3Tree.get_head_object(
s3, from_info.bucket, from_info.path, PartNumber=i
)
part_size = obj["ContentLength"]
@@ -293,9 +293,7 @@ class S3RemoteTree(BaseRemoteTree):
# object is transfered in the same chunks as it was originally.
from boto3.s3.transfer import TransferConfig
obj = S3RemoteTree.get_head_object(
s3, from_info.bucket, from_info.path
)
obj = S3Tree.get_head_object(s3, from_info.bucket, from_info.path)
etag = obj["ETag"].strip('"')
size = obj["ContentLength"]
@@ -315,7 +313,7 @@ class S3RemoteTree(BaseRemoteTree):
Config=TransferConfig(multipart_threshold=size + 1),
)
cached_etag = S3RemoteTree.get_etag(s3, to_info.bucket, to_info.path)
cached_etag = S3Tree.get_etag(s3, to_info.bucket, to_info.path)
if etag != cached_etag:
raise ETagMismatchError(etag, cached_etag)
dvc/tree/ssh/__init__.py
+3 -3
View File
@@ -12,7 +12,7 @@ from funcy import first, memoize, silent, wrap_with
import dvc.prompt as prompt
from dvc.scheme import Schemes
from ..base import BaseRemoteTree
from ..base import BaseTree
from ..pool import get_connection
logger = logging.getLogger(__name__)
@@ -29,7 +29,7 @@ def ask_password(host, user, port):
)
class SSHRemoteTree(BaseRemoteTree):
class SSHTree(BaseTree):
scheme = Schemes.SSH
REQUIRES = {"paramiko": "paramiko"}
JOBS = 4
@@ -99,7 +99,7 @@ class SSHRemoteTree(BaseRemoteTree):
def _load_user_ssh_config(hostname):
import paramiko
user_config_file = SSHRemoteTree.ssh_config_filename()
user_config_file = SSHTree.ssh_config_filename()
user_ssh_config = {}
if hostname and os.path.exists(user_config_file):
ssh_config = paramiko.SSHConfig()
tests/func/remote/test_gdrive.py
+4 -4
View File
@@ -5,18 +5,18 @@ import configobj
from dvc.main import main
from dvc.repo import Repo
from dvc.tree.gdrive import GDriveRemoteTree
from dvc.tree.gdrive import GDriveTree
def test_relative_user_credentials_file_config_setting(tmp_dir, dvc):
# CI sets it to test GDrive, here we want to test the work with file system
# based, regular credentials
if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA):
del os.environ[GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA]
if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
del os.environ[GDriveTree.GDRIVE_CREDENTIALS_DATA]
credentials = os.path.join("secrets", "credentials.json")
# GDriveRemoteTree.credentials_location helper checks for file existence,
# GDriveTree.credentials_location helper checks for file existence,
# create the file
tmp_dir.gen(credentials, "{'token': 'test'}")
tests/func/remote/test_index.py
+5 -5
View File
@@ -5,7 +5,7 @@ import pytest
from dvc.exceptions import DownloadError, UploadError
from dvc.remote.base import Remote
from dvc.remote.index import RemoteIndex
from dvc.remote.local import LocalRemote, LocalRemoteTree
from dvc.remote.local import LocalRemote, LocalTree
from dvc.utils.fs import remove
@@ -16,7 +16,7 @@ def remote(tmp_dir, dvc, tmp_path_factory, mocker):
dvc.config["core"]["remote"] = "upstream"
# patch hashes_exist since the LocalRemote normally overrides
# BaseRemoteTree.hashes_exist.
# BaseTree.hashes_exist.
def hashes_exist(self, *args, **kwargs):
return Remote.hashes_exist(self, *args, **kwargs)
@@ -80,7 +80,7 @@ def test_clear_on_download_err(tmp_dir, dvc, tmp_path_factory, remote, mocker):
remove(dvc.cache.local.cache_dir)
mocked_clear = mocker.patch.object(remote.INDEX_CLS, "clear")
mocker.patch.object(LocalRemoteTree, "_download", side_effect=Exception)
mocker.patch.object(LocalTree, "_download", side_effect=Exception)
with pytest.raises(DownloadError):
dvc.pull()
mocked_clear.assert_called_once_with()
@@ -90,14 +90,14 @@ def test_partial_upload(tmp_dir, dvc, tmp_path_factory, remote, mocker):
tmp_dir.dvc_gen({"foo": "foo content"})
tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})
original = LocalRemoteTree._upload
original = LocalTree._upload
def unreliable_upload(self, from_file, to_info, name=None, **kwargs):
if "baz" in name:
raise Exception("stop baz")
return original(self, from_file, to_info, name, **kwargs)
mocker.patch.object(LocalRemoteTree, "_upload", unreliable_upload)
mocker.patch.object(LocalTree, "_upload", unreliable_upload)
with pytest.raises(UploadError):
dvc.push()
with remote.index:
tests/func/test_add.py
+4 -4
View File
@@ -21,7 +21,7 @@ from dvc.exceptions import (
)
from dvc.main import main
from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError
from dvc.remote.local import LocalRemoteTree
from dvc.remote.local import LocalTree
from dvc.repo import Repo as DvcRepo
from dvc.stage import Stage
from dvc.system import System
@@ -410,7 +410,7 @@ class TestAddCommit(TestDvc):
def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
tmp_dir.gen({"data/data": "foo"})
get_dir_hash_counter = mocker.spy(LocalRemoteTree, "get_dir_hash")
get_dir_hash_counter = mocker.spy(LocalTree, "get_dir_hash")
ret = main(["add", "data"])
assert ret == 0
@@ -652,7 +652,7 @@ def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
dvc.add("dir")
tmp_dir.gen("dir/new_file", "new_file_content")
unprotect_spy = mocker.spy(LocalRemoteTree, "unprotect")
unprotect_spy = mocker.spy(LocalTree, "unprotect")
dvc.add("dir")
assert not unprotect_spy.mock.called
@@ -772,7 +772,7 @@ def test_add_empty_files(tmp_dir, dvc, link):
def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker):
dvc.cache.local.cache_types = ["hardlink"]
tmp_dir.gen({"foo": "", "bar": "", "lorem": "lorem", "ipsum": "ipsum"})
m = mocker.spy(LocalRemoteTree, "is_hardlink")
m = mocker.spy(LocalTree, "is_hardlink")
stages = dvc.add(["foo", "bar", "lorem", "ipsum"])
assert m.call_count == 1
tests/func/test_checkout.py
+6 -6
View File
@@ -21,8 +21,8 @@ from dvc.repo import Repo as DvcRepo
from dvc.stage import Stage
from dvc.stage.exceptions import StageFileDoesNotExistError
from dvc.system import System
from dvc.tree.local import LocalRemoteTree
from dvc.tree.s3 import S3RemoteTree
from dvc.tree.local import LocalTree
from dvc.tree.s3 import S3Tree
from dvc.utils import relpath
from dvc.utils.fs import walk_files
from dvc.utils.yaml import dump_yaml, load_yaml
@@ -308,8 +308,8 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
class TestCheckoutMissingMd5InStageFile(TestRepro):
def test(self):
d = load_yaml(self.file1_stage)
del d[Stage.PARAM_OUTS][0][LocalRemoteTree.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][LocalRemoteTree.PARAM_CHECKSUM]
del d[Stage.PARAM_OUTS][0][LocalTree.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][LocalTree.PARAM_CHECKSUM]
dump_yaml(self.file1_stage, d)
with pytest.raises(CheckoutError):
@@ -759,9 +759,9 @@ def test_checkout_recursive(tmp_dir, dvc):
not S3.should_test(), reason="Only run with S3 credentials"
)
def test_checkout_for_external_outputs(tmp_dir, dvc):
dvc.cache.s3 = CloudCache(S3RemoteTree(dvc, {"url": S3.get_url()}))
dvc.cache.s3 = CloudCache(S3Tree(dvc, {"url": S3.get_url()}))
remote = Remote(S3RemoteTree(dvc, {"url": S3.get_url()}))
remote = Remote(S3Tree(dvc, {"url": S3.get_url()}))
file_path = remote.tree.path_info / "foo"
remote.tree.s3.put_object(
Bucket=remote.tree.path_info.bucket, Key=file_path.path, Body="foo"
tests/func/test_data_cloud.py
+2 -2
View File
@@ -9,7 +9,7 @@ from dvc.cache import NamedCache
from dvc.external_repo import clean_repos
from dvc.main import main
from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
from dvc.remote.local import LocalRemoteTree
from dvc.remote.local import LocalTree
from dvc.stage.exceptions import StageNotFound
from dvc.utils.fs import remove
from dvc.utils.yaml import dump_yaml, load_yaml
@@ -189,7 +189,7 @@ def test_warn_on_outdated_stage(tmp_dir, dvc, local_remote, caplog):
def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
tmp_dir.gen({"foo": "foo"})
test_get_file_hash = mocker.spy(LocalRemoteTree, "get_file_hash")
test_get_file_hash = mocker.spy(LocalTree, "get_file_hash")
ret = main(["config", "cache.type", "hardlink"])
assert ret == 0
ret = main(["add", "foo"])
tests/func/test_external_repo.py
+2 -2
View File
@@ -4,7 +4,7 @@ from mock import patch
from dvc.external_repo import external_repo
from dvc.path_info import PathInfo
from dvc.remote.local import LocalRemoteTree
from dvc.remote.local import LocalTree
from dvc.scm.git import Git
from dvc.utils import relpath
from dvc.utils.fs import remove
@@ -49,7 +49,7 @@ def test_cache_reused(erepo_dir, mocker, local_cloud):
erepo_dir.dvc_gen("file", "text", commit="add file")
erepo_dir.dvc.push()
download_spy = mocker.spy(LocalRemoteTree, "download")
download_spy = mocker.spy(LocalTree, "download")
# Use URL to prevent any fishy optimizations
url = f"file://{erepo_dir}"
tests/func/test_gc.py
+3 -5
View File
@@ -8,7 +8,7 @@ from git import Repo
from dvc.exceptions import CollectCacheError
from dvc.main import main
from dvc.remote.local import LocalRemoteTree
from dvc.remote.local import LocalTree
from dvc.repo import Repo as DvcRepo
from dvc.utils.fs import remove
from tests.basic_env import TestDir, TestDvcGit
@@ -217,7 +217,7 @@ def test_gc_no_unpacked_dir(tmp_dir, dvc):
os.remove("dir.dvc")
unpackeddir = (
dir_stages[0].outs[0].cache_path + LocalRemoteTree.UNPACKED_DIR_SUFFIX
dir_stages[0].outs[0].cache_path + LocalTree.UNPACKED_DIR_SUFFIX
)
# older (pre 1.0) versions of dvc used to generate this dir
@@ -320,9 +320,7 @@ def test_gc_cloud_remove_order(tmp_dir, scm, dvc, tmp_path_factory, mocker):
dvc.remove(dir2.relpath)
dvc.gc(workspace=True)
mocked_remove = mocker.patch.object(
LocalRemoteTree, "remove", autospec=True
)
mocked_remove = mocker.patch.object(LocalTree, "remove", autospec=True)
dvc.gc(workspace=True, cloud=True)
assert len(mocked_remove.mock_calls) == 8
# dir (and unpacked dir) should be first 4 checksums removed from
tests/func/test_ignore.py
+2 -3
View File
@@ -13,7 +13,7 @@ from dvc.ignore import (
)
from dvc.path_info import PathInfo
from dvc.repo import Repo
from dvc.tree.local import LocalRemoteTree
from dvc.tree.local import LocalTree
from dvc.utils import relpath
from dvc.utils.fs import get_mtime_and_size
from tests.dir_helpers import TmpDir
@@ -113,8 +113,7 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
assert ignore_pattern_trie is not None
assert (
DvcIgnorePatterns.from_files(
os.fspath(top_ignore_file),
LocalRemoteTree(None, {"url": dvc.root_dir}),
os.fspath(top_ignore_file), LocalTree(None, {"url": dvc.root_dir}),
)
== ignore_pattern_trie[os.fspath(ignore_file)]
)
tests/func/test_import.py
+1 -1
View File
@@ -234,7 +234,7 @@ def test_download_error_pulling_imported_stage(tmp_dir, dvc, erepo_dir):
remove(dst_cache)
with patch(
"dvc.remote.local.LocalRemoteTree._download", side_effect=Exception
"dvc.remote.local.LocalTree._download", side_effect=Exception
), pytest.raises(DownloadError):
dvc.pull(["foo_imported.dvc"])
tests/func/test_remote.py
+9 -11
View File
@@ -10,8 +10,8 @@ from dvc.config import Config
from dvc.exceptions import DownloadError, UploadError
from dvc.main import main
from dvc.path_info import PathInfo
from dvc.tree.base import BaseRemoteTree, RemoteCacheRequiredError
from dvc.tree.local import LocalRemoteTree
from dvc.tree.base import BaseTree, RemoteCacheRequiredError
from dvc.tree.local import LocalTree
from dvc.utils.fs import remove
from tests.basic_env import TestDvc
from tests.remotes import Local
@@ -151,7 +151,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
path_info = PathInfo("data")
with dvc.state:
with patch.object(
BaseRemoteTree,
BaseTree,
"_collect_dir",
return_value=[
{"relpath": "1", "md5": "1"},
@@ -161,7 +161,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
hash1 = dvc.cache.local.tree.get_hash(path_info)
with patch.object(
BaseRemoteTree,
BaseTree,
"_collect_dir",
return_value=[
{"md5": "1", "relpath": "1"},
@@ -179,14 +179,14 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
baz = tmp_dir.dvc_gen({"baz": {"foo": "baz content"}})[0].outs[0]
# Faulty upload version, failing on foo
original = LocalRemoteTree._upload
original = LocalTree._upload
def unreliable_upload(self, from_file, to_info, name=None, **kwargs):
if "foo" in name:
raise Exception("stop foo")
return original(self, from_file, to_info, name, **kwargs)
with patch.object(LocalRemoteTree, "_upload", unreliable_upload):
with patch.object(LocalTree, "_upload", unreliable_upload):
with pytest.raises(UploadError) as upload_error_info:
dvc.push()
assert upload_error_info.value.amount == 3
@@ -204,7 +204,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
dvc.push()
remove(dvc.cache.local.cache_dir)
with patch.object(LocalRemoteTree, "_download", side_effect=Exception):
with patch.object(LocalTree, "_download", side_effect=Exception):
with pytest.raises(DownloadError) as download_error_info:
dvc.pull()
# error count should be len(.dir + standalone file checksums)
@@ -218,7 +218,7 @@ def test_raise_on_too_many_open_files(
tmp_dir.dvc_gen({"file": "file content"})
mocker.patch.object(
LocalRemoteTree,
LocalTree,
"_upload",
side_effect=OSError(errno.EMFILE, "Too many open files"),
)
@@ -251,9 +251,7 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
tmp_dir.dvc_gen({"foo": {"bar": "bar content"}})
tmp_dir.dvc_gen({"baz": "baz content"})
mocked_upload = mocker.patch.object(
LocalRemoteTree, "_upload", return_value=0
)
mocked_upload = mocker.patch.object(LocalTree, "_upload", return_value=0)
dvc.push()
# last uploaded file should be dir checksum
assert mocked_upload.call_args[0][0].endswith(".dir")
tests/func/test_repro.py
+5 -5
View File
@@ -15,7 +15,7 @@ from dvc.exceptions import (
)
from dvc.main import main
from dvc.output.base import BaseOutput
from dvc.remote.local import LocalRemoteTree
from dvc.remote.local import LocalTree
from dvc.stage import Stage
from dvc.stage.exceptions import StageFileDoesNotExistError
from dvc.system import System
@@ -757,8 +757,8 @@ class TestReproChangedDirData(SingleStageRun, TestDvc):
class TestReproMissingMd5InStageFile(TestRepro):
def test(self):
d = load_yaml(self.file1_stage)
del d[Stage.PARAM_OUTS][0][LocalRemoteTree.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][LocalRemoteTree.PARAM_CHECKSUM]
del d[Stage.PARAM_OUTS][0][LocalTree.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][LocalTree.PARAM_CHECKSUM]
dump_yaml(self.file1_stage, d)
stages = self.dvc.reproduce(self.file1_stage)
@@ -934,9 +934,9 @@ class TestReproAlreadyCached(TestRepro):
self.assertEqual(ret, 0)
patch_download = patch.object(
LocalRemoteTree,
LocalTree,
"download",
side_effect=LocalRemoteTree.download,
side_effect=LocalTree.download,
autospec=True,
)
tests/func/test_s3.py
+8 -11
View File
@@ -6,7 +6,7 @@ import pytest
from moto import mock_s3
from dvc.remote.base import CloudCache
from dvc.tree.s3 import S3RemoteTree
from dvc.tree.s3 import S3Tree
from tests.remotes import S3
# from https://github.com/spulec/moto/blob/v1.3.5/tests/test_s3/test_s3.py#L40
@@ -31,7 +31,7 @@ def reduced_min_part_size(f):
def _get_src_dst():
base_info = S3RemoteTree.PATH_CLS(S3.get_url())
base_info = S3Tree.PATH_CLS(S3.get_url())
return base_info / "from", base_info / "to"
@@ -43,19 +43,16 @@ def test_copy_singlepart_preserve_etag():
s3.create_bucket(Bucket=from_info.bucket)
s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data")
S3RemoteTree._copy(s3, from_info, to_info, {})
S3Tree._copy(s3, from_info, to_info, {})
@mock_s3
@pytest.mark.parametrize(
"base_info",
[
S3RemoteTree.PATH_CLS("s3://bucket/"),
S3RemoteTree.PATH_CLS("s3://bucket/ns/"),
],
[S3Tree.PATH_CLS("s3://bucket/"), S3Tree.PATH_CLS("s3://bucket/ns/")],
)
def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm):
tree = S3RemoteTree(dvc, {"url": str(base_info.parent)})
tree = S3Tree(dvc, {"url": str(base_info.parent)})
cache = CloudCache(tree)
s3 = cache.tree.s3
s3.create_bucket(Bucket=base_info.bucket)
@@ -70,8 +67,8 @@ def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm):
@mock_s3
def test_makedirs_doesnot_try_on_top_level_paths(tmp_dir, dvc, scm):
base_info = S3RemoteTree.PATH_CLS("s3://bucket/")
tree = S3RemoteTree(dvc, {"url": str(base_info)})
base_info = S3Tree.PATH_CLS("s3://bucket/")
tree = S3Tree(dvc, {"url": str(base_info)})
tree.makedirs(base_info)
@@ -113,4 +110,4 @@ def test_copy_multipart_preserve_etag():
s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
_upload_multipart(s3, from_info.bucket, from_info.path)
S3RemoteTree._copy(s3, from_info, to_info, {})
S3Tree._copy(s3, from_info, to_info, {})
tests/func/test_stage.py
+3 -3
View File
@@ -6,7 +6,7 @@ import pytest
from dvc.dvcfile import SingleStageFile
from dvc.main import main
from dvc.output.local import LocalOutput
from dvc.remote.local import LocalRemoteTree
from dvc.remote.local import LocalTree
from dvc.repo import Repo
from dvc.stage import PipelineStage, Stage
from dvc.stage.exceptions import StageFileFormatError
@@ -54,8 +54,8 @@ def test_empty_list():
def test_list():
lst = [
{LocalOutput.PARAM_PATH: "foo", LocalRemoteTree.PARAM_CHECKSUM: "123"},
{LocalOutput.PARAM_PATH: "bar", LocalRemoteTree.PARAM_CHECKSUM: None},
{LocalOutput.PARAM_PATH: "foo", LocalTree.PARAM_CHECKSUM: "123"},
{LocalOutput.PARAM_PATH: "bar", LocalTree.PARAM_CHECKSUM: None},
{LocalOutput.PARAM_PATH: "baz"},
]
d = {Stage.PARAM_DEPS: lst}
tests/func/test_tree.py
+7 -9
View File
@@ -5,16 +5,16 @@ from dvc.path_info import PathInfo
from dvc.repo import Repo
from dvc.repo.tree import RepoTree
from dvc.scm import SCM
from dvc.scm.git import GitTree
from dvc.tree.local import LocalRemoteTree
from dvc.tree.git import GitTree
from dvc.tree.local import LocalTree
from dvc.utils.fs import remove
from tests.basic_env import TestDir, TestGit, TestGitSubmodule
class TestLocalRemoteTree(TestDir):
class TestLocalTree(TestDir):
def setUp(self):
super().setUp()
self.tree = LocalRemoteTree(None, {})
self.tree = LocalTree(None, {})
def test_open(self):
with self.tree.open(self.FOO) as fd:
@@ -108,7 +108,7 @@ class AssertWalkEqualMixin:
class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir):
def test(self):
tree = LocalRemoteTree(None, {"url": self._root_dir})
tree = LocalTree(None, {"url": self._root_dir})
self.assertWalkEqual(
tree.walk(self._root_dir),
[
@@ -127,7 +127,7 @@ class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir):
)
def test_subdir(self):
tree = LocalRemoteTree(None, {"url": self._root_dir})
tree = LocalTree(None, {"url": self._root_dir})
self.assertWalkEqual(
tree.walk(join("data_dir", "data_sub_dir")),
[(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
@@ -136,9 +136,7 @@ class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir):
class TestWalkInGit(AssertWalkEqualMixin, TestGit):
def test_nobranch(self):
tree = LocalRemoteTree(
None, {"url": self._root_dir}, use_dvcignore=True
)
tree = LocalTree(None, {"url": self._root_dir}, use_dvcignore=True)
self.assertWalkEqual(
tree.walk("."),
[
tests/remotes/gdrive.py
+3 -3
View File
@@ -6,7 +6,7 @@ import pytest
from funcy import cached_property
from dvc.path_info import CloudURLInfo
from dvc.tree.gdrive import GDriveRemoteTree
from dvc.tree.gdrive import GDriveTree
from .base import Base
@@ -16,7 +16,7 @@ TEST_GDRIVE_REPO_BUCKET = "root"
class GDrive(Base, CloudURLInfo):
@staticmethod
def should_test():
return os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA) is not None
return os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA) is not None
@cached_property
def config(self):
@@ -46,6 +46,6 @@ def gdrive(make_tmp_dir):
tmp_dir = make_tmp_dir("gdrive", dvc=True)
ret = GDrive(GDrive.get_url())
tree = GDriveRemoteTree(tmp_dir.dvc, ret.config)
tree = GDriveTree(tmp_dir.dvc, ret.config)
tree._gdrive_create_dir("root", tree.path_info.path)
return ret
tests/remotes/ssh.py
+2 -2
View File
@@ -129,9 +129,9 @@ def ssh_connection(ssh_server):
@pytest.fixture
def ssh(ssh_server, monkeypatch):
from dvc.tree.ssh import SSHRemoteTree
from dvc.tree.ssh import SSHTree
# NOTE: see http://github.com/iterative/dvc/pull/3501
monkeypatch.setattr(SSHRemoteTree, "CAN_TRAVERSE", False)
monkeypatch.setattr(SSHTree, "CAN_TRAVERSE", False)
return SSHMocked(SSHMocked.get_url(TEST_SSH_USER, ssh_server.port))
tests/unit/remote/ssh/test_ssh.py
+21 -21
View File
@@ -6,7 +6,7 @@ import pytest
from mock import mock_open, patch
from dvc.system import System
from dvc.tree.ssh import SSHRemoteTree
from dvc.tree.ssh import SSHTree
def test_url(dvc):
@@ -19,20 +19,20 @@ def test_url(dvc):
url = f"ssh://{user}@{host}:{port}{path}"
config = {"url": url}
tree = SSHRemoteTree(dvc, config)
tree = SSHTree(dvc, config)
assert tree.path_info == url
# SCP-like URL ssh://[user@]host.xz:/absolute/path
url = f"ssh://{user}@{host}:{path}"
config = {"url": url}
tree = SSHRemoteTree(dvc, config)
tree = SSHTree(dvc, config)
assert tree.path_info == url
def test_no_path(dvc):
config = {"url": "ssh://127.0.0.1"}
tree = SSHRemoteTree(dvc, config)
tree = SSHTree(dvc, config)
assert tree.path_info.path == ""
@@ -66,10 +66,10 @@ else:
def test_ssh_host_override_from_config(
mock_file, mock_exists, dvc, config, expected_host
):
tree = SSHRemoteTree(dvc, config)
tree = SSHTree(dvc, config)
mock_exists.assert_called_with(SSHRemoteTree.ssh_config_filename())
mock_file.assert_called_with(SSHRemoteTree.ssh_config_filename())
mock_exists.assert_called_with(SSHTree.ssh_config_filename())
mock_file.assert_called_with(SSHTree.ssh_config_filename())
assert tree.path_info.host == expected_host
@@ -94,10 +94,10 @@ def test_ssh_host_override_from_config(
read_data=mock_ssh_config,
)
def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user):
tree = SSHRemoteTree(dvc, config)
tree = SSHTree(dvc, config)
mock_exists.assert_called_with(SSHRemoteTree.ssh_config_filename())
mock_file.assert_called_with(SSHRemoteTree.ssh_config_filename())
mock_exists.assert_called_with(SSHTree.ssh_config_filename())
mock_file.assert_called_with(SSHTree.ssh_config_filename())
assert tree.path_info.user == expected_user
@@ -107,7 +107,7 @@ def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user):
({"url": "ssh://example.com:2222"}, 2222),
({"url": "ssh://example.com"}, 1234),
({"url": "ssh://example.com", "port": 4321}, 4321),
({"url": "ssh://not_in_ssh_config.com"}, SSHRemoteTree.DEFAULT_PORT),
({"url": "ssh://not_in_ssh_config.com"}, SSHTree.DEFAULT_PORT),
({"url": "ssh://not_in_ssh_config.com:2222"}, 2222),
({"url": "ssh://not_in_ssh_config.com:2222", "port": 4321}, 4321),
],
@@ -119,10 +119,10 @@ def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user):
read_data=mock_ssh_config,
)
def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port):
tree = SSHRemoteTree(dvc, config)
tree = SSHTree(dvc, config)
mock_exists.assert_called_with(SSHRemoteTree.ssh_config_filename())
mock_file.assert_called_with(SSHRemoteTree.ssh_config_filename())
mock_exists.assert_called_with(SSHTree.ssh_config_filename())
mock_file.assert_called_with(SSHTree.ssh_config_filename())
assert tree.path_info.port == expected_port
@@ -154,10 +154,10 @@ def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port):
read_data=mock_ssh_config,
)
def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile):
tree = SSHRemoteTree(dvc, config)
tree = SSHTree(dvc, config)
mock_exists.assert_called_with(SSHRemoteTree.ssh_config_filename())
mock_file.assert_called_with(SSHRemoteTree.ssh_config_filename())
mock_exists.assert_called_with(SSHTree.ssh_config_filename())
mock_file.assert_called_with(SSHTree.ssh_config_filename())
assert tree.keyfile == expected_keyfile
@@ -176,15 +176,15 @@ def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile):
read_data=mock_ssh_config,
)
def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth):
tree = SSHRemoteTree(dvc, config)
tree = SSHTree(dvc, config)
mock_exists.assert_called_with(SSHRemoteTree.ssh_config_filename())
mock_file.assert_called_with(SSHRemoteTree.ssh_config_filename())
mock_exists.assert_called_with(SSHTree.ssh_config_filename())
mock_file.assert_called_with(SSHTree.ssh_config_filename())
assert tree.gss_auth == expected_gss_auth
def test_hardlink_optimization(dvc, tmp_dir, ssh):
tree = SSHRemoteTree(dvc, ssh.config)
tree = SSHTree(dvc, ssh.config)
from_info = tree.path_info / "empty"
to_info = tree.path_info / "link"
tests/unit/remote/test_azure.py
+4 -4
View File
@@ -1,5 +1,5 @@
from dvc.path_info import PathInfo
from dvc.tree.azure import AzureRemoteTree
from dvc.tree.azure import AzureTree
container_name = "container-name"
connection_string = (
@@ -15,7 +15,7 @@ def test_init_env_var(monkeypatch, dvc):
monkeypatch.setenv("AZURE_STORAGE_CONNECTION_STRING", connection_string)
config = {"url": "azure://"}
tree = AzureRemoteTree(dvc, config)
tree = AzureTree(dvc, config)
assert tree.path_info == "azure://" + container_name
assert tree._conn_kwargs["connection_string"] == connection_string
@@ -24,7 +24,7 @@ def test_init(dvc):
prefix = "some/prefix"
url = f"azure://{container_name}/{prefix}"
config = {"url": url, "connection_string": connection_string}
tree = AzureRemoteTree(dvc, config)
tree = AzureTree(dvc, config)
assert tree.path_info == url
assert tree._conn_kwargs["connection_string"] == connection_string
@@ -32,7 +32,7 @@ def test_init(dvc):
def test_get_file_hash(tmp_dir, azure):
tmp_dir.gen("foo", "foo")
tree = AzureRemoteTree(None, azure.config)
tree = AzureTree(None, azure.config)
to_info = azure
tree.upload(PathInfo("foo"), to_info)
assert tree.exists(to_info)
tests/unit/remote/test_base.py
+14 -20
View File
@@ -5,11 +5,7 @@ import pytest
from dvc.path_info import PathInfo
from dvc.remote.base import Remote
from dvc.tree.base import (
BaseRemoteTree,
RemoteCmdError,
RemoteMissingDepsError,
)
from dvc.tree.base import BaseTree, RemoteCmdError, RemoteMissingDepsError
class _CallableOrNone:
@@ -24,9 +20,9 @@ CallableOrNone = _CallableOrNone()
def test_missing_deps(dvc):
requires = {"missing": "missing"}
with mock.patch.object(BaseRemoteTree, "REQUIRES", requires):
with mock.patch.object(BaseTree, "REQUIRES", requires):
with pytest.raises(RemoteMissingDepsError):
BaseRemoteTree(dvc, {})
BaseTree(dvc, {})
def test_cmd_error(dvc):
@@ -37,18 +33,16 @@ def test_cmd_error(dvc):
err = "sed: expression #1, char 2: extra characters after command"
with mock.patch.object(
BaseRemoteTree,
"remove",
side_effect=RemoteCmdError("base", cmd, ret, err),
BaseTree, "remove", side_effect=RemoteCmdError("base", cmd, ret, err),
):
with pytest.raises(RemoteCmdError):
BaseRemoteTree(dvc, config).remove("file")
BaseTree(dvc, config).remove("file")
@mock.patch.object(BaseRemoteTree, "list_hashes_traverse")
@mock.patch.object(BaseRemoteTree, "list_hashes_exists")
@mock.patch.object(BaseTree, "list_hashes_traverse")
@mock.patch.object(BaseTree, "list_hashes_exists")
def test_hashes_exist(object_exists, traverse, dvc):
remote = Remote(BaseRemoteTree(dvc, {}))
remote = Remote(BaseTree(dvc, {}))
# remote does not support traverse
remote.tree.CAN_TRAVERSE = False
@@ -101,13 +95,13 @@ def test_hashes_exist(object_exists, traverse, dvc):
@mock.patch.object(
BaseRemoteTree, "list_hashes", return_value=[],
BaseTree, "list_hashes", return_value=[],
)
@mock.patch.object(
BaseRemoteTree, "path_to_hash", side_effect=lambda x: x,
BaseTree, "path_to_hash", side_effect=lambda x: x,
)
def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc):
tree = BaseRemoteTree(dvc, {})
tree = BaseTree(dvc, {})
tree.path_info = PathInfo("foo")
# parallel traverse
@@ -132,7 +126,7 @@ def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc):
def test_list_hashes(dvc):
tree = BaseRemoteTree(dvc, {})
tree = BaseTree(dvc, {})
tree.path_info = PathInfo("foo")
with mock.patch.object(
@@ -143,7 +137,7 @@ def test_list_hashes(dvc):
def test_list_paths(dvc):
tree = BaseRemoteTree(dvc, {})
tree = BaseTree(dvc, {})
tree.path_info = PathInfo("foo")
with mock.patch.object(tree, "walk_files", return_value=[]) as walk_mock:
@@ -161,4 +155,4 @@ def test_list_paths(dvc):
[(None, False), ("", False), ("3456.dir", True), ("3456", False)],
)
def test_is_dir_hash(hash_, result):
assert BaseRemoteTree.is_dir_hash(hash_) == result
assert BaseTree.is_dir_hash(hash_) == result
tests/unit/remote/test_gdrive.py
+7 -7
View File
@@ -2,7 +2,7 @@ import os
import pytest
from dvc.tree.gdrive import GDriveAuthError, GDriveRemoteTree
from dvc.tree.gdrive import GDriveAuthError, GDriveTree
USER_CREDS_TOKEN_REFRESH_ERROR = '{"access_token": "", "client_id": "", "client_secret": "", "refresh_token": "", "token_expiry": "", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "", "expires_in": 3600, "scope": "https://www.googleapis.com/auth/drive.appdata https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive", "https://www.googleapis.com/auth/drive.appdata"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": true, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}' # noqa: E501
@@ -17,21 +17,21 @@ class TestRemoteGDrive:
}
def test_init(self, dvc):
tree = GDriveRemoteTree(dvc, self.CONFIG)
tree = GDriveTree(dvc, self.CONFIG)
assert str(tree.path_info) == self.CONFIG["url"]
def test_drive(self, dvc):
tree = GDriveRemoteTree(dvc, self.CONFIG)
tree = GDriveTree(dvc, self.CONFIG)
os.environ[
GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA
GDriveTree.GDRIVE_CREDENTIALS_DATA
] = USER_CREDS_TOKEN_REFRESH_ERROR
with pytest.raises(GDriveAuthError):
assert tree._drive
os.environ[GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA] = ""
tree = GDriveRemoteTree(dvc, self.CONFIG)
os.environ[GDriveTree.GDRIVE_CREDENTIALS_DATA] = ""
tree = GDriveTree(dvc, self.CONFIG)
os.environ[
GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA
GDriveTree.GDRIVE_CREDENTIALS_DATA
] = USER_CREDS_MISSED_KEY_ERROR
with pytest.raises(GDriveAuthError):
assert tree._drive
tests/unit/remote/test_gs.py
+4 -4
View File
@@ -2,7 +2,7 @@ import mock
import pytest
import requests
from dvc.tree.gs import GSRemoteTree, dynamic_chunk_size
from dvc.tree.gs import GSTree, dynamic_chunk_size
BUCKET = "bucket"
PREFIX = "prefix"
@@ -17,7 +17,7 @@ CONFIG = {
def test_init(dvc):
tree = GSRemoteTree(dvc, CONFIG)
tree = GSTree(dvc, CONFIG)
assert tree.path_info == URL
assert tree.projectname == PROJECT
assert tree.credentialpath == CREDENTIALPATH
@@ -25,7 +25,7 @@ def test_init(dvc):
@mock.patch("google.cloud.storage.Client.from_service_account_json")
def test_gs(mock_client, dvc):
tree = GSRemoteTree(dvc, CONFIG)
tree = GSTree(dvc, CONFIG)
assert tree.credentialpath
assert tree.gs
mock_client.assert_called_once_with(CREDENTIALPATH)
@@ -35,7 +35,7 @@ def test_gs(mock_client, dvc):
def test_gs_no_credspath(mock_client, dvc):
config = CONFIG.copy()
del config["credentialpath"]
tree = GSRemoteTree(dvc, config)
tree = GSTree(dvc, config)
assert tree.gs
mock_client.assert_called_with(PROJECT)
tests/unit/remote/test_http.py
+6 -6
View File
@@ -1,11 +1,11 @@
import pytest
from dvc.exceptions import HTTPError
from dvc.tree.http import HTTPRemoteTree
from dvc.tree.http import HTTPTree
def test_download_fails_on_error_code(dvc, http):
tree = HTTPRemoteTree(dvc, http.config)
tree = HTTPTree(dvc, http.config)
with pytest.raises(HTTPError):
tree._download(http / "missing.txt", "missing.txt")
@@ -19,7 +19,7 @@ def test_public_auth_method(dvc):
"password": "",
}
tree = HTTPRemoteTree(dvc, config)
tree = HTTPTree(dvc, config)
assert tree._auth_method() is None
@@ -38,7 +38,7 @@ def test_basic_auth_method(dvc):
"password": password,
}
tree = HTTPRemoteTree(dvc, config)
tree = HTTPTree(dvc, config)
assert tree._auth_method() == auth
assert isinstance(tree._auth_method(), HTTPBasicAuth)
@@ -58,7 +58,7 @@ def test_digest_auth_method(dvc):
"password": password,
}
tree = HTTPRemoteTree(dvc, config)
tree = HTTPTree(dvc, config)
assert tree._auth_method() == auth
assert isinstance(tree._auth_method(), HTTPDigestAuth)
@@ -75,7 +75,7 @@ def test_custom_auth_method(dvc):
"password": password,
}
tree = HTTPRemoteTree(dvc, config)
tree = HTTPTree(dvc, config)
assert tree._auth_method() is None
assert header in tree.headers
tests/unit/remote/test_local.py
+5 -5
View File
@@ -7,7 +7,7 @@ from dvc.cache import NamedCache
from dvc.path_info import PathInfo
from dvc.remote.index import RemoteIndexNoop
from dvc.remote.local import LocalCache
from dvc.tree.local import LocalRemoteTree
from dvc.tree.local import LocalTree
def test_status_download_optimization(mocker, dvc):
@@ -15,7 +15,7 @@ def test_status_download_optimization(mocker, dvc):
And the desired files to fetch are already on the local cache,
Don't check the existence of the desired files on the remote cache
"""
cache = LocalCache(LocalRemoteTree(dvc, {}))
cache = LocalCache(LocalTree(dvc, {}))
infos = NamedCache()
infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo")
@@ -36,7 +36,7 @@ def test_status_download_optimization(mocker, dvc):
@pytest.mark.parametrize("link_name", ["hardlink", "symlink"])
def test_is_protected(tmp_dir, dvc, link_name):
tree = LocalRemoteTree(dvc, {})
tree = LocalTree(dvc, {})
link_method = getattr(tree, link_name)
(tmp_dir / "foo").write_text("foo")
@@ -69,7 +69,7 @@ def test_is_protected(tmp_dir, dvc, link_name):
def test_protect_ignore_errors(tmp_dir, mocker, err):
tmp_dir.gen("foo", "foo")
foo = PathInfo("foo")
tree = LocalRemoteTree(None, {})
tree = LocalTree(None, {})
tree.protect(foo)
@@ -83,7 +83,7 @@ def test_protect_ignore_errors(tmp_dir, mocker, err):
def test_protect_ignore_erofs(tmp_dir, mocker):
tmp_dir.gen("foo", "foo")
foo = PathInfo("foo")
tree = LocalRemoteTree(None, {})
tree = LocalTree(None, {})
mock_chmod = mocker.patch(
"os.chmod", side_effect=OSError(errno.EROFS, "read-only fs")
tests/unit/remote/test_oss.py
+2 -2
View File
@@ -1,4 +1,4 @@
from dvc.tree.oss import OSSRemoteTree
from dvc.tree.oss import OSSTree
bucket_name = "bucket-name"
endpoint = "endpoint"
@@ -15,7 +15,7 @@ def test_init(dvc):
"oss_key_secret": key_secret,
"oss_endpoint": endpoint,
}
tree = OSSRemoteTree(dvc, config)
tree = OSSTree(dvc, config)
assert tree.path_info == url
assert tree.endpoint == endpoint
assert tree.key_id == key_id
tests/unit/remote/test_remote.py
+3 -3
View File
@@ -1,8 +1,8 @@
import pytest
from dvc.tree import get_cloud_tree
from dvc.tree.gs import GSRemoteTree
from dvc.tree.s3 import S3RemoteTree
from dvc.tree.gs import GSTree
from dvc.tree.s3 import S3Tree
def test_remote_with_hash_jobs(dvc):
@@ -31,7 +31,7 @@ def test_remote_without_hash_jobs_default(dvc):
assert tree.hash_jobs == tree.HASH_JOBS
@pytest.mark.parametrize("tree_cls", [GSRemoteTree, S3RemoteTree])
@pytest.mark.parametrize("tree_cls", [GSTree, S3Tree])
def test_makedirs_not_create_for_top_level_path(tree_cls, dvc, mocker):
url = f"{tree_cls.scheme}://bucket/"
tree = tree_cls(dvc, {"url": url})
tests/unit/remote/test_remote_tree.py
+4 -4
View File
@@ -4,7 +4,7 @@ import pytest
from dvc.path_info import PathInfo
from dvc.remote import get_remote
from dvc.tree.s3 import S3RemoteTree
from dvc.tree.s3 import S3Tree
from dvc.utils.fs import walk_files
remotes = [pytest.lazy_fixture(fix) for fix in ["gs", "s3"]]
@@ -91,15 +91,15 @@ def test_copy_preserve_etag_across_buckets(remote, dvc):
s3 = remote.tree.s3
s3.create_bucket(Bucket="another")
another = S3RemoteTree(dvc, {"url": "s3://another", "region": "us-east-1"})
another = S3Tree(dvc, {"url": "s3://another", "region": "us-east-1"})
from_info = remote.tree.path_info / "foo"
to_info = another.path_info / "foo"
remote.tree.copy(from_info, to_info)
from_etag = S3RemoteTree.get_etag(s3, from_info.bucket, from_info.path)
to_etag = S3RemoteTree.get_etag(s3, "another", "foo")
from_etag = S3Tree.get_etag(s3, from_info.bucket, from_info.path)
to_etag = S3Tree.get_etag(s3, "another", "foo")
assert from_etag == to_etag
tests/unit/remote/test_s3.py
+6 -6
View File
@@ -1,7 +1,7 @@
import pytest
from dvc.config import ConfigError
from dvc.tree.s3 import S3RemoteTree
from dvc.tree.s3 import S3Tree
bucket_name = "bucket-name"
prefix = "some/prefix"
@@ -22,7 +22,7 @@ def grants():
def test_init(dvc):
config = {"url": url}
tree = S3RemoteTree(dvc, config)
tree = S3Tree(dvc, config)
assert tree.path_info == url
@@ -35,7 +35,7 @@ def test_grants(dvc):
"grant_write_acp": "id=write-acp-permission-id",
"grant_full_control": "id=full-control-permission-id",
}
tree = S3RemoteTree(dvc, config)
tree = S3Tree(dvc, config)
assert (
tree.extra_args["GrantRead"]
@@ -53,16 +53,16 @@ def test_grants_mutually_exclusive_acl_error(dvc, grants):
config = {"url": url, "acl": "public-read", grant_option: grant_value}
with pytest.raises(ConfigError):
S3RemoteTree(dvc, config)
S3Tree(dvc, config)
def test_sse_kms_key_id(dvc):
tree = S3RemoteTree(dvc, {"url": url, "sse_kms_key_id": "key"})
tree = S3Tree(dvc, {"url": url, "sse_kms_key_id": "key"})
assert tree.extra_args["SSEKMSKeyId"] == "key"
def test_key_id_and_secret(dvc):
tree = S3RemoteTree(
tree = S3Tree(
dvc,
{"url": url, "access_key_id": key_id, "secret_access_key": key_secret},
)
tests/unit/utils/test_fs.py
+3 -7
View File
@@ -8,7 +8,7 @@ from mock import patch
import dvc
from dvc.path_info import PathInfo
from dvc.system import System
from dvc.tree.local import LocalRemoteTree
from dvc.tree.local import LocalTree
from dvc.utils import relpath
from dvc.utils.fs import (
BasePathNotInCheckedPathException,
@@ -28,9 +28,7 @@ from tests.basic_env import TestDir
class TestMtimeAndSize(TestDir):
def test(self):
tree = LocalRemoteTree(
None, {"url": self.root_dir}, use_dvcignore=True
)
tree = LocalTree(None, {"url": self.root_dir}, use_dvcignore=True)
file_time, file_size = get_mtime_and_size(self.DATA, tree)
dir_time, dir_size = get_mtime_and_size(self.DATA_DIR, tree)
@@ -131,9 +129,7 @@ def test_path_object_and_str_are_valid_types_get_mtime_and_size(tmp_dir):
tmp_dir.gen(
{"dir": {"dir_file": "dir file content"}, "file": "file_content"}
)
tree = LocalRemoteTree(
None, {"url": os.fspath(tmp_dir)}, use_dvcignore=True
)
tree = LocalTree(None, {"url": os.fspath(tmp_dir)}, use_dvcignore=True)
time, size = get_mtime_and_size("dir", tree)
object_time, object_size = get_mtime_and_size(PathInfo("dir"), tree)