dvc: bump to 1.2.3

* [WIP] gdrive: add workaround for downloading empty files

Better to solve it on PyDrive's side:
https://github.com/iterative/dvc/issues/4286

Fixes #4286

* deps: limit flatten_json

```
pylint...................................................................Failed
- hook id: pylint
- exit code: 2

Registered custom plugin. Some checks will be disabled for tests.
************* Module dvc.command.experiments
dvc/command/experiments.py:14:4: E0401: Unable to import 'flatten_json'
(import-error)
dvc/command/experiments.py:39:4: E0401: Unable to import 'flatten_json'
(import-error)

-----------------------------------
Your code has been rated at 9.99/10

Registered custom plugin. Some checks will be disabled for tests.
************* Module dvc.utils.diff
dvc/utils/diff.py:38:8: E0401: Unable to import 'flatten_json'
(import-error)
************* Module dvc.command.metrics
dvc/command/metrics.py:17:4: E0401: Unable to import 'flatten_json'
(import-error)

-----------------------------------
```

.codeclimate.yml

@@ -1,19 +0,0 @@
-version: "2"
-checks:
-    argument-count:
-        config:
-            threshold: 8
-    complex-logic:
-        config:
-            threshold: 6
-    file-lines:
-        config:
-            threshold: 350
-    method-complexity:
-        config:
-            threshold: 15
-    method-count:
-        config:
-            threshold: 30
-exclude_patterns:
-    - tests

.deepsource.toml

@@ -1,11 +0,0 @@
-version = 1
-
-test_patterns = [
-  'tests/**'
-]
-
-[[analyzers]]
-name = 'python'
-enabled = true
-runtime_version = '3.x.x'
-

.dvc/.gitignore

@@ -9,3 +9,4 @@
 /pkg
 /repos
 /tmp
+/experiments

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,6 +2,4 @@
 
 * [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
 
-* [ ] ❌ I will check DeepSource, CodeClimate, and other sanity checks below. (We consider them recommendatory and don't expect everything to be addressed. Please fix things that actually improve code or fix bugs.)
-
 Thank you for the contribution - we'll try to review it as soon as possible. 🙏

.travis.yml

@@ -160,11 +160,3 @@ deploy:
       condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL"
       repo: iterative/dvc
       stage: build
-  - provider: script
-    skip_cleanup: true
-    script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL_MAJOR || echo timed out"
-    on:
-      all_branches: true
-      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL_MAJOR"
-      repo: iterative/dvc
-      stage: build

README.rst

@@ -131,8 +131,6 @@ Conda (Anaconda)
 
    conda install -c conda-forge dvc
 
-Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
-
 pip (PyPI)
 ----------
 

dvc/api.py

@@ -30,7 +30,7 @@ def get_url(path, repo=None, rev=None, remote=None):
             raise UrlNotDvcRepoError(_repo.url)  # pylint: disable=no-member
         out = _repo.find_out_by_relpath(path)
         remote_obj = _repo.cloud.get_remote(remote)
-        return str(remote_obj.hash_to_path_info(out.checksum))
+        return str(remote_obj.tree.hash_to_path_info(out.checksum))
 
 
 def open(  # noqa, pylint: disable=redefined-builtin

dvc/cache/__init__.py

@@ -28,8 +28,8 @@ def _make_remote_property(name):
     """
 
     def getter(self):
-        from dvc.remote import get_cloud_tree
-        from dvc.remote.base import CloudCache
+        from ..tree import get_cloud_tree
+        from .base import CloudCache
 
         remote = self.config.get(name)
         if not remote:
@@ -52,8 +52,8 @@ class Cache:
     CACHE_DIR = "cache"
 
     def __init__(self, repo):
-        from dvc.remote import get_cloud_tree
-        from dvc.remote.local import LocalCache
+        from ..tree import get_cloud_tree
+        from .local import LocalCache
 
         self.repo = repo
         self.config = config = repo.config["cache"]

dvc/cache/base.py

@@ -0,0 +1,554 @@
+import json
+import logging
+from copy import copy
+
+from shortuuid import uuid
+
+import dvc.prompt as prompt
+from dvc.exceptions import CheckoutError, ConfirmRemoveError, DvcException
+from dvc.path_info import WindowsPathInfo
+from dvc.progress import Tqdm
+from dvc.remote.slow_link_detection import slow_link_guard
+
+from ..tree.base import RemoteActionNotImplemented
+
+logger = logging.getLogger(__name__)
+
+STATUS_OK = 1
+STATUS_MISSING = 2
+STATUS_NEW = 3
+STATUS_DELETED = 4
+
+STATUS_MAP = {
+    # (local_exists, remote_exists)
+    (True, True): STATUS_OK,
+    (False, False): STATUS_MISSING,
+    (True, False): STATUS_NEW,
+    (False, True): STATUS_DELETED,
+}
+
+
+class DirCacheError(DvcException):
+    def __init__(self, hash_):
+        super().__init__(
+            f"Failed to load dir cache for hash value: '{hash_}'."
+        )
+
+
+class CloudCache:
+    """Cloud cache class."""
+
+    DEFAULT_CACHE_TYPES = ["copy"]
+    CACHE_MODE = None
+
+    def __init__(self, tree):
+        self.tree = tree
+        self.repo = tree.repo
+
+        self.cache_types = tree.config.get("type") or copy(
+            self.DEFAULT_CACHE_TYPES
+        )
+        self.cache_type_confirmed = False
+        self._dir_info = {}
+
+    def get_dir_cache(self, hash_):
+        assert hash_
+
+        dir_info = self._dir_info.get(hash_)
+        if dir_info:
+            return dir_info
+
+        try:
+            dir_info = self.load_dir_cache(hash_)
+        except DirCacheError:
+            dir_info = []
+
+        self._dir_info[hash_] = dir_info
+        return dir_info
+
+    def load_dir_cache(self, hash_):
+        path_info = self.tree.hash_to_path_info(hash_)
+
+        try:
+            with self.tree.open(path_info, "r") as fobj:
+                d = json.load(fobj)
+        except (ValueError, FileNotFoundError) as exc:
+            raise DirCacheError(hash_) from exc
+
+        if not isinstance(d, list):
+            logger.error(
+                "dir cache file format error '%s' [skipping the file]",
+                path_info,
+            )
+            return []
+
+        if self.tree.PATH_CLS == WindowsPathInfo:
+            # only need to convert it for Windows
+            for info in d:
+                # NOTE: here is a BUG, see comment to .as_posix() below
+                relpath = info[self.tree.PARAM_RELPATH]
+                info[self.tree.PARAM_RELPATH] = relpath.replace(
+                    "/", self.tree.PATH_CLS.sep
+                )
+
+        return d
+
+    def changed(self, path_info, hash_info):
+        """Checks if data has changed.
+
+        A file is considered changed if:
+            - It doesn't exist on the working directory (was unlinked)
+            - Hash value is not computed (saving a new file)
+            - The hash value stored is different from the given one
+            - There's no file in the cache
+
+        Args:
+            path_info: dict with path information.
+            hash: expected hash value for this data.
+
+        Returns:
+            bool: True if data has changed, False otherwise.
+        """
+
+        logger.debug(
+            "checking if '%s'('%s') has changed.", path_info, hash_info
+        )
+
+        if not self.tree.exists(path_info):
+            logger.debug("'%s' doesn't exist.", path_info)
+            return True
+
+        hash_ = hash_info.get(self.tree.PARAM_CHECKSUM)
+        if hash_ is None:
+            logger.debug("hash value for '%s' is missing.", path_info)
+            return True
+
+        if self.changed_cache(hash_):
+            logger.debug("cache for '%s'('%s') has changed.", path_info, hash_)
+            return True
+
+        actual = self.tree.get_hash(path_info)
+        if hash_ != actual:
+            logger.debug(
+                "hash value '%s' for '%s' has changed (actual '%s').",
+                hash_,
+                actual,
+                path_info,
+            )
+            return True
+
+        logger.debug("'%s' hasn't changed.", path_info)
+        return False
+
+    def link(self, from_info, to_info):
+        self._link(from_info, to_info, self.cache_types)
+
+    def _link(self, from_info, to_info, link_types):
+        assert self.tree.isfile(from_info)
+
+        self.tree.makedirs(to_info.parent)
+
+        self._try_links(from_info, to_info, link_types)
+
+    def _verify_link(self, path_info, link_type):
+        if self.cache_type_confirmed:
+            return
+
+        is_link = getattr(self.tree, f"is_{link_type}", None)
+        if is_link and not is_link(path_info):
+            self.tree.remove(path_info)
+            raise DvcException(f"failed to verify {link_type}")
+
+        self.cache_type_confirmed = True
+
+    @slow_link_guard
+    def _try_links(self, from_info, to_info, link_types):
+        while link_types:
+            link_method = getattr(self.tree, link_types[0])
+            try:
+                self._do_link(from_info, to_info, link_method)
+                self._verify_link(to_info, link_types[0])
+                return
+
+            except DvcException as exc:
+                logger.debug(
+                    "Cache type '%s' is not supported: %s", link_types[0], exc
+                )
+                del link_types[0]
+
+        raise DvcException("no possible cache types left to try out.")
+
+    def _do_link(self, from_info, to_info, link_method):
+        if self.tree.exists(to_info):
+            raise DvcException(f"Link '{to_info}' already exists!")
+
+        link_method(from_info, to_info)
+
+        logger.debug(
+            "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
+        )
+
+    def _save_file(self, path_info, tree, hash_, save_link=True, **kwargs):
+        assert hash_
+
+        cache_info = self.tree.hash_to_path_info(hash_)
+        if tree == self.tree:
+            if self.changed_cache(hash_):
+                self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
+                self.link(cache_info, path_info)
+            elif self.tree.iscopy(path_info) and self._cache_is_copy(
+                path_info
+            ):
+                # Default relink procedure involves unneeded copy
+                self.tree.unprotect(path_info)
+            else:
+                self.tree.remove(path_info)
+                self.link(cache_info, path_info)
+
+            if save_link:
+                self.tree.state.save_link(path_info)
+            # we need to update path and cache, since in case of reflink,
+            # or copy cache type moving original file results in updates on
+            # next executed command, which causes md5 recalculation
+            self.tree.state.save(path_info, hash_)
+        else:
+            if self.changed_cache(hash_):
+                with tree.open(path_info, mode="rb") as fobj:
+                    # if tree has fetch enabled, DVC out will be fetched on
+                    # open and we do not need to read/copy any data
+                    if not (
+                        tree.isdvc(path_info, strict=False) and tree.fetch
+                    ):
+                        self.tree.copy_fobj(fobj, cache_info)
+                callback = kwargs.get("download_callback")
+                if callback:
+                    callback(1)
+
+        self.tree.state.save(cache_info, hash_)
+        return {self.tree.PARAM_CHECKSUM: hash_}
+
+    def _cache_is_copy(self, path_info):
+        """Checks whether cache uses copies."""
+        if self.cache_type_confirmed:
+            return self.cache_types[0] == "copy"
+
+        if set(self.cache_types) <= {"copy"}:
+            return True
+
+        workspace_file = path_info.with_name("." + uuid())
+        test_cache_file = self.tree.path_info / ".cache_type_test_file"
+        if not self.tree.exists(test_cache_file):
+            with self.tree.open(test_cache_file, "wb") as fobj:
+                fobj.write(bytes(1))
+        try:
+            self.link(test_cache_file, workspace_file)
+        finally:
+            self.tree.remove(workspace_file)
+            self.tree.remove(test_cache_file)
+
+        self.cache_type_confirmed = True
+        return self.cache_types[0] == "copy"
+
+    def _save_dir(self, path_info, tree, hash_, save_link=True, **kwargs):
+        dir_info = self.get_dir_cache(hash_)
+        for entry in Tqdm(
+            dir_info, desc="Saving " + path_info.name, unit="file"
+        ):
+            entry_info = path_info / entry[self.tree.PARAM_RELPATH]
+            entry_hash = entry[self.tree.PARAM_CHECKSUM]
+            self._save_file(
+                entry_info, tree, entry_hash, save_link=False, **kwargs
+            )
+
+        if save_link:
+            self.tree.state.save_link(path_info)
+        if self.tree.exists(path_info):
+            self.tree.state.save(path_info, hash_)
+
+        cache_info = self.tree.hash_to_path_info(hash_)
+        self.tree.state.save(cache_info, hash_)
+        return {self.tree.PARAM_CHECKSUM: hash_}
+
+    def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        if path_info.scheme != self.tree.scheme:
+            raise RemoteActionNotImplemented(
+                f"save {path_info.scheme} -> {self.tree.scheme}",
+                self.tree.scheme,
+            )
+
+        if not hash_info:
+            hash_info = self.tree.save_info(path_info, tree=tree, **kwargs)
+        hash_ = hash_info[self.tree.PARAM_CHECKSUM]
+        return self._save(path_info, tree, hash_, save_link, **kwargs)
+
+    def _save(self, path_info, tree, hash_, save_link=True, **kwargs):
+        to_info = self.tree.hash_to_path_info(hash_)
+        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
+
+        if tree.isdir(path_info):
+            return self._save_dir(path_info, tree, hash_, save_link, **kwargs)
+        return self._save_file(path_info, tree, hash_, save_link, **kwargs)
+
+    # Override to return path as a string instead of PathInfo for clouds
+    # which support string paths (see local)
+    def hash_to_path(self, hash_):
+        return self.tree.hash_to_path_info(hash_)
+
+    def changed_cache_file(self, hash_):
+        """Compare the given hash with the (corresponding) actual one.
+
+        - Use `State` as a cache for computed hashes
+            + The entries are invalidated by taking into account the following:
+                * mtime
+                * inode
+                * size
+                * hash
+
+        - Remove the file from cache if it doesn't match the actual hash
+        """
+        # Prefer string path over PathInfo when possible due to performance
+        cache_info = self.hash_to_path(hash_)
+        if self.tree.is_protected(cache_info):
+            logger.debug(
+                "Assuming '%s' is unchanged since it is read-only", cache_info
+            )
+            return False
+
+        actual = self.tree.get_hash(cache_info)
+
+        logger.debug(
+            "cache '%s' expected '%s' actual '%s'", cache_info, hash_, actual,
+        )
+
+        if not hash_ or not actual:
+            return True
+
+        if actual.split(".")[0] == hash_.split(".")[0]:
+            # making cache file read-only so we don't need to check it
+            # next time
+            self.tree.protect(cache_info)
+            return False
+
+        if self.tree.exists(cache_info):
+            logger.warning("corrupted cache file '%s'.", cache_info)
+            self.tree.remove(cache_info)
+
+        return True
+
+    def _changed_dir_cache(self, hash_, path_info=None, filter_info=None):
+        if self.changed_cache_file(hash_):
+            return True
+
+        for entry in self.get_dir_cache(hash_):
+            entry_hash = entry[self.tree.PARAM_CHECKSUM]
+
+            if path_info and filter_info:
+                entry_info = path_info / entry[self.tree.PARAM_RELPATH]
+                if not entry_info.isin_or_eq(filter_info):
+                    continue
+
+            if self.changed_cache_file(entry_hash):
+                return True
+
+        return False
+
+    def changed_cache(self, hash_, path_info=None, filter_info=None):
+        if self.tree.is_dir_hash(hash_):
+            return self._changed_dir_cache(
+                hash_, path_info=path_info, filter_info=filter_info
+            )
+        return self.changed_cache_file(hash_)
+
+    def already_cached(self, path_info):
+        current = self.tree.get_hash(path_info)
+
+        if not current:
+            return False
+
+        return not self.changed_cache(current)
+
+    def safe_remove(self, path_info, force=False):
+        if not self.tree.exists(path_info):
+            return
+
+        if not force and not self.already_cached(path_info):
+            msg = (
+                "file '{}' is going to be removed."
+                " Are you sure you want to proceed?".format(str(path_info))
+            )
+
+            if not prompt.confirm(msg):
+                raise ConfirmRemoveError(str(path_info))
+
+        self.tree.remove(path_info)
+
+    def _checkout_file(
+        self, path_info, hash_, force, progress_callback=None, relink=False
+    ):
+        """The file is changed we need to checkout a new copy"""
+        added, modified = True, False
+        cache_info = self.tree.hash_to_path_info(hash_)
+        if self.tree.exists(path_info):
+            logger.debug("data '%s' will be replaced.", path_info)
+            self.safe_remove(path_info, force=force)
+            added, modified = False, True
+
+        self.link(cache_info, path_info)
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_)
+        if progress_callback:
+            progress_callback(str(path_info))
+
+        return added, modified and not relink
+
+    def _checkout_dir(
+        self,
+        path_info,
+        hash_,
+        force,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        added, modified = False, False
+        # Create dir separately so that dir is created
+        # even if there are no files in it
+        if not self.tree.exists(path_info):
+            added = True
+            self.tree.makedirs(path_info)
+
+        dir_info = self.get_dir_cache(hash_)
+
+        logger.debug("Linking directory '%s'.", path_info)
+
+        for entry in dir_info:
+            relative_path = entry[self.tree.PARAM_RELPATH]
+            entry_hash = entry[self.tree.PARAM_CHECKSUM]
+            entry_cache_info = self.tree.hash_to_path_info(entry_hash)
+            entry_info = path_info / relative_path
+
+            if filter_info and not entry_info.isin_or_eq(filter_info):
+                continue
+
+            entry_hash_info = {self.tree.PARAM_CHECKSUM: entry_hash}
+            if relink or self.changed(entry_info, entry_hash_info):
+                modified = True
+                self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+                self.tree.state.save(entry_info, entry_hash)
+            if progress_callback:
+                progress_callback(str(entry_info))
+
+        modified = (
+            self._remove_redundant_files(path_info, dir_info, force)
+            or modified
+        )
+
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_)
+
+        # relink is not modified, assume it as nochange
+        return added, not added and modified and not relink
+
+    def _remove_redundant_files(self, path_info, dir_info, force):
+        existing_files = set(self.tree.walk_files(path_info))
+
+        needed_files = {
+            path_info / entry[self.tree.PARAM_RELPATH] for entry in dir_info
+        }
+        redundant_files = existing_files - needed_files
+        for path in redundant_files:
+            self.safe_remove(path, force)
+
+        return bool(redundant_files)
+
+    def checkout(
+        self,
+        path_info,
+        hash_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        if path_info.scheme not in ["local", self.tree.scheme]:
+            raise NotImplementedError
+
+        hash_ = hash_info.get(self.tree.PARAM_CHECKSUM)
+        failed = None
+        skip = False
+        if not hash_:
+            logger.warning(
+                "No file hash info found for '%s'. " "It won't be created.",
+                path_info,
+            )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        elif not relink and not self.changed(path_info, hash_info):
+            logger.debug("Data '%s' didn't change.", path_info)
+            skip = True
+
+        elif self.changed_cache(
+            hash_, path_info=path_info, filter_info=filter_info
+        ):
+            logger.warning(
+                "Cache '%s' not found. File '%s' won't be created.",
+                hash_,
+                path_info,
+            )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        if failed or skip:
+            if progress_callback:
+                progress_callback(
+                    str(path_info),
+                    self.get_files_number(
+                        self.tree.path_info, hash_, filter_info
+                    ),
+                )
+            if failed:
+                raise CheckoutError([failed])
+            return
+
+        logger.debug("Checking out '%s' with cache '%s'.", path_info, hash_)
+
+        return self._checkout(
+            path_info, hash_, force, progress_callback, relink, filter_info,
+        )
+
+    def _checkout(
+        self,
+        path_info,
+        hash_,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        if not self.tree.is_dir_hash(hash_):
+            return self._checkout_file(
+                path_info, hash_, force, progress_callback, relink
+            )
+
+        return self._checkout_dir(
+            path_info, hash_, force, progress_callback, relink, filter_info
+        )
+
+    def get_files_number(self, path_info, hash_, filter_info):
+        from funcy.py3 import ilen
+
+        if not hash_:
+            return 0
+
+        if not self.tree.is_dir_hash(hash_):
+            return 1
+
+        if not filter_info:
+            return len(self.get_dir_cache(hash_))
+
+        return ilen(
+            filter_info.isin_or_eq(path_info / entry[self.tree.PARAM_CHECKSUM])
+            for entry in self.get_dir_cache(hash_)
+        )

dvc/cache/local.py

@@ -0,0 +1,431 @@
+import errno
+import logging
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import partial, wraps
+
+from funcy import cached_property, concat
+
+from dvc.exceptions import DownloadError, UploadError
+from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
+
+from ..remote.base import index_locked
+from ..tree.local import LocalTree
+from .base import (
+    STATUS_DELETED,
+    STATUS_MAP,
+    STATUS_MISSING,
+    STATUS_NEW,
+    CloudCache,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _log_exceptions(func, operation):
+    @wraps(func)
+    def wrapper(from_info, to_info, *args, **kwargs):
+        try:
+            func(from_info, to_info, *args, **kwargs)
+            return 0
+        except Exception as exc:  # pylint: disable=broad-except
+            # NOTE: this means we ran out of file descriptors and there is no
+            # reason to try to proceed, as we will hit this error anyways.
+            # pylint: disable=no-member
+            if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+                raise
+
+            logger.exception(
+                "failed to %s '%s' to '%s'", operation, from_info, to_info
+            )
+            return 1
+
+    return wrapper
+
+
+class LocalCache(CloudCache):
+    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
+    CACHE_MODE = LocalTree.CACHE_MODE
+
+    def __init__(self, tree):
+        super().__init__(tree)
+        self.cache_dir = tree.config.get("url")
+
+    @property
+    def cache_dir(self):
+        return self.tree.path_info.fspath if self.tree.path_info else None
+
+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.tree.path_info = PathInfo(value) if value else None
+
+    @classmethod
+    def supported(cls, config):  # pylint: disable=unused-argument
+        return True
+
+    @cached_property
+    def cache_path(self):
+        return os.path.abspath(self.cache_dir)
+
+    def hash_to_path(self, hash_):
+        # NOTE: `self.cache_path` is already normalized so we can simply use
+        # `os.sep` instead of `os.path.join`. This results in this helper
+        # being ~5.5 times faster.
+        return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
+
+    def hashes_exist(
+        self, hashes, jobs=None, name=None
+    ):  # pylint: disable=unused-argument
+        return [
+            hash_
+            for hash_ in Tqdm(
+                hashes,
+                unit="file",
+                desc="Querying "
+                + ("cache in " + name if name else "local cache"),
+            )
+            if not self.changed_cache_file(hash_)
+        ]
+
+    def already_cached(self, path_info):
+        assert path_info.scheme in ["", "local"]
+
+        current_md5 = self.tree.get_hash(path_info)
+
+        if not current_md5:
+            return False
+
+        return not self.changed_cache(current_md5)
+
+    def _verify_link(self, path_info, link_type):
+        if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
+            return
+
+        super()._verify_link(path_info, link_type)
+
+    @index_locked
+    def status(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        # Return flattened dict containing all status info
+        dir_status, file_status, _ = self._status(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
+        )
+        return dict(dir_status, **file_status)
+
+    def _status(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        """Return a tuple of (dir_status_info, file_status_info, dir_contents).
+
+        dir_status_info contains status for .dir files, file_status_info
+        contains status for all other files, and dir_contents is a dict of
+        {dir_hash: set(file_hash, ...)} which can be used to map
+        a .dir file to its file contents.
+        """
+        logger.debug(
+            f"Preparing to collect status from {remote.tree.path_info}"
+        )
+        md5s = set(named_cache.scheme_keys(self.tree.scheme))
+
+        logger.debug("Collecting information from local cache...")
+        local_exists = frozenset(
+            self.hashes_exist(md5s, jobs=jobs, name=self.cache_dir)
+        )
+
+        # This is a performance optimization. We can safely assume that,
+        # if the resources that we want to fetch are already cached,
+        # there's no need to check the remote storage for the existence of
+        # those files.
+        if download and local_exists == md5s:
+            remote_exists = local_exists
+        else:
+            logger.debug("Collecting information from remote cache...")
+            remote_exists = set()
+            dir_md5s = set(named_cache.dir_keys(self.tree.scheme))
+            if dir_md5s:
+                remote_exists.update(
+                    self._indexed_dir_hashes(named_cache, remote, dir_md5s)
+                )
+                md5s.difference_update(remote_exists)
+            if md5s:
+                remote_exists.update(
+                    remote.hashes_exist(
+                        md5s, jobs=jobs, name=str(remote.tree.path_info)
+                    )
+                )
+        return self._make_status(
+            named_cache, show_checksums, local_exists, remote_exists
+        )
+
+    def _make_status(
+        self, named_cache, show_checksums, local_exists, remote_exists
+    ):
+        def make_names(hash_, names):
+            return {"name": hash_ if show_checksums else " ".join(names)}
+
+        dir_status = {}
+        file_status = {}
+        dir_contents = {}
+        for hash_, item in named_cache[self.tree.scheme].items():
+            if item.children:
+                dir_status[hash_] = make_names(hash_, item.names)
+                dir_contents[hash_] = set()
+                for child_hash, child in item.children.items():
+                    file_status[child_hash] = make_names(
+                        child_hash, child.names
+                    )
+                    dir_contents[hash_].add(child_hash)
+            else:
+                file_status[hash_] = make_names(hash_, item.names)
+
+        self._fill_statuses(dir_status, local_exists, remote_exists)
+        self._fill_statuses(file_status, local_exists, remote_exists)
+
+        self._log_missing_caches(dict(dir_status, **file_status))
+
+        return dir_status, file_status, dir_contents
+
+    def _indexed_dir_hashes(self, named_cache, remote, dir_md5s):
+        # Validate our index by verifying all indexed .dir hashes
+        # still exist on the remote
+        indexed_dirs = set(remote.index.dir_hashes())
+        indexed_dir_exists = set()
+        if indexed_dirs:
+            indexed_dir_exists.update(
+                remote.tree.list_hashes_exists(indexed_dirs)
+            )
+            missing_dirs = indexed_dirs.difference(indexed_dir_exists)
+            if missing_dirs:
+                logger.debug(
+                    "Remote cache missing indexed .dir hashes '{}', "
+                    "clearing remote index".format(", ".join(missing_dirs))
+                )
+                remote.index.clear()
+
+        # Check if non-indexed (new) dir hashes exist on remote
+        dir_exists = dir_md5s.intersection(indexed_dir_exists)
+        dir_exists.update(
+            remote.tree.list_hashes_exists(dir_md5s - dir_exists)
+        )
+
+        # If .dir hash exists on the remote, assume directory contents
+        # still exists on the remote
+        for dir_hash in dir_exists:
+            file_hashes = list(
+                named_cache.child_keys(self.tree.scheme, dir_hash)
+            )
+            if dir_hash not in remote.index:
+                logger.debug(
+                    "Indexing new .dir '{}' with '{}' nested files".format(
+                        dir_hash, len(file_hashes)
+                    )
+                )
+                remote.index.update([dir_hash], file_hashes)
+            yield dir_hash
+            yield from file_hashes
+
+    @staticmethod
+    def _fill_statuses(hash_info_dir, local_exists, remote_exists):
+        # Using sets because they are way faster for lookups
+        local = set(local_exists)
+        remote = set(remote_exists)
+
+        for md5, info in hash_info_dir.items():
+            status = STATUS_MAP[(md5 in local, md5 in remote)]
+            info["status"] = status
+
+    def _get_plans(self, download, remote, status_info, status):
+        cache = []
+        path_infos = []
+        names = []
+        hashes = []
+        for md5, info in Tqdm(
+            status_info.items(), desc="Analysing status", unit="file"
+        ):
+            if info["status"] == status:
+                cache.append(self.tree.hash_to_path_info(md5))
+                path_infos.append(remote.tree.hash_to_path_info(md5))
+                names.append(info["name"])
+                hashes.append(md5)
+
+        if download:
+            to_infos = cache
+            from_infos = path_infos
+        else:
+            to_infos = path_infos
+            from_infos = cache
+
+        return from_infos, to_infos, names, hashes
+
+    def _process(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        logger.debug(
+            "Preparing to {} '{}'".format(
+                "download data from" if download else "upload data to",
+                remote.tree.path_info,
+            )
+        )
+
+        if download:
+            func = partial(
+                _log_exceptions(remote.tree.download, "download"),
+                dir_mode=self.tree.dir_mode,
+                file_mode=self.tree.file_mode,
+            )
+            status = STATUS_DELETED
+            desc = "Downloading"
+        else:
+            func = _log_exceptions(remote.tree.upload, "upload")
+            status = STATUS_NEW
+            desc = "Uploading"
+
+        if jobs is None:
+            jobs = remote.tree.JOBS
+
+        dir_status, file_status, dir_contents = self._status(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
+        )
+
+        dir_plans = self._get_plans(download, remote, dir_status, status)
+        file_plans = self._get_plans(download, remote, file_status, status)
+
+        total = len(dir_plans[0]) + len(file_plans[0])
+        if total == 0:
+            return 0
+
+        with Tqdm(total=total, unit="file", desc=desc) as pbar:
+            func = pbar.wrap_fn(func)
+            with ThreadPoolExecutor(max_workers=jobs) as executor:
+                if download:
+                    from_infos, to_infos, names, _ = (
+                        d + f for d, f in zip(dir_plans, file_plans)
+                    )
+                    fails = sum(
+                        executor.map(func, from_infos, to_infos, names)
+                    )
+                else:
+                    # for uploads, push files first, and any .dir files last
+
+                    file_futures = {}
+                    for from_info, to_info, name, hash_ in zip(*file_plans):
+                        file_futures[hash_] = executor.submit(
+                            func, from_info, to_info, name
+                        )
+                    dir_futures = {}
+                    for from_info, to_info, name, dir_hash in zip(*dir_plans):
+                        wait_futures = {
+                            future
+                            for file_hash, future in file_futures.items()
+                            if file_hash in dir_contents[dir_hash]
+                        }
+                        dir_futures[dir_hash] = executor.submit(
+                            self._dir_upload,
+                            func,
+                            wait_futures,
+                            from_info,
+                            to_info,
+                            name,
+                        )
+                    fails = sum(
+                        future.result()
+                        for future in concat(
+                            file_futures.values(), dir_futures.values()
+                        )
+                    )
+
+        if fails:
+            if download:
+                remote.index.clear()
+                raise DownloadError(fails)
+            raise UploadError(fails)
+
+        if not download:
+            # index successfully pushed dirs
+            for dir_hash, future in dir_futures.items():
+                if future.result() == 0:
+                    file_hashes = dir_contents[dir_hash]
+                    logger.debug(
+                        "Indexing pushed dir '{}' with "
+                        "'{}' nested files".format(dir_hash, len(file_hashes))
+                    )
+                    remote.index.update([dir_hash], file_hashes)
+
+        return len(dir_plans[0]) + len(file_plans[0])
+
+    @staticmethod
+    def _dir_upload(func, futures, from_info, to_info, name):
+        for future in as_completed(futures):
+            if future.result():
+                # do not upload this .dir file if any file in this
+                # directory failed to upload
+                logger.debug(
+                    "failed to upload full contents of '{}', "
+                    "aborting .dir file upload".format(name)
+                )
+                logger.error(f"failed to upload '{from_info}' to '{to_info}'")
+                return 1
+        return func(from_info, to_info, name)
+
+    @index_locked
+    def push(self, named_cache, remote, jobs=None, show_checksums=False):
+        return self._process(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=False,
+        )
+
+    @index_locked
+    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
+        return self._process(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=True,
+        )
+
+    @staticmethod
+    def _log_missing_caches(hash_info_dict):
+        missing_caches = [
+            (md5, info)
+            for md5, info in hash_info_dict.items()
+            if info["status"] == STATUS_MISSING
+        ]
+        if missing_caches:
+            missing_desc = "\n".join(
+                "name: {}, md5: {}".format(info["name"], md5)
+                for md5, info in missing_caches
+            )
+            msg = (
+                "Some of the cache files do not exist neither locally "
+                "nor on remote. Missing cache files:\n{}".format(missing_desc)
+            )
+            logger.warning(msg)

dvc/cli.py

@@ -15,6 +15,7 @@ from .command import (
     data_sync,
     destroy,
     diff,
+    experiments,
     freeze,
     gc,
     get,
@@ -77,37 +78,40 @@ COMMANDS = [
     update,
     git_hook,
     plots,
+    experiments,
 ]
 
 
+def _find_parser(parser, cmd_cls):
+    defaults = parser._defaults  # pylint: disable=protected-access
+    if not cmd_cls or cmd_cls == defaults.get("func"):
+        parser.print_help()
+        raise DvcParserError()
+
+    actions = parser._actions  # pylint: disable=protected-access
+    for action in actions:
+        if not isinstance(action.choices, dict):
+            # NOTE: we are only interested in subparsers
+            continue
+        for subparser in action.choices.values():
+            _find_parser(subparser, cmd_cls)
+
+
 class DvcParser(argparse.ArgumentParser):
     """Custom parser class for dvc CLI."""
 
-    def error(self, message, command=None):  # pylint: disable=arguments-differ
-        """Custom error method.
-        Args:
-            message (str): error message.
-            command (str): subcommand name for help message
-        Raises:
-            dvc.exceptions.DvcParser: dvc parser exception.
-
-        """
+    def error(self, message, cmd_cls=None):  # pylint: disable=arguments-differ
         logger.error(message)
-        if command is not None:
-            for action in self._actions:
-                if action.dest == "cmd" and command in action.choices:
-                    subparser = action.choices[command]
-                    subparser.print_help()
-                    raise DvcParserError()
-        self.print_help()
-        raise DvcParserError()
+        _find_parser(self, cmd_cls)
 
-    # override this to send subcommand name to error method
     def parse_args(self, args=None, namespace=None):
+        # NOTE: overriding to provide a more granular help message.
+        # E.g. `dvc plots diff --bad-flag` would result in a `dvc plots diff`
+        # help message instead of generic `dvc` usage.
         args, argv = self.parse_known_args(args, namespace)
         if argv:
             msg = "unrecognized arguments: %s"
-            self.error(msg % " ".join(argv), args.cmd)
+            self.error(msg % " ".join(argv), getattr(args, "func", None))
         return args
 
 

dvc/command/add.py

@@ -63,10 +63,6 @@ def add_parser(subparsers, parent_parser):
         metavar="<filename>",
     )
     parser.add_argument(
-        "targets",
-        nargs="+",
-        help="Input files/directories to add.",
-        metavar="targets",
-        choices=completion.Required.FILE,
-    )
+        "targets", nargs="+", help="Input files/directories to add.",
+    ).complete = completion.FILE
     parser.set_defaults(func=CmdAdd)

dvc/command/cache.py

@@ -56,7 +56,5 @@ def add_parser(subparsers, parent_parser):
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
         "config file location.",
-        metavar="value",
-        choices=completion.Required.DIR,
-    )
+    ).complete = completion.DIR
     cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/checkout.py

@@ -112,9 +112,9 @@ def add_parser(subparsers, parent_parser):
     checkout_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC-files to checkout. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
-        metavar="targets",
-        choices=completion.Optional.DVC_FILE,
-    )
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVC_FILE
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -67,7 +67,5 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="DVC-files to commit. Optional. "
         "(Finds all DVC-files in the workspace by default.)",
-        metavar="targets",
-        choices=completion.Optional.DVC_FILE,
-    )
+    ).complete = completion.DVC_FILE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -6,9 +6,11 @@ import shtab
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
-CHOICE_FUNCTIONS = {
-    "bash": {"DVCFile": "_dvc_compgen_DVCFiles"},
-    "zsh": {"DVCFile": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'"},
+FILE = shtab.FILE
+DIR = shtab.DIRECTORY
+DVC_FILE = {
+    "bash": "_dvc_compgen_DVCFiles",
+    "zsh": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'",
 }
 PREAMBLE = {
     "bash": """
@@ -24,26 +26,13 @@ _dvc_compgen_DVCFiles() {
 }
 
 
-class Optional(shtab.Optional):
-    DVC_FILE = [shtab.Choice("DVCFile", required=False)]
-
-
-class Required(shtab.Required):
-    DVC_FILE = [shtab.Choice("DVCFile", required=True)]
-
-
 class CmdCompletion(CmdBaseNoRepo):
     def run(self):
         from dvc.cli import get_main_parser
 
         parser = get_main_parser()
         shell = self.args.shell
-        script = shtab.complete(
-            parser,
-            shell=shell,
-            preamble=PREAMBLE[shell],
-            choice_functions=CHOICE_FUNCTIONS[shell],
-        )
+        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
         print(script)
         return 0
 

dvc/command/daemon.py

@@ -66,9 +66,6 @@ def add_parser(subparsers, parent_parser):
         help=DAEMON_ANALYTICS_HELP,
     )
     daemon_analytics_parser.add_argument(
-        "target",
-        help="Analytics file.",
-        metavar="target",
-        choices=completion.Required.FILE,
-    )
+        "target", help="Analytics file.",
+    ).complete = completion.FILE
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/data_sync.py

@@ -103,11 +103,11 @@ def shared_parent_parser():
     parent_parser.add_argument(
         "targets",
         nargs="*",
-        help="Limit command scope to these DVC-files. "
-        "Using -R, directories to search DVC-files in can also be given.",
-        metavar="targets",
-        choices=completion.Optional.DVC_FILE,
-    )
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVC_FILE
 
     return parent_parser
 

dvc/command/experiments.py

@@ -0,0 +1,354 @@
+import argparse
+import io
+import logging
+from collections import OrderedDict
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.command.metrics import DEFAULT_PRECISION
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _update_names(names, items):
+    from flatten_json import flatten
+
+    for name, item in items:
+        if isinstance(item, dict):
+            item = flatten(item, ".")
+            names.update(item.keys())
+        else:
+            names.add(name)
+
+
+def _collect_names(all_experiments):
+    metric_names = set()
+    param_names = set()
+
+    for _, experiments in all_experiments.items():
+        for exp in experiments.values():
+            _update_names(metric_names, exp.get("metrics", {}).items())
+            _update_names(param_names, exp.get("params", {}).items())
+
+    return sorted(metric_names), sorted(param_names)
+
+
+def _collect_rows(
+    base_rev, experiments, metric_names, param_names, precision=None
+):
+    from flatten_json import flatten
+
+    if precision is None:
+        precision = DEFAULT_PRECISION
+
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
+
+        return val
+
+    def _extend(row, names, items):
+        for fname, item in items:
+            if isinstance(item, dict):
+                item = flatten(item, ".")
+            else:
+                item = {fname: item}
+            for name in names:
+                if name in item:
+                    row.append(str(_round(item[name])))
+                else:
+                    row.append("-")
+
+    for i, (rev, exp) in enumerate(experiments.items()):
+        row = []
+        style = None
+        if rev == "baseline":
+            row.append(f"{base_rev}")
+            style = "bold"
+        elif i < len(experiments) - 1:
+            row.append(f"├── {rev[:7]}")
+        else:
+            row.append(f"└── {rev[:7]}")
+
+        _extend(row, metric_names, exp.get("metrics", {}).items())
+        _extend(row, param_names, exp.get("params", {}).items())
+
+        yield row, style
+
+
+def _show_experiments(all_experiments, console, precision=None):
+    from rich.table import Table
+    from dvc.scm.git import Git
+
+    metric_names, param_names = _collect_names(all_experiments)
+
+    table = Table()
+    table.add_column("Experiment")
+    for name in metric_names:
+        table.add_column(name, justify="right")
+    for name in param_names:
+        table.add_column(name, justify="left")
+
+    for base_rev, experiments in all_experiments.items():
+        if Git.is_sha(base_rev):
+            base_rev = base_rev[:7]
+
+        for row, _, in _collect_rows(
+            base_rev,
+            experiments,
+            metric_names,
+            param_names,
+            precision=precision,
+        ):
+            table.add_row(*row)
+
+    console.print(table)
+
+
+class CmdExperimentsShow(CmdBase):
+    def run(self):
+        from rich.console import Console
+        from dvc.utils.pager import pager
+
+        if not self.repo.experiments:
+            return 0
+
+        try:
+            all_experiments = self.repo.experiments.show(
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+            )
+
+            # Note: rich does not currently include a native way to force
+            # infinite width for use with a pager
+            console = Console(
+                file=io.StringIO(), force_terminal=True, width=9999
+            )
+
+            _show_experiments(all_experiments, console)
+
+            pager(console.file.getvalue())
+        except DvcException:
+            logger.exception("failed to show experiments")
+            return 1
+
+        return 0
+
+
+class CmdExperimentsCheckout(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        self.repo.experiments.checkout(
+            self.args.experiment, force=self.args.force
+        )
+
+        return 0
+
+
+def _show_diff(
+    diff, title="", markdown=False, no_path=False, old=False, precision=None
+):
+    from dvc.utils.diff import table
+
+    if precision is None:
+        precision = DEFAULT_PRECISION
+
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
+
+        return val
+
+    rows = []
+    for fname, diff_ in diff.items():
+        sorted_diff = OrderedDict(sorted(diff_.items()))
+        for item, change in sorted_diff.items():
+            row = [] if no_path else [fname]
+            row.append(item)
+            if old:
+                row.append(_round(change.get("old")))
+            row.append(_round(change["new"]))
+            row.append(_round(change.get("diff", "diff not supported")))
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append(title)
+    if old:
+        header.extend(["Old", "New"])
+    else:
+        header.append("Value")
+    header.append("Change")
+
+    return table(header, rows, markdown)
+
+
+class CmdExperimentsDiff(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        try:
+            diff = self.repo.experiments.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                all=self.args.all,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                diffs = [("metrics", "Metric"), ("params", "Param")]
+                for key, title in diffs:
+                    table = _show_diff(
+                        diff[key],
+                        title=title,
+                        markdown=self.args.show_md,
+                        no_path=self.args.no_path,
+                        old=self.args.old,
+                        precision=self.args.precision,
+                    )
+                    if table:
+                        logger.info(table)
+                        logger.info("")
+
+        except DvcException:
+            logger.exception("failed to show experiments diff")
+            return 1
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    EXPERIMENTS_HELP = "Commands to display and compare experiments."
+
+    experiments_parser = subparsers.add_parser(
+        "experiments",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_HELP, "experiments"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    experiments_subparsers = experiments_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc experiments CMD --help` to display "
+        "command-specific help.",
+    )
+
+    fix_subparsers(experiments_subparsers)
+
+    EXPERIMENTS_SHOW_HELP = "Print experiments."
+    experiments_show_parser = experiments_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_SHOW_HELP, "experiments/show"),
+        help=EXPERIMENTS_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_show_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Show metrics for all branches.",
+    )
+    experiments_show_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Show metrics for all tags.",
+    )
+    experiments_show_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show metrics for all commits.",
+    )
+    experiments_show_parser.set_defaults(func=CmdExperimentsShow)
+
+    EXPERIMENTS_CHECKOUT_HELP = "Checkout experiments."
+    experiments_checkout_parser = experiments_subparsers.add_parser(
+        "checkout",
+        parents=[parent_parser],
+        description=append_doc_link(
+            EXPERIMENTS_CHECKOUT_HELP, "experiments/checkout"
+        ),
+        help=EXPERIMENTS_CHECKOUT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_checkout_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Overwrite your current workspace with changes from the "
+        "experiment.",
+    )
+    experiments_checkout_parser.add_argument(
+        "experiment", help="Checkout this experiment.",
+    )
+    experiments_checkout_parser.set_defaults(func=CmdExperimentsCheckout)
+
+    EXPERIMENTS_DIFF_HELP = (
+        "Show changes between experiments in the DVC repository."
+    )
+    experiments_diff_parser = experiments_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_DIFF_HELP, "experiments/diff"),
+        help=EXPERIMENTS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_diff_parser.add_argument(
+        "a_rev", nargs="?", help="Old experiment to compare (defaults to HEAD)"
+    )
+    experiments_diff_parser.add_argument(
+        "b_rev",
+        nargs="?",
+        help="New experiment to compare (defaults to the current workspace)",
+    )
+    experiments_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged metrics/params as well.",
+    )
+    experiments_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    experiments_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    experiments_diff_parser.add_argument(
+        "--old",
+        action="store_true",
+        default=False,
+        help="Show old metric/param value.",
+    )
+    experiments_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show metric/param path.",
+    )
+    experiments_diff_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics/params to `n` digits precision after the decimal "
+            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
+    experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)

dvc/command/freeze.py

@@ -40,12 +40,8 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     freeze_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="Stages or .dvc files to freeze",
-        metavar="targets",
-        choices=completion.Required.DVC_FILE,
-    )
+        "targets", nargs="+", help="Stages or .dvc files to freeze",
+    ).complete = completion.DVC_FILE
     freeze_parser.set_defaults(func=CmdFreeze)
 
     UNFREEZE_HELP = "Unfreeze stages or .dvc files."
@@ -57,10 +53,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unfreeze_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="Stages or .dvc files to unfreeze",
-        metavar="targets",
-        choices=completion.Required.DVC_FILE,
-    )
+        "targets", nargs="+", help="Stages or .dvc files to unfreeze",
+    ).complete = completion.DVC_FILE
     unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/get.py

@@ -63,19 +63,15 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the repository",
-        metavar="path",
-        choices=completion.Required.FILE,
-    )
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
     get_parser.add_argument(
         "-o",
         "--out",
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     get_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/get_url.py

@@ -34,10 +34,6 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out",
-        nargs="?",
-        help="Destination path to put data to.",
-        metavar="out",
-        choices=completion.Optional.DIR,
-    )
+        "out", nargs="?", help="Destination path to put data to.",
+    ).complete = completion.DIR
     get_parser.set_defaults(func=CmdGetUrl)

dvc/command/imp.py

@@ -44,19 +44,15 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the repository",
-        metavar="path",
-        choices=completion.Required.FILE,
-    )
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
     import_parser.add_argument(
         "-o",
         "--out",
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     import_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/imp_url.py

@@ -58,18 +58,13 @@ def add_parser(subparsers, parent_parser):
         "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out",
-        nargs="?",
-        help="Destination path to put files to.",
-        metavar="out",
-        choices=completion.Optional.DIR,
-    )
+        "out", nargs="?", help="Destination path to put files to.",
+    ).complete = completion.DIR
     import_parser.add_argument(
         "--file",
         help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     import_parser.add_argument(
         "--no-exec",
         action="store_true",

dvc/command/ls/__init__.py

@@ -75,7 +75,5 @@ def add_parser(subparsers, parent_parser):
         "path",
         nargs="?",
         help="Path to directory within the repository to list outputs for",
-        metavar="path",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     list_parser.set_defaults(func=CmdList)

dvc/command/metrics.py

@@ -170,9 +170,7 @@ def add_parser(subparsers, parent_parser):
             "Limit command scope to these metric files. Using -R, "
             "directories to search metric files in can also be given."
         ),
-        metavar="targets",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     metrics_show_parser.add_argument(
         "-a",
         "--all-branches",
@@ -238,8 +236,7 @@ def add_parser(subparsers, parent_parser):
             "directories to search metric files in can also be given."
         ),
         metavar="<paths>",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     metrics_diff_parser.add_argument(
         "-R",
         "--recursive",

dvc/command/move.py

@@ -37,15 +37,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src",
-        help="Source path to a data file or directory.",
-        metavar="src",
-        choices=completion.Required.FILE,
-    )
+        "src", help="Source path to a data file or directory.",
+    ).complete = completion.FILE
     move_parser.add_argument(
-        "dst",
-        help="Destination path.",
-        metavar="dst",
-        choices=completion.Required.FILE,
-    )
+        "dst", help="Destination path.",
+    ).complete = completion.FILE
     move_parser.set_defaults(func=CmdMove)

dvc/command/plots.py

@@ -129,9 +129,7 @@ def add_parser(subparsers, parent_parser):
         "targets",
         nargs="*",
         help="Plots files to visualize. Shows all plots by default.",
-        metavar="targets",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     _add_props_arguments(plots_show_parser)
     _add_output_arguments(plots_show_parser)
     plots_show_parser.set_defaults(func=CmdPlotsShow)
@@ -152,8 +150,7 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="Plots file to visualize. Shows all plots by default.",
         metavar="<path>",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     plots_diff_parser.add_argument(
         "revisions", nargs="*", default=None, help="Git commits to plot from",
     )
@@ -170,11 +167,8 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_modify_parser.add_argument(
-        "target",
-        help="Metric file to set properties to",
-        metavar="target",
-        choices=completion.Required.FILE,
-    )
+        "target", help="Metric file to set properties to",
+    ).complete = completion.FILE
     _add_props_arguments(plots_modify_parser)
     plots_modify_parser.add_argument(
         "--unset",
@@ -198,8 +192,7 @@ def _add_props_arguments(parser):
             )
         ),
         metavar="<path>",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     parser.add_argument(
         "-x", default=None, help="Field name for X axis.", metavar="<field>"
     )
@@ -231,8 +224,7 @@ def _add_output_arguments(parser):
         default=None,
         help="Destination path to save plots to",
         metavar="<path>",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     parser.add_argument(
         "--show-vega",
         action="store_true",

dvc/command/remove.py

@@ -37,10 +37,6 @@ def add_parser(subparsers, parent_parser):
         help="Remove outputs as well.",
     )
     remove_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="DVC-files to remove.",
-        metavar="targets",
-        choices=completion.Required.DVC_FILE,
-    )
+        "targets", nargs="+", help="DVC-files to remove.",
+    ).complete = completion.DVC_FILE
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -40,6 +40,7 @@ class CmdRepro(CmdBase):
                     downstream=self.args.downstream,
                     recursive=self.args.recursive,
                     force_downstream=self.args.force_downstream,
+                    experiment=self.args.experiment,
                 )
 
                 if len(stages) == 0:
@@ -72,9 +73,7 @@ def add_parser(subparsers, parent_parser):
         "targets",
         nargs="*",
         help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
-        metavar="targets",
-        choices=completion.Optional.DVC_FILE,
-    )
+    ).complete = completion.DVC_FILE
     repro_parser.add_argument(
         "-f",
         "--force",
@@ -168,4 +167,11 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Start from the specified stages when reproducing pipelines.",
     )
+    repro_parser.add_argument(
+        "-e",
+        "--experiment",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/unprotect.py

@@ -33,10 +33,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="Data files/directories to unprotect.",
-        metavar="targets",
-        choices=completion.Required.FILE,
-    )
+        "targets", nargs="+", help="Data files/directories to unprotect.",
+    ).complete = completion.FILE
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -33,12 +33,8 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="DVC-files to update.",
-        metavar="targets",
-        choices=completion.Required.DVC_FILE,
-    )
+        "targets", nargs="+", help="DVC-files to update.",
+    ).complete = completion.DVC_FILE
     update_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/version.py

@@ -10,7 +10,7 @@ from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.scm.base import SCMError
 from dvc.system import System
-from dvc.utils import is_binary, relpath
+from dvc.utils import relpath
 from dvc.utils.pkg import PKG
 from dvc.version import __version__
 
@@ -27,13 +27,18 @@ class CmdVersion(CmdBaseNoRepo):
     def run(self):
         from dvc.repo import Repo
 
+        package = PKG
+        if PKG is None:
+            package = ""
+        else:
+            package = f"({PKG})"
+
         info = [
-            f"DVC version: {__version__}",
-            f"Python version: {platform.python_version()}",
-            f"Platform: {platform.platform()}",
-            f"Binary: {is_binary()}",
-            f"Package: {PKG}",
-            f"Supported remotes: {self.get_supported_remotes()}",
+            f"DVC version: {__version__} {package}",
+            "---------------------------------",
+            f"Platform: Python {platform.python_version()} on "
+            f"{platform.platform()}",
+            f"Supports: {self.get_supported_remotes()}",
         ]
 
         try:
@@ -46,13 +51,13 @@ class CmdVersion(CmdBaseNoRepo):
             # `dvc config cache.shared group`.
             if os.path.exists(repo.cache.local.cache_dir):
                 info.append(
-                    "Cache: {}".format(self.get_linktype_support_info(repo))
+                    "Cache types: {}".format(
+                        self.get_linktype_support_info(repo)
+                    )
                 )
                 if psutil:
                     fs_type = self.get_fs_type(repo.cache.local.cache_dir)
-                    info.append(
-                        f"Filesystem type (cache directory): {fs_type}"
-                    )
+                    info.append(f"Cache directory: {fs_type}")
             else:
                 logger.warning(
                     "Unable to detect supported link types, as cache "
@@ -68,11 +73,11 @@ class CmdVersion(CmdBaseNoRepo):
             root_directory = os.getcwd()
             info.append("Repo: dvc, git (broken)")
         else:
-            info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
+            if psutil:
+                fs_root = self.get_fs_type(os.path.abspath(root_directory))
+                info.append(f"Workspace directory: {fs_root}")
 
-        if psutil:
-            fs_root = self.get_fs_type(os.path.abspath(root_directory))
-            info.append(f"Filesystem type (workspace): {fs_root}")
+            info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
 
         logger.info("\n".join(info))
         return 0
@@ -80,7 +85,7 @@ class CmdVersion(CmdBaseNoRepo):
     @staticmethod
     def get_fs_type(path):
         partition = {
-            pathlib.Path(part.mountpoint): (part.fstype, part.device)
+            pathlib.Path(part.mountpoint): (part.fstype + " on " + part.device)
             for part in psutil.disk_partitions(all=True)
         }
 
@@ -115,20 +120,28 @@ class CmdVersion(CmdBaseNoRepo):
                 os.unlink(dst)
             except DvcException:
                 status = "not supported"
-            cache.append(f"{name} - {status}")
+
+            if status == "supported":
+                cache.append(name)
         os.remove(src)
 
         return ", ".join(cache)
 
     @staticmethod
     def get_supported_remotes():
-        from dvc.remote import TREES
+        from dvc.tree import TREES
 
         supported_remotes = []
         for tree_cls in TREES:
             if not tree_cls.get_missing_deps():
                 supported_remotes.append(tree_cls.scheme)
 
+        if len(supported_remotes) == len(TREES):
+            return "All remotes"
+
+        if len(supported_remotes) == 1:
+            return supported_remotes
+
         return ", ".join(supported_remotes)
 
 

dvc/config.py

@@ -127,6 +127,7 @@ SCHEMA = {
         Optional("analytics", default=True): Bool,
         Optional("hardlink_lock", default=False): Bool,
         Optional("no_scm", default=False): Bool,
+        Optional("experiments", default=False): Bool,
     },
     "cache": {
         "local": str,
@@ -148,6 +149,8 @@ SCHEMA = {
                     "profile": str,
                     "credentialpath": str,
                     "endpointurl": str,
+                    "access_key_id": str,
+                    "secret_access_key": str,
                     Optional("listobjects", default=False): Bool,
                     Optional("use_ssl", default=True): Bool,
                     "sse": str,
@@ -232,8 +235,10 @@ class Config(dict):
     CONFIG_LOCAL = "config.local"
 
     def __init__(
-        self, dvc_dir=None, validate=True
+        self, dvc_dir=None, validate=True, tree=None,
     ):  # pylint: disable=super-init-not-called
+        from dvc.tree.local import LocalTree
+
         self.dvc_dir = dvc_dir
 
         if not dvc_dir:
@@ -246,6 +251,9 @@ class Config(dict):
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
+        self.wtree = LocalTree(None, {"url": self.dvc_dir})
+        self.tree = tree or self.wtree
+
         self.load(validate=validate)
 
     @classmethod
@@ -304,8 +312,30 @@ class Config(dict):
         if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
+    def _load_config(self, level):
+        filename = self.files[level]
+
+        if self.tree.exists(filename, use_dvcignore=False):
+            with self.tree.open(filename) as fobj:
+                conf_obj = configobj.ConfigObj(fobj)
+        else:
+            conf_obj = configobj.ConfigObj()
+        return _parse_remotes(_lower_keys(conf_obj.dict()))
+
+    def _save_config(self, level, conf_dict):
+        filename = self.files[level]
+
+        logger.debug(f"Writing '{filename}'.")
+
+        self.tree.makedirs(os.path.dirname(filename))
+
+        config = configobj.ConfigObj(_pack_remotes(conf_dict))
+        with self.tree.open(filename, "wb") as fobj:
+            config.write(fobj)
+        config.filename = filename
+
     def load_one(self, level):
-        conf = _load_config(self.files[level])
+        conf = self._load_config(level)
         conf = self._load_paths(conf, self.files[level])
 
         # Auto-verify sections
@@ -375,7 +405,7 @@ class Config(dict):
         _merge(merged_conf, conf)
         self.validate(merged_conf)
 
-        _save_config(self.files[level], conf)
+        self._save_config(level, conf)
         self.load()
 
     @staticmethod
@@ -386,20 +416,6 @@ class Config(dict):
             raise ConfigError(str(exc)) from None
 
 
-def _load_config(filename):
-    conf_obj = configobj.ConfigObj(filename)
-    return _parse_remotes(_lower_keys(conf_obj.dict()))
-
-
-def _save_config(filename, conf_dict):
-    logger.debug(f"Writing '{filename}'.")
-    os.makedirs(os.path.dirname(filename), exist_ok=True)
-
-    config = configobj.ConfigObj(_pack_remotes(conf_dict))
-    config.filename = filename
-    config.write()
-
-
 def _parse_remotes(conf):
     result = {"remote": {}}
 

dvc/daemon.py

@@ -3,6 +3,7 @@
 import inspect
 import logging
 import os
+import platform
 import sys
 from subprocess import Popen
 
@@ -15,6 +16,14 @@ CREATE_NEW_PROCESS_GROUP = 0x00000200
 DETACHED_PROCESS = 0x00000008
 
 
+def _popen(cmd, **kwargs):
+    prefix = [sys.executable]
+    if not is_binary():
+        prefix += [sys.argv[0]]
+
+    return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
+
+
 def _spawn_windows(cmd, env):
     from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
 
@@ -23,17 +32,14 @@ def _spawn_windows(cmd, env):
     startupinfo = STARTUPINFO()
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
-    Popen(
-        cmd,
-        env=env,
-        close_fds=True,
-        shell=False,
-        creationflags=creationflags,
-        startupinfo=startupinfo,
+    _popen(
+        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo,
     ).communicate()
 
 
 def _spawn_posix(cmd, env):
+    from dvc.main import main
+
     # NOTE: using os._exit instead of sys.exit, because dvc built
     # with PyInstaller has trouble with SystemExit exception and throws
     # errors such as "[26338] Failed to execute script __main__"
@@ -59,7 +65,13 @@ def _spawn_posix(cmd, env):
     sys.stdout.close()
     sys.stderr.close()
 
-    Popen(cmd, env=env, close_fds=True, shell=False).communicate()
+    if platform.system() == "Darwin":
+        # workaround for MacOS bug
+        # https://github.com/iterative/dvc/issues/4294
+        _popen(cmd, env=env).communicate()
+    else:
+        os.environ.update(env)
+        main(cmd)
 
     os._exit(0)  # pylint: disable=protected-access
 
@@ -87,10 +99,7 @@ def daemon(args):
         logger.debug("skipping launching a new daemon.")
         return
 
-    cmd = [sys.executable]
-    if not is_binary():
-        cmd += [sys.argv[0]]
-    cmd += ["daemon", "-q"] + args
+    cmd = ["daemon", "-q"] + args
 
     env = fix_env()
     file_path = os.path.abspath(inspect.stack()[0][1])

dvc/data_cloud.py

@@ -92,7 +92,7 @@ class DataCloud:
 
     def _save_pulled_checksums(self, cache):
         for checksum in cache.scheme_keys("local"):
-            cache_file = self.repo.cache.local.hash_to_path_info(checksum)
+            cache_file = self.repo.cache.local.tree.hash_to_path_info(checksum)
             if self.repo.cache.local.tree.exists(cache_file):
                 # We can safely save here, as existing corrupted files will
                 # be removed upon status, while files corrupted during

dvc/dependency/__init__.py

@@ -12,9 +12,9 @@ from dvc.dependency.param import ParamsDependency
 from dvc.dependency.s3 import S3Dependency
 from dvc.dependency.ssh import SSHDependency
 from dvc.output.base import BaseOutput
-from dvc.remote import get_remote
 from dvc.scheme import Schemes
 
+from ..tree import get_cloud_tree
 from .repo import RepoDependency
 
 DEPS = [
@@ -54,8 +54,8 @@ SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
 def _get(stage, p, info):
     parsed = urlparse(p) if p else None
     if parsed and parsed.scheme == "remote":
-        remote = get_remote(stage.repo, name=parsed.netloc)
-        return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
+        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
+        return DEP_MAP[tree.scheme](stage, p, info, tree=tree)
 
     if info and info.get(RepoDependency.PARAM_REPO):
         repo = info.pop(RepoDependency.PARAM_REPO)

dvc/dependency/azure.py

@@ -1,7 +1,8 @@
 from dvc.dependency.base import BaseDependency
 from dvc.output.base import BaseOutput
-from dvc.remote.azure import AzureRemoteTree
+
+from ..tree.azure import AzureTree
 
 
 class AzureDependency(BaseDependency, BaseOutput):
-    TREE_CLS = AzureRemoteTree
+    TREE_CLS = AzureTree

dvc/dependency/http.py

@@ -1,7 +1,8 @@
 from dvc.dependency.base import BaseDependency
 from dvc.output.base import BaseOutput
-from dvc.remote.http import HTTPRemoteTree
+
+from ..tree.http import HTTPTree
 
 
 class HTTPDependency(BaseDependency, BaseOutput):
-    TREE_CLS = HTTPRemoteTree
+    TREE_CLS = HTTPTree

dvc/dependency/https.py

@@ -1,7 +1,6 @@
-from dvc.remote.https import HTTPSRemoteTree
-
+from ..tree.https import HTTPSTree
 from .http import HTTPDependency
 
 
 class HTTPSDependency(HTTPDependency):
-    TREE_CLS = HTTPSRemoteTree
+    TREE_CLS = HTTPSTree

dvc/dependency/param.py

@@ -2,6 +2,7 @@ import os
 from collections import defaultdict
 
 import dpath.util
+import toml
 import yaml
 from voluptuous import Any
 
@@ -21,6 +22,8 @@ class ParamsDependency(LocalDependency):
     PARAM_PARAMS = "params"
     PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
     DEFAULT_PARAMS_FILE = "params.yaml"
+    PARAMS_FILE_LOADERS = defaultdict(lambda: yaml.safe_load)
+    PARAMS_FILE_LOADERS.update({".toml": toml.load})
 
     def __init__(self, stage, path, params):
         info = {}
@@ -45,9 +48,8 @@ class ParamsDependency(LocalDependency):
         if not values:
             return
         for param in self.params:
-            value = values.get(param)
-            if value:
-                self.info[param] = value
+            if param in values:
+                self.info[param] = values[param]
 
     def save(self):
         super().save()
@@ -88,8 +90,10 @@ class ParamsDependency(LocalDependency):
 
         with self.repo.tree.open(self.path_info, "r") as fobj:
             try:
-                config = yaml.safe_load(fobj)
-            except yaml.YAMLError as exc:
+                config = self.PARAMS_FILE_LOADERS[
+                    self.path_info.suffix.lower()
+                ](fobj)
+            except (yaml.YAMLError, toml.TomlDecodeError) as exc:
                 raise BadParamFileError(
                     f"Unable to read parameters from '{self}'"
                 ) from exc

dvc/dependency/repo.py

@@ -26,7 +26,7 @@ class RepoDependency(LocalDependency):
         self.def_repo = def_repo
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, remote, path):
+    def _parse_path(self, tree, path):
         return None
 
     @property
@@ -64,7 +64,7 @@ class RepoDependency(LocalDependency):
 
                 # We are polluting our repo cache with some dir listing here
                 if tree.isdir(path):
-                    return self.repo.cache.local.get_hash(path, tree)
+                    return self.repo.cache.local.tree.get_hash(path, tree=tree)
                 return tree.get_file_hash(path)
 
     def status(self):
@@ -98,3 +98,9 @@ class RepoDependency(LocalDependency):
 
         with self._make_repo(locked=False) as repo:
             self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
+
+    def changed_checksum(self):
+        # From current repo point of view what describes RepoDependency is its
+        # origin project url and rev_lock, and it makes RepoDependency
+        # immutable, hence its impossible for checksum to change.
+        return False

dvc/dvcfile.py

@@ -233,7 +233,11 @@ class PipelineFile(FileMixin):
 
         logger.debug("Removing '%s' from '%s'", stage.name, self.path)
         del d["stages"][stage.name]
-        dump_yaml(self.path, d)
+
+        if d["stages"]:
+            dump_yaml(self.path, d)
+        else:
+            super().remove()
 
 
 class Lockfile(FileMixin):
@@ -286,7 +290,10 @@ class Lockfile(FileMixin):
         logger.debug("Removing '%s' from '%s'", stage.name, self.path)
         del d[stage.name]
 
-        dump_yaml(self.path, d)
+        if d:
+            dump_yaml(self.path, d)
+        else:
+            self.remove()
 
 
 class Dvcfile:

dvc/external_repo.py

@@ -19,8 +19,9 @@ from dvc.exceptions import (
 from dvc.path_info import PathInfo
 from dvc.repo import Repo
 from dvc.repo.tree import RepoTree
+from dvc.scm.base import CloneError
 from dvc.scm.git import Git
-from dvc.scm.tree import is_working_tree
+from dvc.tree.local import LocalTree
 from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
@@ -31,7 +32,10 @@ def external_repo(url, rev=None, for_write=False):
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
     if not rev:
-        rev = "HEAD"
+        # Local HEAD points to the tip of whatever branch we first cloned from
+        # (which may not be the default branch), use origin/HEAD here to get
+        # the tip of the default branch
+        rev = "refs/remotes/origin/HEAD"
     try:
         repo = ExternalRepo(path, url, rev, for_write=for_write)
     except NotDvcRepoError:
@@ -59,7 +63,7 @@ CACHE_DIRS = {}
 
 def clean_repos():
     # Outside code should not see cache while we are removing
-    paths = list(CLONES.values()) + list(CACHE_DIRS.values())
+    paths = [path for path, _ in CLONES.values()] + list(CACHE_DIRS.values())
     CLONES.clear()
     CACHE_DIRS.clear()
 
@@ -72,6 +76,9 @@ class BaseExternalRepo:
 
     _local_cache = None
 
+    def __str__(self):
+        return self.url
+
     @property
     def local_cache(self):
         if hasattr(self, "cache"):
@@ -99,10 +106,8 @@ class BaseExternalRepo:
         return RepoTree(self, fetch=True)
 
     def get_rev(self):
-        if is_working_tree(self.tree):
+        if isinstance(self.tree, LocalTree):
             return self.scm.get_rev()
-        if hasattr(self.tree, "tree"):
-            return self.tree.tree.rev
         return self.tree.rev
 
     def fetch_external(self, paths: Iterable, **kwargs):
@@ -250,10 +255,10 @@ def _cached_clone(url, rev, for_write=False):
     """
     # even if we have already cloned this repo, we may need to
     # fetch/fast-forward to get specified rev
-    clone_path = _clone_default_branch(url, rev)
+    clone_path, shallow = _clone_default_branch(url, rev, for_write=for_write)
 
     if not for_write and (url) in CLONES:
-        return CLONES[url]
+        return CLONES[url][0]
 
     # Copy to a new dir to keep the clone clean
     repo_path = tempfile.mkdtemp("dvc-erepo")
@@ -264,17 +269,17 @@ def _cached_clone(url, rev, for_write=False):
     if for_write:
         _git_checkout(repo_path, rev)
     else:
-        CLONES[url] = repo_path
+        CLONES[url] = (repo_path, shallow)
     return repo_path
 
 
 @wrap_with(threading.Lock())
-def _clone_default_branch(url, rev):
+def _clone_default_branch(url, rev, for_write=False):
     """Get or create a clean clone of the url.
 
     The cloned is reactualized with git pull unless rev is a known sha.
     """
-    clone_path = CLONES.get(url)
+    clone_path, shallow = CLONES.get(url, (None, False))
 
     git = None
     try:
@@ -282,18 +287,55 @@ def _clone_default_branch(url, rev):
             git = Git(clone_path)
             # Do not pull for known shas, branches and tags might move
             if not Git.is_sha(rev) or not git.has_rev(rev):
-                logger.debug("erepo: git pull %s", url)
-                git.pull()
+                if shallow:
+                    # If we are missing a rev in a shallow clone, fallback to
+                    # a full (unshallowed) clone. Since fetching specific rev
+                    # SHAs is only available in certain git versions, if we
+                    # have need to reference multiple specific revs for a
+                    # given repo URL it is easier/safer for us to work with
+                    # full clones in this case.
+                    logger.debug("erepo: unshallowing clone for '%s'", url)
+                    _unshallow(git)
+                    shallow = False
+                    CLONES[url] = (clone_path, shallow)
+                else:
+                    logger.debug("erepo: git pull '%s'", url)
+                    git.pull()
         else:
-            logger.debug("erepo: git clone %s to a temporary dir", url)
+            logger.debug("erepo: git clone '%s' to a temporary dir", url)
             clone_path = tempfile.mkdtemp("dvc-clone")
-            git = Git.clone(url, clone_path)
-            CLONES[url] = clone_path
+            if not for_write and rev and not Git.is_sha(rev):
+                # If rev is a tag or branch name try shallow clone first
+                try:
+                    git = Git.clone(url, clone_path, shallow_branch=rev)
+                    shallow = True
+                    logger.debug(
+                        "erepo: using shallow clone for branch '%s'", rev
+                    )
+                except CloneError:
+                    pass
+            if not git:
+                git = Git.clone(url, clone_path)
+                shallow = False
+            CLONES[url] = (clone_path, shallow)
     finally:
         if git:
             git.close()
 
-    return clone_path
+    return clone_path, shallow
+
+
+def _unshallow(git):
+    if git.repo.head.is_detached:
+        # If this is a detached head (i.e. we shallow cloned a tag) switch to
+        # the default branch
+        origin_refs = git.repo.remotes["origin"].refs
+        ref = origin_refs["HEAD"].reference
+        branch_name = ref.name.split("/")[-1]
+        branch = git.repo.create_head(branch_name, ref)
+        branch.set_tracking_branch(ref)
+        branch.checkout()
+    git.pull(unshallow=True)
 
 
 def _git_checkout(repo_path, rev):

dvc/ignore.py

@@ -1,14 +1,14 @@
 import logging
 import os
 import re
-from itertools import groupby
+from itertools import groupby, takewhile
 
-from funcy import cached_property
 from pathspec.patterns import GitWildMatchPattern
 from pathspec.util import normalize_file
+from pygtrie import StringTrie
 
 from dvc.path_info import PathInfo
-from dvc.scm.tree import BaseTree
+from dvc.pathspec_math import merge_patterns
 from dvc.system import System
 from dvc.utils import relpath
 
@@ -23,39 +23,46 @@ class DvcIgnore:
 
 
 class DvcIgnorePatterns(DvcIgnore):
-    def __init__(self, ignore_file_path, tree):
+    def __init__(self, pattern_list, dirname):
+
+        self.pattern_list = pattern_list
+        self.dirname = dirname
+        self.prefix = self.dirname + os.sep
+
+        regex_pattern_list = map(
+            GitWildMatchPattern.pattern_to_regex, pattern_list
+        )
+
+        self.ignore_spec = [
+            (ignore, re.compile("|".join(item[0] for item in group)))
+            for ignore, group in groupby(regex_pattern_list, lambda x: x[1])
+            if ignore is not None
+        ]
+
+    @classmethod
+    def from_files(cls, ignore_file_path, tree):
         assert os.path.isabs(ignore_file_path)
-
-        self.ignore_file_path = ignore_file_path
-        self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
-
+        dirname = os.path.normpath(os.path.dirname(ignore_file_path))
         with tree.open(ignore_file_path, encoding="utf-8") as fobj:
-            path_spec_lines = fobj.readlines()
-            regex_pattern_list = map(
-                GitWildMatchPattern.pattern_to_regex, path_spec_lines
-            )
-            self.ignore_spec = [
-                (ignore, re.compile("|".join(item[0] for item in group)))
-                for ignore, group in groupby(
-                    regex_pattern_list, lambda x: x[1]
-                )
-                if ignore is not None
+            path_spec_lines = [
+                line for line in map(str.strip, fobj.readlines()) if line
             ]
 
+        return cls(path_spec_lines, dirname)
+
     def __call__(self, root, dirs, files):
         files = [f for f in files if not self.matches(root, f)]
-        dirs = [d for d in dirs if not self.matches(root, d)]
+        dirs = [d for d in dirs if not self.matches(root, d, True)]
 
         return dirs, files
 
-    def matches(self, dirname, basename):
+    def matches(self, dirname, basename, is_dir=False):
         # NOTE: `relpath` is too slow, so we have to assume that both
         # `dirname` and `self.dirname` are relative or absolute together.
-        prefix = self.dirname + os.sep
         if dirname == self.dirname:
             path = basename
-        elif dirname.startswith(prefix):
-            rel = dirname[len(prefix) :]
+        elif dirname.startswith(self.prefix):
+            rel = dirname[len(self.prefix) :]
             # NOTE: `os.path.join` is ~x5.5 slower
             path = f"{rel}{os.sep}{basename}"
         else:
@@ -63,188 +70,178 @@ class DvcIgnorePatterns(DvcIgnore):
 
         if not System.is_unix():
             path = normalize_file(path)
-        return self.ignore(path)
+        return self.ignore(path, is_dir)
 
-    def ignore(self, path):
+    def ignore(self, path, is_dir):
         result = False
-        for ignore, pattern in self.ignore_spec:
-            if pattern.match(path):
-                result = ignore
+        if is_dir:
+            path_dir = f"{path}/"
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path) or pattern.match(path_dir):
+                    result = ignore
+        else:
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path):
+                    result = ignore
         return result
 
     def __hash__(self):
-        return hash(self.ignore_file_path)
+        return hash(self.dirname + ":" + "\n".join(self.pattern_list))
 
     def __eq__(self, other):
         if not isinstance(other, DvcIgnorePatterns):
             return NotImplemented
+        return (self.dirname == other.dirname) & (
+            self.pattern_list == other.pattern_list
+        )
 
-        return self.ignore_file_path == other.ignore_file_path
+    def __bool__(self):
+        return bool(self.pattern_list)
 
 
-class DvcIgnoreDirs(DvcIgnore):
-    def __init__(self, basenames):
-        self.basenames = set(basenames)
+class DvcIgnoreFilterNoop:
+    def __init__(self, tree, root_dir):
+        pass
 
     def __call__(self, root, dirs, files):
-        dirs = [d for d in dirs if d not in self.basenames]
-
         return dirs, files
 
-    def __hash__(self):
-        return hash(tuple(self.basenames))
+    def is_ignored_dir(self, _):
+        return False
 
-    def __eq__(self, other):
-        if not isinstance(other, DvcIgnoreDirs):
-            return NotImplemented
-
-        return self.basenames == other.basenames
-
-
-class DvcIgnoreRepo(DvcIgnore):
-    def __call__(self, root, dirs, files):
-        def is_dvc_repo(directory):
-            from dvc.repo import Repo
-
-            return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
-
-        dirs = [d for d in dirs if not is_dvc_repo(d)]
-
-        return dirs, files
+    def is_ignored_file(self, _):
+        return False
 
 
 class DvcIgnoreFilter:
-    def __init__(self, tree, root_dir):
-        self.tree = tree
-        self.root_dir = root_dir
-        self.ignores = {
-            DvcIgnoreDirs([".git", ".hg", ".dvc"]),
-            DvcIgnoreRepo(),
-        }
-        for root, dirs, files in self.tree.walk(self.root_dir):
-            self._update(root)
-            dirs[:], files[:] = self(root, dirs, files)
-
-    def _update(self, dirname):
-        ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if self.tree.exists(ignore_file_path):
-            self.ignores.add(DvcIgnorePatterns(ignore_file_path, self.tree))
-
-    def __call__(self, root, dirs, files):
-        for ignore in self.ignores:
-            dirs, files = ignore(root, dirs, files)
-
-        return dirs, files
-
-
-class CleanTree(BaseTree):
-    def __init__(self, tree, tree_root=None):
-        self.tree = tree
-        if tree_root:
-            self._tree_root = tree_root
-        else:
-            self._tree_root = self.tree.tree_root
-
-    @cached_property
-    def dvcignore(self):
-        return DvcIgnoreFilter(self.tree, self.tree_root)
-
-    @property
-    def tree_root(self):
-        return self._tree_root
-
-    def open(self, path, mode="r", encoding="utf-8"):
-        if self.isfile(path):
-            return self.tree.open(path, mode, encoding)
-        raise FileNotFoundError
-
-    def exists(self, path):
-        if self.tree.exists(path) and self._parents_exist(path):
-            if self.tree.isdir(path):
-                return self._valid_dirname(path)
-            return self._valid_filename(path)
-        return False
-
-    def isdir(self, path):
-        return (
-            self.tree.isdir(path)
-            and self._parents_exist(path)
-            and self._valid_dirname(path)
-        )
-
-    def _valid_dirname(self, path):
-        path = os.path.abspath(path)
-        if path == self.tree_root:
-            return True
-        dirname, basename = os.path.split(path)
-        dirs, _ = self.dvcignore(dirname, [basename], [])
-        if dirs:
-            return True
-        return False
-
-    def isfile(self, path):
-        return (
-            self.tree.isfile(path)
-            and self._parents_exist(path)
-            and self._valid_filename(path)
-        )
-
-    def _valid_filename(self, path):
-        dirname, basename = os.path.split(os.path.normpath(path))
-        _, files = self.dvcignore(os.path.abspath(dirname), [], [basename])
-        if files:
-            return True
-        return False
-
-    def isexec(self, path):
-        return self.exists(path) and self.tree.isexec(path)
-
-    def _parents_exist(self, path):
+    @staticmethod
+    def _is_dvc_repo(root, directory):
         from dvc.repo import Repo
 
+        return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
+
+    def __init__(self, tree, root_dir):
+        from dvc.repo import Repo
+
+        default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
+
+        self.tree = tree
+        self.root_dir = root_dir
+        self.ignores_trie_tree = StringTrie(separator=os.sep)
+        self.ignores_trie_tree[root_dir] = DvcIgnorePatterns(
+            default_ignore_patterns, root_dir
+        )
+        self._update(self.root_dir)
+
+    def _update(self, dirname):
+        old_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
+        matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
+
+        ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
+        if not matches and self.tree.exists(
+            ignore_file_path, use_dvcignore=False
+        ):
+            new_pattern = DvcIgnorePatterns.from_files(
+                ignore_file_path, self.tree
+            )
+            if old_pattern:
+                self.ignores_trie_tree[dirname] = DvcIgnorePatterns(
+                    *merge_patterns(
+                        old_pattern.pattern_list,
+                        old_pattern.dirname,
+                        new_pattern.pattern_list,
+                        new_pattern.dirname,
+                    )
+                )
+            else:
+                self.ignores_trie_tree[dirname] = new_pattern
+        elif old_pattern:
+            self.ignores_trie_tree[dirname] = old_pattern
+
+        # NOTE: using `walk` + `break` because tree doesn't have `listdir()`
+        for root, dirs, _ in self.tree.walk(dirname, use_dvcignore=False):
+            self._update_sub_repo(root, dirs)
+            break
+
+    def _update_sub_repo(self, root, dirs):
+        for d in dirs:
+            if self._is_dvc_repo(root, d):
+                old_pattern = self.ignores_trie_tree.longest_prefix(root).value
+                if old_pattern:
+                    self.ignores_trie_tree[root] = DvcIgnorePatterns(
+                        *merge_patterns(
+                            old_pattern.pattern_list,
+                            old_pattern.dirname,
+                            ["/{}/".format(d)],
+                            root,
+                        )
+                    )
+                else:
+                    self.ignores_trie_tree[root] = DvcIgnorePatterns(
+                        ["/{}/".format(d)], root
+                    )
+
+    def __call__(self, root, dirs, files):
+        ignore_pattern = self._get_trie_pattern(root)
+        if ignore_pattern:
+            return ignore_pattern(root, dirs, files)
+        else:
+            return dirs, files
+
+    def _get_trie_pattern(self, dirname):
+        ignore_pattern = self.ignores_trie_tree.get(dirname)
+        if ignore_pattern:
+            return ignore_pattern
+
+        prefix = self.ignores_trie_tree.longest_prefix(dirname).key
+        if not prefix:
+            # outside of the repo
+            return None
+
+        dirs = list(
+            takewhile(
+                lambda path: path != prefix,
+                (parent.fspath for parent in PathInfo(dirname).parents),
+            )
+        )
+        dirs.reverse()
+        dirs.append(dirname)
+
+        for parent in dirs:
+            self._update(parent)
+
+        return self.ignores_trie_tree.get(dirname)
+
+    def _is_ignored(self, path, is_dir=False):
+        if self._outside_repo(path):
+            return True
+        dirname, basename = os.path.split(os.path.normpath(path))
+        ignore_pattern = self._get_trie_pattern(dirname)
+        if ignore_pattern:
+            return ignore_pattern.matches(dirname, basename, is_dir)
+        else:
+            return False
+
+    def is_ignored_dir(self, path):
+        path = os.path.abspath(path)
+        if path == self.root_dir:
+            return False
+
+        return self._is_ignored(path, True)
+
+    def is_ignored_file(self, path):
+        return self._is_ignored(path, False)
+
+    def _outside_repo(self, path):
         path = PathInfo(path)
 
-        # if parent is tree_root or inside a .dvc dir we can skip this check
-        if path.parent == self.tree_root or Repo.DVC_DIR in path.parts:
-            return True
-
-        # paths outside of the CleanTree root should be ignored
-        path = relpath(path, self.tree_root)
+        # paths outside of the repo should be ignored
+        path = relpath(path, self.root_dir)
         if path.startswith("..") or (
             os.name == "nt"
             and not os.path.commonprefix(
-                [os.path.abspath(path), self.tree_root]
+                [os.path.abspath(path), self.root_dir]
             )
         ):
-            return False
-
-        # check if parent directories are in our ignores, starting from
-        # tree_root
-        for parent_dir in reversed(PathInfo(path).parents):
-            dirname, basename = os.path.split(parent_dir)
-            if basename == ".":
-                # parent_dir == tree_root
-                continue
-            dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
-            if not dirs:
-                return False
-        return True
-
-    def walk(self, top, topdown=True, onerror=None):
-        for root, dirs, files in self.tree.walk(
-            top, topdown=topdown, onerror=onerror
-        ):
-            dirs[:], files[:] = self.dvcignore(
-                os.path.abspath(root), dirs, files
-            )
-
-            yield root, dirs, files
-
-    def stat(self, path):
-        if self.exists(path):
-            return self.tree.stat(path)
-        raise FileNotFoundError
-
-    @property
-    def hash_jobs(self):
-        return self.tree.hash_jobs
+            return True
+        return False

dvc/main.py

@@ -9,7 +9,7 @@ from dvc.config import ConfigError
 from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
 from dvc.external_repo import clean_repos
 from dvc.logger import FOOTER, disable_other_loggers
-from dvc.remote.pool import close_pools
+from dvc.tree.pool import close_pools
 from dvc.utils import format_link
 
 # Workaround for CPython bug. See [1] and [2] for more info.

dvc/output/__init__.py

@@ -10,12 +10,13 @@ from dvc.output.hdfs import HDFSOutput
 from dvc.output.local import LocalOutput
 from dvc.output.s3 import S3Output
 from dvc.output.ssh import SSHOutput
-from dvc.remote import get_remote
-from dvc.remote.hdfs import HDFSRemoteTree
-from dvc.remote.local import LocalRemoteTree
-from dvc.remote.s3 import S3RemoteTree
 from dvc.scheme import Schemes
 
+from ..tree import get_cloud_tree
+from ..tree.hdfs import HDFSTree
+from ..tree.local import LocalTree
+from ..tree.s3 import S3Tree
+
 OUTS = [
     HDFSOutput,
     S3Output,
@@ -47,9 +48,9 @@ CHECKSUM_SCHEMA = Any(
 # so when a few types of outputs share the same name, we only need
 # specify it once.
 CHECKSUMS_SCHEMA = {
-    LocalRemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    S3RemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    HDFSRemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    LocalTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    S3Tree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    HDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
 }
 
 SCHEMA = CHECKSUMS_SCHEMA.copy()
@@ -66,13 +67,13 @@ def _get(
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":
-        remote = get_remote(stage.repo, name=parsed.netloc)
-        return OUTS_MAP[remote.scheme](
+        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
+        return OUTS_MAP[tree.scheme](
             stage,
             p,
             info,
             cache=cache,
-            remote=remote,
+            tree=tree,
             metric=metric,
             plot=plot,
             persist=persist,
@@ -85,7 +86,7 @@ def _get(
                 p,
                 info,
                 cache=cache,
-                remote=None,
+                tree=None,
                 metric=metric,
                 plot=plot,
                 persist=persist,
@@ -95,7 +96,7 @@ def _get(
         p,
         info,
         cache=cache,
-        remote=None,
+        tree=None,
         metric=metric,
         plot=plot,
         persist=persist,

dvc/output/base.py

@@ -12,7 +12,8 @@ from dvc.exceptions import (
     DvcException,
     RemoteCacheRequiredError,
 )
-from dvc.remote.base import BaseRemoteTree, Remote
+
+from ..tree.base import BaseTree
 
 logger = logging.getLogger(__name__)
 
@@ -47,8 +48,7 @@ class OutputIsStageFileError(DvcException):
 class BaseOutput:
     IS_DEPENDENCY = False
 
-    REMOTE_CLS = Remote
-    TREE_CLS = BaseRemoteTree
+    TREE_CLS = BaseTree
 
     PARAM_PATH = "path"
     PARAM_CACHE = "cache"
@@ -85,7 +85,7 @@ class BaseOutput:
         stage,
         path,
         info=None,
-        remote=None,
+        tree=None,
         cache=True,
         metric=False,
         plot=False,
@@ -106,24 +106,23 @@ class BaseOutput:
         self.repo = stage.repo if stage else None
         self.def_path = path
         self.info = info
-        if remote:
-            self.remote = remote
+        if tree:
+            self.tree = tree
         else:
-            tree = self.TREE_CLS(self.repo, {})
-            self.remote = self.REMOTE_CLS(tree)
+            self.tree = self.TREE_CLS(self.repo, {})
         self.use_cache = False if self.IS_DEPENDENCY else cache
         self.metric = False if self.IS_DEPENDENCY else metric
         self.plot = False if self.IS_DEPENDENCY else plot
         self.persist = persist
 
-        self.path_info = self._parse_path(remote, path)
+        self.path_info = self._parse_path(tree, path)
         if self.use_cache and self.cache is None:
             raise RemoteCacheRequiredError(self.path_info)
 
-    def _parse_path(self, remote, path):
-        if remote:
+    def _parse_path(self, tree, path):
+        if tree:
             parsed = urlparse(path)
-            return remote.path_info / parsed.path.lstrip("/")
+            return tree.path_info / parsed.path.lstrip("/")
         return self.TREE_CLS.PATH_CLS(path)
 
     def __repr__(self):
@@ -163,33 +162,33 @@ class BaseOutput:
 
     @property
     def cache_path(self):
-        return self.cache.hash_to_path_info(self.checksum).url
+        return self.cache.tree.hash_to_path_info(self.checksum).url
 
     @property
     def checksum_type(self):
-        return self.remote.tree.PARAM_CHECKSUM
+        return self.tree.PARAM_CHECKSUM
 
     @property
     def checksum(self):
-        return self.info.get(self.remote.tree.PARAM_CHECKSUM)
+        return self.info.get(self.tree.PARAM_CHECKSUM)
 
     @checksum.setter
     def checksum(self, checksum):
-        self.info[self.remote.tree.PARAM_CHECKSUM] = checksum
+        self.info[self.tree.PARAM_CHECKSUM] = checksum
 
     def get_checksum(self):
-        return self.remote.get_hash(self.path_info)
+        return self.tree.get_hash(self.path_info)
 
     @property
     def is_dir_checksum(self):
-        return self.remote.is_dir_hash(self.checksum)
+        return self.tree.is_dir_hash(self.checksum)
 
     @property
     def exists(self):
-        return self.remote.tree.exists(self.path_info)
+        return self.tree.exists(self.path_info)
 
     def save_info(self):
-        return self.remote.save_info(self.path_info)
+        return self.tree.save_info(self.path_info)
 
     def changed_checksum(self):
         return self.checksum != self.get_checksum()
@@ -222,13 +221,13 @@ class BaseOutput:
 
     @property
     def is_empty(self):
-        return self.remote.tree.is_empty(self.path_info)
+        return self.tree.is_empty(self.path_info)
 
     def isdir(self):
-        return self.remote.tree.isdir(self.path_info)
+        return self.tree.isdir(self.path_info)
 
     def isfile(self):
-        return self.remote.tree.isfile(self.path_info)
+        return self.tree.isfile(self.path_info)
 
     # pylint: disable=no-member
 
@@ -316,7 +315,7 @@ class BaseOutput:
         raise DvcException(f"verify metric is not supported for {self.scheme}")
 
     def download(self, to):
-        self.remote.tree.download(self.path_info, to.path_info)
+        self.tree.download(self.path_info, to.path_info)
 
     def checkout(
         self,
@@ -342,7 +341,7 @@ class BaseOutput:
         )
 
     def remove(self, ignore_remove=False):
-        self.remote.tree.remove(self.path_info)
+        self.tree.remove(self.path_info)
         if self.scheme != "local":
             return
 
@@ -354,7 +353,7 @@ class BaseOutput:
         if self.scheme == "local" and self.use_scm_ignore:
             self.repo.scm.ignore_remove(self.fspath)
 
-        self.remote.tree.move(self.path_info, out.path_info)
+        self.tree.move(self.path_info, out.path_info)
         self.def_path = out.def_path
         self.path_info = out.path_info
         self.save()
@@ -373,7 +372,7 @@ class BaseOutput:
 
     def unprotect(self):
         if self.exists:
-            self.remote.tree.unprotect(self.path_info)
+            self.tree.unprotect(self.path_info)
 
     def get_dir_cache(self, **kwargs):
         if not self.is_dir_checksum:
@@ -433,8 +432,8 @@ class BaseOutput:
         filter_path = str(filter_info) if filter_info else None
         is_win = os.name == "nt"
         for entry in self.dir_cache:
-            checksum = entry[self.remote.tree.PARAM_CHECKSUM]
-            entry_relpath = entry[self.remote.tree.PARAM_RELPATH]
+            checksum = entry[self.tree.PARAM_CHECKSUM]
+            entry_relpath = entry[self.tree.PARAM_RELPATH]
             if is_win:
                 entry_relpath = entry_relpath.replace("/", os.sep)
             entry_path = os.path.join(path, entry_relpath)

dvc/output/gs.py

@@ -1,6 +1,7 @@
 from dvc.output.s3 import S3Output
-from dvc.remote.gs import GSRemoteTree
+
+from ..tree.gs import GSTree
 
 
 class GSOutput(S3Output):
-    TREE_CLS = GSRemoteTree
+    TREE_CLS = GSTree

dvc/output/hdfs.py

@@ -1,6 +1,7 @@
 from dvc.output.base import BaseOutput
-from dvc.remote.hdfs import HDFSRemoteTree
+
+from ..tree.hdfs import HDFSTree
 
 
 class HDFSOutput(BaseOutput):
-    TREE_CLS = HDFSRemoteTree
+    TREE_CLS = HDFSTree

dvc/output/local.py

@@ -5,16 +5,16 @@ from urllib.parse import urlparse
 from dvc.exceptions import DvcException
 from dvc.istextfile import istextfile
 from dvc.output.base import BaseOutput
-from dvc.remote.local import LocalRemote, LocalRemoteTree
 from dvc.utils import relpath
 from dvc.utils.fs import path_isin
 
+from ..tree.local import LocalTree
+
 logger = logging.getLogger(__name__)
 
 
 class LocalOutput(BaseOutput):
-    REMOTE_CLS = LocalRemote
-    TREE_CLS = LocalRemoteTree
+    TREE_CLS = LocalTree
     sep = os.sep
 
     def __init__(self, stage, path, *args, **kwargs):
@@ -22,11 +22,17 @@ class LocalOutput(BaseOutput):
             path = relpath(path, stage.wdir)
 
         super().__init__(stage, path, *args, **kwargs)
+        if (
+            self.is_in_repo
+            and self.repo
+            and isinstance(self.repo.tree, LocalTree)
+        ):
+            self.tree = self.repo.tree
 
-    def _parse_path(self, remote, path):
+    def _parse_path(self, tree, path):
         parsed = urlparse(path)
         if parsed.scheme == "remote":
-            p = remote.path_info / parsed.path.lstrip("/")
+            p = tree.path_info / parsed.path.lstrip("/")
         else:
             # NOTE: we can path either from command line or .dvc file,
             # so we should expect both posix and windows style paths.

dvc/output/s3.py

@@ -1,6 +1,7 @@
 from dvc.output.base import BaseOutput
-from dvc.remote.s3 import S3RemoteTree
+
+from ..tree.s3 import S3Tree
 
 
 class S3Output(BaseOutput):
-    TREE_CLS = S3RemoteTree
+    TREE_CLS = S3Tree

dvc/output/ssh.py

@@ -1,7 +1,7 @@
 from dvc.output.base import BaseOutput
-from dvc.remote.ssh import SSHRemote, SSHRemoteTree
+
+from ..tree.ssh import SSHTree
 
 
 class SSHOutput(BaseOutput):
-    REMOTE_CLS = SSHRemote
-    TREE_CLS = SSHRemoteTree
+    TREE_CLS = SSHTree

dvc/pathspec_math.py

@@ -0,0 +1,87 @@
+# Path Specification Pattern Math
+# Including changing base dir of path specification patterns and merging
+# of two path specification patterns with different base
+# All the operations follow the documents of `gitignore`
+import os
+
+from pathspec.util import normalize_file
+
+from dvc.utils import relpath
+
+
+def _not_ignore(rule):
+    return (True, rule[1:]) if rule.startswith("!") else (False, rule)
+
+
+def _is_comment(rule):
+    return rule.startswith("#")
+
+
+def _remove_slash(rule):
+    if rule.startswith("\\"):
+        return rule[1:]
+    return rule
+
+
+def _match_all_level(rule):
+    if rule[:-1].find("/") >= 0 and not rule.startswith("**/"):
+        if rule.startswith("/"):
+            rule = rule[1:]
+        return False, rule
+    if rule.startswith("**/"):
+        rule = rule[3:]
+    return True, rule
+
+
+def change_rule(rule, rel):
+    rule = rule.strip()
+    if _is_comment(rule):
+        return rule
+    not_ignore, rule = _not_ignore(rule)
+    match_all, rule = _match_all_level(rule)
+    rule = _remove_slash(rule)
+    if not match_all:
+        rule = f"/{rule}"
+    else:
+        rule = f"/**/{rule}"
+    if not_ignore:
+        rule = f"!/{rel}{rule}"
+    else:
+        rule = f"/{rel}{rule}"
+    rule = normalize_file(rule)
+    return rule
+
+
+def _change_dirname(dirname, pattern_list, new_dirname):
+    if new_dirname == dirname:
+        return pattern_list
+    rel = relpath(dirname, new_dirname)
+    if rel.startswith(".."):
+        raise ValueError("change dirname can only change to parent path")
+
+    return [change_rule(rule, rel) for rule in pattern_list]
+
+
+def merge_patterns(pattern_a, prefix_a, pattern_b, prefix_b):
+    """
+    Merge two path specification patterns.
+
+    This implementation merge two path specification patterns on different
+    bases. It returns the longest common parent directory, and the patterns
+    based on this new base directory.
+    """
+    if not pattern_a:
+        return pattern_b, prefix_b
+    elif not pattern_b:
+        return pattern_a, prefix_a
+
+    longest_common_dir = os.path.commonpath([prefix_a, prefix_b])
+    new_pattern_a = _change_dirname(prefix_a, pattern_a, longest_common_dir)
+    new_pattern_b = _change_dirname(prefix_b, pattern_b, longest_common_dir)
+
+    if len(prefix_a) <= len(prefix_b):
+        merged_pattern = new_pattern_a + new_pattern_b
+    else:
+        merged_pattern = new_pattern_b + new_pattern_a
+
+    return merged_pattern, longest_common_dir

dvc/remote/__init__.py

@@ -1,82 +1,7 @@
-import posixpath
-from urllib.parse import urlparse
-
-from dvc.remote.azure import AzureRemoteTree
-from dvc.remote.base import Remote
-from dvc.remote.gdrive import GDriveRemoteTree
-from dvc.remote.gs import GSRemoteTree
-from dvc.remote.hdfs import HDFSRemoteTree
-from dvc.remote.http import HTTPRemoteTree
-from dvc.remote.https import HTTPSRemoteTree
-from dvc.remote.local import LocalRemote, LocalRemoteTree
-from dvc.remote.oss import OSSRemoteTree
-from dvc.remote.s3 import S3RemoteTree
-from dvc.remote.ssh import SSHRemote, SSHRemoteTree
-
-TREES = [
-    AzureRemoteTree,
-    GDriveRemoteTree,
-    GSRemoteTree,
-    HDFSRemoteTree,
-    HTTPRemoteTree,
-    HTTPSRemoteTree,
-    S3RemoteTree,
-    SSHRemoteTree,
-    OSSRemoteTree,
-    # NOTE: LocalRemoteTree is the default
-]
-
-
-def _get_tree(remote_conf):
-    for tree_cls in TREES:
-        if tree_cls.supported(remote_conf):
-            return tree_cls
-    return LocalRemoteTree
-
-
-def _get_conf(repo, **kwargs):
-    name = kwargs.get("name")
-    if name:
-        remote_conf = repo.config["remote"][name.lower()]
-    else:
-        remote_conf = kwargs
-    return _resolve_remote_refs(repo.config, remote_conf)
-
-
-def _resolve_remote_refs(config, remote_conf):
-    # Support for cross referenced remotes.
-    # This will merge the settings, shadowing base ref with remote_conf.
-    # For example, having:
-    #
-    #       dvc remote add server ssh://localhost
-    #       dvc remote modify server user root
-    #       dvc remote modify server ask_password true
-    #
-    #       dvc remote add images remote://server/tmp/pictures
-    #       dvc remote modify images user alice
-    #       dvc remote modify images ask_password false
-    #       dvc remote modify images password asdf1234
-    #
-    # Results on a config dictionary like:
-    #
-    #       {
-    #           "url": "ssh://localhost/tmp/pictures",
-    #           "user": "alice",
-    #           "password": "asdf1234",
-    #           "ask_password": False,
-    #       }
-    parsed = urlparse(remote_conf["url"])
-    if parsed.scheme != "remote":
-        return remote_conf
-
-    base = config["remote"][parsed.netloc]
-    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
-    return {**base, **remote_conf, "url": url}
-
-
-def get_cloud_tree(repo, **kwargs):
-    remote_conf = _get_conf(repo, **kwargs)
-    return _get_tree(remote_conf)(repo, remote_conf)
+from ..tree import get_cloud_tree
+from .base import Remote
+from .local import LocalRemote
+from .ssh import SSHRemote
 
 
 def get_remote(repo, **kwargs):

dvc/remote/https.py

@@ -1,7 +0,0 @@
-from dvc.scheme import Schemes
-
-from .http import HTTPRemoteTree
-
-
-class HTTPSRemoteTree(HTTPRemoteTree):  # pylint:disable=abstract-method
-    scheme = Schemes.HTTPS

dvc/remote/local.py

@@ -1,729 +1,6 @@
-import errno
-import logging
-import os
-import stat
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial, wraps
-
-from funcy import cached_property, concat
-from shortuuid import uuid
-
-from dvc.exceptions import DownloadError, DvcException, UploadError
-from dvc.path_info import PathInfo
-from dvc.progress import Tqdm
-from dvc.remote.base import (
-    STATUS_DELETED,
-    STATUS_MAP,
-    STATUS_MISSING,
-    STATUS_NEW,
-    BaseRemoteTree,
-    CloudCache,
-    Remote,
-    index_locked,
-)
-from dvc.remote.index import RemoteIndexNoop
-from dvc.scheme import Schemes
-from dvc.scm.tree import WorkingTree, is_working_tree
-from dvc.system import System
-from dvc.utils import file_md5, relpath, tmp_fname
-from dvc.utils.fs import (
-    copy_fobj_to_file,
-    copyfile,
-    makedirs,
-    move,
-    remove,
-    walk_files,
-)
-
-logger = logging.getLogger(__name__)
-
-
-class LocalRemoteTree(BaseRemoteTree):
-    scheme = Schemes.LOCAL
-    PATH_CLS = PathInfo
-    PARAM_CHECKSUM = "md5"
-    PARAM_PATH = "path"
-    TRAVERSE_PREFIX_LEN = 2
-    UNPACKED_DIR_SUFFIX = ".unpacked"
-
-    CACHE_MODE = 0o444
-    SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
-
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-        url = config.get("url")
-        self.path_info = self.PATH_CLS(url) if url else None
-
-    @property
-    def state(self):
-        from dvc.state import StateNoop
-
-        return self.repo.state if self.repo else StateNoop()
-
-    @cached_property
-    def work_tree(self):
-        # When using repo.brancher, repo.tree may change to/from WorkingTree to
-        # GitTree arbitarily. When repo.tree is GitTree, local cache needs to
-        # use its own WorkingTree instance.
-        if self.repo:
-            return WorkingTree(self.repo.root_dir)
-        return None
-
-    @staticmethod
-    def open(path_info, mode="r", encoding=None):
-        return open(path_info, mode=mode, encoding=encoding)
-
-    def exists(self, path_info):
-        assert isinstance(path_info, str) or path_info.scheme == "local"
-        if not self.repo:
-            return os.path.exists(path_info)
-        return self.work_tree.exists(path_info)
-
-    def isfile(self, path_info):
-        if not self.repo:
-            return os.path.isfile(path_info)
-        return self.work_tree.isfile(path_info)
-
-    def isdir(self, path_info):
-        if not self.repo:
-            return os.path.isdir(path_info)
-        return self.work_tree.isdir(path_info)
-
-    def iscopy(self, path_info):
-        return not (
-            System.is_symlink(path_info) or System.is_hardlink(path_info)
-        )
-
-    def walk_files(self, path_info, **kwargs):
-        for fname in self.work_tree.walk_files(path_info):
-            yield PathInfo(fname)
-
-    def is_empty(self, path_info):
-        path = path_info.fspath
-
-        if self.isfile(path_info) and os.path.getsize(path) == 0:
-            return True
-
-        if self.isdir(path_info) and len(os.listdir(path)) == 0:
-            return True
-
-        return False
-
-    def remove(self, path_info):
-        if isinstance(path_info, PathInfo):
-            if path_info.scheme != "local":
-                raise NotImplementedError
-            path = path_info.fspath
-        else:
-            path = path_info
-
-        if self.exists(path):
-            remove(path)
-
-    def makedirs(self, path_info):
-        makedirs(path_info, exist_ok=True, mode=self.dir_mode)
-
-    def move(self, from_info, to_info, mode=None):
-        if from_info.scheme != "local" or to_info.scheme != "local":
-            raise NotImplementedError
-
-        self.makedirs(to_info.parent)
-
-        if mode is None:
-            if self.isfile(from_info):
-                mode = self.file_mode
-            else:
-                mode = self.dir_mode
-
-        move(from_info, to_info, mode=mode)
-
-    def copy(self, from_info, to_info):
-        tmp_info = to_info.parent / tmp_fname(to_info.name)
-        try:
-            System.copy(from_info, tmp_info)
-            os.chmod(tmp_info, self.file_mode)
-            os.rename(tmp_info, to_info)
-        except Exception:
-            self.remove(tmp_info)
-            raise
-
-    def copy_fobj(self, fobj, to_info):
-        self.makedirs(to_info.parent)
-        tmp_info = to_info.parent / tmp_fname(to_info.name)
-        try:
-            copy_fobj_to_file(fobj, tmp_info)
-            os.chmod(tmp_info, self.file_mode)
-            os.rename(tmp_info, to_info)
-        except Exception:
-            self.remove(tmp_info)
-            raise
-
-    @staticmethod
-    def symlink(from_info, to_info):
-        System.symlink(from_info, to_info)
-
-    @staticmethod
-    def is_symlink(path_info):
-        return System.is_symlink(path_info)
-
-    def hardlink(self, from_info, to_info):
-        # If there are a lot of empty files (which happens a lot in datasets),
-        # and the cache type is `hardlink`, we might reach link limits and
-        # will get something like: `too many links error`
-        #
-        # This is because all those empty files will have the same hash
-        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
-        # linked to the same file in the cache.
-        #
-        # From https://en.wikipedia.org/wiki/Hard_link
-        #   * ext4 limits the number of hard links on a file to 65,000
-        #   * Windows with NTFS has a limit of 1024 hard links on a file
-        #
-        # That's why we simply create an empty file rather than a link.
-        if self.getsize(from_info) == 0:
-            self.open(to_info, "w").close()
-
-            logger.debug(
-                "Created empty file: {src} -> {dest}".format(
-                    src=str(from_info), dest=str(to_info)
-                )
-            )
-            return
-
-        System.hardlink(from_info, to_info)
-
-    @staticmethod
-    def is_hardlink(path_info):
-        return System.is_hardlink(path_info)
-
-    def reflink(self, from_info, to_info):
-        tmp_info = to_info.parent / tmp_fname(to_info.name)
-        System.reflink(from_info, tmp_info)
-        # NOTE: reflink has its own separate inode, so you can set permissions
-        # that are different from the source.
-        os.chmod(tmp_info, self.file_mode)
-        os.rename(tmp_info, to_info)
-
-    def _unprotect_file(self, path):
-        if System.is_symlink(path) or System.is_hardlink(path):
-            logger.debug(f"Unprotecting '{path}'")
-            tmp = os.path.join(os.path.dirname(path), "." + uuid())
-
-            # The operations order is important here - if some application
-            # would access the file during the process of copyfile then it
-            # would get only the part of file. So, at first, the file should be
-            # copied with the temporary name, and then original file should be
-            # replaced by new.
-            copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
-            remove(path)
-            os.rename(tmp, path)
-
-        else:
-            logger.debug(
-                "Skipping copying for '{}', since it is not "
-                "a symlink or a hardlink.".format(path)
-            )
-
-        os.chmod(path, self.file_mode)
-
-    def _unprotect_dir(self, path):
-        assert is_working_tree(self.repo.tree)
-
-        for fname in self.repo.tree.walk_files(path):
-            self._unprotect_file(fname)
-
-    def unprotect(self, path_info):
-        path = path_info.fspath
-        if not os.path.exists(path):
-            raise DvcException(f"can't unprotect non-existing data '{path}'")
-
-        if os.path.isdir(path):
-            self._unprotect_dir(path)
-        else:
-            self._unprotect_file(path)
-
-    def protect(self, path_info):
-        path = os.fspath(path_info)
-        mode = self.CACHE_MODE
-
-        try:
-            os.chmod(path, mode)
-        except OSError as exc:
-            # There is nothing we need to do in case of a read-only file system
-            if exc.errno == errno.EROFS:
-                return
-
-            # In shared cache scenario, we might not own the cache file, so we
-            # need to check if cache file is already protected.
-            if exc.errno not in [errno.EPERM, errno.EACCES]:
-                raise
-
-            actual = stat.S_IMODE(os.stat(path).st_mode)
-            if actual != mode:
-                raise
-
-    def is_protected(self, path_info):
-        try:
-            mode = os.stat(path_info).st_mode
-        except FileNotFoundError:
-            return False
-
-        return stat.S_IMODE(mode) == self.CACHE_MODE
-
-    def get_file_hash(self, path_info):
-        return file_md5(path_info)[0]
-
-    @staticmethod
-    def getsize(path_info):
-        return os.path.getsize(path_info)
-
-    def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
-    ):
-        makedirs(to_info.parent, exist_ok=True)
-
-        tmp_file = tmp_fname(to_info)
-        copyfile(
-            from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
-        )
-
-        self.protect(tmp_file)
-        os.rename(tmp_file, to_info)
-
-    @staticmethod
-    def _download(
-        from_info, to_file, name=None, no_progress_bar=False, **_kwargs
-    ):
-        copyfile(
-            from_info, to_file, no_progress_bar=no_progress_bar, name=name
-        )
-
-    def list_paths(self, prefix=None, progress_callback=None):
-        assert self.path_info is not None
-        if prefix:
-            path_info = self.path_info / prefix[:2]
-            if not self.exists(path_info):
-                return
-        else:
-            path_info = self.path_info
-        # NOTE: use utils.fs walk_files since tree.walk_files will not follow
-        # symlinks
-        if progress_callback:
-            for path in walk_files(path_info):
-                progress_callback()
-                yield path
-        else:
-            yield from walk_files(path_info)
-
-    def _remove_unpacked_dir(self, hash_):
-        info = self.hash_to_path_info(hash_)
-        path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
-        self.remove(path_info)
-
-
-def _log_exceptions(func, operation):
-    @wraps(func)
-    def wrapper(from_info, to_info, *args, **kwargs):
-        try:
-            func(from_info, to_info, *args, **kwargs)
-            return 0
-        except Exception as exc:  # pylint: disable=broad-except
-            # NOTE: this means we ran out of file descriptors and there is no
-            # reason to try to proceed, as we will hit this error anyways.
-            # pylint: disable=no-member
-            if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
-                raise
-
-            logger.exception(
-                "failed to %s '%s' to '%s'", operation, from_info, to_info
-            )
-            return 1
-
-    return wrapper
+from .base import Remote
+from .index import RemoteIndexNoop
 
 
 class LocalRemote(Remote):
     INDEX_CLS = RemoteIndexNoop
-
-
-class LocalCache(CloudCache):
-    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
-    CACHE_MODE = LocalRemoteTree.CACHE_MODE
-
-    def __init__(self, tree):
-        super().__init__(tree)
-        self.cache_dir = tree.config.get("url")
-
-    @property
-    def cache_dir(self):
-        return self.tree.path_info.fspath if self.tree.path_info else None
-
-    @cache_dir.setter
-    def cache_dir(self, value):
-        self.tree.path_info = PathInfo(value) if value else None
-
-    @classmethod
-    def supported(cls, config):  # pylint: disable=unused-argument
-        return True
-
-    @cached_property
-    def cache_path(self):
-        return os.path.abspath(self.cache_dir)
-
-    def hash_to_path(self, hash_):
-        # NOTE: `self.cache_path` is already normalized so we can simply use
-        # `os.sep` instead of `os.path.join`. This results in this helper
-        # being ~5.5 times faster.
-        return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
-
-    def hashes_exist(
-        self, hashes, jobs=None, name=None
-    ):  # pylint: disable=unused-argument
-        return [
-            hash_
-            for hash_ in Tqdm(
-                hashes,
-                unit="file",
-                desc="Querying "
-                + ("cache in " + name if name else "local cache"),
-            )
-            if not self.changed_cache_file(hash_)
-        ]
-
-    def already_cached(self, path_info):
-        assert path_info.scheme in ["", "local"]
-
-        current_md5 = self.get_hash(path_info)
-
-        if not current_md5:
-            return False
-
-        return not self.changed_cache(current_md5)
-
-    def _verify_link(self, path_info, link_type):
-        if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
-            return
-
-        super()._verify_link(path_info, link_type)
-
-    @index_locked
-    def status(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        # Return flattened dict containing all status info
-        dir_status, file_status, _ = self._status(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=download,
-        )
-        return dict(dir_status, **file_status)
-
-    def _status(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        """Return a tuple of (dir_status_info, file_status_info, dir_contents).
-
-        dir_status_info contains status for .dir files, file_status_info
-        contains status for all other files, and dir_contents is a dict of
-        {dir_hash: set(file_hash, ...)} which can be used to map
-        a .dir file to its file contents.
-        """
-        logger.debug(f"Preparing to collect status from {remote.path_info}")
-        md5s = set(named_cache.scheme_keys(self.scheme))
-
-        logger.debug("Collecting information from local cache...")
-        local_exists = frozenset(
-            self.hashes_exist(md5s, jobs=jobs, name=self.cache_dir)
-        )
-
-        # This is a performance optimization. We can safely assume that,
-        # if the resources that we want to fetch are already cached,
-        # there's no need to check the remote storage for the existence of
-        # those files.
-        if download and local_exists == md5s:
-            remote_exists = local_exists
-        else:
-            logger.debug("Collecting information from remote cache...")
-            remote_exists = set()
-            dir_md5s = set(named_cache.dir_keys(self.scheme))
-            if dir_md5s:
-                remote_exists.update(
-                    self._indexed_dir_hashes(named_cache, remote, dir_md5s)
-                )
-                md5s.difference_update(remote_exists)
-            if md5s:
-                remote_exists.update(
-                    remote.hashes_exist(
-                        md5s, jobs=jobs, name=str(remote.path_info)
-                    )
-                )
-        return self._make_status(
-            named_cache, show_checksums, local_exists, remote_exists
-        )
-
-    def _make_status(
-        self, named_cache, show_checksums, local_exists, remote_exists
-    ):
-        def make_names(hash_, names):
-            return {"name": hash_ if show_checksums else " ".join(names)}
-
-        dir_status = {}
-        file_status = {}
-        dir_contents = {}
-        for hash_, item in named_cache[self.scheme].items():
-            if item.children:
-                dir_status[hash_] = make_names(hash_, item.names)
-                dir_contents[hash_] = set()
-                for child_hash, child in item.children.items():
-                    file_status[child_hash] = make_names(
-                        child_hash, child.names
-                    )
-                    dir_contents[hash_].add(child_hash)
-            else:
-                file_status[hash_] = make_names(hash_, item.names)
-
-        self._fill_statuses(dir_status, local_exists, remote_exists)
-        self._fill_statuses(file_status, local_exists, remote_exists)
-
-        self._log_missing_caches(dict(dir_status, **file_status))
-
-        return dir_status, file_status, dir_contents
-
-    def _indexed_dir_hashes(self, named_cache, remote, dir_md5s):
-        # Validate our index by verifying all indexed .dir hashes
-        # still exist on the remote
-        indexed_dirs = set(remote.index.dir_hashes())
-        indexed_dir_exists = set()
-        if indexed_dirs:
-            indexed_dir_exists.update(
-                remote.tree.list_hashes_exists(indexed_dirs)
-            )
-            missing_dirs = indexed_dirs.difference(indexed_dir_exists)
-            if missing_dirs:
-                logger.debug(
-                    "Remote cache missing indexed .dir hashes '{}', "
-                    "clearing remote index".format(", ".join(missing_dirs))
-                )
-                remote.index.clear()
-
-        # Check if non-indexed (new) dir hashes exist on remote
-        dir_exists = dir_md5s.intersection(indexed_dir_exists)
-        dir_exists.update(
-            remote.tree.list_hashes_exists(dir_md5s - dir_exists)
-        )
-
-        # If .dir hash exists on the remote, assume directory contents
-        # still exists on the remote
-        for dir_hash in dir_exists:
-            file_hashes = list(named_cache.child_keys(self.scheme, dir_hash))
-            if dir_hash not in remote.index:
-                logger.debug(
-                    "Indexing new .dir '{}' with '{}' nested files".format(
-                        dir_hash, len(file_hashes)
-                    )
-                )
-                remote.index.update([dir_hash], file_hashes)
-            yield dir_hash
-            yield from file_hashes
-
-    @staticmethod
-    def _fill_statuses(hash_info_dir, local_exists, remote_exists):
-        # Using sets because they are way faster for lookups
-        local = set(local_exists)
-        remote = set(remote_exists)
-
-        for md5, info in hash_info_dir.items():
-            status = STATUS_MAP[(md5 in local, md5 in remote)]
-            info["status"] = status
-
-    def _get_plans(self, download, remote, status_info, status):
-        cache = []
-        path_infos = []
-        names = []
-        hashes = []
-        for md5, info in Tqdm(
-            status_info.items(), desc="Analysing status", unit="file"
-        ):
-            if info["status"] == status:
-                cache.append(self.hash_to_path_info(md5))
-                path_infos.append(remote.hash_to_path_info(md5))
-                names.append(info["name"])
-                hashes.append(md5)
-
-        if download:
-            to_infos = cache
-            from_infos = path_infos
-        else:
-            to_infos = path_infos
-            from_infos = cache
-
-        return from_infos, to_infos, names, hashes
-
-    def _process(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        logger.debug(
-            "Preparing to {} '{}'".format(
-                "download data from" if download else "upload data to",
-                remote.path_info,
-            )
-        )
-
-        if download:
-            func = partial(
-                _log_exceptions(remote.tree.download, "download"),
-                dir_mode=self.tree.dir_mode,
-                file_mode=self.tree.file_mode,
-            )
-            status = STATUS_DELETED
-            desc = "Downloading"
-        else:
-            func = _log_exceptions(remote.tree.upload, "upload")
-            status = STATUS_NEW
-            desc = "Uploading"
-
-        if jobs is None:
-            jobs = remote.tree.JOBS
-
-        dir_status, file_status, dir_contents = self._status(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=download,
-        )
-
-        dir_plans = self._get_plans(download, remote, dir_status, status)
-        file_plans = self._get_plans(download, remote, file_status, status)
-
-        total = len(dir_plans[0]) + len(file_plans[0])
-        if total == 0:
-            return 0
-
-        with Tqdm(total=total, unit="file", desc=desc) as pbar:
-            func = pbar.wrap_fn(func)
-            with ThreadPoolExecutor(max_workers=jobs) as executor:
-                if download:
-                    from_infos, to_infos, names, _ = (
-                        d + f for d, f in zip(dir_plans, file_plans)
-                    )
-                    fails = sum(
-                        executor.map(func, from_infos, to_infos, names)
-                    )
-                else:
-                    # for uploads, push files first, and any .dir files last
-
-                    file_futures = {}
-                    for from_info, to_info, name, hash_ in zip(*file_plans):
-                        file_futures[hash_] = executor.submit(
-                            func, from_info, to_info, name
-                        )
-                    dir_futures = {}
-                    for from_info, to_info, name, dir_hash in zip(*dir_plans):
-                        wait_futures = {
-                            future
-                            for file_hash, future in file_futures.items()
-                            if file_hash in dir_contents[dir_hash]
-                        }
-                        dir_futures[dir_hash] = executor.submit(
-                            self._dir_upload,
-                            func,
-                            wait_futures,
-                            from_info,
-                            to_info,
-                            name,
-                        )
-                    fails = sum(
-                        future.result()
-                        for future in concat(
-                            file_futures.values(), dir_futures.values()
-                        )
-                    )
-
-        if fails:
-            if download:
-                remote.index.clear()
-                raise DownloadError(fails)
-            raise UploadError(fails)
-
-        if not download:
-            # index successfully pushed dirs
-            for dir_hash, future in dir_futures.items():
-                if future.result() == 0:
-                    file_hashes = dir_contents[dir_hash]
-                    logger.debug(
-                        "Indexing pushed dir '{}' with "
-                        "'{}' nested files".format(dir_hash, len(file_hashes))
-                    )
-                    remote.index.update([dir_hash], file_hashes)
-
-        return len(dir_plans[0]) + len(file_plans[0])
-
-    @staticmethod
-    def _dir_upload(func, futures, from_info, to_info, name):
-        for future in as_completed(futures):
-            if future.result():
-                # do not upload this .dir file if any file in this
-                # directory failed to upload
-                logger.debug(
-                    "failed to upload full contents of '{}', "
-                    "aborting .dir file upload".format(name)
-                )
-                logger.error(f"failed to upload '{from_info}' to '{to_info}'")
-                return 1
-        return func(from_info, to_info, name)
-
-    @index_locked
-    def push(self, named_cache, remote, jobs=None, show_checksums=False):
-        return self._process(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=False,
-        )
-
-    @index_locked
-    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
-        return self._process(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=True,
-        )
-
-    @staticmethod
-    def _log_missing_caches(hash_info_dict):
-        missing_caches = [
-            (md5, info)
-            for md5, info in hash_info_dict.items()
-            if info["status"] == STATUS_MISSING
-        ]
-        if missing_caches:
-            missing_desc = "\n".join(
-                "name: {}, md5: {}".format(info["name"], md5)
-                for md5, info in missing_caches
-            )
-            msg = (
-                "Some of the cache files do not exist neither locally "
-                "nor on remote. Missing cache files:\n{}".format(missing_desc)
-            )
-            logger.warning(msg)

dvc/remote/ssh.py

@@ -0,0 +1,73 @@
+import errno
+import itertools
+import logging
+from concurrent.futures import ThreadPoolExecutor
+
+from dvc.progress import Tqdm
+from dvc.utils import to_chunks
+
+from .base import Remote
+
+logger = logging.getLogger(__name__)
+
+
+class SSHRemote(Remote):
+    def batch_exists(self, path_infos, callback):
+        def _exists(chunk_and_channel):
+            chunk, channel = chunk_and_channel
+            ret = []
+            for path in chunk:
+                try:
+                    channel.stat(path)
+                    ret.append(True)
+                except OSError as exc:
+                    if exc.errno != errno.ENOENT:
+                        raise
+                    ret.append(False)
+                callback(path)
+            return ret
+
+        with self.tree.ssh(path_infos[0]) as ssh:
+            channels = ssh.open_max_sftp_channels()
+            max_workers = len(channels)
+
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                paths = [path_info.path for path_info in path_infos]
+                chunks = to_chunks(paths, num_chunks=max_workers)
+                chunks_and_channels = zip(chunks, channels)
+                outcome = executor.map(_exists, chunks_and_channels)
+                results = list(itertools.chain.from_iterable(outcome))
+
+            return results
+
+    def hashes_exist(self, hashes, jobs=None, name=None):
+        """This is older implementation used in remote/base.py
+        We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
+        faster than current approach (relying on exists(path_info)) applied in
+        remote/base.
+        """
+        if not self.tree.CAN_TRAVERSE:
+            return list(set(hashes) & set(self.tree.all()))
+
+        # possibly prompt for credentials before "Querying" progress output
+        self.tree.ensure_credentials()
+
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(hashes),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(chunks):
+                return self.batch_exists(chunks, callback=pbar.update_msg)
+
+            with ThreadPoolExecutor(
+                max_workers=jobs or self.tree.JOBS
+            ) as executor:
+                path_infos = [self.tree.hash_to_path_info(x) for x in hashes]
+                chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
+                results = executor.map(exists_with_progress, chunks)
+                in_remote = itertools.chain.from_iterable(results)
+                ret = list(itertools.compress(hashes, in_remote))
+                return ret

dvc/repo/__init__.py

@@ -14,10 +14,8 @@ from dvc.exceptions import (
     NotDvcRepoError,
     OutputNotFoundError,
 )
-from dvc.ignore import CleanTree
 from dvc.path_info import PathInfo
 from dvc.repo.tree import RepoTree
-from dvc.scm.tree import is_working_tree
 from dvc.utils.fs import path_isin
 
 from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
@@ -74,31 +72,38 @@ class Repo:
         from dvc.scm import SCM
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
+        from dvc.repo.experiments import Experiments
         from dvc.repo.metrics import Metrics
         from dvc.repo.plots import Plots
         from dvc.repo.params import Params
-        from dvc.scm.tree import WorkingTree
+        from dvc.tree.local import LocalTree
         from dvc.utils.fs import makedirs
         from dvc.stage.cache import StageCache
 
         if scm:
-            # use GitTree instead of WorkingTree as default repo tree instance
             tree = scm.get_tree(rev)
             self.root_dir = self.find_root(root_dir, tree)
             self.scm = scm
-            self.tree = tree
+            self.tree = scm.get_tree(
+                rev, use_dvcignore=True, dvcignore_root=self.root_dir
+            )
             self.state = StateNoop()
         else:
             root_dir = self.find_root(root_dir)
             self.root_dir = os.path.abspath(os.path.realpath(root_dir))
+            self.tree = LocalTree(
+                self,
+                {"url": self.root_dir},
+                use_dvcignore=True,
+                dvcignore_root=self.root_dir,
+            )
 
         self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
-        self.config = Config(self.dvc_dir)
+        self.config = Config(self.dvc_dir, tree=self.tree)
 
         if not scm:
             no_scm = self.config["core"].get("no_scm", False)
             self.scm = SCM(self.root_dir, no_scm=no_scm)
-            self.tree = WorkingTree(self.root_dir)
 
         self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
         self.index_dir = os.path.join(self.tmp_dir, "index")
@@ -127,6 +132,11 @@ class Repo:
         self.plots = Plots(self)
         self.params = Params(self)
 
+        try:
+            self.experiments = Experiments(self)
+        except NotImplementedError:
+            self.experiments = None
+
         self._ignore()
 
     @property
@@ -135,13 +145,7 @@ class Repo:
 
     @tree.setter
     def tree(self, tree):
-        if is_working_tree(tree) or tree.tree_root == self.root_dir:
-            root = None
-        else:
-            root = self.root_dir
-        self._tree = (
-            tree if isinstance(tree, CleanTree) else CleanTree(tree, root)
-        )
+        self._tree = tree
         # Our graph cache is no longer valid, as it was based on the previous
         # tree.
         self._reset()
@@ -191,7 +195,12 @@ class Repo:
         return self.cache.local.tree.unprotect(PathInfo(target))
 
     def _ignore(self):
-        flist = [self.config.files["local"], self.tmp_dir]
+        flist = [
+            self.config.files["local"],
+            self.tmp_dir,
+        ]
+        if self.experiments:
+            flist.append(self.experiments.exp_dir)
 
         if path_isin(self.cache.local.cache_dir, self.root_dir):
             flist += [self.cache.local.cache_dir]
@@ -606,4 +615,4 @@ class Repo:
         self.__dict__.pop("graph", None)
         self.__dict__.pop("stages", None)
         self.__dict__.pop("pipelines", None)
-        self.__dict__.pop("dvcignore", None)
+        self.tree.__dict__.pop("dvcignore", None)

dvc/repo/add.py

@@ -109,17 +109,17 @@ def add(
 def _find_all_targets(repo, target, recursive):
     if os.path.isdir(target) and recursive:
         return [
-            fname
-            for fname in Tqdm(
+            os.fspath(path)
+            for path in Tqdm(
                 repo.tree.walk_files(target),
                 desc="Searching " + target,
                 bar_format=Tqdm.BAR_FMT_NOTOTAL,
                 unit="file",
             )
-            if not repo.is_dvc_internal(fname)
-            if not is_dvc_file(fname)
-            if not repo.scm.belongs_to_scm(fname)
-            if not repo.scm.is_tracked(fname)
+            if not repo.is_dvc_internal(os.fspath(path))
+            if not is_dvc_file(os.fspath(path))
+            if not repo.scm.belongs_to_scm(os.fspath(path))
+            if not repo.scm.is_tracked(os.fspath(path))
         ]
     return [target]
 

dvc/repo/brancher.py

@@ -1,6 +1,6 @@
 from funcy import group_by
 
-from dvc.scm.tree import WorkingTree
+from dvc.tree.local import LocalTree
 
 
 def brancher(  # noqa: E302
@@ -29,7 +29,7 @@ def brancher(  # noqa: E302
 
     scm = self.scm
 
-    self.tree = WorkingTree(self.root_dir)
+    self.tree = LocalTree(self, {"url": self.root_dir}, use_dvcignore=True)
     yield "workspace"
 
     if revs and "workspace" in revs:
@@ -47,7 +47,9 @@ def brancher(  # noqa: E302
     try:
         if revs:
             for sha, names in group_by(scm.resolve_rev, revs).items():
-                self.tree = scm.get_tree(sha)
+                self.tree = scm.get_tree(
+                    sha, use_dvcignore=True, dvcignore_root=self.root_dir
+                )
                 # ignore revs that don't contain repo root
                 # (i.e. revs from before a subdir=True repo was init'ed)
                 if self.tree.exists(self.root_dir):

dvc/repo/diff.py

@@ -1,7 +1,7 @@
 import os
 
 from dvc.repo import locked
-from dvc.scm.tree import is_working_tree
+from dvc.tree.local import LocalTree
 
 
 @locked
@@ -33,11 +33,11 @@ def diff(self, a_rev="HEAD", b_rev=None):
                 else os.path.join(str(output), "")
             )
 
-        on_working_tree = is_working_tree(self.tree)
+        on_working_tree = isinstance(self.tree, LocalTree)
 
         def _to_checksum(output):
             if on_working_tree:
-                return self.cache.local.get_hash(output.path_info)
+                return self.cache.local.tree.get_hash(output.path_info)
             return output.checksum
 
         def _exists(output):

dvc/repo/experiments/__init__.py

@@ -0,0 +1,197 @@
+import logging
+import os
+import tempfile
+from contextlib import contextmanager
+
+from funcy import cached_property
+
+from dvc.exceptions import DvcException
+from dvc.scm.git import Git
+from dvc.stage.serialize import to_lockfile
+from dvc.utils import dict_sha256, env2bool, relpath
+from dvc.utils.fs import remove
+
+logger = logging.getLogger(__name__)
+
+
+class UnchangedExperimentError(DvcException):
+    pass
+
+
+class Experiments:
+    """Class that manages experiments in a DVC repo.
+
+    Args:
+        repo (dvc.repo.Repo): repo instance that these experiments belong to.
+    """
+
+    EXPERIMENTS_DIR = "experiments"
+
+    def __init__(self, repo):
+        if not (
+            env2bool("DVC_TEST")
+            or repo.config["core"].get("experiments", False)
+        ):
+            raise NotImplementedError
+
+        self.repo = repo
+
+    @cached_property
+    def exp_dir(self):
+        return os.path.join(self.repo.dvc_dir, self.EXPERIMENTS_DIR)
+
+    @cached_property
+    def scm(self):
+        """Experiments clone scm instance."""
+        if os.path.exists(self.exp_dir):
+            return Git(self.exp_dir)
+        return self._init_clone()
+
+    @cached_property
+    def exp_dvc_dir(self):
+        dvc_dir = relpath(self.repo.dvc_dir, self.repo.scm.root_dir)
+        return os.path.join(self.exp_dir, dvc_dir)
+
+    @cached_property
+    def exp_dvc(self):
+        """Return clone dvc Repo instance."""
+        from dvc.repo import Repo
+
+        return Repo(self.exp_dvc_dir)
+
+    @staticmethod
+    def exp_hash(stages):
+        exp_data = {}
+        for stage in stages:
+            exp_data.update(to_lockfile(stage))
+        return dict_sha256(exp_data)
+
+    @contextmanager
+    def chdir(self):
+        cwd = os.getcwd()
+        os.chdir(self.exp_dvc.root_dir)
+        yield
+        os.chdir(cwd)
+
+    def _init_clone(self):
+        src_dir = self.repo.scm.root_dir
+        logger.debug("Initializing experiments clone")
+        git = Git.clone(src_dir, self.exp_dir)
+        self._config_clone()
+        return git
+
+    def _config_clone(self):
+        dvc_dir = relpath(self.repo.dvc_dir, self.repo.scm.root_dir)
+        local_config = os.path.join(self.exp_dir, dvc_dir, "config.local")
+        cache_dir = self.repo.cache.local.cache_dir
+        logger.debug("Writing experiments local config '%s'", local_config)
+        with open(local_config, "w") as fobj:
+            fobj.write(f"[cache]\n    dir = {cache_dir}")
+
+    def _scm_checkout(self, rev):
+        self.scm.repo.git.reset(hard=True)
+        if not Git.is_sha(rev) or not self.scm.has_rev(rev):
+            self.scm.fetch(all=True)
+        logger.debug("Checking out base experiment commit '%s'", rev)
+        self.scm.checkout(rev)
+
+    def _patch_exp(self):
+        """Create a patch based on the current (parent) workspace and apply it
+        to the experiment workspace.
+        """
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        try:
+            self.repo.scm.repo.git.diff(patch=True, output=tmp)
+            if os.path.getsize(tmp):
+                logger.debug("Patching experiment workspace")
+                self.scm.repo.git.apply(tmp)
+            else:
+                raise UnchangedExperimentError(
+                    "Experiment identical to baseline commit."
+                )
+        finally:
+            remove(tmp)
+
+    def _commit(self, stages, check_exists=True, branch=True, rev=None):
+        """Commit stages as an experiment and return the commit SHA."""
+        hash_ = self.exp_hash(stages)
+        exp_name = f"{rev[:7]}-{hash_}"
+        if branch:
+            if check_exists and exp_name in self.scm.list_branches():
+                logger.debug("Using existing experiment branch '%s'", exp_name)
+                return self.scm.resolve_rev(exp_name)
+            self.scm.checkout(exp_name, create_new=True)
+        logger.debug("Commit new experiment branch '%s'", exp_name)
+        self.scm.repo.git.add(A=True)
+        self.scm.commit(f"Add experiment {exp_name}")
+        return self.scm.get_rev()
+
+    def _reproduce(self, *args, **kwargs):
+        """Run `dvc repro` inside the experiments workspace."""
+        with self.chdir():
+            return self.exp_dvc.reproduce(*args, **kwargs)
+
+    def new(self, *args, workspace=True, **kwargs):
+        """Create a new experiment.
+
+        Experiment will be reproduced and checked out into the user's
+        workspace.
+        """
+        rev = self.repo.scm.get_rev()
+        self._scm_checkout(rev)
+        if workspace:
+            try:
+                self._patch_exp()
+            except UnchangedExperimentError as exc:
+                logger.info("Reproducing existing experiment '%s'.", rev[:7])
+                raise exc
+        else:
+            # configure params via command line here
+            pass
+        self.exp_dvc.checkout()
+        stages = self._reproduce(*args, **kwargs)
+        exp_rev = self._commit(stages, rev=rev)
+        self.checkout_exp(exp_rev, force=True)
+        logger.info("Generated experiment '%s'.", exp_rev[:7])
+        return stages
+
+    def checkout_exp(self, rev, force=False):
+        """Checkout an experiment to the user's workspace."""
+        from git.exc import GitCommandError
+        from dvc.repo.checkout import _checkout as dvc_checkout
+
+        if force:
+            self.repo.scm.repo.git.reset(hard=True)
+        logger.debug(f"checkout {rev}")
+        self._scm_checkout(rev)
+
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        self.scm.repo.head.commit.diff("HEAD~1", patch=True, output=tmp)
+        try:
+            if os.path.getsize(tmp):
+                logger.debug("Patching local workspace")
+                self.repo.scm.repo.git.apply(tmp, reverse=True)
+            dvc_checkout(self.repo)
+        except GitCommandError:
+            raise DvcException(
+                "Checkout failed, experiment contains changes which "
+                "conflict with your current workspace. To overwrite "
+                "your workspace, use `dvc experiments checkout --force`."
+            )
+        finally:
+            remove(tmp)
+
+    def checkout(self, *args, **kwargs):
+        from dvc.repo.experiments.checkout import checkout
+
+        return checkout(self.repo, *args, **kwargs)
+
+    def diff(self, *args, **kwargs):
+        from dvc.repo.experiments.diff import diff
+
+        return diff(self.repo, *args, **kwargs)
+
+    def show(self, *args, **kwargs):
+        from dvc.repo.experiments.show import show
+
+        return show(self.repo, *args, **kwargs)

dvc/repo/experiments/checkout.py

@@ -0,0 +1,17 @@
+import logging
+
+from dvc.repo import locked
+from dvc.repo.scm_context import scm_context
+
+logger = logging.getLogger(__name__)
+
+
+@locked
+@scm_context
+def checkout(repo, rev, *args, **kwargs):
+    repo.experiments.checkout_exp(rev, *args, **kwargs)
+    logger.info(
+        "Changes for experiment '%s' have been applied to your current "
+        "workspace.",
+        rev,
+    )

dvc/repo/experiments/diff.py

@@ -0,0 +1,36 @@
+import logging
+
+from dvc.utils.diff import diff as _diff
+from dvc.utils.diff import format_dict
+
+logger = logging.getLogger(__name__)
+
+
+def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
+    from dvc.repo.experiments.show import _collect_experiment
+
+    if repo.scm.no_commits:
+        return {}
+
+    if a_rev:
+        with repo.experiments.chdir():
+            old = _collect_experiment(repo.experiments.exp_dvc, a_rev)
+    else:
+        old = _collect_experiment(repo, "HEAD")
+
+    if b_rev:
+        with repo.experiments.chdir():
+            new = _collect_experiment(repo.experiments.exp_dvc, b_rev)
+    else:
+        new = _collect_experiment(repo, "workspace")
+
+    with_unchanged = kwargs.pop("all", False)
+
+    return {
+        key: _diff(
+            format_dict(old[key]),
+            format_dict(new[key]),
+            with_unchanged=with_unchanged,
+        )
+        for key in ["metrics", "params"]
+    }

dvc/repo/experiments/show.py

@@ -0,0 +1,64 @@
+import logging
+import re
+from collections import OrderedDict, defaultdict
+
+from dvc.repo import locked
+from dvc.repo.metrics.show import _collect_metrics, _read_metrics
+from dvc.repo.params.show import _collect_configs, _read_params
+
+logger = logging.getLogger(__name__)
+
+
+EXP_RE = re.compile(r"(?P<rev_sha>[a-f0-9]{7})-(?P<exp_sha>[a-f0-9]+)")
+
+
+def _collect_experiment(repo, branch):
+    res = defaultdict(dict)
+    for rev in repo.brancher(revs=[branch]):
+        configs = _collect_configs(repo)
+        params = _read_params(repo, configs, rev)
+        if params:
+            res["params"] = params
+
+        metrics = _collect_metrics(repo, None, False)
+        vals = _read_metrics(repo, metrics, rev)
+        if vals:
+            res["metrics"] = vals
+
+    return res
+
+
+@locked
+def show(
+    repo, all_branches=False, all_tags=False, revs=None, all_commits=False
+):
+    res = defaultdict(OrderedDict)
+
+    if revs is None:
+        revs = [repo.scm.get_rev()]
+
+    revs = set(
+        repo.brancher(
+            revs=revs,
+            all_branches=all_branches,
+            all_tags=all_tags,
+            all_commits=all_commits,
+        )
+    )
+
+    for rev in revs:
+        res[rev]["baseline"] = _collect_experiment(repo, rev)
+
+    for exp_branch in repo.experiments.scm.list_branches():
+        m = re.match(EXP_RE, exp_branch)
+        if m:
+            rev = repo.scm.resolve_rev(m.group("rev_sha"))
+            if rev in revs:
+                exp_rev = repo.experiments.scm.resolve_rev(exp_branch)
+                with repo.experiments.chdir():
+                    experiment = _collect_experiment(
+                        repo.experiments.exp_dvc, exp_branch
+                    )
+                res[rev][exp_rev] = experiment
+
+    return res

dvc/repo/params/show.py

@@ -1,5 +1,6 @@
 import logging
 
+import toml
 import yaml
 
 from dvc.dependency.param import ParamsDependency
@@ -34,8 +35,10 @@ def _read_params(repo, configs, rev):
 
         with repo.tree.open(config, "r") as fobj:
             try:
-                res[str(config)] = yaml.safe_load(fobj)
-            except yaml.YAMLError:
+                res[str(config)] = ParamsDependency.PARAMS_FILE_LOADERS[
+                    config.suffix.lower()
+                ](fobj)
+            except (yaml.YAMLError, toml.TomlDecodeError):
                 logger.debug(
                     "failed to read '%s' on '%s'", config, rev, exc_info=True
                 )

dvc/repo/reproduce.py

@@ -1,6 +1,7 @@
 import logging
 
 from dvc.exceptions import InvalidArgumentError, ReproductionError
+from dvc.repo.experiments import UnchangedExperimentError
 from dvc.repo.scm_context import scm_context
 
 from . import locked
@@ -69,6 +70,19 @@ def reproduce(
             "Neither `target` nor `--all-pipelines` are specified."
         )
 
+    experiment = kwargs.pop("experiment", False)
+    if experiment and self.experiments:
+        try:
+            return self.experiments.new(
+                target=target,
+                recursive=recursive,
+                all_pipelines=all_pipelines,
+                **kwargs
+            )
+        except UnchangedExperimentError:
+            # If experiment contains no changes, just run regular repro
+            pass
+
     interactive = kwargs.get("interactive", False)
     if not interactive:
         kwargs["interactive"] = self.config["core"].get("interactive", False)

dvc/repo/status.py

@@ -74,7 +74,7 @@ def _cloud_status(
 
             { "bar": "deleted" }
     """
-    import dvc.remote.base as cloud
+    import dvc.cache.base as cloud
 
     used = self.used_cache(
         targets,

dvc/repo/tree.py

@@ -4,15 +4,14 @@ import os
 from dvc.dvcfile import is_valid_filename
 from dvc.exceptions import OutputNotFoundError
 from dvc.path_info import PathInfo
-from dvc.remote.base import RemoteActionNotImplemented
-from dvc.scm.tree import BaseTree
+from dvc.tree.base import BaseTree, RemoteActionNotImplemented
 from dvc.utils import file_md5
 from dvc.utils.fs import copy_fobj_to_file, makedirs
 
 logger = logging.getLogger(__name__)
 
 
-class DvcTree(BaseTree):
+class DvcTree(BaseTree):  # pylint:disable=abstract-method
     """DVC repo tree.
 
     Args:
@@ -27,7 +26,7 @@ class DvcTree(BaseTree):
     """
 
     def __init__(self, repo, fetch=False, stream=False):
-        self.repo = repo
+        super().__init__(repo, {"url": repo.root_dir})
         self.fetch = fetch
         self.stream = stream
 
@@ -44,15 +43,16 @@ class DvcTree(BaseTree):
         return outs
 
     def _get_granular_checksum(self, path, out, remote=None):
+        assert isinstance(path, PathInfo)
         if not self.fetch and not self.stream:
             raise FileNotFoundError
         dir_cache = out.get_dir_cache(remote=remote)
         for entry in dir_cache:
-            entry_relpath = entry[out.remote.tree.PARAM_RELPATH]
+            entry_relpath = entry[out.tree.PARAM_RELPATH]
             if os.name == "nt":
                 entry_relpath = entry_relpath.replace("/", os.sep)
             if path == out.path_info / entry_relpath:
-                return entry[out.remote.tree.PARAM_CHECKSUM]
+                return entry[out.tree.PARAM_CHECKSUM]
         raise FileNotFoundError
 
     def open(
@@ -84,8 +84,8 @@ class DvcTree(BaseTree):
                 else:
                     checksum = out.checksum
                 try:
-                    remote_info = remote_obj.hash_to_path_info(checksum)
-                    return remote_obj.open(
+                    remote_info = remote_obj.tree.hash_to_path_info(checksum)
+                    return remote_obj.tree.open(
                         remote_info, mode=mode, encoding=encoding
                     )
                 except RemoteActionNotImplemented:
@@ -95,19 +95,19 @@ class DvcTree(BaseTree):
 
         if out.is_dir_checksum:
             checksum = self._get_granular_checksum(path, out)
-            cache_path = out.cache.hash_to_path_info(checksum).url
+            cache_path = out.cache.tree.hash_to_path_info(checksum).url
         else:
             cache_path = out.cache_path
         return open(cache_path, mode=mode, encoding=encoding)
 
-    def exists(self, path):
+    def exists(self, path):  # pylint: disable=arguments-differ
         try:
             self._find_outs(path, strict=False, recursive=True)
             return True
         except OutputNotFoundError:
             return False
 
-    def isdir(self, path):
+    def isdir(self, path):  # pylint: disable=arguments-differ
         if not self.exists(path):
             return False
 
@@ -128,12 +128,12 @@ class DvcTree(BaseTree):
         # for dir checksum, we need to check if this is a file inside the
         # directory
         try:
-            self._get_granular_checksum(path, out)
+            self._get_granular_checksum(path_info, out)
             return False
         except FileNotFoundError:
             return True
 
-    def isfile(self, path):
+    def isfile(self, path):  # pylint: disable=arguments-differ
         if not self.exists(path):
             return False
 
@@ -155,7 +155,7 @@ class DvcTree(BaseTree):
                     download_callback(downloaded)
 
         for entry in dir_cache:
-            entry_relpath = entry[out.remote.tree.PARAM_RELPATH]
+            entry_relpath = entry[out.tree.PARAM_RELPATH]
             if os.name == "nt":
                 entry_relpath = entry_relpath.replace("/", os.sep)
             path_info = out.path_info / entry_relpath
@@ -236,7 +236,7 @@ class DvcTree(BaseTree):
         return out.checksum
 
 
-class RepoTree(BaseTree):
+class RepoTree(BaseTree):  # pylint:disable=abstract-method
     """DVC + git-tracked files tree.
 
     Args:
@@ -246,7 +246,7 @@ class RepoTree(BaseTree):
     """
 
     def __init__(self, repo, **kwargs):
-        self.repo = repo
+        super().__init__(repo, {"url": repo.root_dir})
         if hasattr(repo, "dvc_dir"):
             self.dvctree = DvcTree(repo, **kwargs)
         else:
@@ -265,7 +265,9 @@ class RepoTree(BaseTree):
             return self.dvctree.stream
         return False
 
-    def open(self, path, mode="r", encoding="utf-8", **kwargs):
+    def open(
+        self, path, mode="r", encoding="utf-8", **kwargs
+    ):  # pylint: disable=arguments-differ
         if "b" in mode:
             encoding = None
 
@@ -275,12 +277,14 @@ class RepoTree(BaseTree):
             )
         return self.repo.tree.open(path, mode=mode, encoding=encoding)
 
-    def exists(self, path):
+    def exists(
+        self, path, use_dvcignore=True
+    ):  # pylint: disable=arguments-differ
         return self.repo.tree.exists(path) or (
             self.dvctree and self.dvctree.exists(path)
         )
 
-    def isdir(self, path):
+    def isdir(self, path):  # pylint: disable=arguments-differ
         return self.repo.tree.isdir(path) or (
             self.dvctree and self.dvctree.isdir(path)
         )
@@ -288,7 +292,7 @@ class RepoTree(BaseTree):
     def isdvc(self, path, **kwargs):
         return self.dvctree is not None and self.dvctree.isdvc(path, **kwargs)
 
-    def isfile(self, path):
+    def isfile(self, path):  # pylint: disable=arguments-differ
         return self.repo.tree.isfile(path) or (
             self.dvctree and self.dvctree.isfile(path)
         )
@@ -392,7 +396,7 @@ class RepoTree(BaseTree):
         repo_walk = self.repo.tree.walk(top, topdown=topdown)
         yield from self._walk(dvc_walk, repo_walk, dvcfiles=dvcfiles)
 
-    def walk_files(self, top, **kwargs):
+    def walk_files(self, top, **kwargs):  # pylint: disable=arguments-differ
         for root, _, files in self.walk(top, **kwargs):
             for fname in files:
                 yield PathInfo(root) / fname
@@ -436,5 +440,5 @@ class RepoTree(BaseTree):
                     copy_fobj_to_file(fobj, dest_dir / fname)
 
     @property
-    def hash_jobs(self):
+    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
         return self.repo.tree.hash_jobs

dvc/scm/git.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import shlex
+from functools import partial
 
 import yaml
 from funcy import cached_property
@@ -17,7 +18,6 @@ from dvc.scm.base import (
     RevError,
     SCMError,
 )
-from dvc.scm.git.tree import GitTree
 from dvc.utils import fix_env, is_binary, relpath
 from dvc.utils.fs import path_isin
 
@@ -93,7 +93,7 @@ class Git(Base):
         return self.repo.working_tree_dir
 
     @staticmethod
-    def clone(url, to_path, rev=None):
+    def clone(url, to_path, rev=None, shallow_branch=None):
         import git
 
         ld_key = "LD_LIBRARY_PATH"
@@ -110,14 +110,23 @@ class Git(Base):
             env[ld_key] = ""
 
         try:
+            if shallow_branch is not None and os.path.exists(url):
+                # git disables --depth for local clones unless file:// url
+                # scheme is used
+                url = f"file://{url}"
             with TqdmGit(desc="Cloning", unit="obj") as pbar:
-                tmp_repo = git.Repo.clone_from(
+                clone_from = partial(
+                    git.Repo.clone_from,
                     url,
                     to_path,
                     env=env,  # needed before we can fix it in __init__
                     no_single_branch=True,
                     progress=pbar.update_git,
                 )
+                if shallow_branch is None:
+                    tmp_repo = clone_from()
+                else:
+                    tmp_repo = clone_from(branch=shallow_branch, depth=1)
             tmp_repo.close()
         except git.exc.GitCommandError as exc:  # pylint: disable=no-member
             raise CloneError(url, to_path) from exc
@@ -251,8 +260,8 @@ class Git(Base):
         else:
             self.repo.git.checkout(branch)
 
-    def pull(self):
-        infos = self.repo.remote().pull()
+    def pull(self, **kwargs):
+        infos = self.repo.remote().pull(**kwargs)
         for info in infos:
             if info.flags & info.ERROR:
                 raise SCMError(f"pull failed: {info.note}")
@@ -375,8 +384,10 @@ class Git(Base):
         path_parts = os.path.normpath(path).split(os.path.sep)
         return basename == self.ignore_file or Git.GIT_DIR in path_parts
 
-    def get_tree(self, rev):
-        return GitTree(self.repo, self.resolve_rev(rev))
+    def get_tree(self, rev, **kwargs):
+        from dvc.tree.git import GitTree
+
+        return GitTree(self.repo, self.resolve_rev(rev), **kwargs)
 
     def get_rev(self):
         return self.repo.rev_parse("HEAD").hexsha

dvc/scm/tree.py

@@ -1,97 +0,0 @@
-import os
-import stat
-from multiprocessing import cpu_count
-
-from funcy import cached_property
-
-
-class BaseTree:
-    """Abstract class to represent access to files"""
-
-    @property
-    def tree_root(self):
-        pass
-
-    def open(self, path, mode="r", encoding="utf-8"):
-        """Open file and return a stream."""
-
-    def exists(self, path):
-        """Test whether a path exists."""
-
-    def isdir(self, path):
-        """Return true if the pathname refers to an existing directory."""
-
-    def isfile(self, path):
-        """Test whether a path is a regular file"""
-
-    def walk(self, top, topdown=True, onerror=None):
-        """Directory tree generator.
-
-        See `os.walk` for the docs. Differences:
-        - no support for symlinks
-        """
-
-    def walk_files(self, top):
-        for root, _, files in self.walk(top):
-            for file in files:
-                # NOTE: os.path.join is ~5.5 times slower
-                yield f"{root}{os.sep}{file}"
-
-
-class WorkingTree(BaseTree):
-    """Proxies the repo file access methods to working tree files"""
-
-    def __init__(self, repo_root=None):
-        repo_root = repo_root or os.getcwd()
-        self.repo_root = repo_root
-
-    @property
-    def tree_root(self):
-        return self.repo_root
-
-    def open(self, path, mode="r", encoding="utf-8"):
-        """Open file and return a stream."""
-        if "b" in mode:
-            encoding = None
-        return open(path, mode=mode, encoding=encoding)
-
-    def exists(self, path):
-        """Test whether a path exists."""
-        return os.path.lexists(path)
-
-    def isdir(self, path):
-        """Return true if the pathname refers to an existing directory."""
-        return os.path.isdir(path)
-
-    def isfile(self, path):
-        """Test whether a path is a regular file"""
-        return os.path.isfile(path)
-
-    def walk(self, top, topdown=True, onerror=None):
-        """Directory tree generator.
-
-        See `os.walk` for the docs. Differences:
-        - no support for symlinks
-        """
-        for root, dirs, files in os.walk(
-            top, topdown=topdown, onerror=onerror
-        ):
-            yield os.path.normpath(root), dirs, files
-
-    def isexec(self, path):
-        mode = os.stat(path).st_mode
-        return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
-
-    @staticmethod
-    def stat(path):
-        return os.stat(path)
-
-    @cached_property
-    def hash_jobs(self):
-        return max(1, min(4, cpu_count() // 2))
-
-
-def is_working_tree(tree):
-    return isinstance(tree, WorkingTree) or isinstance(
-        getattr(tree, "tree", None), WorkingTree
-    )

dvc/stage/cache.py

@@ -6,7 +6,7 @@ import yaml
 from funcy import first
 from voluptuous import Invalid
 
-from dvc.remote.local import _log_exceptions
+from dvc.cache.local import _log_exceptions
 from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA
 from dvc.utils import dict_sha256, relpath
 from dvc.utils.fs import makedirs

dvc/stage/utils.py

@@ -8,8 +8,8 @@ from dvc import dependency, output
 from dvc.utils.fs import path_isin
 
 from ..dependency import ParamsDependency
-from ..remote.local import LocalRemoteTree
-from ..remote.s3 import S3RemoteTree
+from ..tree.local import LocalTree
+from ..tree.s3 import S3Tree
 from ..utils import dict_md5, format_link, relpath
 from .exceptions import (
     MissingDataSource,
@@ -133,8 +133,8 @@ def stage_dump_eq(stage_cls, old_d, new_d):
     new_d.pop(stage_cls.PARAM_MD5, None)
     outs = old_d.get(stage_cls.PARAM_OUTS, [])
     for out in outs:
-        out.pop(LocalRemoteTree.PARAM_CHECKSUM, None)
-        out.pop(S3RemoteTree.PARAM_CHECKSUM, None)
+        out.pop(LocalTree.PARAM_CHECKSUM, None)
+        out.pop(S3Tree.PARAM_CHECKSUM, None)
 
     # outs and deps are lists of dicts. To check equality, we need to make
     # them independent of the order, so, we convert them to dicts.

dvc/state.py

@@ -89,10 +89,12 @@ class State:  # pylint: disable=too-many-instance-attributes
     MAX_UINT = 2 ** 64 - 2
 
     def __init__(self, local_cache):
+        from dvc.tree.local import LocalTree
+
         repo = local_cache.repo
         self.repo = repo
         self.root_dir = repo.root_dir
-        self.tree = local_cache.tree.work_tree
+        self.tree = LocalTree(None, {"url": self.root_dir})
 
         state_config = repo.config.get("state", {})
         self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
@@ -394,8 +396,8 @@ class State:  # pylint: disable=too-many-instance-attributes
         assert isinstance(path_info, str) or path_info.scheme == "local"
         path = os.fspath(path_info)
 
-        # NOTE: use os.path.exists instead of WorkingTree.exists
-        # WorkingTree.exists uses lexists() and will return True for broken
+        # NOTE: use os.path.exists instead of LocalTree.exists
+        # because it uses lexists() and will return True for broken
         # symlinks that we cannot stat() in get_mtime_and_size
         if not os.path.exists(path):
             return None

dvc/tree/__init__.py

@@ -0,0 +1,78 @@
+import posixpath
+from urllib.parse import urlparse
+
+from .azure import AzureTree
+from .gdrive import GDriveTree
+from .gs import GSTree
+from .hdfs import HDFSTree
+from .http import HTTPTree
+from .https import HTTPSTree
+from .local import LocalTree
+from .oss import OSSTree
+from .s3 import S3Tree
+from .ssh import SSHTree
+
+TREES = [
+    AzureTree,
+    GDriveTree,
+    GSTree,
+    HDFSTree,
+    HTTPTree,
+    HTTPSTree,
+    S3Tree,
+    SSHTree,
+    OSSTree,
+    # NOTE: LocalTree is the default
+]
+
+
+def _get_tree(remote_conf):
+    for tree_cls in TREES:
+        if tree_cls.supported(remote_conf):
+            return tree_cls
+    return LocalTree
+
+
+def _get_conf(repo, **kwargs):
+    name = kwargs.get("name")
+    if name:
+        remote_conf = repo.config["remote"][name.lower()]
+    else:
+        remote_conf = kwargs
+    return _resolve_remote_refs(repo.config, remote_conf)
+
+
+def _resolve_remote_refs(config, remote_conf):
+    # Support for cross referenced remotes.
+    # This will merge the settings, shadowing base ref with remote_conf.
+    # For example, having:
+    #
+    #       dvc remote add server ssh://localhost
+    #       dvc remote modify server user root
+    #       dvc remote modify server ask_password true
+    #
+    #       dvc remote add images remote://server/tmp/pictures
+    #       dvc remote modify images user alice
+    #       dvc remote modify images ask_password false
+    #       dvc remote modify images password asdf1234
+    #
+    # Results on a config dictionary like:
+    #
+    #       {
+    #           "url": "ssh://localhost/tmp/pictures",
+    #           "user": "alice",
+    #           "password": "asdf1234",
+    #           "ask_password": False,
+    #       }
+    parsed = urlparse(remote_conf["url"])
+    if parsed.scheme != "remote":
+        return remote_conf
+
+    base = config["remote"][parsed.netloc]
+    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
+    return {**base, **remote_conf, "url": url}
+
+
+def get_cloud_tree(repo, **kwargs):
+    remote_conf = _get_conf(repo, **kwargs)
+    return _get_tree(remote_conf)(repo, remote_conf)

dvc/tree/azure.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import threading
 from datetime import datetime, timedelta
 
@@ -6,49 +7,57 @@ from funcy import cached_property, wrap_prop
 
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree
 from dvc.scheme import Schemes
 
+from .base import BaseTree
+
 logger = logging.getLogger(__name__)
 
 
-class AzureRemoteTree(BaseRemoteTree):
+class AzureTree(BaseTree):
     scheme = Schemes.AZURE
     PATH_CLS = CloudURLInfo
     REQUIRES = {
         "azure-storage-blob": "azure.storage.blob",
-        "azure-cli-core": "azure.cli.core",
+        "knack": "knack",
     }
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
     LIST_OBJECT_PAGE_SIZE = 5000
 
     def __init__(self, repo, config):
-        from azure.cli.core import get_default_cli
-
         super().__init__(repo, config)
 
-        # NOTE: az_config takes care of env vars
-        az_config = get_default_cli().config
-
         url = config.get("url", "azure://")
         self.path_info = self.PATH_CLS(url)
 
         if not self.path_info.bucket:
-            container = az_config.get("storage", "container_name", None)
+            container = self._az_config.get("storage", "container_name", None)
             self.path_info = self.PATH_CLS(f"azure://{container}")
 
         self._conn_kwargs = {
-            opt: config.get(opt) or az_config.get("storage", opt, None)
+            opt: config.get(opt) or self._az_config.get("storage", opt, None)
             for opt in ["connection_string", "sas_token"]
         }
-        self._conn_kwargs["account_name"] = az_config.get(
+        self._conn_kwargs["account_name"] = self._az_config.get(
             "storage", "account", None
         )
-        self._conn_kwargs["account_key"] = az_config.get(
+        self._conn_kwargs["account_key"] = self._az_config.get(
             "storage", "key", None
         )
 
+    @cached_property
+    def _az_config(self):
+        # NOTE: ideally we would've used get_default_cli().config from
+        # azure.cli.core, but azure-cli-core has a lot of conflicts with other
+        # dependencies. So instead we are just use knack directly
+        from knack.config import CLIConfig
+
+        config_dir = os.getenv(
+            "AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure"))
+        )
+        return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE")
+
     @wrap_prop(threading.Lock())
     @cached_property
     def blob_service(self):
@@ -92,7 +101,7 @@ class AzureRemoteTree(BaseRemoteTree):
         )
         return download_url
 
-    def exists(self, path_info):
+    def exists(self, path_info, use_dvcignore=True):
         paths = self._list_paths(path_info.bucket, path_info.path)
         return any(path_info.path == path for path in paths)
 
@@ -113,6 +122,8 @@ class AzureRemoteTree(BaseRemoteTree):
             next_marker = blobs.next_marker
 
     def walk_files(self, path_info, **kwargs):
+        if not kwargs.pop("prefix", False):
+            path_info = path_info / ""
         for fname in self._list_paths(
             path_info.bucket, path_info.path, **kwargs
         ):

dvc/tree/base.py

@@ -0,0 +1,681 @@
+import itertools
+import json
+import logging
+import tempfile
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import partial
+from multiprocessing import cpu_count
+from operator import itemgetter
+from urllib.parse import urlparse
+
+from funcy import cached_property
+
+from dvc.exceptions import (
+    DvcException,
+    DvcIgnoreInCollectedDirError,
+    RemoteCacheRequiredError,
+)
+from dvc.ignore import DvcIgnore
+from dvc.path_info import PathInfo, URLInfo
+from dvc.progress import Tqdm
+from dvc.state import StateNoop
+from dvc.utils import tmp_fname
+from dvc.utils.fs import makedirs, move
+from dvc.utils.http import open_url
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteCmdError(DvcException):
+    def __init__(self, remote, cmd, ret, err):
+        super().__init__(
+            "{remote} command '{cmd}' finished with non-zero return code"
+            " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
+        )
+
+
+class RemoteActionNotImplemented(DvcException):
+    def __init__(self, action, scheme):
+        m = f"{action} is not supported for {scheme} remotes"
+        super().__init__(m)
+
+
+class RemoteMissingDepsError(DvcException):
+    pass
+
+
+class BaseTree:
+    scheme = "base"
+    REQUIRES = {}
+    PATH_CLS = URLInfo
+    JOBS = 4 * cpu_count()
+
+    PARAM_RELPATH = "relpath"
+    CHECKSUM_DIR_SUFFIX = ".dir"
+    HASH_JOBS = max(1, min(4, cpu_count() // 2))
+    DEFAULT_VERIFY = False
+    LIST_OBJECT_PAGE_SIZE = 1000
+    TRAVERSE_WEIGHT_MULTIPLIER = 5
+    TRAVERSE_PREFIX_LEN = 3
+    TRAVERSE_THRESHOLD_SIZE = 500000
+    CAN_TRAVERSE = True
+
+    CACHE_MODE = None
+    SHARED_MODE_MAP = {None: (None, None), "group": (None, None)}
+    PARAM_CHECKSUM = None
+
+    state = StateNoop()
+
+    def __init__(self, repo, config):
+        self.repo = repo
+        self.config = config
+
+        self._check_requires(config)
+
+        shared = config.get("shared")
+        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
+
+        self.verify = config.get("verify", self.DEFAULT_VERIFY)
+        self.path_info = None
+
+    @cached_property
+    def hash_jobs(self):
+        return (
+            self.config.get("hash_jobs")
+            or (self.repo and self.repo.config["core"].get("hash_jobs"))
+            or self.HASH_JOBS
+        )
+
+    @classmethod
+    def get_missing_deps(cls):
+        import importlib
+
+        missing = []
+        for package, module in cls.REQUIRES.items():
+            try:
+                importlib.import_module(module)
+            except ImportError:
+                missing.append(package)
+
+        return missing
+
+    def _check_requires(self, config):
+        missing = self.get_missing_deps()
+        if not missing:
+            return
+
+        url = config.get("url", f"{self.scheme}://")
+        msg = (
+            "URL '{}' is supported but requires these missing "
+            "dependencies: {}. If you have installed dvc using pip, "
+            "choose one of these options to proceed: \n"
+            "\n"
+            "    1) Install specific missing dependencies:\n"
+            "        pip install {}\n"
+            "    2) Install dvc package that includes those missing "
+            "dependencies: \n"
+            "        pip install 'dvc[{}]'\n"
+            "    3) Install dvc package with all possible "
+            "dependencies included: \n"
+            "        pip install 'dvc[all]'\n"
+            "\n"
+            "If you have installed dvc from a binary package and you "
+            "are still seeing this message, please report it to us "
+            "using https://github.com/iterative/dvc/issues. Thank you!"
+        ).format(url, missing, " ".join(missing), self.scheme)
+        raise RemoteMissingDepsError(msg)
+
+    @classmethod
+    def supported(cls, config):
+        if isinstance(config, (str, bytes)):
+            url = config
+        else:
+            url = config["url"]
+
+        # NOTE: silently skipping remote, calling code should handle that
+        parsed = urlparse(url)
+        return parsed.scheme == cls.scheme
+
+    @property
+    def file_mode(self):
+        return self._file_mode
+
+    @property
+    def dir_mode(self):
+        return self._dir_mode
+
+    @property
+    def cache(self):
+        return getattr(self.repo.cache, self.scheme)
+
+    def open(self, path_info, mode="r", encoding=None):
+        if hasattr(self, "_generate_download_url"):
+            func = self._generate_download_url  # noqa,pylint:disable=no-member
+            get_url = partial(func, path_info)
+            return open_url(get_url, mode=mode, encoding=encoding)
+
+        raise RemoteActionNotImplemented("open", self.scheme)
+
+    def exists(self, path_info, use_dvcignore=True):
+        raise NotImplementedError
+
+    # pylint: disable=unused-argument
+
+    def isdir(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return False
+
+    def isfile(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return True
+
+    def iscopy(self, path_info):
+        """Check if this file is an independent copy."""
+        return False  # We can't be sure by default
+
+    def walk_files(self, path_info, **kwargs):
+        """Return a generator with `PathInfo`s to all the files.
+
+        Optional kwargs:
+            prefix (bool): If true `path_info` will be treated as a prefix
+                rather than directory path.
+        """
+        raise NotImplementedError
+
+    def is_empty(self, path_info):
+        return False
+
+    def remove(self, path_info):
+        raise RemoteActionNotImplemented("remove", self.scheme)
+
+    def makedirs(self, path_info):
+        """Optional: Implement only if the remote needs to create
+        directories before copying/linking/moving data
+        """
+
+    def move(self, from_info, to_info, mode=None):
+        assert mode is None
+        self.copy(from_info, to_info)
+        self.remove(from_info)
+
+    def copy(self, from_info, to_info):
+        raise RemoteActionNotImplemented("copy", self.scheme)
+
+    def copy_fobj(self, fobj, to_info):
+        raise RemoteActionNotImplemented("copy_fobj", self.scheme)
+
+    def symlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("symlink", self.scheme)
+
+    def hardlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("hardlink", self.scheme)
+
+    def reflink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("reflink", self.scheme)
+
+    @staticmethod
+    def protect(path_info):
+        pass
+
+    def is_protected(self, path_info):
+        return False
+
+    # pylint: enable=unused-argument
+
+    @staticmethod
+    def unprotect(path_info):
+        pass
+
+    @classmethod
+    def is_dir_hash(cls, hash_):
+        if not hash_:
+            return False
+        return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
+
+    def get_hash(self, path_info, tree=None, **kwargs):
+        assert path_info and (
+            isinstance(path_info, str) or path_info.scheme == self.scheme
+        )
+
+        if not tree:
+            tree = self
+
+        if not tree.exists(path_info):
+            return None
+
+        if tree == self:
+            # pylint: disable=assignment-from-none
+            hash_ = self.state.get(path_info)
+        else:
+            hash_ = None
+        # If we have dir hash in state db, but dir cache file is lost,
+        # then we need to recollect the dir via .get_dir_hash() call below,
+        # see https://github.com/iterative/dvc/issues/2219 for context
+        if (
+            hash_
+            and self.is_dir_hash(hash_)
+            and not self.cache.tree.exists(
+                self.cache.tree.hash_to_path_info(hash_)
+            )
+        ):
+            hash_ = None
+
+        if hash_:
+            return hash_
+
+        if tree.isdir(path_info):
+            hash_ = self.get_dir_hash(path_info, tree, **kwargs)
+        else:
+            hash_ = tree.get_file_hash(path_info)
+
+        if hash_ and self.exists(path_info):
+            self.state.save(path_info, hash_)
+
+        return hash_
+
+    def get_file_hash(self, path_info):
+        raise NotImplementedError
+
+    def get_dir_hash(self, path_info, tree, **kwargs):
+        if not self.cache:
+            raise RemoteCacheRequiredError(path_info)
+
+        dir_info = self._collect_dir(path_info, tree, **kwargs)
+        return self._save_dir_info(dir_info, path_info)
+
+    def hash_to_path_info(self, hash_):
+        return self.path_info / hash_[0:2] / hash_[2:]
+
+    def path_to_hash(self, path):
+        parts = self.PATH_CLS(path).parts[-2:]
+
+        if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
+            raise ValueError(f"Bad cache file path '{path}'")
+
+        return "".join(parts)
+
+    def save_info(self, path_info, tree=None, **kwargs):
+        return {
+            self.PARAM_CHECKSUM: self.get_hash(path_info, tree=tree, **kwargs)
+        }
+
+    @staticmethod
+    def _calculate_hashes(file_infos, tree):
+        file_infos = list(file_infos)
+        with Tqdm(
+            total=len(file_infos),
+            unit="md5",
+            desc="Computing file/dir hashes (only done once)",
+        ) as pbar:
+            worker = pbar.wrap_fn(tree.get_file_hash)
+            with ThreadPoolExecutor(max_workers=tree.hash_jobs) as executor:
+                tasks = executor.map(worker, file_infos)
+                hashes = dict(zip(file_infos, tasks))
+        return hashes
+
+    def _collect_dir(self, path_info, tree, **kwargs):
+        file_infos = set()
+
+        for fname in tree.walk_files(path_info, **kwargs):
+            if DvcIgnore.DVCIGNORE_FILE == fname.name:
+                raise DvcIgnoreInCollectedDirError(fname.parent)
+
+            file_infos.add(fname)
+
+        hashes = {fi: self.state.get(fi) for fi in file_infos}
+        not_in_state = {fi for fi, hash_ in hashes.items() if hash_ is None}
+
+        new_hashes = self._calculate_hashes(not_in_state, tree)
+        hashes.update(new_hashes)
+
+        result = [
+            {
+                self.PARAM_CHECKSUM: hashes[fi],
+                # NOTE: this is lossy transformation:
+                #   "hey\there" -> "hey/there"
+                #   "hey/there" -> "hey/there"
+                # The latter is fine filename on Windows, which
+                # will transform to dir/file on back transform.
+                #
+                # Yes, this is a BUG, as long as we permit "/" in
+                # filenames on Windows and "\" on Unix
+                self.PARAM_RELPATH: fi.relative_to(path_info).as_posix(),
+            }
+            for fi in file_infos
+        ]
+
+        # Sorting the list by path to ensure reproducibility
+        return sorted(result, key=itemgetter(self.PARAM_RELPATH))
+
+    def _save_dir_info(self, dir_info, path_info):
+        hash_, tmp_info = self._get_dir_info_hash(dir_info)
+        new_info = self.cache.tree.hash_to_path_info(hash_)
+        if self.cache.changed_cache_file(hash_):
+            self.cache.tree.makedirs(new_info.parent)
+            self.cache.tree.move(
+                tmp_info, new_info, mode=self.cache.CACHE_MODE
+            )
+
+        if self.exists(path_info):
+            self.state.save(path_info, hash_)
+        self.state.save(new_info, hash_)
+
+        return hash_
+
+    def _get_dir_info_hash(self, dir_info):
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        with open(tmp, "w+") as fobj:
+            json.dump(dir_info, fobj, sort_keys=True)
+
+        tree = self.cache.tree
+        from_info = PathInfo(tmp)
+        to_info = tree.path_info / tmp_fname("")
+        tree.upload(from_info, to_info, no_progress_bar=True)
+
+        hash_ = tree.get_file_hash(to_info) + self.CHECKSUM_DIR_SUFFIX
+        return hash_, to_info
+
+    def upload(self, from_info, to_info, name=None, no_progress_bar=False):
+        if not hasattr(self, "_upload"):
+            raise RemoteActionNotImplemented("upload", self.scheme)
+
+        if to_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        if from_info.scheme != "local":
+            raise NotImplementedError
+
+        logger.debug("Uploading '%s' to '%s'", from_info, to_info)
+
+        name = name or from_info.name
+
+        self._upload(  # noqa, pylint: disable=no-member
+            from_info.fspath,
+            to_info,
+            name=name,
+            no_progress_bar=no_progress_bar,
+        )
+
+    def download(
+        self,
+        from_info,
+        to_info,
+        name=None,
+        no_progress_bar=False,
+        file_mode=None,
+        dir_mode=None,
+    ):
+        if not hasattr(self, "_download"):
+            raise RemoteActionNotImplemented("download", self.scheme)
+
+        if from_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        if to_info.scheme == self.scheme != "local":
+            self.copy(from_info, to_info)
+            return 0
+
+        if to_info.scheme != "local":
+            raise NotImplementedError
+
+        if self.isdir(from_info):
+            return self._download_dir(
+                from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+            )
+        return self._download_file(
+            from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+        )
+
+    def _download_dir(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        from_infos = list(self.walk_files(from_info))
+        to_infos = (
+            to_info / info.relative_to(from_info) for info in from_infos
+        )
+
+        with Tqdm(
+            total=len(from_infos),
+            desc="Downloading directory",
+            unit="Files",
+            disable=no_progress_bar,
+        ) as pbar:
+            download_files = pbar.wrap_fn(
+                partial(
+                    self._download_file,
+                    name=name,
+                    no_progress_bar=True,
+                    file_mode=file_mode,
+                    dir_mode=dir_mode,
+                )
+            )
+            with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
+                futures = [
+                    executor.submit(download_files, from_info, to_info)
+                    for from_info, to_info in zip(from_infos, to_infos)
+                ]
+
+                # NOTE: unlike pulling/fetching cache, where we need to
+                # download everything we can, not raising an error here might
+                # turn very ugly, as the user might think that he has
+                # downloaded a complete directory, while having a partial one,
+                # which might cause unexpected results in his pipeline.
+                for future in as_completed(futures):
+                    # NOTE: executor won't let us raise until all futures that
+                    # it has are finished, so we need to cancel them ourselves
+                    # before re-raising.
+                    exc = future.exception()
+                    if exc:
+                        for entry in futures:
+                            entry.cancel()
+                        raise exc
+
+    def _download_file(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
+
+        logger.debug("Downloading '%s' to '%s'", from_info, to_info)
+        name = name or to_info.name
+
+        tmp_file = tmp_fname(to_info)
+
+        self._download(  # noqa, pylint: disable=no-member
+            from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
+        )
+
+        move(tmp_file, to_info, mode=file_mode)
+
+    def list_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            if len(prefix) > 2:
+                path_info = self.path_info / prefix[:2] / prefix[2:]
+            else:
+                path_info = self.path_info / prefix[:2]
+            prefix = True
+        else:
+            path_info = self.path_info
+            prefix = False
+        if progress_callback:
+            for file_info in self.walk_files(path_info, prefix=prefix):
+                progress_callback()
+                yield file_info.path
+        else:
+            yield from self.walk_files(path_info, prefix=prefix)
+
+    def list_hashes(self, prefix=None, progress_callback=None):
+        """Iterate over hashes in this tree.
+
+        If `prefix` is specified, only hashes which begin with `prefix`
+        will be returned.
+        """
+        for path in self.list_paths(prefix, progress_callback):
+            try:
+                yield self.path_to_hash(path)
+            except ValueError:
+                logger.debug(
+                    "'%s' doesn't look like a cache file, skipping", path
+                )
+
+    def all(self, jobs=None, name=None):
+        """Iterate over all hashes in this tree.
+
+        Hashes will be fetched in parallel threads according to prefix
+        (except for small remotes) and a progress bar will be displayed.
+        """
+        logger.debug(
+            "Fetching all hashes from '{}'".format(
+                name if name else "remote cache"
+            )
+        )
+
+        if not self.CAN_TRAVERSE:
+            return self.list_hashes()
+
+        remote_size, remote_hashes = self.estimate_remote_size(name=name)
+        return self.list_hashes_traverse(
+            remote_size, remote_hashes, jobs, name
+        )
+
+    def _hashes_with_limit(self, limit, prefix=None, progress_callback=None):
+        count = 0
+        for hash_ in self.list_hashes(prefix, progress_callback):
+            yield hash_
+            count += 1
+            if count > limit:
+                logger.debug(
+                    "`list_hashes()` returned max '{}' hashes, "
+                    "skipping remaining results".format(limit)
+                )
+                return
+
+    def _max_estimation_size(self, hashes):
+        # Max remote size allowed for us to use traverse method
+        return max(
+            self.TRAVERSE_THRESHOLD_SIZE,
+            len(hashes)
+            / self.TRAVERSE_WEIGHT_MULTIPLIER
+            * self.LIST_OBJECT_PAGE_SIZE,
+        )
+
+    def estimate_remote_size(self, hashes=None, name=None):
+        """Estimate tree size based on number of entries beginning with
+        "00..." prefix.
+        """
+        prefix = "0" * self.TRAVERSE_PREFIX_LEN
+        total_prefixes = pow(16, self.TRAVERSE_PREFIX_LEN)
+        if hashes:
+            max_hashes = self._max_estimation_size(hashes)
+        else:
+            max_hashes = None
+
+        with Tqdm(
+            desc="Estimating size of "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            unit="file",
+        ) as pbar:
+
+            def update(n=1):
+                pbar.update(n * total_prefixes)
+
+            if max_hashes:
+                hashes = self._hashes_with_limit(
+                    max_hashes / total_prefixes, prefix, update
+                )
+            else:
+                hashes = self.list_hashes(prefix, update)
+
+            remote_hashes = set(hashes)
+            if remote_hashes:
+                remote_size = total_prefixes * len(remote_hashes)
+            else:
+                remote_size = total_prefixes
+            logger.debug(f"Estimated remote size: {remote_size} files")
+        return remote_size, remote_hashes
+
+    def list_hashes_traverse(
+        self, remote_size, remote_hashes, jobs=None, name=None
+    ):
+        """Iterate over all hashes found in this tree.
+        Hashes are fetched in parallel according to prefix, except in
+        cases where the remote size is very small.
+
+        All hashes from the remote (including any from the size
+        estimation step passed via the `remote_hashes` argument) will be
+        returned.
+
+        NOTE: For large remotes the list of hashes will be very
+        big(e.g. 100M entries, md5 for each is 32 bytes, so ~3200Mb list)
+        and we don't really need all of it at the same time, so it makes
+        sense to use a generator to gradually iterate over it, without
+        keeping all of it in memory.
+        """
+        num_pages = remote_size / self.LIST_OBJECT_PAGE_SIZE
+        if num_pages < 256 / self.JOBS:
+            # Fetching prefixes in parallel requires at least 255 more
+            # requests, for small enough remotes it will be faster to fetch
+            # entire cache without splitting it into prefixes.
+            #
+            # NOTE: this ends up re-fetching hashes that were already
+            # fetched during remote size estimation
+            traverse_prefixes = [None]
+            initial = 0
+        else:
+            yield from remote_hashes
+            initial = len(remote_hashes)
+            traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
+            if self.TRAVERSE_PREFIX_LEN > 2:
+                traverse_prefixes += [
+                    "{0:0{1}x}".format(i, self.TRAVERSE_PREFIX_LEN)
+                    for i in range(1, pow(16, self.TRAVERSE_PREFIX_LEN - 2))
+                ]
+        with Tqdm(
+            desc="Querying "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            total=remote_size,
+            initial=initial,
+            unit="file",
+        ) as pbar:
+
+            def list_with_update(prefix):
+                return list(
+                    self.list_hashes(
+                        prefix=prefix, progress_callback=pbar.update
+                    )
+                )
+
+            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
+                in_remote = executor.map(list_with_update, traverse_prefixes,)
+                yield from itertools.chain.from_iterable(in_remote)
+
+    def list_hashes_exists(self, hashes, jobs=None, name=None):
+        """Return list of the specified hashes which exist in this tree.
+        Hashes will be queried individually.
+        """
+        logger.debug(
+            "Querying {} hashes via object_exists".format(len(hashes))
+        )
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(hashes),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(path_info):
+                ret = self.exists(path_info)
+                pbar.update_msg(str(path_info))
+                return ret
+
+            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
+                path_infos = map(self.hash_to_path_info, hashes)
+                in_remote = executor.map(exists_with_progress, path_infos)
+                ret = list(itertools.compress(hashes, in_remote))
+                return ret
+
+    def _remove_unpacked_dir(self, hash_):
+        pass

dvc/tree/gdrive.py

@@ -14,11 +14,12 @@ from funcy.py3 import cat
 from dvc.exceptions import DvcException, FileMissingError
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree
 from dvc.scheme import Schemes
 from dvc.utils import format_link, tmp_fname
 from dvc.utils.stream import IterStream
 
+from .base import BaseTree
+
 logger = logging.getLogger(__name__)
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
 
@@ -84,7 +85,7 @@ class GDriveURLInfo(CloudURLInfo):
         self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
 
 
-class GDriveRemoteTree(BaseRemoteTree):
+class GDriveTree(BaseTree):
     scheme = Schemes.GDRIVE
     PATH_CLS = GDriveURLInfo
     REQUIRES = {"pydrive2": "pydrive2"}
@@ -129,7 +130,7 @@ class GDriveRemoteTree(BaseRemoteTree):
         self._validate_config()
         self._gdrive_user_credentials_path = (
             tmp_fname(os.path.join(self.repo.tmp_dir, ""))
-            if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA)
+            if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA)
             else config.get(
                 "gdrive_user_credentials_file",
                 os.path.join(
@@ -171,8 +172,8 @@ class GDriveRemoteTree(BaseRemoteTree):
         Useful for tests, exception messages, etc. Returns either env variable
         name if it's set or actual path to the credentials file.
         """
-        if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA):
-            return GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA
+        if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
+            return GDriveTree.GDRIVE_CREDENTIALS_DATA
         if os.path.exists(self._gdrive_user_credentials_path):
             return self._gdrive_user_credentials_path
         return None
@@ -186,7 +187,7 @@ class GDriveRemoteTree(BaseRemoteTree):
         DVC config client id or secret but forgets to remove the cached
         credentials file.
         """
-        if not os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA):
+        if not os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
             if (
                 settings["client_config"]["client_id"]
                 != auth.credentials.client_id
@@ -209,11 +210,9 @@ class GDriveRemoteTree(BaseRemoteTree):
         from pydrive2.auth import GoogleAuth
         from pydrive2.drive import GoogleDrive
 
-        if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA):
+        if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
             with open(self._gdrive_user_credentials_path, "w") as cred_file:
-                cred_file.write(
-                    os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA)
-                )
+                cred_file.write(os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA))
 
         auth_settings = {
             "client_config_backend": "settings",
@@ -259,7 +258,7 @@ class GDriveRemoteTree(BaseRemoteTree):
                 gauth.ServiceAuth()
             else:
                 gauth.CommandLineAuth()
-                GDriveRemoteTree._validate_credentials(gauth, auth_settings)
+                GDriveTree._validate_credentials(gauth, auth_settings)
 
         # Handle AuthenticationError, RefreshError and other auth failures
         # It's hard to come up with a narrow exception, since PyDrive throws
@@ -268,7 +267,7 @@ class GDriveRemoteTree(BaseRemoteTree):
         except Exception as exc:
             raise GDriveAuthError(self.credentials_location) from exc
         finally:
-            if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA):
+            if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
                 os.remove(self._gdrive_user_credentials_path)
 
         return GoogleDrive(gauth)
@@ -382,6 +381,9 @@ class GDriveRemoteTree(BaseRemoteTree):
         # it does not create a file on the remote
         gdrive_file = self._drive.CreateFile(param)
 
+        gdrive_file.FetchMetadata(fields="fileSize")
+        size = gdrive_file["fileSize"]
+
         with Tqdm(
             desc=progress_desc,
             disable=no_progress_bar,
@@ -389,7 +391,13 @@ class GDriveRemoteTree(BaseRemoteTree):
             # explicit `bar_format` as `total` will be set by `update_to`
             bar_format=Tqdm.BAR_FMT_DEFAULT,
         ) as pbar:
-            gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
+            if size:
+                gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
+            else:
+                # PyDrive doesn't like downloading empty files
+                # https://github.com/iterative/dvc/issues/4286
+                with open(to_file, "w"):
+                    pass
 
     @contextmanager
     @_gdrive_retry
@@ -523,7 +531,7 @@ class GDriveRemoteTree(BaseRemoteTree):
         assert not create
         raise FileMissingError(path_info, hint)
 
-    def exists(self, path_info):
+    def exists(self, path_info, use_dvcignore=True):
         try:
             self._get_item_id(path_info)
         except FileMissingError:
@@ -553,7 +561,8 @@ class GDriveRemoteTree(BaseRemoteTree):
             )
 
     def walk_files(self, path_info, **kwargs):
-        if path_info == self.path_info:
+        use_prefix = kwargs.pop("prefix", False)
+        if path_info == self.path_info or not use_prefix:
             prefix = None
         else:
             prefix = path_info.path

dvc/tree/git.py

@@ -3,8 +3,10 @@ import io
 import os
 import stat
 
+from funcy import cached_property
+
 from dvc.exceptions import DvcException
-from dvc.scm.tree import BaseTree
+from dvc.tree.base import BaseTree
 from dvc.utils import relpath
 
 # see git-fast-import(1)
@@ -19,24 +21,37 @@ def _item_basename(item):
     return os.path.basename(item.path)
 
 
-class GitTree(BaseTree):
+class GitTree(BaseTree):  # pylint:disable=abstract-method
     """Proxies the repo file access methods to Git objects"""
 
-    def __init__(self, git, rev):
+    def __init__(self, git, rev, use_dvcignore=False, dvcignore_root=None):
         """Create GitTree instance
 
         Args:
             git (dvc.scm.Git):
             branch:
         """
+        super().__init__(None, {})
         self.git = git
         self.rev = rev
+        self.use_dvcignore = use_dvcignore
+        self.dvcignore_root = dvcignore_root
 
     @property
     def tree_root(self):
         return self.git.working_dir
 
-    def open(self, path, mode="r", encoding="utf-8"):
+    @cached_property
+    def dvcignore(self):
+        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
+
+        root = self.dvcignore_root or self.tree_root
+        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
+        return cls(self, root)
+
+    def open(
+        self, path, mode="r", encoding="utf-8"
+    ):  # pylint: disable=arguments-differ
         assert mode in {"r", "rb"}
 
         relative_path = relpath(path, self.git.working_dir)
@@ -57,21 +72,34 @@ class GitTree(BaseTree):
             return io.BytesIO(data)
         return io.StringIO(data.decode(encoding))
 
-    def exists(self, path):
-        return self._git_object_by_path(path) is not None
+    def exists(
+        self, path, use_dvcignore=True
+    ):  # pylint: disable=arguments-differ
+        if self._git_object_by_path(path) is None:
+            return False
+        if not use_dvcignore:
+            return True
 
-    def isdir(self, path):
+        return not self.dvcignore.is_ignored_file(
+            path
+        ) and not self.dvcignore.is_ignored_dir(path)
+
+    def isdir(self, path):  # pylint: disable=arguments-differ
         obj = self._git_object_by_path(path)
         if obj is None:
             return False
-        return obj.mode == GIT_MODE_DIR
+        if obj.mode != GIT_MODE_DIR:
+            return False
+        return not self.dvcignore.is_ignored_dir(path)
 
-    def isfile(self, path):
+    def isfile(self, path):  # pylint: disable=arguments-differ
         obj = self._git_object_by_path(path)
         if obj is None:
             return False
         # according to git-fast-import(1) file mode could be 644 or 755
-        return obj.mode & GIT_MODE_FILE == GIT_MODE_FILE
+        if obj.mode & GIT_MODE_FILE != GIT_MODE_FILE:
+            return False
+        return not self.dvcignore.is_ignored_file(path)
 
     @staticmethod
     def _is_tree_and_contains(obj, path):
@@ -128,7 +156,7 @@ class GitTree(BaseTree):
         if not topdown:
             yield os.path.normpath(tree.abspath), dirs, nondirs
 
-    def walk(self, top, topdown=True, onerror=None):
+    def walk(self, top, topdown=True, onerror=None, use_dvcignore=True):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
@@ -145,7 +173,12 @@ class GitTree(BaseTree):
                 onerror(NotADirectoryError(top))
             return
 
-        yield from self._walk(tree, topdown=topdown)
+        for root, dirs, files in self._walk(tree, topdown=topdown):
+            if use_dvcignore:
+                dirs[:], files[:] = self.dvcignore(
+                    os.path.abspath(root), dirs, files
+                )
+            yield root, dirs, files
 
     def isexec(self, path):
         if not self.exists(path):
@@ -185,7 +218,13 @@ class GitTree(BaseTree):
         )
 
     @property
-    def hash_jobs(self):
+    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
         # NOTE: gitpython is not threadsafe. See
         # https://github.com/iterative/dvc/issues/4079
         return 1
+
+    def walk_files(self, top):  # pylint: disable=arguments-differ
+        for root, _, files in self.walk(top):
+            for file in files:
+                # NOTE: os.path.join is ~5.5 times slower
+                yield f"{root}{os.sep}{file}"

dvc/tree/gs.py

@@ -9,9 +9,10 @@ from funcy import cached_property, wrap_prop
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree
 from dvc.scheme import Schemes
 
+from .base import BaseTree
+
 logger = logging.getLogger(__name__)
 
 
@@ -65,7 +66,7 @@ def _upload_to_bucket(
             blob.upload_from_file(wrapped)
 
 
-class GSRemoteTree(BaseRemoteTree):
+class GSTree(BaseTree):
     scheme = Schemes.GS
     PATH_CLS = CloudURLInfo
     REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
@@ -118,7 +119,7 @@ class GSRemoteTree(BaseRemoteTree):
         )
         return signing_credentials.signer.sign(blob)
 
-    def exists(self, path_info):
+    def exists(self, path_info, use_dvcignore=True):
         """Check if the blob exists. If it does not exist,
         it could be a part of a directory path.
 
@@ -144,7 +145,9 @@ class GSRemoteTree(BaseRemoteTree):
             yield blob.name
 
     def walk_files(self, path_info, **kwargs):
-        for fname in self._list_paths(path_info / "", **kwargs):
+        if not kwargs.pop("prefix", False):
+            path_info = path_info / ""
+        for fname in self._list_paths(path_info, **kwargs):
             # skip nested empty directories
             if fname.endswith("/"):
                 continue

dvc/tree/hdfs.py

@@ -10,13 +10,13 @@ from urllib.parse import urlparse
 from dvc.scheme import Schemes
 from dvc.utils import fix_env, tmp_fname
 
-from .base import BaseRemoteTree, RemoteCmdError
+from .base import BaseTree, RemoteCmdError
 from .pool import get_connection
 
 logger = logging.getLogger(__name__)
 
 
-class HDFSRemoteTree(BaseRemoteTree):
+class HDFSTree(BaseTree):
     scheme = Schemes.HDFS
     REQUIRES = {"pyarrow": "pyarrow"}
     REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
@@ -72,7 +72,7 @@ class HDFSRemoteTree(BaseRemoteTree):
                 raise FileNotFoundError(*e.args)
             raise
 
-    def exists(self, path_info):
+    def exists(self, path_info, use_dvcignore=True):
         assert not isinstance(path_info, list)
         assert path_info.scheme == "hdfs"
         with self.hdfs(path_info) as hdfs:
@@ -165,7 +165,7 @@ class HDFSRemoteTree(BaseRemoteTree):
         # NOTE: pyarrow doesn't support checksum, so we need to use hadoop
         regex = r".*\t.*\t(?P<checksum>.*)"
         stdout = self.hadoop_fs(
-            f"checksum {path_info.path}", user=path_info.user
+            f"checksum {path_info.url}", user=path_info.user
         )
         return self._group(regex, stdout, "checksum")
 

dvc/tree/http.py

@@ -8,9 +8,10 @@ import dvc.prompt as prompt
 from dvc.exceptions import DvcException, HTTPError
 from dvc.path_info import HTTPURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree
 from dvc.scheme import Schemes
 
+from .base import BaseTree
+
 logger = logging.getLogger(__name__)
 
 
@@ -23,7 +24,7 @@ def ask_password(host, user):
     )
 
 
-class HTTPRemoteTree(BaseRemoteTree):  # pylint:disable=abstract-method
+class HTTPTree(BaseTree):  # pylint:disable=abstract-method
     scheme = Schemes.HTTP
     PATH_CLS = HTTPURLInfo
     PARAM_CHECKSUM = "etag"
@@ -121,7 +122,7 @@ class HTTPRemoteTree(BaseRemoteTree):  # pylint:disable=abstract-method
         except requests.exceptions.RequestException:
             raise DvcException(f"could not perform a {method} request")
 
-    def exists(self, path_info):
+    def exists(self, path_info, use_dvcignore=True):
         return bool(self.request("HEAD", path_info.url))
 
     def get_file_hash(self, path_info):

dvc/tree/https.py

@@ -0,0 +1,7 @@
+from dvc.scheme import Schemes
+
+from .http import HTTPTree
+
+
+class HTTPSTree(HTTPTree):  # pylint:disable=abstract-method
+    scheme = Schemes.HTTPS

dvc/tree/local.py

@@ -0,0 +1,348 @@
+import errno
+import logging
+import os
+import stat
+
+from funcy import cached_property
+from shortuuid import uuid
+
+from dvc.exceptions import DvcException
+from dvc.path_info import PathInfo
+from dvc.scheme import Schemes
+from dvc.system import System
+from dvc.utils import file_md5, relpath, tmp_fname
+from dvc.utils.fs import (
+    copy_fobj_to_file,
+    copyfile,
+    makedirs,
+    move,
+    remove,
+    walk_files,
+)
+
+from .base import BaseTree
+
+logger = logging.getLogger(__name__)
+
+
+class LocalTree(BaseTree):
+    scheme = Schemes.LOCAL
+    PATH_CLS = PathInfo
+    PARAM_CHECKSUM = "md5"
+    PARAM_PATH = "path"
+    TRAVERSE_PREFIX_LEN = 2
+    UNPACKED_DIR_SUFFIX = ".unpacked"
+
+    CACHE_MODE = 0o444
+    SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
+
+    def __init__(self, repo, config, use_dvcignore=False, dvcignore_root=None):
+        super().__init__(repo, config)
+        url = config.get("url")
+        self.path_info = self.PATH_CLS(url) if url else None
+        self.use_dvcignore = use_dvcignore
+        self.dvcignore_root = dvcignore_root
+
+    @property
+    def tree_root(self):
+        return self.config.get("url")
+
+    @property
+    def state(self):
+        from dvc.state import StateNoop
+
+        return self.repo.state if self.repo else StateNoop()
+
+    @cached_property
+    def dvcignore(self):
+        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
+
+        root = self.dvcignore_root or self.tree_root
+        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
+        return cls(self, root)
+
+    @staticmethod
+    def open(path_info, mode="r", encoding=None):
+        return open(path_info, mode=mode, encoding=encoding)
+
+    def exists(self, path_info, use_dvcignore=True):
+        assert isinstance(path_info, str) or path_info.scheme == "local"
+        if self.repo:
+            ret = os.path.lexists(path_info)
+        else:
+            ret = os.path.exists(path_info)
+        if not ret:
+            return False
+        if not use_dvcignore:
+            return True
+
+        return not self.dvcignore.is_ignored_file(
+            path_info
+        ) and not self.dvcignore.is_ignored_dir(path_info)
+
+    def isfile(self, path_info):
+        if not os.path.isfile(path_info):
+            return False
+
+        return not self.dvcignore.is_ignored_file(path_info)
+
+    def isdir(self, path_info):
+        if not os.path.isdir(path_info):
+            return False
+
+        return not self.dvcignore.is_ignored_dir(path_info)
+
+    def iscopy(self, path_info):
+        return not (
+            System.is_symlink(path_info) or System.is_hardlink(path_info)
+        )
+
+    def walk(self, top, topdown=True, onerror=None, use_dvcignore=True):
+        """Directory tree generator.
+
+        See `os.walk` for the docs. Differences:
+        - no support for symlinks
+        """
+        for root, dirs, files in os.walk(
+            top, topdown=topdown, onerror=onerror
+        ):
+            if use_dvcignore:
+                dirs[:], files[:] = self.dvcignore(
+                    os.path.abspath(root), dirs, files
+                )
+
+            yield os.path.normpath(root), dirs, files
+
+    def walk_files(self, path_info, **kwargs):
+        for root, _, files in self.walk(path_info):
+            for file in files:
+                # NOTE: os.path.join is ~5.5 times slower
+                yield PathInfo(f"{root}{os.sep}{file}")
+
+    def is_empty(self, path_info):
+        path = path_info.fspath
+
+        if self.isfile(path_info) and os.path.getsize(path) == 0:
+            return True
+
+        if self.isdir(path_info) and len(os.listdir(path)) == 0:
+            return True
+
+        return False
+
+    def remove(self, path_info):
+        if isinstance(path_info, PathInfo):
+            if path_info.scheme != "local":
+                raise NotImplementedError
+            path = path_info.fspath
+        else:
+            path = path_info
+
+        if self.exists(path):
+            remove(path)
+
+    def makedirs(self, path_info):
+        makedirs(path_info, exist_ok=True, mode=self.dir_mode)
+
+    def isexec(self, path):
+        mode = os.stat(path).st_mode
+        return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+    @staticmethod
+    def stat(path):
+        return os.stat(path)
+
+    def move(self, from_info, to_info, mode=None):
+        if from_info.scheme != "local" or to_info.scheme != "local":
+            raise NotImplementedError
+
+        self.makedirs(to_info.parent)
+
+        if mode is None:
+            if self.isfile(from_info):
+                mode = self.file_mode
+            else:
+                mode = self.dir_mode
+
+        move(from_info, to_info, mode=mode)
+
+    def copy(self, from_info, to_info):
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        try:
+            System.copy(from_info, tmp_info)
+            os.chmod(tmp_info, self.file_mode)
+            os.rename(tmp_info, to_info)
+        except Exception:
+            self.remove(tmp_info)
+            raise
+
+    def copy_fobj(self, fobj, to_info):
+        self.makedirs(to_info.parent)
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        try:
+            copy_fobj_to_file(fobj, tmp_info)
+            os.chmod(tmp_info, self.file_mode)
+            os.rename(tmp_info, to_info)
+        except Exception:
+            self.remove(tmp_info)
+            raise
+
+    @staticmethod
+    def symlink(from_info, to_info):
+        System.symlink(from_info, to_info)
+
+    @staticmethod
+    def is_symlink(path_info):
+        return System.is_symlink(path_info)
+
+    def hardlink(self, from_info, to_info):
+        # If there are a lot of empty files (which happens a lot in datasets),
+        # and the cache type is `hardlink`, we might reach link limits and
+        # will get something like: `too many links error`
+        #
+        # This is because all those empty files will have the same hash
+        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
+        # linked to the same file in the cache.
+        #
+        # From https://en.wikipedia.org/wiki/Hard_link
+        #   * ext4 limits the number of hard links on a file to 65,000
+        #   * Windows with NTFS has a limit of 1024 hard links on a file
+        #
+        # That's why we simply create an empty file rather than a link.
+        if self.getsize(from_info) == 0:
+            self.open(to_info, "w").close()
+
+            logger.debug(
+                "Created empty file: {src} -> {dest}".format(
+                    src=str(from_info), dest=str(to_info)
+                )
+            )
+            return
+
+        System.hardlink(from_info, to_info)
+
+    @staticmethod
+    def is_hardlink(path_info):
+        return System.is_hardlink(path_info)
+
+    def reflink(self, from_info, to_info):
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        System.reflink(from_info, tmp_info)
+        # NOTE: reflink has its own separate inode, so you can set permissions
+        # that are different from the source.
+        os.chmod(tmp_info, self.file_mode)
+        os.rename(tmp_info, to_info)
+
+    def _unprotect_file(self, path):
+        if System.is_symlink(path) or System.is_hardlink(path):
+            logger.debug(f"Unprotecting '{path}'")
+            tmp = os.path.join(os.path.dirname(path), "." + uuid())
+
+            # The operations order is important here - if some application
+            # would access the file during the process of copyfile then it
+            # would get only the part of file. So, at first, the file should be
+            # copied with the temporary name, and then original file should be
+            # replaced by new.
+            copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
+            remove(path)
+            os.rename(tmp, path)
+
+        else:
+            logger.debug(
+                "Skipping copying for '{}', since it is not "
+                "a symlink or a hardlink.".format(path)
+            )
+
+        os.chmod(path, self.file_mode)
+
+    def _unprotect_dir(self, path):
+        for fname in self.walk_files(path):
+            self._unprotect_file(fname)
+
+    def unprotect(self, path_info):
+        path = path_info.fspath
+        if not os.path.exists(path):
+            raise DvcException(f"can't unprotect non-existing data '{path}'")
+
+        if os.path.isdir(path):
+            self._unprotect_dir(path)
+        else:
+            self._unprotect_file(path)
+
+    def protect(self, path_info):
+        path = os.fspath(path_info)
+        mode = self.CACHE_MODE
+
+        try:
+            os.chmod(path, mode)
+        except OSError as exc:
+            # There is nothing we need to do in case of a read-only file system
+            if exc.errno == errno.EROFS:
+                return
+
+            # In shared cache scenario, we might not own the cache file, so we
+            # need to check if cache file is already protected.
+            if exc.errno not in [errno.EPERM, errno.EACCES]:
+                raise
+
+            actual = stat.S_IMODE(os.stat(path).st_mode)
+            if actual != mode:
+                raise
+
+    def is_protected(self, path_info):
+        try:
+            mode = os.stat(path_info).st_mode
+        except FileNotFoundError:
+            return False
+
+        return stat.S_IMODE(mode) == self.CACHE_MODE
+
+    def get_file_hash(self, path_info):
+        return file_md5(path_info)[0]
+
+    @staticmethod
+    def getsize(path_info):
+        return os.path.getsize(path_info)
+
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+    ):
+        makedirs(to_info.parent, exist_ok=True)
+
+        tmp_file = tmp_fname(to_info)
+        copyfile(
+            from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
+        )
+
+        self.protect(tmp_file)
+        os.rename(tmp_file, to_info)
+
+    @staticmethod
+    def _download(
+        from_info, to_file, name=None, no_progress_bar=False, **_kwargs
+    ):
+        copyfile(
+            from_info, to_file, no_progress_bar=no_progress_bar, name=name
+        )
+
+    def list_paths(self, prefix=None, progress_callback=None):
+        assert self.path_info is not None
+        if prefix:
+            path_info = self.path_info / prefix[:2]
+            if not self.exists(path_info):
+                return
+        else:
+            path_info = self.path_info
+        # NOTE: use utils.fs walk_files since tree.walk_files will not follow
+        # symlinks
+        if progress_callback:
+            for path in walk_files(path_info):
+                progress_callback()
+                yield path
+        else:
+            yield from walk_files(path_info)
+
+    def _remove_unpacked_dir(self, hash_):
+        info = self.hash_to_path_info(hash_)
+        path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
+        self.remove(path_info)

dvc/tree/oss.py

@@ -6,13 +6,14 @@ from funcy import cached_property, wrap_prop
 
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree
 from dvc.scheme import Schemes
 
+from .base import BaseTree
+
 logger = logging.getLogger(__name__)
 
 
-class OSSRemoteTree(BaseRemoteTree):  # pylint:disable=abstract-method
+class OSSTree(BaseTree):  # pylint:disable=abstract-method
     """
     oss2 document:
     https://www.alibabacloud.com/help/doc-detail/32026.htm
@@ -87,7 +88,7 @@ class OSSRemoteTree(BaseRemoteTree):  # pylint:disable=abstract-method
 
         return self.oss_service.sign_url("GET", path_info.path, expires)
 
-    def exists(self, path_info):
+    def exists(self, path_info, use_dvcignore=True):
         paths = self._list_paths(path_info)
         return any(path_info.path == path for path in paths)
 
@@ -100,6 +101,8 @@ class OSSRemoteTree(BaseRemoteTree):  # pylint:disable=abstract-method
             yield blob.key
 
     def walk_files(self, path_info, **kwargs):
+        if not kwargs.pop("prefix", False):
+            path_info = path_info / ""
         for fname in self._list_paths(path_info):
             if fname.endswith("/"):
                 continue

dvc/tree/s3.py

@@ -8,13 +8,14 @@ from dvc.config import ConfigError
 from dvc.exceptions import DvcException, ETagMismatchError
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree
 from dvc.scheme import Schemes
 
+from .base import BaseTree
+
 logger = logging.getLogger(__name__)
 
 
-class S3RemoteTree(BaseRemoteTree):
+class S3Tree(BaseTree):
     scheme = Schemes.S3
     PATH_CLS = CloudURLInfo
     REQUIRES = {"boto3": "boto3"}
@@ -53,6 +54,9 @@ class S3RemoteTree(BaseRemoteTree):
 
         self._append_aws_grants_to_extra_args(config)
 
+        self.access_key_id = config.get("access_key_id")
+        self.secret_access_key = config.get("secret_access_key")
+
         shared_creds = config.get("credentialpath")
         if shared_creds:
             os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
@@ -62,9 +66,14 @@ class S3RemoteTree(BaseRemoteTree):
     def s3(self):
         import boto3
 
-        session = boto3.session.Session(
-            profile_name=self.profile, region_name=self.region
-        )
+        session_opts = dict(profile_name=self.profile, region_name=self.region)
+
+        if self.access_key_id:
+            session_opts["aws_access_key_id"] = self.access_key_id
+        if self.secret_access_key:
+            session_opts["aws_secret_access_key"] = self.secret_access_key
+
+        session = boto3.session.Session(**session_opts)
 
         return session.client(
             "s3", endpoint_url=self.endpoint_url, use_ssl=self.use_ssl
@@ -122,7 +131,7 @@ class S3RemoteTree(BaseRemoteTree):
             ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
         )
 
-    def exists(self, path_info):
+    def exists(self, path_info, use_dvcignore=True):
         """Check if the blob exists. If it does not exist,
         it could be a part of a directory path.
 
@@ -187,7 +196,9 @@ class S3RemoteTree(BaseRemoteTree):
         )
 
     def walk_files(self, path_info, **kwargs):
-        for fname in self._list_paths(path_info / "", **kwargs):
+        if not kwargs.pop("prefix", False):
+            path_info = path_info / ""
+        for fname in self._list_paths(path_info, **kwargs):
             if fname.endswith("/"):
                 continue
 
@@ -227,7 +238,7 @@ class S3RemoteTree(BaseRemoteTree):
         parts = []
         byte_position = 0
         for i in range(1, n_parts + 1):
-            obj = S3RemoteTree.get_head_object(
+            obj = S3Tree.get_head_object(
                 s3, from_info.bucket, from_info.path, PartNumber=i
             )
             part_size = obj["ContentLength"]
@@ -282,9 +293,7 @@ class S3RemoteTree(BaseRemoteTree):
         # object is transfered in the same chunks as it was originally.
         from boto3.s3.transfer import TransferConfig
 
-        obj = S3RemoteTree.get_head_object(
-            s3, from_info.bucket, from_info.path
-        )
+        obj = S3Tree.get_head_object(s3, from_info.bucket, from_info.path)
         etag = obj["ETag"].strip('"')
         size = obj["ContentLength"]
 
@@ -304,7 +313,7 @@ class S3RemoteTree(BaseRemoteTree):
                 Config=TransferConfig(multipart_threshold=size + 1),
             )
 
-        cached_etag = S3RemoteTree.get_etag(s3, to_info.bucket, to_info.path)
+        cached_etag = S3Tree.get_etag(s3, to_info.bucket, to_info.path)
         if etag != cached_etag:
             raise ETagMismatchError(etag, cached_etag)
 

dvc/tree/ssh/__init__.py

@@ -1,23 +1,19 @@
-import errno
 import getpass
 import io
-import itertools
 import logging
 import os
 import posixpath
 import threading
-from concurrent.futures import ThreadPoolExecutor
 from contextlib import closing, contextmanager
 from urllib.parse import urlparse
 
 from funcy import first, memoize, silent, wrap_with
 
 import dvc.prompt as prompt
-from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree, Remote
-from dvc.remote.pool import get_connection
 from dvc.scheme import Schemes
-from dvc.utils import to_chunks
+
+from ..base import BaseTree
+from ..pool import get_connection
 
 logger = logging.getLogger(__name__)
 
@@ -33,7 +29,7 @@ def ask_password(host, user, port):
     )
 
 
-class SSHRemoteTree(BaseRemoteTree):
+class SSHTree(BaseTree):
     scheme = Schemes.SSH
     REQUIRES = {"paramiko": "paramiko"}
     JOBS = 4
@@ -103,7 +99,7 @@ class SSHRemoteTree(BaseRemoteTree):
     def _load_user_ssh_config(hostname):
         import paramiko
 
-        user_config_file = SSHRemoteTree.ssh_config_filename()
+        user_config_file = SSHTree.ssh_config_filename()
         user_ssh_config = {}
         if hostname and os.path.exists(user_config_file):
             ssh_config = paramiko.SSHConfig()
@@ -160,7 +156,7 @@ class SSHRemoteTree(BaseRemoteTree):
             else:
                 yield io.TextIOWrapper(fd, encoding=encoding)
 
-    def exists(self, path_info):
+    def exists(self, path_info, use_dvcignore=True):
         with self.ssh(path_info) as ssh:
             return ssh.exists(path_info.path)
 
@@ -283,65 +279,3 @@ class SSHRemoteTree(BaseRemoteTree):
                     yield path
             else:
                 yield from ssh.walk_files(root)
-
-
-class SSHRemote(Remote):
-    def batch_exists(self, path_infos, callback):
-        def _exists(chunk_and_channel):
-            chunk, channel = chunk_and_channel
-            ret = []
-            for path in chunk:
-                try:
-                    channel.stat(path)
-                    ret.append(True)
-                except OSError as exc:
-                    if exc.errno != errno.ENOENT:
-                        raise
-                    ret.append(False)
-                callback(path)
-            return ret
-
-        with self.tree.ssh(path_infos[0]) as ssh:
-            channels = ssh.open_max_sftp_channels()
-            max_workers = len(channels)
-
-            with ThreadPoolExecutor(max_workers=max_workers) as executor:
-                paths = [path_info.path for path_info in path_infos]
-                chunks = to_chunks(paths, num_chunks=max_workers)
-                chunks_and_channels = zip(chunks, channels)
-                outcome = executor.map(_exists, chunks_and_channels)
-                results = list(itertools.chain.from_iterable(outcome))
-
-            return results
-
-    def hashes_exist(self, hashes, jobs=None, name=None):
-        """This is older implementation used in remote/base.py
-        We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
-        faster than current approach (relying on exists(path_info)) applied in
-        remote/base.
-        """
-        if not self.tree.CAN_TRAVERSE:
-            return list(set(hashes) & set(self.tree.all()))
-
-        # possibly prompt for credentials before "Querying" progress output
-        self.tree.ensure_credentials()
-
-        with Tqdm(
-            desc="Querying "
-            + ("cache in " + name if name else "remote cache"),
-            total=len(hashes),
-            unit="file",
-        ) as pbar:
-
-            def exists_with_progress(chunks):
-                return self.batch_exists(chunks, callback=pbar.update_msg)
-
-            with ThreadPoolExecutor(
-                max_workers=jobs or self.tree.JOBS
-            ) as executor:
-                path_infos = [self.hash_to_path_info(x) for x in hashes]
-                chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
-                results = executor.map(exists_with_progress, chunks)
-                in_remote = itertools.chain.from_iterable(results)
-                ret = list(itertools.compress(hashes, in_remote))
-                return ret

dvc/tree/ssh/connection.py

@@ -9,9 +9,10 @@ from funcy import cached_property
 
 from dvc.exceptions import DvcException
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteCmdError
 from dvc.utils import tmp_fname
 
+from ..base import RemoteCmdError
+
 try:
     import paramiko
 except ImportError:

dvc/utils/.gitignore

@@ -0,0 +1 @@
+/build.py

dvc/utils/fs.py

@@ -44,7 +44,7 @@ def get_mtime_and_size(path, tree):
                     raise
                 continue
             size += stats.st_size
-            files_mtimes[file_path] = stats.st_mtime
+            files_mtimes[os.fspath(file_path)] = stats.st_mtime
 
         # We track file changes and moves, which cannot be detected with simply
         # max(mtime(f) for f in non_ignored_files)
@@ -96,8 +96,7 @@ def move(src, dst, mode=None):
     tmp = f"{dst}.{uuid()}"
 
     if os.path.islink(src):
-        # readlink does not accept path-like obj for Windows in Python <3.8
-        shutil.copy(os.readlink(os.fspath(src)), tmp)
+        shutil.copy(src, tmp)
         os.unlink(src)
     else:
         shutil.move(src, tmp)

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "1.1.2"
+_BASE_VERSION = "1.2.3"
 
 
 def _generate_version(base_version):

scripts/ci/before_install.sh

@@ -16,27 +16,6 @@ fi
 echo >env.sh
 
 if [[ "$TRAVIS_BUILD_STAGE_NAME" == "test" ]]; then
-  if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then
-    # NOTE: ssh keys for ssh test to be able to ssh to the localhost
-    ssh-keygen -t rsa -N "" -f mykey
-    mkdir -p ~/.ssh
-    cp mykey ~/.ssh/id_rsa
-    cp mykey.pub ~/.ssh/id_rsa.pub
-    cat mykey.pub >>~/.ssh/authorized_keys
-    ssh-keyscan localhost >>~/.ssh/known_hosts
-    ssh localhost ls &>/dev/null
-    ssh-keyscan 127.0.0.1 >>~/.ssh/known_hosts
-    ssh 127.0.0.1 ls &>/dev/null
-    ssh-keyscan 0.0.0.0 >>~/.ssh/known_hosts
-    ssh 0.0.0.0 ls &>/dev/null
-  fi
-
-  if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-    bash "$scriptdir/install_azurite.sh"
-    bash "$scriptdir/install_oss.sh"
-    bash "$scriptdir/install_hadoop.sh"
-  fi
-
   if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
     brew install openssl
     $scriptdir/retry.sh brew cask install google-cloud-sdk
@@ -61,17 +40,14 @@ if [ "$TRAVIS_OS_NAME" == "linux" ]; then
 
   if [[ -n "$TRAVIS_TAG" ]]; then
     if [[ $(echo "$TRAVIS_TAG" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$') ]]; then
-      echo "export SNAP_CHANNEL=stable" >>env.sh
-      echo "export SNAP_CHANNEL_MAJOR=v$TAG_MAJOR/stable" >>env.sh
+      SNAP_CHANNEL=stable
     else
-      echo "export SNAP_CHANNEL=beta" >>env.sh
-      echo "export SNAP_CHANNEL_MAJOR=v$TAG_MAJOR/beta" >>env.sh
+      SNAP_CHANNEL=beta
     fi
   else
-    echo "export SNAP_CHANNEL=edge" >>env.sh
-    echo "export SNAP_CHANNEL_MAJOR=v$TAG_MAJOR/edge" >>env.sh
+    SNAP_CHANNEL=edge
   fi
-
+  echo "export SNAP_CHANNEL=${SNAP_CHANNEL},v${TAG_MAJOR}/${SNAP_CHANNEL}" >>env.sh
   # NOTE: after deprecating this major version branch, uncomment this line
-  # echo "unset SNAP_CHANNEL" >>env.sh
+  # echo "export SNAP_CHANNEL=v${TAG_MAJOR}/${SNAP_CHANNEL}" >>env.sh
 fi

scripts/ci/core-site.xml

@@ -1,6 +0,0 @@
-<configuration>
-<property>
-<name>fs.defaultFS</name>
-<value>hdfs://localhost:8020</value>
-</property>
-</configuration>

scripts/ci/hdfs-site.xml

@@ -1,6 +0,0 @@
-<configuration>
-<property>
-<name>dfs.replication</name>
-<value>1</value>
-</property>
-</configuration>

scripts/ci/install_azurite.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -x
-
-# run azurite
-sudo docker run -d --restart always -e executable=blob -p 10000:10000 \
-  --tmpfs /opt/azurite/folder \
-  mcr.microsoft.com/azure-storage/azurite:3.3.0-preview \
-  azurite -l /data --blobHost 0.0.0.0
-
-# save secrets
-echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >>env.sh
-echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >>env.sh

scripts/ci/install_hadoop.sh

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-# NOTE: based on http://sharafjaffri.blogspot.com/2015/04/installing-single-node-hadoop-26-using.html
-
-set -x
-set -e
-
-sudo add-apt-repository -y ppa:openjdk-r/ppa
-sudo apt-get update -y
-sudo apt-get install openjdk-7-jdk
-java -version
-
-pushd /usr/local
-sudo wget https://s3-us-east-2.amazonaws.com/dvc-public/travis-test/hadoop-2.6.5.tar.gz
-sudo tar xzf hadoop-2.6.5.tar.gz
-sudo mkdir hadoop
-sudo mv hadoop-2.6.5/* hadoop/
-popd
-
-echo "export HADOOP_HOME=/usr/local/hadoop" >>env.sh
-echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >>env.sh
-echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >>env.sh
-echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >>env.sh
-echo "export YARN_HOME=/usr/local/hadoop" >>env.sh
-echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >>env.sh
-
-# PyArrow==0.16.0 regression https://issues.apache.org/jira/browse/ARROW-7841
-# retrieves native library from $HADOOP_HOME directory instead of
-# `$HADOOP_HOME/lib/native`.
-# Fix: force search for `libhdfs.so` inside `$HADOOP_HOME/lib/native`.
-# Note: not needed for PyArrow==0.17.0.
-echo "export ARROW_LIBHDFS_DIR=/usr/local/hadoop/lib/native" >> env.sh
-
-echo "export JAVA_HOME=/usr/" >>env.sh
-echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >>env.sh
-
-source env.sh
-
-sudo bash -c 'echo "export JAVA_HOME=/usr/" >> /usr/local/hadoop/etc/hadoop/hadoop-env.sh'
-
-sudo cp scripts/ci/core-site.xml /usr/local/hadoop/etc/hadoop
-sudo cp scripts/ci/hdfs-site.xml /usr/local/hadoop/etc/hadoop
-
-hadoop fs -help
-pushd ~
-sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/
-sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/
-sudo chmod 777 -R /usr/local/hadoop
-hdfs namenode -format
-start-dfs.sh
-jps
-hadoop fs -ls /
-hadoop fs -ls hdfs://127.0.0.1/
-hadoop fs -mkdir hdfs://127.0.0.1/ololo
-popd

scripts/ci/install_oss.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-git clone https://github.com/iterative/oss-emulator.git
-sudo docker image build -t oss:1.0 oss-emulator
-sudo docker run --detach --restart always -p 8880:8880 --name oss-emulator oss:1.0
-echo "export OSS_ENDPOINT='localhost:8880'" >>env.sh
-echo "export OSS_ACCESS_KEY_ID='AccessKeyID'" >>env.sh
-echo "export OSS_ACCESS_KEY_SECRET='AccessKeySecret'" >>env.sh