dvc: bump to 1.6.2

* s3: provide more helpful messages on common errors

Fix #4478

* utils: introduce error_link

* Update dvc/tree/s3.py

Co-authored-by: Jorge Orpinel <[email protected]>

* fix tests

* dont forget the link

* remove redundant message

Co-authored-by: Jorge Orpinel <[email protected]>

.codeclimate.yml

@@ -1,5 +0,0 @@
-languages:
-    Python: true
-
-exclude_paths:
-    - tests/

.deepsource.toml

@@ -1,15 +0,0 @@
-version = 1
-
-test_patterns = [
-  'tests/**'
-]
-
-exclude_patterns = [
-  'dvc/utils/compat.py'
-]
-
-[[analyzers]]
-name = 'python'
-enabled = true
-runtime_version = '3.x.x'
-

.dvc/.gitignore

@@ -9,3 +9,4 @@
 /pkg
 /repos
 /tmp
+/experiments

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,10 +1,18 @@
 ---
-name: Bug report
-about: Create a report to help us improve
-
+name: "\U0001F41B Bug Report"
+about: Create a bug report to help us improve DVC
 ---
 
+## Bug Report
 
-**Please provide information about your setup**
-DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
+### Please provide information about your setup
 
+**Output of `dvc version`:**
+
+```console
+$ dvc version
+```
+
+**Additional Information (if any):**
+
+If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: "🤗 Need help?"
+    url: https://dvc.org/chat
+    about: If you have a question, ask us on Discord. Please join with this invite 👉

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,5 @@
 ---
-name: Feature request
+name: "\U0001F680 Feature Request"
 about: Suggest an idea for this project
-
 ---
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,5 @@
 * [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
 
-* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here. If the CLI API is changed, I have updated [tab completion scripts](https://github.com/iterative/dvc/tree/master/scripts/completion).
-
-* [ ] ❌ I will check DeepSource, CodeClimate, and other sanity checks below. (We consider them recommendatory and don't expect everything to be addressed. Please fix things that actually improve code or fix bugs.)
+* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
 
 Thank you for the contribution - we'll try to review it as soon as possible. 🙏

.github/release-drafter.yml

@@ -1,5 +1,48 @@
 # Config for https://github.com/apps/release-drafter
-branches:
-  - master
+name-template: "$RESOLVED_VERSION 🦉 "
+tag-template: "$RESOLVED_VERSION"
+categories:
+  - title: "🚀 New Features and Enhancements "
+    labels:
+      - "feature"
+      - "enhancement"
+  - title: "💣 Breaking Changes "
+    labels:
+      - "breaking-change"
+  - title: "🏇 Optimizations "
+    labels:
+      - "optimize"
+      - "performance"
+  - title: "🐛 Bug Fixes "
+    labels:
+      - "bugfix"
+      - "bug"
+  - title: "🔨 Maintenance "
+    labels:
+      - "maintenance"
+      - "refactoring"
+      - "chore"
+      - "build"
+      - "ci"
+change-template: "- $TITLE (#$NUMBER) @$AUTHOR"
+version-resolver:
+  major:
+    labels:
+      - "major"
+  minor:
+    labels:
+      - "minor"
+  patch:
+    labels:
+      - "patch"
+  default: patch
+exclude-labels:
+  - "skip-changelog"
 template: |
+  > Refer to https://dvc.org/doc/install for installation instructions.
+
+  ## Changes
+
   $CHANGES
+
+  Thanks again to $CONTRIBUTORS for the contributions! 🎉

.github/workflows/check-patch.yaml

@@ -0,0 +1,19 @@
+name: Lint Python
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Install requirements
+      run: bash ./scripts/ci/install.sh
+    - name: Check Patch
+      run: ./scripts/ci/check_patch.sh

.pre-commit-config.yaml

@@ -1,44 +1,71 @@
 repos:
-- hooks:
-  - id: black
-    language_version: python3
-  repo: https://github.com/ambv/black
-  rev: 19.10b0
--   repo: https://github.com/asottile/seed-isort-config
-    rev: v2.1.1
+  - hooks:
+      - id: black
+        language_version: python3
+    repo: https://github.com/ambv/black
+    rev: 19.10b0
+  - hooks:
+      - id: isort
+        language_version: python3
+    repo: https://github.com/timothycrosley/isort
+    rev: 5.4.2
+  - hooks:
+      - id: flake8
+        language_version: python3
+        additional_dependencies:
+          - flake8-bugbear
+          - flake8-comprehensions
+          - flake8-debugger
+          - flake8-string-format
+    repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.3
+  - repo: local
     hooks:
-    -   id: seed-isort-config
-- hooks:
-  - id: isort
-    language_version: python3
-  repo: https://github.com/timothycrosley/isort
-  rev: 4.3.21
-- hooks:
-  - id: flake8
-    language_version: python3
-  repo: https://gitlab.com/pycqa/flake8
-  rev: 3.7.9
-- hooks:
-  - args:
-    - -i
-    - '2'
-    id: beautysh
-    language_version: python3
-  repo: https://github.com/lovesegfault/beautysh
-  rev: master
-- hooks:
-  - id: dvc-pre-commit
-    language_version: python3
-    stages:
-    - commit
-  - id: dvc-pre-push
-    language_version: python3
-    stages:
-    - push
-  - always_run: true
-    id: dvc-post-checkout
-    language_version: python3
-    stages:
-    - post-checkout
-  repo: https://github.com/iterative/dvc
-  rev: master
+    - id: pylint
+      name: pylint
+      entry: pylint
+      language: system
+      types: [python]
+  - hooks:
+      - args:
+          - -i
+          - "2"
+        id: beautysh
+        language_version: python3
+    repo: https://github.com/lovesegfault/beautysh
+    rev: master
+  - hooks:
+      - args:
+          - git-hook
+          - pre-commit
+        entry: dvc
+        id: dvc-pre-commit
+        language: system
+        name: DVC pre-commit
+        stages:
+          - commit
+        verbose: true
+        require_serial: true
+      - args:
+          - git-hook
+          - pre-push
+        entry: dvc
+        id: dvc-pre-push
+        language: system
+        name: DVC pre-push
+        stages:
+          - push
+        require_serial: true
+      - always_run: true
+        args:
+          - git-hook
+          - post-checkout
+        entry: dvc
+        id: dvc-post-checkout
+        language: system
+        minimum_pre_commit_version: 2.2.0
+        name: DVC post-checkout
+        stages:
+          - post-checkout
+        require_serial: true
+    repo: local

.pre-commit-hooks.yaml

@@ -6,6 +6,7 @@
   language: python
   language_version: python3
   name: DVC pre-commit
+  require_serial: true
   stages:
   - commit
   verbose: true
@@ -17,6 +18,7 @@
   language: python
   language_version: python3
   name: DVC pre-push
+  require_serial: true
   stages:
   - push
 - always_run: true
@@ -29,5 +31,6 @@
   language_version: python3
   minimum_pre_commit_version: 2.2.0
   name: DVC post-checkout
+  require_serial: true
   stages:
   - post-checkout

.pylintrc

@@ -15,7 +15,11 @@ ignore-patterns=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-#init-hook=
+init-hook="import sys; \
+           from pathlib import Path; \
+           from pylint.config import find_pylintrc; \
+           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
+
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 # number of processors available to use.
@@ -28,7 +32,7 @@ limit-inference-results=100
 
 # List of plugins (as comma separated values of python modules names) to load,
 # usually to register additional checkers.
-load-plugins=
+load-plugins=pylint_pytest,pylint_plugin_disable
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -60,7 +64,16 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use "--disable=all --enable=classes
 # --disable=W".
-disable=print-statement,
+disable=format,
+        # ^ most of them are covered by flake8,
+        # and are conventions rather than warnings or errors
+        refactoring,  # will be noisy if it starts giving suggestions
+        spelling,
+        # design is too-few-* and too-many-* checks, too-much-* for us right now
+        design,
+        # following were generated by default
+        # most of them are redundant Py3 checks for Py2+3 codebase
+        print-statement,
         parameter-unpacking,
         unpacking-in-except,
         old-raise-syntax,
@@ -139,9 +152,33 @@ disable=print-statement,
         deprecated-sys-function,
         exception-escape,
         comprehension-escape,
-        useless-object-inheritance,
-        missing-docstring,
-        bad-continuation
+        # custom disables start here
+        no-self-use,
+        invalid-name,
+        # already a check enabled for `wildcard-import`
+        unused-wildcard-import,
+        misplaced-comparison-constant,
+        # probably soon: https://github.com/iterative/dvc/issues/1843
+        logging-format-interpolation,
+        logging-fstring-interpolation,
+        # we don't have proper docstrings yet
+        # even if we do, we'll be doing it gradually
+        missing-function-docstring,
+        missing-module-docstring,
+        missing-class-docstring,
+        duplicate-code,
+        # too noisy, and detects issues even when import is inside a function
+        # see: https://github.com/PyCQA/pylint/issues/850
+        cyclic-import,
+        # isort should cover these:
+        wrong-import-order,
+        wrong-import-position,
+        ungrouped-imports,
+        multiple-imports,
+        # we use it to speedup/optimize
+        import-outside-toplevel,
+        fixme,
+
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -228,7 +265,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=
+generated-members=pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # mixin class is detected if its name ends with "mixin" (case insensitive).
@@ -249,7 +286,7 @@ ignore-on-opaque-inference=yes
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
 # qualified names.
-ignored-classes=optparse.Values,thread._local,_thread._local
+ignored-classes=optparse.Values,thread._local,_thread._local,Dvcfile
 
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
@@ -290,7 +327,7 @@ dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
 
 # Argument names that match this expression will be ignored. Default to name
 # with leading underscore.
-ignored-argument-names=_.*|^ignored_|^unused_
+ignored-argument-names=_.*|^ignored_|^unused_|args|kwargs
 
 # Tells whether we should check for unused import in __init__ files.
 init-import=no

.restyled.yaml

@@ -10,7 +10,7 @@ restylers:
     interpreters:
     - python
   - name: isort
-    image: restyled/restyler-isort:v4.3.21
+    image: restyled/restyler-isort:v5.4.2
     command:
     - isort
     arguments: []

.travis.yml

@@ -29,7 +29,7 @@ jobs:
     os: linux
     language: python
     python: 3.7
-    before_install:
+    before_install: bash ./scripts/ci/install.sh
     install:
     script: ./scripts/ci/check_patch.sh
   # test jobs
@@ -98,12 +98,15 @@ jobs:
      - name: lxd
        channel: stable
     env:
-    - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}'
+    - SNAPCRAFT_IMAGE_INFO: |
+        '{"build_url": "$TRAVIS_JOB_WEB_URL"}'
+    - SNAPCRAFT_BUILD_ENVIRONMENT: lxd
+    - SNAPCRAFT_BUILD_INFO: 1  # https://snapcraft.io/blog/introducing-developer-notifications-for-snap-security-updates
     install:
+    - sudo usermod --append --groups lxd $USER
     - sudo /snap/bin/lxd.migrate -yes
     - sudo /snap/bin/lxd waitready
     - sudo /snap/bin/lxd init --auto
-    - git fetch --tags
     script:
     - ./scripts/build_snap.sh
     after_failure:
@@ -154,6 +157,6 @@ deploy:
     script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
     on:
       all_branches: true
-      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG)"
+      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL"
       repo: iterative/dvc
       stage: build

README.rst

@@ -10,7 +10,7 @@
 
 |Release| |CI| |Maintainability| |Coverage| |Donate| |DOI|
 
-|Snap| |Choco| |Brew| |Conda| |PyPI| |Packages|
+|PyPI| |Packages| |Brew| |Conda| |Choco| |Snap|
 
 |
 
@@ -101,7 +101,8 @@ Snap (Snapcraft/Linux)
    snap install dvc --classic
 
 This corresponds to the latest tagged release.
-Add ``--edge`` for the latest ``master`` version.
+Add ``--beta`` for the latest tagged release candidate,
+or ``--edge`` for the latest ``master`` version.
 
 Choco (Chocolatey/Windows)
 --------------------------
@@ -130,8 +131,6 @@ Conda (Anaconda)
 
    conda install -c conda-forge dvc
 
-Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
-
 pip (PyPI)
 ----------
 

dvc/api.py

@@ -27,13 +27,15 @@ def get_url(path, repo=None, rev=None, remote=None):
     """
     with _make_repo(repo, rev=rev) as _repo:
         if not isinstance(_repo, Repo):
-            raise UrlNotDvcRepoError(_repo.url)
+            raise UrlNotDvcRepoError(_repo.url)  # pylint: disable=no-member
         out = _repo.find_out_by_relpath(path)
         remote_obj = _repo.cloud.get_remote(remote)
-        return str(remote_obj.checksum_to_path_info(out.checksum))
+        return str(remote_obj.tree.hash_to_path_info(out.checksum))
 
 
-def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
+def open(  # noqa, pylint: disable=redefined-builtin
+    path, repo=None, rev=None, remote=None, mode="r", encoding=None
+):
     """
     Open file in the supplied path tracked in a repo (both DVC projects and
     plain Git repos are supported). For Git repos, HEAD is used unless a rev
@@ -58,7 +60,9 @@ def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
 
 
 class _OpenContextManager(GCM):
-    def __init__(self, func, args, kwds):
+    def __init__(
+        self, func, args, kwds
+    ):  # pylint: disable=super-init-not-called
         self.gen = func(*args, **kwds)
         self.func, self.args, self.kwds = func, args, kwds
 

dvc/cache/__init__.py

@@ -28,13 +28,15 @@ def _make_remote_property(name):
     """
 
     def getter(self):
-        from dvc.remote import Remote
+        from ..tree import get_cloud_tree
+        from .base import CloudCache
 
         remote = self.config.get(name)
         if not remote:
             return None
 
-        return Remote(self.repo, name=remote)
+        tree = get_cloud_tree(self.repo, name=remote)
+        return CloudCache(tree)
 
     getter.__name__ = name
     return cached_property(getter)
@@ -50,7 +52,8 @@ class Cache:
     CACHE_DIR = "cache"
 
     def __init__(self, repo):
-        from dvc.remote import Remote
+        from ..tree import get_cloud_tree
+        from .local import LocalCache
 
         self.repo = repo
         self.config = config = repo.config["cache"]
@@ -62,7 +65,8 @@ class Cache:
         else:
             settings = {**config, "url": config["dir"]}
 
-        self.local = Remote(repo, **settings)
+        tree = get_cloud_tree(repo, **settings)
+        self.local = LocalCache(tree)
 
     s3 = _make_remote_property("s3")
     gs = _make_remote_property("gs")
@@ -102,6 +106,7 @@ class NamedCacheItem:
 
 
 class NamedCache:
+    # pylint: disable=protected-access
     def __init__(self):
         self._items = defaultdict(lambda: defaultdict(NamedCacheItem))
         self.external = defaultdict(set)

dvc/cache/base.py

@@ -0,0 +1,638 @@
+import json
+import logging
+from copy import copy
+
+from shortuuid import uuid
+
+import dvc.prompt as prompt
+from dvc.exceptions import (
+    CheckoutError,
+    ConfirmRemoveError,
+    DvcException,
+    MergeError,
+)
+from dvc.path_info import WindowsPathInfo
+from dvc.progress import Tqdm
+from dvc.remote.slow_link_detection import slow_link_guard
+
+from ..tree.base import RemoteActionNotImplemented
+
+logger = logging.getLogger(__name__)
+
+STATUS_OK = 1
+STATUS_MISSING = 2
+STATUS_NEW = 3
+STATUS_DELETED = 4
+
+STATUS_MAP = {
+    # (local_exists, remote_exists)
+    (True, True): STATUS_OK,
+    (False, False): STATUS_MISSING,
+    (True, False): STATUS_NEW,
+    (False, True): STATUS_DELETED,
+}
+
+
+class DirCacheError(DvcException):
+    def __init__(self, hash_):
+        super().__init__(
+            f"Failed to load dir cache for hash value: '{hash_}'."
+        )
+
+
+class CloudCache:
+    """Cloud cache class."""
+
+    DEFAULT_CACHE_TYPES = ["copy"]
+    CACHE_MODE = None
+
+    def __init__(self, tree):
+        self.tree = tree
+        self.repo = tree.repo
+
+        self.cache_types = tree.config.get("type") or copy(
+            self.DEFAULT_CACHE_TYPES
+        )
+        self.cache_type_confirmed = False
+        self._dir_info = {}
+
+    def get_dir_cache(self, hash_):
+        assert hash_
+
+        dir_info = self._dir_info.get(hash_)
+        if dir_info:
+            return dir_info
+
+        try:
+            dir_info = self.load_dir_cache(hash_)
+        except DirCacheError:
+            dir_info = []
+
+        self._dir_info[hash_] = dir_info
+        return dir_info
+
+    def load_dir_cache(self, hash_):
+        path_info = self.tree.hash_to_path_info(hash_)
+
+        try:
+            with self.tree.open(path_info, "r") as fobj:
+                d = json.load(fobj)
+        except (ValueError, FileNotFoundError) as exc:
+            raise DirCacheError(hash_) from exc
+
+        if not isinstance(d, list):
+            logger.error(
+                "dir cache file format error '%s' [skipping the file]",
+                path_info,
+            )
+            return []
+
+        if self.tree.PATH_CLS == WindowsPathInfo:
+            # only need to convert it for Windows
+            for info in d:
+                # NOTE: here is a BUG, see comment to .as_posix() below
+                relpath = info[self.tree.PARAM_RELPATH]
+                info[self.tree.PARAM_RELPATH] = relpath.replace(
+                    "/", self.tree.PATH_CLS.sep
+                )
+
+        return d
+
+    def changed(self, path_info, hash_info):
+        """Checks if data has changed.
+
+        A file is considered changed if:
+            - It doesn't exist on the working directory (was unlinked)
+            - Hash value is not computed (saving a new file)
+            - The hash value stored is different from the given one
+            - There's no file in the cache
+
+        Args:
+            path_info: dict with path information.
+            hash: expected hash value for this data.
+
+        Returns:
+            bool: True if data has changed, False otherwise.
+        """
+
+        logger.debug(
+            "checking if '%s'('%s') has changed.", path_info, hash_info
+        )
+
+        if not self.tree.exists(path_info):
+            logger.debug("'%s' doesn't exist.", path_info)
+            return True
+
+        hash_ = hash_info.get(self.tree.PARAM_CHECKSUM)
+        if hash_ is None:
+            logger.debug("hash value for '%s' is missing.", path_info)
+            return True
+
+        if self.changed_cache(hash_):
+            logger.debug("cache for '%s'('%s') has changed.", path_info, hash_)
+            return True
+
+        typ, actual = self.tree.get_hash(path_info)
+        assert typ == self.tree.PARAM_CHECKSUM
+
+        if hash_ != actual:
+            logger.debug(
+                "hash value '%s' for '%s' has changed (actual '%s').",
+                hash_,
+                actual,
+                path_info,
+            )
+            return True
+
+        logger.debug("'%s' hasn't changed.", path_info)
+        return False
+
+    def link(self, from_info, to_info):
+        self._link(from_info, to_info, self.cache_types)
+
+    def _link(self, from_info, to_info, link_types):
+        assert self.tree.isfile(from_info)
+
+        self.tree.makedirs(to_info.parent)
+
+        self._try_links(from_info, to_info, link_types)
+
+    def _verify_link(self, path_info, link_type):
+        if self.cache_type_confirmed:
+            return
+
+        is_link = getattr(self.tree, f"is_{link_type}", None)
+        if is_link and not is_link(path_info):
+            self.tree.remove(path_info)
+            raise DvcException(f"failed to verify {link_type}")
+
+        self.cache_type_confirmed = True
+
+    @slow_link_guard
+    def _try_links(self, from_info, to_info, link_types):
+        while link_types:
+            link_method = getattr(self.tree, link_types[0])
+            try:
+                self._do_link(from_info, to_info, link_method)
+                self._verify_link(to_info, link_types[0])
+                return
+
+            except DvcException as exc:
+                logger.debug(
+                    "Cache type '%s' is not supported: %s", link_types[0], exc
+                )
+                del link_types[0]
+
+        raise DvcException("no possible cache types left to try out.")
+
+    def _do_link(self, from_info, to_info, link_method):
+        if self.tree.exists(to_info):
+            raise DvcException(f"Link '{to_info}' already exists!")
+
+        link_method(from_info, to_info)
+
+        logger.debug(
+            "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
+        )
+
+    def _save_file(self, path_info, tree, hash_, save_link=True, **kwargs):
+        assert hash_
+
+        cache_info = self.tree.hash_to_path_info(hash_)
+        if tree == self.tree:
+            if self.changed_cache(hash_):
+                self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
+                self.link(cache_info, path_info)
+            elif self.tree.iscopy(path_info) and self._cache_is_copy(
+                path_info
+            ):
+                # Default relink procedure involves unneeded copy
+                self.tree.unprotect(path_info)
+            else:
+                self.tree.remove(path_info)
+                self.link(cache_info, path_info)
+
+            if save_link:
+                self.tree.state.save_link(path_info)
+            # we need to update path and cache, since in case of reflink,
+            # or copy cache type moving original file results in updates on
+            # next executed command, which causes md5 recalculation
+            self.tree.state.save(path_info, hash_)
+        else:
+            if self.changed_cache(hash_):
+                with tree.open(path_info, mode="rb") as fobj:
+                    # if tree has fetch enabled, DVC out will be fetched on
+                    # open and we do not need to read/copy any data
+                    if not (
+                        tree.isdvc(path_info, strict=False) and tree.fetch
+                    ):
+                        self.tree.copy_fobj(fobj, cache_info)
+                callback = kwargs.get("download_callback")
+                if callback:
+                    callback(1)
+
+        self.tree.state.save(cache_info, hash_)
+        return {self.tree.PARAM_CHECKSUM: hash_}
+
+    def _cache_is_copy(self, path_info):
+        """Checks whether cache uses copies."""
+        if self.cache_type_confirmed:
+            return self.cache_types[0] == "copy"
+
+        if set(self.cache_types) <= {"copy"}:
+            return True
+
+        workspace_file = path_info.with_name("." + uuid())
+        test_cache_file = self.tree.path_info / ".cache_type_test_file"
+        if not self.tree.exists(test_cache_file):
+            with self.tree.open(test_cache_file, "wb") as fobj:
+                fobj.write(bytes(1))
+        try:
+            self.link(test_cache_file, workspace_file)
+        finally:
+            self.tree.remove(workspace_file)
+            self.tree.remove(test_cache_file)
+
+        self.cache_type_confirmed = True
+        return self.cache_types[0] == "copy"
+
+    def _save_dir(self, path_info, tree, hash_, save_link=True, **kwargs):
+        dir_info = self.get_dir_cache(hash_)
+        for entry in Tqdm(
+            dir_info, desc="Saving " + path_info.name, unit="file"
+        ):
+            entry_info = path_info / entry[self.tree.PARAM_RELPATH]
+            entry_hash = entry[self.tree.PARAM_CHECKSUM]
+            self._save_file(
+                entry_info, tree, entry_hash, save_link=False, **kwargs
+            )
+
+        if save_link:
+            self.tree.state.save_link(path_info)
+        if self.tree.exists(path_info):
+            self.tree.state.save(path_info, hash_)
+
+        cache_info = self.tree.hash_to_path_info(hash_)
+        self.tree.state.save(cache_info, hash_)
+        return {self.tree.PARAM_CHECKSUM: hash_}
+
+    def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        if path_info.scheme != self.tree.scheme:
+            raise RemoteActionNotImplemented(
+                f"save {path_info.scheme} -> {self.tree.scheme}",
+                self.tree.scheme,
+            )
+
+        hash_ = hash_info[self.tree.PARAM_CHECKSUM]
+        return self._save(path_info, tree, hash_, save_link, **kwargs)
+
+    def _save(self, path_info, tree, hash_, save_link=True, **kwargs):
+        to_info = self.tree.hash_to_path_info(hash_)
+        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
+
+        if tree.isdir(path_info):
+            return self._save_dir(path_info, tree, hash_, save_link, **kwargs)
+        return self._save_file(path_info, tree, hash_, save_link, **kwargs)
+
+    # Override to return path as a string instead of PathInfo for clouds
+    # which support string paths (see local)
+    def hash_to_path(self, hash_):
+        return self.tree.hash_to_path_info(hash_)
+
+    def changed_cache_file(self, hash_):
+        """Compare the given hash with the (corresponding) actual one.
+
+        - Use `State` as a cache for computed hashes
+            + The entries are invalidated by taking into account the following:
+                * mtime
+                * inode
+                * size
+                * hash
+
+        - Remove the file from cache if it doesn't match the actual hash
+        """
+        # Prefer string path over PathInfo when possible due to performance
+        cache_info = self.hash_to_path(hash_)
+        if self.tree.is_protected(cache_info):
+            logger.debug(
+                "Assuming '%s' is unchanged since it is read-only", cache_info
+            )
+            return False
+
+        _, actual = self.tree.get_hash(cache_info)
+
+        logger.debug(
+            "cache '%s' expected '%s' actual '%s'", cache_info, hash_, actual,
+        )
+
+        if not hash_ or not actual:
+            return True
+
+        if actual.split(".")[0] == hash_.split(".")[0]:
+            # making cache file read-only so we don't need to check it
+            # next time
+            self.tree.protect(cache_info)
+            return False
+
+        if self.tree.exists(cache_info):
+            logger.warning("corrupted cache file '%s'.", cache_info)
+            self.tree.remove(cache_info)
+
+        return True
+
+    def _changed_dir_cache(self, hash_, path_info=None, filter_info=None):
+        if self.changed_cache_file(hash_):
+            return True
+
+        for entry in self.get_dir_cache(hash_):
+            entry_hash = entry[self.tree.PARAM_CHECKSUM]
+
+            if path_info and filter_info:
+                entry_info = path_info / entry[self.tree.PARAM_RELPATH]
+                if not entry_info.isin_or_eq(filter_info):
+                    continue
+
+            if self.changed_cache_file(entry_hash):
+                return True
+
+        return False
+
+    def changed_cache(self, hash_, path_info=None, filter_info=None):
+        if self.tree.is_dir_hash(hash_):
+            return self._changed_dir_cache(
+                hash_, path_info=path_info, filter_info=filter_info
+            )
+        return self.changed_cache_file(hash_)
+
+    def already_cached(self, path_info):
+        _, current = self.tree.get_hash(path_info)
+
+        if not current:
+            return False
+
+        return not self.changed_cache(current)
+
+    def safe_remove(self, path_info, force=False):
+        if not self.tree.exists(path_info):
+            return
+
+        if not force and not self.already_cached(path_info):
+            msg = (
+                "file '{}' is going to be removed."
+                " Are you sure you want to proceed?".format(str(path_info))
+            )
+
+            if not prompt.confirm(msg):
+                raise ConfirmRemoveError(str(path_info))
+
+        self.tree.remove(path_info)
+
+    def _checkout_file(
+        self, path_info, hash_, force, progress_callback=None, relink=False
+    ):
+        """The file is changed we need to checkout a new copy"""
+        added, modified = True, False
+        cache_info = self.tree.hash_to_path_info(hash_)
+        if self.tree.exists(path_info):
+            logger.debug("data '%s' will be replaced.", path_info)
+            self.safe_remove(path_info, force=force)
+            added, modified = False, True
+
+        self.link(cache_info, path_info)
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_)
+        if progress_callback:
+            progress_callback(str(path_info))
+
+        return added, modified and not relink
+
+    def _checkout_dir(
+        self,
+        path_info,
+        hash_,
+        force,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        added, modified = False, False
+        # Create dir separately so that dir is created
+        # even if there are no files in it
+        if not self.tree.exists(path_info):
+            added = True
+            self.tree.makedirs(path_info)
+
+        dir_info = self.get_dir_cache(hash_)
+
+        logger.debug("Linking directory '%s'.", path_info)
+
+        for entry in dir_info:
+            relative_path = entry[self.tree.PARAM_RELPATH]
+            entry_hash = entry[self.tree.PARAM_CHECKSUM]
+            entry_cache_info = self.tree.hash_to_path_info(entry_hash)
+            entry_info = path_info / relative_path
+
+            if filter_info and not entry_info.isin_or_eq(filter_info):
+                continue
+
+            entry_hash_info = {self.tree.PARAM_CHECKSUM: entry_hash}
+            if relink or self.changed(entry_info, entry_hash_info):
+                modified = True
+                self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+                self.tree.state.save(entry_info, entry_hash)
+            if progress_callback:
+                progress_callback(str(entry_info))
+
+        modified = (
+            self._remove_redundant_files(path_info, dir_info, force)
+            or modified
+        )
+
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_)
+
+        # relink is not modified, assume it as nochange
+        return added, not added and modified and not relink
+
+    def _remove_redundant_files(self, path_info, dir_info, force):
+        existing_files = set(self.tree.walk_files(path_info))
+
+        needed_files = {
+            path_info / entry[self.tree.PARAM_RELPATH] for entry in dir_info
+        }
+        redundant_files = existing_files - needed_files
+        for path in redundant_files:
+            self.safe_remove(path, force)
+
+        return bool(redundant_files)
+
+    def checkout(
+        self,
+        path_info,
+        hash_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        if path_info.scheme not in ["local", self.tree.scheme]:
+            raise NotImplementedError
+
+        hash_ = hash_info.get(self.tree.PARAM_CHECKSUM)
+        failed = None
+        skip = False
+        if not hash_:
+            logger.warning(
+                "No file hash info found for '%s'. " "It won't be created.",
+                path_info,
+            )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        elif not relink and not self.changed(path_info, hash_info):
+            logger.debug("Data '%s' didn't change.", path_info)
+            skip = True
+
+        elif self.changed_cache(
+            hash_, path_info=path_info, filter_info=filter_info
+        ):
+            logger.warning(
+                "Cache '%s' not found. File '%s' won't be created.",
+                hash_,
+                path_info,
+            )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        if failed or skip:
+            if progress_callback:
+                progress_callback(
+                    str(path_info),
+                    self.get_files_number(
+                        self.tree.path_info, hash_, filter_info
+                    ),
+                )
+            if failed:
+                raise CheckoutError([failed])
+            return
+
+        logger.debug("Checking out '%s' with cache '%s'.", path_info, hash_)
+
+        return self._checkout(
+            path_info, hash_, force, progress_callback, relink, filter_info,
+        )
+
+    def _checkout(
+        self,
+        path_info,
+        hash_,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        if not self.tree.is_dir_hash(hash_):
+            return self._checkout_file(
+                path_info, hash_, force, progress_callback, relink
+            )
+
+        return self._checkout_dir(
+            path_info, hash_, force, progress_callback, relink, filter_info
+        )
+
+    def get_files_number(self, path_info, hash_, filter_info):
+        from funcy.py3 import ilen
+
+        if not hash_:
+            return 0
+
+        if not self.tree.is_dir_hash(hash_):
+            return 1
+
+        if not filter_info:
+            return len(self.get_dir_cache(hash_))
+
+        return ilen(
+            filter_info.isin_or_eq(path_info / entry[self.tree.PARAM_CHECKSUM])
+            for entry in self.get_dir_cache(hash_)
+        )
+
+    def _to_dict(self, dir_info):
+        return {
+            entry[self.tree.PARAM_RELPATH]: entry[self.tree.PARAM_CHECKSUM]
+            for entry in dir_info
+        }
+
+    def _from_dict(self, dir_dict):
+        return [
+            {
+                self.tree.PARAM_RELPATH: relpath,
+                self.tree.PARAM_CHECKSUM: checksum,
+            }
+            for relpath, checksum in dir_dict.items()
+        ]
+
+    @staticmethod
+    def _diff(ancestor, other, allow_removed=False):
+        from dictdiffer import diff
+
+        allowed = ["add"]
+        if allow_removed:
+            allowed.append("remove")
+
+        result = list(diff(ancestor, other))
+        for typ, _, _ in result:
+            if typ not in allowed:
+                raise MergeError(
+                    "unable to auto-merge directories with diff that contains "
+                    f"'{typ}'ed files"
+                )
+        return result
+
+    def _merge_dirs(self, ancestor_info, our_info, their_info):
+        from operator import itemgetter
+
+        from dictdiffer import patch
+
+        ancestor = self._to_dict(ancestor_info)
+        our = self._to_dict(our_info)
+        their = self._to_dict(their_info)
+
+        our_diff = self._diff(ancestor, our)
+        if not our_diff:
+            return self._from_dict(their)
+
+        their_diff = self._diff(ancestor, their)
+        if not their_diff:
+            return self._from_dict(our)
+
+        # make sure there are no conflicting files
+        self._diff(our, their, allow_removed=True)
+
+        merged = patch(our_diff + their_diff, ancestor, in_place=True)
+
+        # Sorting the list by path to ensure reproducibility
+        return sorted(
+            self._from_dict(merged), key=itemgetter(self.tree.PARAM_RELPATH)
+        )
+
+    def merge(self, ancestor_info, our_info, their_info):
+        assert our_info
+        assert their_info
+
+        if ancestor_info:
+            ancestor_hash = ancestor_info[self.tree.PARAM_CHECKSUM]
+            ancestor = self.get_dir_cache(ancestor_hash)
+        else:
+            ancestor = []
+
+        our_hash = our_info[self.tree.PARAM_CHECKSUM]
+        our = self.get_dir_cache(our_hash)
+
+        their_hash = their_info[self.tree.PARAM_CHECKSUM]
+        their = self.get_dir_cache(their_hash)
+
+        merged = self._merge_dirs(ancestor, our, their)
+        typ, merged_hash = self.tree.save_dir_info(merged)
+        return {typ: merged_hash}

dvc/cache/local.py

@@ -0,0 +1,464 @@
+import errno
+import logging
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import partial, wraps
+
+from funcy import cached_property, concat
+
+from dvc.exceptions import DownloadError, UploadError
+from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
+
+from ..remote.base import index_locked
+from ..tree.local import LocalTree
+from .base import (
+    STATUS_DELETED,
+    STATUS_MAP,
+    STATUS_MISSING,
+    STATUS_NEW,
+    CloudCache,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _log_exceptions(func, operation):
+    @wraps(func)
+    def wrapper(from_info, to_info, *args, **kwargs):
+        try:
+            func(from_info, to_info, *args, **kwargs)
+            return 0
+        except Exception as exc:  # pylint: disable=broad-except
+            # NOTE: this means we ran out of file descriptors and there is no
+            # reason to try to proceed, as we will hit this error anyways.
+            # pylint: disable=no-member
+            if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+                raise
+
+            logger.exception(
+                "failed to %s '%s' to '%s'", operation, from_info, to_info
+            )
+            return 1
+
+    return wrapper
+
+
+class LocalCache(CloudCache):
+    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
+    CACHE_MODE = LocalTree.CACHE_MODE
+
+    def __init__(self, tree):
+        super().__init__(tree)
+        self.cache_dir = tree.config.get("url")
+
+    @property
+    def cache_dir(self):
+        return self.tree.path_info.fspath if self.tree.path_info else None
+
+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.tree.path_info = PathInfo(value) if value else None
+
+    @classmethod
+    def supported(cls, config):  # pylint: disable=unused-argument
+        return True
+
+    @cached_property
+    def cache_path(self):
+        return os.path.abspath(self.cache_dir)
+
+    def hash_to_path(self, hash_):
+        # NOTE: `self.cache_path` is already normalized so we can simply use
+        # `os.sep` instead of `os.path.join`. This results in this helper
+        # being ~5.5 times faster.
+        return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
+
+    def hashes_exist(
+        self, hashes, jobs=None, name=None
+    ):  # pylint: disable=unused-argument
+        return [
+            hash_
+            for hash_ in Tqdm(
+                hashes,
+                unit="file",
+                desc="Querying "
+                + ("cache in " + name if name else "local cache"),
+            )
+            if not self.changed_cache_file(hash_)
+        ]
+
+    def already_cached(self, path_info):
+        assert path_info.scheme in ["", "local"]
+
+        typ, current_md5 = self.tree.get_hash(path_info)
+
+        assert typ == "md5"
+
+        if not current_md5:
+            return False
+
+        return not self.changed_cache(current_md5)
+
+    def _verify_link(self, path_info, link_type):
+        if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
+            return
+
+        super()._verify_link(path_info, link_type)
+
+    @index_locked
+    def status(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+        log_missing=True,
+    ):
+        # Return flattened dict containing all status info
+        dir_status, file_status, _ = self._status(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
+            log_missing=log_missing,
+        )
+        return dict(dir_status, **file_status)
+
+    def _status(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+        log_missing=True,
+    ):
+        """Return a tuple of (dir_status_info, file_status_info, dir_contents).
+
+        dir_status_info contains status for .dir files, file_status_info
+        contains status for all other files, and dir_contents is a dict of
+        {dir_hash: set(file_hash, ...)} which can be used to map
+        a .dir file to its file contents.
+        """
+        logger.debug(
+            f"Preparing to collect status from {remote.tree.path_info}"
+        )
+        md5s = set(named_cache.scheme_keys(self.tree.scheme))
+
+        logger.debug("Collecting information from local cache...")
+        local_exists = frozenset(
+            self.hashes_exist(md5s, jobs=jobs, name=self.cache_dir)
+        )
+
+        # This is a performance optimization. We can safely assume that,
+        # if the resources that we want to fetch are already cached,
+        # there's no need to check the remote storage for the existence of
+        # those files.
+        if download and local_exists == md5s:
+            remote_exists = local_exists
+        else:
+            logger.debug("Collecting information from remote cache...")
+            remote_exists = set()
+            dir_md5s = set(named_cache.dir_keys(self.tree.scheme))
+            if dir_md5s:
+                remote_exists.update(
+                    self._indexed_dir_hashes(named_cache, remote, dir_md5s)
+                )
+                md5s.difference_update(remote_exists)
+            if md5s:
+                remote_exists.update(
+                    remote.hashes_exist(
+                        md5s, jobs=jobs, name=str(remote.tree.path_info)
+                    )
+                )
+        return self._make_status(
+            named_cache,
+            show_checksums,
+            local_exists,
+            remote_exists,
+            log_missing,
+        )
+
+    def _make_status(
+        self,
+        named_cache,
+        show_checksums,
+        local_exists,
+        remote_exists,
+        log_missing,
+    ):
+        def make_names(hash_, names):
+            return {"name": hash_ if show_checksums else " ".join(names)}
+
+        dir_status = {}
+        file_status = {}
+        dir_contents = {}
+        for hash_, item in named_cache[self.tree.scheme].items():
+            if item.children:
+                dir_status[hash_] = make_names(hash_, item.names)
+                dir_contents[hash_] = set()
+                for child_hash, child in item.children.items():
+                    file_status[child_hash] = make_names(
+                        child_hash, child.names
+                    )
+                    dir_contents[hash_].add(child_hash)
+            else:
+                file_status[hash_] = make_names(hash_, item.names)
+
+        self._fill_statuses(dir_status, local_exists, remote_exists)
+        self._fill_statuses(file_status, local_exists, remote_exists)
+
+        if log_missing:
+            self._log_missing_caches(dict(dir_status, **file_status))
+
+        return dir_status, file_status, dir_contents
+
+    def _indexed_dir_hashes(self, named_cache, remote, dir_md5s):
+        # Validate our index by verifying all indexed .dir hashes
+        # still exist on the remote
+        indexed_dirs = set(remote.index.dir_hashes())
+        indexed_dir_exists = set()
+        if indexed_dirs:
+            indexed_dir_exists.update(
+                remote.tree.list_hashes_exists(indexed_dirs)
+            )
+            missing_dirs = indexed_dirs.difference(indexed_dir_exists)
+            if missing_dirs:
+                logger.debug(
+                    "Remote cache missing indexed .dir hashes '{}', "
+                    "clearing remote index".format(", ".join(missing_dirs))
+                )
+                remote.index.clear()
+
+        # Check if non-indexed (new) dir hashes exist on remote
+        dir_exists = dir_md5s.intersection(indexed_dir_exists)
+        dir_exists.update(
+            remote.tree.list_hashes_exists(dir_md5s - dir_exists)
+        )
+
+        # If .dir hash exists on the remote, assume directory contents
+        # still exists on the remote
+        for dir_hash in dir_exists:
+            file_hashes = list(
+                named_cache.child_keys(self.tree.scheme, dir_hash)
+            )
+            if dir_hash not in remote.index:
+                logger.debug(
+                    "Indexing new .dir '{}' with '{}' nested files".format(
+                        dir_hash, len(file_hashes)
+                    )
+                )
+                remote.index.update([dir_hash], file_hashes)
+            yield dir_hash
+            yield from file_hashes
+
+    @staticmethod
+    def _fill_statuses(hash_info_dir, local_exists, remote_exists):
+        # Using sets because they are way faster for lookups
+        local = set(local_exists)
+        remote = set(remote_exists)
+
+        for md5, info in hash_info_dir.items():
+            status = STATUS_MAP[(md5 in local, md5 in remote)]
+            info["status"] = status
+
+    def _get_plans(self, download, remote, status_info, status):
+        cache = []
+        path_infos = []
+        names = []
+        hashes = []
+        missing = []
+        for md5, info in Tqdm(
+            status_info.items(), desc="Analysing status", unit="file"
+        ):
+            if info["status"] == status:
+                cache.append(self.tree.hash_to_path_info(md5))
+                path_infos.append(remote.tree.hash_to_path_info(md5))
+                names.append(info["name"])
+                hashes.append(md5)
+            elif info["status"] == STATUS_MISSING:
+                missing.append(md5)
+
+        if download:
+            to_infos = cache
+            from_infos = path_infos
+        else:
+            to_infos = path_infos
+            from_infos = cache
+
+        return (from_infos, to_infos, names, hashes), missing
+
+    def _process(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        logger.debug(
+            "Preparing to {} '{}'".format(
+                "download data from" if download else "upload data to",
+                remote.tree.path_info,
+            )
+        )
+
+        if download:
+            func = partial(
+                _log_exceptions(remote.tree.download, "download"),
+                dir_mode=self.tree.dir_mode,
+                file_mode=self.tree.file_mode,
+            )
+            status = STATUS_DELETED
+            desc = "Downloading"
+        else:
+            func = _log_exceptions(remote.tree.upload, "upload")
+            status = STATUS_NEW
+            desc = "Uploading"
+
+        if jobs is None:
+            jobs = remote.tree.JOBS
+
+        dir_status, file_status, dir_contents = self._status(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
+        )
+
+        dir_plans, _ = self._get_plans(download, remote, dir_status, status)
+        file_plans, missing_files = self._get_plans(
+            download, remote, file_status, status
+        )
+
+        total = len(dir_plans[0]) + len(file_plans[0])
+        if total == 0:
+            return 0
+
+        with Tqdm(total=total, unit="file", desc=desc) as pbar:
+            func = pbar.wrap_fn(func)
+            with ThreadPoolExecutor(max_workers=jobs) as executor:
+                if download:
+                    from_infos, to_infos, names, _ = (
+                        d + f for d, f in zip(dir_plans, file_plans)
+                    )
+                    fails = sum(
+                        executor.map(func, from_infos, to_infos, names)
+                    )
+                else:
+                    # for uploads, push files first, and any .dir files last
+
+                    file_futures = {}
+                    for from_info, to_info, name, hash_ in zip(*file_plans):
+                        file_futures[hash_] = executor.submit(
+                            func, from_info, to_info, name
+                        )
+                    dir_futures = {}
+                    for from_info, to_info, name, dir_hash in zip(*dir_plans):
+                        # if for some reason a file contained in this dir is
+                        # missing both locally and in the remote, we want to
+                        # push whatever file content we have, but should not
+                        # push .dir file
+                        for file_hash in missing_files:
+                            if file_hash in dir_contents[dir_hash]:
+                                logger.debug(
+                                    "directory '%s' contains missing files,"
+                                    "skipping .dir file upload",
+                                    name,
+                                )
+                                break
+                        else:
+                            wait_futures = {
+                                future
+                                for file_hash, future in file_futures.items()
+                                if file_hash in dir_contents[dir_hash]
+                            }
+                            dir_futures[dir_hash] = executor.submit(
+                                self._dir_upload,
+                                func,
+                                wait_futures,
+                                from_info,
+                                to_info,
+                                name,
+                            )
+                    fails = sum(
+                        future.result()
+                        for future in concat(
+                            file_futures.values(), dir_futures.values()
+                        )
+                    )
+
+        if fails:
+            if download:
+                remote.index.clear()
+                raise DownloadError(fails)
+            raise UploadError(fails)
+
+        if not download:
+            # index successfully pushed dirs
+            for dir_hash, future in dir_futures.items():
+                if future.result() == 0:
+                    file_hashes = dir_contents[dir_hash]
+                    logger.debug(
+                        "Indexing pushed dir '{}' with "
+                        "'{}' nested files".format(dir_hash, len(file_hashes))
+                    )
+                    remote.index.update([dir_hash], file_hashes)
+
+        return len(dir_plans[0]) + len(file_plans[0])
+
+    @staticmethod
+    def _dir_upload(func, futures, from_info, to_info, name):
+        for future in as_completed(futures):
+            if future.result():
+                # do not upload this .dir file if any file in this
+                # directory failed to upload
+                logger.debug(
+                    "failed to upload full contents of '{}', "
+                    "aborting .dir file upload".format(name)
+                )
+                logger.error(f"failed to upload '{from_info}' to '{to_info}'")
+                return 1
+        return func(from_info, to_info, name)
+
+    @index_locked
+    def push(self, named_cache, remote, jobs=None, show_checksums=False):
+        return self._process(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=False,
+        )
+
+    @index_locked
+    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
+        return self._process(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=True,
+        )
+
+    @staticmethod
+    def _log_missing_caches(hash_info_dict):
+        missing_caches = [
+            (md5, info)
+            for md5, info in hash_info_dict.items()
+            if info["status"] == STATUS_MISSING
+        ]
+        if missing_caches:
+            missing_desc = "\n".join(
+                "name: {}, md5: {}".format(info["name"], md5)
+                for md5, info in missing_caches
+            )
+            msg = (
+                "Some of the cache files do not exist neither locally "
+                "nor on remote. Missing cache files:\n{}".format(missing_desc)
+            )
+            logger.warning(msg)

dvc/cli.py

@@ -6,13 +6,18 @@ import sys
 from .command import (
     add,
     cache,
+    check_ignore,
     checkout,
     commit,
+    completion,
     config,
     daemon,
+    dag,
     data_sync,
     destroy,
     diff,
+    experiments,
+    freeze,
     gc,
     get,
     get_url,
@@ -21,12 +26,10 @@ from .command import (
     imp_url,
     init,
     install,
-    lock,
     ls,
     metrics,
     move,
     params,
-    pipeline,
     plots,
     remote,
     remove,
@@ -66,32 +69,52 @@ COMMANDS = [
     install,
     root,
     ls,
-    lock,
-    pipeline,
+    freeze,
+    dag,
     daemon,
     commit,
+    completion,
     diff,
     version,
     update,
     git_hook,
     plots,
+    experiments,
+    check_ignore,
 ]
 
 
+def _find_parser(parser, cmd_cls):
+    defaults = parser._defaults  # pylint: disable=protected-access
+    if not cmd_cls or cmd_cls == defaults.get("func"):
+        parser.print_help()
+        raise DvcParserError()
+
+    actions = parser._actions  # pylint: disable=protected-access
+    for action in actions:
+        if not isinstance(action.choices, dict):
+            # NOTE: we are only interested in subparsers
+            continue
+        for subparser in action.choices.values():
+            _find_parser(subparser, cmd_cls)
+
+
 class DvcParser(argparse.ArgumentParser):
     """Custom parser class for dvc CLI."""
 
-    def error(self, message):
-        """Custom error method.
-        Args:
-            message (str): error message.
-
-        Raises:
-            dvc.exceptions.DvcParser: dvc parser exception.
-        """
+    def error(self, message, cmd_cls=None):  # pylint: disable=arguments-differ
         logger.error(message)
-        self.print_help()
-        raise DvcParserError()
+        _find_parser(self, cmd_cls)
+
+    def parse_args(self, args=None, namespace=None):
+        # NOTE: overriding to provide a more granular help message.
+        # E.g. `dvc plots diff --bad-flag` would result in a `dvc plots diff`
+        # help message instead of generic `dvc` usage.
+        args, argv = self.parse_known_args(args, namespace)
+        if argv:
+            msg = "unrecognized arguments: %s"
+            self.error(msg % " ".join(argv), getattr(args, "func", None))
+        return args
 
 
 class VersionAction(argparse.Action):  # pragma: no cover
@@ -114,30 +137,26 @@ def get_parent_parser():
     """
     parent_parser = argparse.ArgumentParser(add_help=False)
 
-    log_level_group = parent_parser.add_mutually_exclusive_group()
-    log_level_group.add_argument(
-        "-q", "--quiet", action="store_true", default=False, help="Be quiet."
-    )
-    log_level_group.add_argument(
-        "-v",
-        "--verbose",
+    parent_parser.add_argument(
+        "--cprofile",
         action="store_true",
         default=False,
-        help="Be verbose.",
+        help=argparse.SUPPRESS,
+    )
+    parent_parser.add_argument("--cprofile-dump", help=argparse.SUPPRESS)
+
+    log_level_group = parent_parser.add_mutually_exclusive_group()
+    log_level_group.add_argument(
+        "-q", "--quiet", action="count", default=0, help="Be quiet."
+    )
+    log_level_group.add_argument(
+        "-v", "--verbose", action="count", default=0, help="Be verbose."
     )
 
     return parent_parser
 
 
-def parse_args(argv=None):
-    """Parses CLI arguments.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Raises:
-        dvc.exceptions.DvcParserError: raised for argument parsing errors.
-    """
+def get_main_parser():
     parent_parser = get_parent_parser()
 
     # Main parser
@@ -185,6 +204,18 @@ def parse_args(argv=None):
     for cmd in COMMANDS:
         cmd.add_parser(subparsers, parent_parser)
 
-    args = parser.parse_args(argv)
+    return parser
 
+
+def parse_args(argv=None):
+    """Parses CLI arguments.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Raises:
+        dvc.exceptions.DvcParserError: raised for argument parsing errors.
+    """
+    parser = get_main_parser()
+    args = parser.parse_args(argv)
     return args

dvc/command/add.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
@@ -18,6 +19,7 @@ class CmdAdd(CmdBase):
                 recursive=self.args.recursive,
                 no_commit=self.args.no_commit,
                 fname=self.args.file,
+                external=self.args.external,
             )
 
         except DvcException:
@@ -50,12 +52,17 @@ def add_parser(subparsers, parent_parser):
         help="Don't put files/directories into cache.",
     )
     parser.add_argument(
-        "-f",
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow targets that are outside of the DVC repository.",
+    )
+    parser.add_argument(
         "--file",
         help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
     )
     parser.add_argument(
-        "targets", nargs="+", help="Input files/directories to add."
-    )
+        "targets", nargs="+", help="Input files/directories to add.",
+    ).complete = completion.FILE
     parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -1,4 +1,5 @@
 import logging
+from abc import ABC, abstractmethod
 
 logger = logging.getLogger(__name__)
 
@@ -26,7 +27,7 @@ def append_doc_link(help_message, path):
     )
 
 
-class CmdBase:
+class CmdBase(ABC):
     def __init__(self, args):
         from dvc.repo import Repo
         from dvc.updater import Updater
@@ -46,11 +47,11 @@ class CmdBase:
         logger.debug(f"assuming default target '{PIPELINE_FILE}'.")
         return [PIPELINE_FILE]
 
-    # Abstract methods that have to be implemented by any inheritance class
+    @abstractmethod
     def run(self):
-        raise NotImplementedError
+        pass
 
 
 class CmdBaseNoRepo(CmdBase):
-    def __init__(self, args):
+    def __init__(self, args):  # pylint: disable=super-init-not-called
         self.args = args

dvc/command/cache.py

@@ -1,5 +1,6 @@
 import argparse
 
+from dvc.command import completion
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
 
@@ -55,5 +56,5 @@ def add_parser(subparsers, parent_parser):
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
         "config file location.",
-    )
+    ).complete = completion.DIR
     cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/check_ignore.py

@@ -0,0 +1,126 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+class CmdCheckIgnore(CmdBase):
+    def __init__(self, args):
+        super().__init__(args)
+        self.ignore_filter = self.repo.tree.dvcignore
+
+    def _show_results(self, result):
+        if not result.match and not self.args.non_matching:
+            return
+
+        if self.args.details:
+            patterns = result.patterns
+            if not self.args.all:
+                patterns = patterns[-1:]
+
+            for pattern in patterns:
+                logger.info("{}\t{}".format(pattern, result.file))
+        else:
+            logger.info(result.file)
+
+    def _check_one_file(self, target):
+        result = self.ignore_filter.check_ignore(target)
+        self._show_results(result)
+        if result.match:
+            return 0
+        return 1
+
+    def _interactive_mode(self):
+        ret = 1
+        while True:
+            try:
+                target = input()
+            except (KeyboardInterrupt, EOFError):
+                break
+            if target == "":
+                break
+            if not self._check_one_file(target):
+                ret = 0
+        return ret
+
+    def _normal_mode(self):
+        ret = 1
+        for target in self.args.targets:
+            if not self._check_one_file(target):
+                ret = 0
+        return ret
+
+    def _check_args(self):
+        if not self.args.stdin and not self.args.targets:
+            raise DvcException("`targets` or `--stdin` needed")
+
+        if self.args.stdin and self.args.targets:
+            raise DvcException("cannot have both `targets` and `--stdin`")
+
+        if self.args.non_matching and not self.args.details:
+            raise DvcException(
+                "`--non-matching` is only valid with `--details`"
+            )
+
+        if self.args.all and not self.args.details:
+            raise DvcException("`--all` is only valid with `--details`")
+
+        if self.args.quiet and self.args.details:
+            raise DvcException("cannot use both `--details` and `--quiet`")
+
+    def run(self):
+        self._check_args()
+        if self.args.stdin:
+            return self._interactive_mode()
+        return self._normal_mode()
+
+
+def add_parser(subparsers, parent_parser):
+    ADD_HELP = (
+        "Check whether files or directories are excluded due to `.dvcignore`."
+    )
+
+    parser = subparsers.add_parser(
+        "check-ignore",
+        parents=[parent_parser],
+        description=append_doc_link(ADD_HELP, "check-ignore"),
+        help=ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "-d",
+        "--details",
+        action="store_true",
+        default=False,
+        help="Show the exclude patterns along with each target path.",
+    )
+    parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        default=False,
+        help="Include the target paths which don’t match any pattern "
+        "in the `--details` list.",
+    )
+    parser.add_argument(
+        "-n",
+        "--non-matching",
+        action="store_true",
+        default=False,
+        help="Include the target paths which don’t match any pattern "
+        "in the `--details` list.",
+    )
+    parser.add_argument(
+        "--stdin",
+        action="store_true",
+        default=False,
+        help="Read paths from standard input instead of providing `targets`.",
+    )
+    parser.add_argument(
+        "targets", nargs="*", help="File or directory paths to check",
+    ).complete = completion.FILE
+    parser.set_defaults(func=CmdCheckIgnore)

dvc/command/checkout.py

@@ -4,6 +4,7 @@ import operator
 
 import colorama
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import CheckoutError
 from dvc.utils.humanize import get_summary
@@ -111,7 +112,9 @@ def add_parser(subparsers, parent_parser):
     checkout_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC-files to checkout. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
-    )
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVC_FILE
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -66,5 +67,5 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="DVC-files to commit. Optional. "
         "(Finds all DVC-files in the workspace by default.)",
-    )
+    ).complete = completion.DVC_FILE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -0,0 +1,57 @@
+import argparse
+import logging
+
+import shtab
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+
+logger = logging.getLogger(__name__)
+FILE = shtab.FILE
+DIR = shtab.DIRECTORY
+DVC_FILE = {
+    "bash": "_dvc_compgen_DVCFiles",
+    "zsh": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'",
+}
+PREAMBLE = {
+    "bash": """
+# $1=COMP_WORDS[1]
+_dvc_compgen_DVCFiles() {
+  compgen -d -S '/' -- $1  # recurse into subdirs
+  compgen -f -X '!*?.dvc' -- $1
+  compgen -f -X '!*Dvcfile' -- $1
+  compgen -f -X '!*dvc.yaml' -- $1
+}
+""",
+    "zsh": "",
+}
+
+
+class CmdCompletion(CmdBaseNoRepo):
+    def run(self):
+        from dvc.cli import get_main_parser
+
+        parser = get_main_parser()
+        shell = self.args.shell
+        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
+        print(script)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    COMPLETION_HELP = "Generate shell tab completion."
+    COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
+    completion_parser = subparsers.add_parser(
+        "completion",
+        parents=[parent_parser],
+        description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
+        help=COMPLETION_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    completion_parser.add_argument(
+        "-s",
+        "--shell",
+        help="Shell syntax for completions.",
+        default="bash",
+        choices=["bash", "zsh"],
+    )
+    completion_parser.set_defaults(func=CmdCompletion)

dvc/command/config.py

@@ -50,21 +50,22 @@ class CmdConfig(CmdBaseNoRepo):
 
 
 parent_config_parser = argparse.ArgumentParser(add_help=False)
-parent_config_parser.add_argument(
+level_group = parent_config_parser.add_mutually_exclusive_group()
+level_group.add_argument(
     "--global",
     dest="level",
     action="store_const",
     const="global",
     help="Use global config.",
 )
-parent_config_parser.add_argument(
+level_group.add_argument(
     "--system",
     dest="level",
     action="store_const",
     const="system",
     help="Use system config.",
 )
-parent_config_parser.add_argument(
+level_group.add_argument(
     "--local",
     dest="level",
     action="store_const",

dvc/command/daemon.py

@@ -1,3 +1,4 @@
+from dvc.command import completion
 from dvc.command.base import CmdBaseNoRepo, fix_subparsers
 
 
@@ -8,8 +9,9 @@ class CmdDaemonBase(CmdBaseNoRepo):
 class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
-        from dvc.repo import Repo
+
         from dvc.config import Config
+        from dvc.repo import Repo
         from dvc.updater import Updater
 
         root_dir = Repo.find_root()
@@ -64,5 +66,7 @@ def add_parser(subparsers, parent_parser):
         description=DAEMON_ANALYTICS_HELP,
         help=DAEMON_ANALYTICS_HELP,
     )
-    daemon_analytics_parser.add_argument("target", help="Analytics file.")
+    daemon_analytics_parser.add_argument(
+        "target", help="Analytics file.",
+    ).complete = completion.FILE
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/dag.py

@@ -0,0 +1,117 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _show_ascii(G):
+    from dvc.dagascii import draw
+    from dvc.repo.graph import get_pipelines
+
+    pipelines = get_pipelines(G)
+
+    ret = []
+    for pipeline in pipelines:
+        ret.append(draw(pipeline.nodes, pipeline.edges))
+
+    return "\n".join(ret)
+
+
+def _show_dot(G):
+    import io
+
+    from networkx.drawing.nx_pydot import write_dot
+
+    dot_file = io.StringIO()
+    write_dot(G, dot_file)
+    return dot_file.getvalue()
+
+
+def _build(G, target=None, full=False):
+    import networkx as nx
+
+    from dvc.repo.graph import get_pipeline, get_pipelines
+
+    if target:
+        H = get_pipeline(get_pipelines(G), target)
+        if not full:
+            descendants = nx.descendants(G, target)
+            descendants.add(target)
+            H.remove_nodes_from(set(G.nodes()) - descendants)
+    else:
+        H = G
+
+    def _relabel(stage):
+        return stage.addressing
+
+    return nx.relabel_nodes(H, _relabel, copy=False)
+
+
+class CmdDAG(CmdBase):
+    def run(self):
+        try:
+            target = None
+            if self.args.target:
+                stages = self.repo.collect(self.args.target)
+                if len(stages) > 1:
+                    logger.error(
+                        f"'{self.args.target}' contains more than one stage "
+                        "{stages}, please specify one stage"
+                    )
+                    return 1
+                target = stages[0]
+
+            G = _build(self.repo.graph, target=target, full=self.args.full,)
+
+            if self.args.dot:
+                logger.info(_show_dot(G))
+            else:
+                from dvc.utils.pager import pager
+
+                pager(_show_ascii(G))
+
+            return 0
+        except DvcException:
+            msg = "failed to show "
+            if self.args.target:
+                msg += f"a pipeline for '{target}'"
+            else:
+                msg += "pipelines"
+            logger.exception(msg)
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    DAG_HELP = "Visualize DVC project DAG."
+    dag_parser = subparsers.add_parser(
+        "dag",
+        parents=[parent_parser],
+        description=append_doc_link(DAG_HELP, "dag"),
+        help=DAG_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    dag_parser.add_argument(
+        "--dot",
+        action="store_true",
+        default=False,
+        help="Print DAG with .dot format.",
+    )
+    dag_parser.add_argument(
+        "--full",
+        action="store_true",
+        default=False,
+        help=(
+            "Show full DAG that the target belongs too, instead of "
+            "showing DAG consisting only of ancestors."
+        ),
+    )
+    dag_parser.add_argument(
+        "target",
+        nargs="?",
+        help="Stage or output to show pipeline for (optional). "
+        "Finds all stages in the workspace by default.",
+    )
+    dag_parser.set_defaults(func=CmdDAG)

dvc/command/data_sync.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.command.checkout import log_changes
 from dvc.exceptions import CheckoutError, DvcException
@@ -96,15 +97,21 @@ def shared_parent_parser():
         "-j",
         "--jobs",
         type=int,
-        help="Number of jobs to run simultaneously.",
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
         metavar="<number>",
     )
     parent_parser.add_argument(
         "targets",
         nargs="*",
-        help="Limit command scope to these DVC-files. "
-        "Using -R, directories to search DVC-files in can also be given.",
-    )
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVC_FILE
 
     return parent_parser
 
@@ -170,7 +177,7 @@ def add_parser(subparsers, _parent_parser):
         "--run-cache",
         action="store_true",
         default=False,
-        help=argparse.SUPPRESS,
+        help="Fetch run history for all stages.",
     )
     pull_parser.set_defaults(func=CmdDataPull)
 
@@ -225,7 +232,7 @@ def add_parser(subparsers, _parent_parser):
         "--run-cache",
         action="store_true",
         default=False,
-        help=argparse.SUPPRESS,
+        help="Push run history for all stages.",
     )
     push_parser.set_defaults(func=CmdDataPush)
 
@@ -286,7 +293,7 @@ def add_parser(subparsers, _parent_parser):
         "--run-cache",
         action="store_true",
         default=False,
-        help=argparse.SUPPRESS,
+        help="Fetch run history for all stages.",
     )
     fetch_parser.set_defaults(func=CmdDataFetch)
 
@@ -363,5 +370,11 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Show status of all stages in the specified directory.",
     )
+    status_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show status in JSON format.",
+    )
 
     status_parser.set_defaults(func=CmdDataStatus)

dvc/command/destroy.py

@@ -25,7 +25,7 @@ class CmdDestroy(CmdBase):
                 )
 
             self.repo.destroy()
-        except Exception:
+        except Exception:  # noqa, pylint: disable=broad-except
             logger.exception("failed to destroy DVC")
             return 1
         return 0

dvc/command/diff.py

@@ -11,19 +11,30 @@ from dvc.exceptions import DvcException
 logger = logging.getLogger(__name__)
 
 
-def _show_md(diff):
+def _digest(checksum):
+    if isinstance(checksum, str):
+        return checksum[0:8]
+    return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
+
+
+def _show_md(diff, show_hash=False):
     from dvc.utils.diff import table
 
+    header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
     rows = []
     for status in ["added", "deleted", "modified"]:
         entries = diff.get(status, [])
         if not entries:
             continue
-        paths = sorted([entry["path"] for entry in entries])
-        for path in paths:
-            rows.append([status, path])
+        for entry in entries:
+            path = entry["path"]
+            if show_hash:
+                check_sum = _digest(entry.get("hash", ""))
+                rows.append([status, check_sum, path])
+            else:
+                rows.append([status, path])
 
-    return table(["Status", "Path"], rows, True)
+    return table(header, rows, True)
 
 
 class CmdDiff(CmdBase):
@@ -33,8 +44,7 @@ class CmdDiff(CmdBase):
         Given a diff structure, generate a string of paths separated
         by new lines and grouped together by their state.
 
-        A group's header is colored and its entries are sorted to enhance
-        readability, for example:
+        A group's header is colored to enhance readability, for example:
 
             Added:
                 another_file.txt
@@ -55,11 +65,6 @@ class CmdDiff(CmdBase):
         At the bottom, include a summary with the number of files per state.
         """
 
-        def _digest(checksum):
-            if type(checksum) is str:
-                return checksum[0:8]
-            return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
-
         colors = {
             "added": colorama.Fore.GREEN,
             "modified": colorama.Fore.YELLOW,
@@ -110,16 +115,19 @@ class CmdDiff(CmdBase):
     def run(self):
         try:
             diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
+            show_hash = self.args.show_hash
 
-            if not self.args.show_hash:
-                for _, entries in diff.items():
+            for key, entries in diff.items():
+                entries = sorted(entries, key=lambda entry: entry["path"])
+                if not show_hash:
                     for entry in entries:
                         del entry["hash"]
+                diff[key] = entries
 
             if self.args.show_json:
                 logger.info(json.dumps(diff))
             elif self.args.show_md:
-                logger.info(_show_md(diff))
+                logger.info(_show_md(diff, show_hash))
             elif diff:
                 logger.info(self._format(diff))
 

dvc/command/experiments.py

@@ -0,0 +1,491 @@
+import argparse
+import io
+import logging
+from collections import OrderedDict
+from itertools import groupby
+from typing import Iterable, Optional
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.command.metrics import DEFAULT_PRECISION
+from dvc.exceptions import DvcException, InvalidArgumentError
+
+logger = logging.getLogger(__name__)
+
+
+def _filter_names(
+    names: Iterable,
+    label: str,
+    include: Optional[Iterable],
+    exclude: Optional[Iterable],
+):
+    if include and exclude:
+        intersection = set(include) & set(exclude)
+        if intersection:
+            values = ", ".join(intersection)
+            raise InvalidArgumentError(
+                f"'{values}' specified in both --include-{label} and"
+                f" --exclude-{label}"
+            )
+
+    names = {tuple(name.split(".")) for name in names}
+
+    def _filter(filters, update_func):
+        filters = {tuple(name.split(".")) for name in filters}
+        for length, groups in groupby(filters, len):
+            for group in groups:
+                matches = [name for name in names if name[:length] == group]
+                if not matches:
+                    name = ".".join(group)
+                    raise InvalidArgumentError(
+                        f"'{name}' does not match any known {label}"
+                    )
+                update_func(matches)
+
+    if include:
+        ret = set()
+        _filter(include, ret.update)
+    else:
+        ret = set(names)
+
+    if exclude:
+        _filter(exclude, ret.difference_update)
+
+    return {".".join(name) for name in ret}
+
+
+def _update_names(names, items):
+    from flatten_json import flatten
+
+    for name, item in items:
+        if isinstance(item, dict):
+            item = flatten(item, ".")
+            names.update(item.keys())
+        else:
+            names.add(name)
+
+
+def _collect_names(all_experiments, **kwargs):
+    metric_names = set()
+    param_names = set()
+
+    for _, experiments in all_experiments.items():
+        for exp in experiments.values():
+            _update_names(metric_names, exp.get("metrics", {}).items())
+            _update_names(param_names, exp.get("params", {}).items())
+
+    metric_names = _filter_names(
+        metric_names,
+        "metrics",
+        kwargs.get("include_metrics"),
+        kwargs.get("exclude_metrics"),
+    )
+    param_names = _filter_names(
+        param_names,
+        "params",
+        kwargs.get("include_params"),
+        kwargs.get("exclude_params"),
+    )
+
+    return sorted(metric_names), sorted(param_names)
+
+
+def _collect_rows(
+    base_rev,
+    experiments,
+    metric_names,
+    param_names,
+    precision=DEFAULT_PRECISION,
+    no_timestamp=False,
+):
+    for i, (rev, exp) in enumerate(experiments.items()):
+        row = []
+        style = None
+        queued = "*" if exp.get("queued", False) else ""
+        if no_timestamp:
+            timestamp = ""
+        else:
+            commit_time = exp.get("timestamp")
+            timestamp = f" ({commit_time})" if commit_time is not None else ""
+
+        if rev == "baseline":
+            row.append(f"{base_rev}{timestamp}")
+            style = "bold"
+        elif i < len(experiments) - 1:
+            row.append(f"├── {queued}{rev[:7]}{timestamp}")
+        else:
+            row.append(f"└── {queued}{rev[:7]}{timestamp}")
+
+        _extend_row(
+            row, metric_names, exp.get("metrics", {}).items(), precision
+        )
+        _extend_row(row, param_names, exp.get("params", {}).items(), precision)
+
+        yield row, style
+
+
+def _extend_row(row, names, items, precision):
+    from flatten_json import flatten
+
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
+
+        return val
+
+    if not items:
+        row.extend(["-"] * len(names))
+        return
+
+    for fname, item in items:
+        if isinstance(item, dict):
+            item = flatten(item, ".")
+        else:
+            item = {fname: item}
+        for name in names:
+            if name in item:
+                value = item[name]
+                text = str(_round(value)) if value is not None else "-"
+                row.append(text)
+            else:
+                row.append("-")
+
+
+def _parse_list(param_list):
+    ret = []
+    for param_str in param_list:
+        # we don't care about filename prefixes for show, silently
+        # ignore it if provided to keep usage consistent with other
+        # metric/param list command options
+        _, _, param_str = param_str.rpartition(":")
+        ret.extend(param_str.split(","))
+    return ret
+
+
+def _show_experiments(all_experiments, console, **kwargs):
+    from rich.table import Table
+
+    from dvc.scm.git import Git
+
+    include_metrics = _parse_list(kwargs.pop("include_metrics", []))
+    exclude_metrics = _parse_list(kwargs.pop("exclude_metrics", []))
+    include_params = _parse_list(kwargs.pop("include_params", []))
+    exclude_params = _parse_list(kwargs.pop("exclude_params", []))
+
+    metric_names, param_names = _collect_names(
+        all_experiments,
+        include_metrics=include_metrics,
+        exclude_metrics=exclude_metrics,
+        include_params=include_params,
+        exclude_params=exclude_params,
+    )
+
+    table = Table()
+    table.add_column("Experiment", no_wrap=True)
+    for name in metric_names:
+        table.add_column(name, justify="right", no_wrap=True)
+    for name in param_names:
+        table.add_column(name, justify="left")
+
+    for base_rev, experiments in all_experiments.items():
+        if Git.is_sha(base_rev):
+            base_rev = base_rev[:7]
+
+        for row, _, in _collect_rows(
+            base_rev, experiments, metric_names, param_names, **kwargs,
+        ):
+            table.add_row(*row)
+
+    console.print(table)
+
+
+class CmdExperimentsShow(CmdBase):
+    def run(self):
+        from rich.console import Console
+
+        from dvc.utils.pager import pager
+
+        if not self.repo.experiments:
+            return 0
+
+        try:
+            all_experiments = self.repo.experiments.show(
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+            )
+
+            if self.args.no_pager:
+                console = Console()
+            else:
+                # Note: rich does not currently include a native way to force
+                # infinite width for use with a pager
+                console = Console(
+                    file=io.StringIO(), force_terminal=True, width=9999
+                )
+
+            _show_experiments(
+                all_experiments,
+                console,
+                include_metrics=self.args.include_metrics,
+                exclude_metrics=self.args.exclude_metrics,
+                include_params=self.args.include_params,
+                exclude_params=self.args.exclude_params,
+                no_timestamp=self.args.no_timestamp,
+            )
+
+            if not self.args.no_pager:
+                pager(console.file.getvalue())
+        except DvcException:
+            logger.exception("failed to show experiments")
+            return 1
+
+        return 0
+
+
+class CmdExperimentsCheckout(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        self.repo.experiments.checkout(self.args.experiment)
+
+        return 0
+
+
+def _show_diff(
+    diff, title="", markdown=False, no_path=False, old=False, precision=None
+):
+    from dvc.utils.diff import table
+
+    if precision is None:
+        precision = DEFAULT_PRECISION
+
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
+
+        return val
+
+    rows = []
+    for fname, diff_ in diff.items():
+        sorted_diff = OrderedDict(sorted(diff_.items()))
+        for item, change in sorted_diff.items():
+            row = [] if no_path else [fname]
+            row.append(item)
+            if old:
+                row.append(_round(change.get("old")))
+            row.append(_round(change["new"]))
+            row.append(_round(change.get("diff", "diff not supported")))
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append(title)
+    if old:
+        header.extend(["Old", "New"])
+    else:
+        header.append("Value")
+    header.append("Change")
+
+    return table(header, rows, markdown)
+
+
+class CmdExperimentsDiff(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        try:
+            diff = self.repo.experiments.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                all=self.args.all,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                diffs = [("metrics", "Metric"), ("params", "Param")]
+                for key, title in diffs:
+                    table = _show_diff(
+                        diff[key],
+                        title=title,
+                        markdown=self.args.show_md,
+                        no_path=self.args.no_path,
+                        old=self.args.old,
+                        precision=self.args.precision,
+                    )
+                    if table:
+                        logger.info(table)
+                        logger.info("")
+
+        except DvcException:
+            logger.exception("failed to show experiments diff")
+            return 1
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    EXPERIMENTS_HELP = "Commands to display and compare experiments."
+
+    experiments_parser = subparsers.add_parser(
+        "experiments",
+        parents=[parent_parser],
+        aliases=["exp"],
+        description=append_doc_link(EXPERIMENTS_HELP, "experiments"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    experiments_subparsers = experiments_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc experiments CMD --help` to display "
+        "command-specific help.",
+    )
+
+    fix_subparsers(experiments_subparsers)
+
+    EXPERIMENTS_SHOW_HELP = "Print experiments."
+    experiments_show_parser = experiments_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_SHOW_HELP, "experiments/show"),
+        help=EXPERIMENTS_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_show_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Show metrics for all branches.",
+    )
+    experiments_show_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Show metrics for all tags.",
+    )
+    experiments_show_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show metrics for all commits.",
+    )
+    experiments_show_parser.add_argument(
+        "--no-pager",
+        action="store_true",
+        default=False,
+        help="Do not pipe output into a pager.",
+    )
+    experiments_show_parser.add_argument(
+        "--include-metrics",
+        action="append",
+        default=[],
+        help="Include the specified metrics in output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-metrics",
+        action="append",
+        default=[],
+        help="Exclude the specified metrics from output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--include-params",
+        action="append",
+        default=[],
+        help="Include the specified params in output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-params",
+        action="append",
+        default=[],
+        help="Exclude the specified params from output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--no-timestamp",
+        action="store_true",
+        default=False,
+        help="Do not show experiment timestamps.",
+    )
+    experiments_show_parser.set_defaults(func=CmdExperimentsShow)
+
+    EXPERIMENTS_CHECKOUT_HELP = "Checkout experiments."
+    experiments_checkout_parser = experiments_subparsers.add_parser(
+        "checkout",
+        parents=[parent_parser],
+        description=append_doc_link(
+            EXPERIMENTS_CHECKOUT_HELP, "experiments/checkout"
+        ),
+        help=EXPERIMENTS_CHECKOUT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_checkout_parser.add_argument(
+        "experiment", help="Checkout this experiment.",
+    )
+    experiments_checkout_parser.set_defaults(func=CmdExperimentsCheckout)
+
+    EXPERIMENTS_DIFF_HELP = (
+        "Show changes between experiments in the DVC repository."
+    )
+    experiments_diff_parser = experiments_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_DIFF_HELP, "experiments/diff"),
+        help=EXPERIMENTS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_diff_parser.add_argument(
+        "a_rev", nargs="?", help="Old experiment to compare (defaults to HEAD)"
+    )
+    experiments_diff_parser.add_argument(
+        "b_rev",
+        nargs="?",
+        help="New experiment to compare (defaults to the current workspace)",
+    )
+    experiments_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged metrics/params as well.",
+    )
+    experiments_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    experiments_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    experiments_diff_parser.add_argument(
+        "--old",
+        action="store_true",
+        default=False,
+        help="Show old metric/param value.",
+    )
+    experiments_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show metric/param path.",
+    )
+    experiments_diff_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics/params to `n` digits precision after the decimal "
+            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
+    experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)

dvc/command/freeze.py

@@ -0,0 +1,58 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+class CmdFreezeBase(CmdBase):
+    def _run(self, func, name):
+        ret = 0
+        for target in self.args.targets:
+            try:
+                func(target)
+            except DvcException:
+                logger.exception(f"failed to {name} '{target}'")
+                ret = 1
+        return ret
+
+
+class CmdFreeze(CmdFreezeBase):
+    def run(self):
+        return self._run(self.repo.freeze, "freeze")
+
+
+class CmdUnfreeze(CmdFreezeBase):
+    def run(self):
+        return self._run(self.repo.unfreeze, "unfreeze")
+
+
+def add_parser(subparsers, parent_parser):
+    FREEZE_HELP = "Freeze stages or .dvc files."
+    freeze_parser = subparsers.add_parser(
+        "freeze",
+        parents=[parent_parser],
+        description=append_doc_link(FREEZE_HELP, "freeze"),
+        help=FREEZE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    freeze_parser.add_argument(
+        "targets", nargs="+", help="Stages or .dvc files to freeze",
+    ).complete = completion.DVC_FILE
+    freeze_parser.set_defaults(func=CmdFreeze)
+
+    UNFREEZE_HELP = "Unfreeze stages or .dvc files."
+    unfreeze_parser = subparsers.add_parser(
+        "unfreeze",
+        parents=[parent_parser],
+        description=append_doc_link(UNFREEZE_HELP, "unfreeze"),
+        help=UNFREEZE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    unfreeze_parser.add_argument(
+        "targets", nargs="+", help="Stages or .dvc files to unfreeze",
+    ).complete = completion.DVC_FILE
+    unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/gc.py

@@ -21,7 +21,7 @@ class CmdGC(CmdBase):
 
         msg = "This will remove all cache except items used in "
 
-        msg += "the working tree"
+        msg += "the workspace"
         if self.args.all_commits:
             msg += " and all git commits"
         elif self.args.all_branches and self.args.all_tags:
@@ -77,7 +77,7 @@ def add_parser(subparsers, parent_parser):
         "--workspace",
         action="store_true",
         default=False,
-        help="Keep data files used in the current workspace (working tree).",
+        help="Keep data files used in the current workspace.",
     )
     gc_parser.add_argument(
         "-a",
@@ -123,7 +123,11 @@ def add_parser(subparsers, parent_parser):
         "-j",
         "--jobs",
         type=int,
-        help="Number of jobs to run simultaneously.",
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
         metavar="<number>",
     )
     gc_parser.add_argument(

dvc/command/get.py

@@ -3,6 +3,7 @@ import logging
 
 from dvc.exceptions import DvcException
 
+from . import completion
 from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
@@ -62,15 +63,15 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path", help="Path to a file or directory within the repository"
-    )
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
     get_parser.add_argument(
         "-o",
         "--out",
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
-    )
+    ).complete = completion.DIR
     get_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/get_url.py

@@ -3,6 +3,7 @@ import logging
 
 from dvc.exceptions import DvcException
 
+from . import completion
 from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
@@ -33,6 +34,6 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out", nargs="?", help="Destination path to put data to."
-    )
+        "out", nargs="?", help="Destination path to put data to.",
+    ).complete = completion.DIR
     get_parser.set_defaults(func=CmdGetUrl)

dvc/command/git_hook.py

@@ -19,6 +19,9 @@ class CmdHookBase(CmdBaseNoRepo):
 
         return self._run()
 
+    def _run(self):
+        raise NotImplementedError
+
 
 class CmdPreCommit(CmdHookBase):
     def _run(self):
@@ -28,7 +31,7 @@ class CmdPreCommit(CmdHookBase):
 
 
 class CmdPostCheckout(CmdHookBase):
-    def run(self):
+    def _run(self):
         # when we are running from pre-commit tool, it doesn't provide CLI
         # flags, but instead provides respective env vars that we could use.
         flag = os.environ.get("PRE_COMMIT_CHECKOUT_TYPE")
@@ -52,12 +55,32 @@ class CmdPostCheckout(CmdHookBase):
 
 
 class CmdPrePush(CmdHookBase):
-    def run(self):
+    def _run(self):
         from dvc.main import main
 
         return main(["push"])
 
 
+class CmdMergeDriver(CmdHookBase):
+    def _run(self):
+        from dvc.dvcfile import Dvcfile
+        from dvc.repo import Repo
+
+        dvc = Repo()
+
+        try:
+            with dvc.state:
+                ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
+                our = Dvcfile(dvc, self.args.our, verify=False)
+                their = Dvcfile(dvc, self.args.their, verify=False)
+
+                our.merge(ancestor, their)
+
+                return 0
+        finally:
+            dvc.close()
+
+
 def add_parser(subparsers, parent_parser):
     GIT_HOOK_HELP = "Run GIT hook."
 
@@ -110,3 +133,27 @@ def add_parser(subparsers, parent_parser):
         "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
     )
     pre_push_parser.set_defaults(func=CmdPrePush)
+
+    MERGE_DRIVER_HELP = "Run GIT merge driver."
+    merge_driver_parser = git_hook_subparsers.add_parser(
+        "merge-driver",
+        parents=[parent_parser],
+        description=MERGE_DRIVER_HELP,
+        help=MERGE_DRIVER_HELP,
+    )
+    merge_driver_parser.add_argument(
+        "--ancestor",
+        required=True,
+        help="Ancestor's version of the conflicting file.",
+    )
+    merge_driver_parser.add_argument(
+        "--our",
+        required=True,
+        help="Current version of the conflicting file.",
+    )
+    merge_driver_parser.add_argument(
+        "--their",
+        required=True,
+        help="Other branch's version of the conflicting file.",
+    )
+    merge_driver_parser.set_defaults(func=CmdMergeDriver)

dvc/command/imp.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -43,15 +44,15 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path", help="Path to a file or directory within the repository"
-    )
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
     import_parser.add_argument(
         "-o",
         "--out",
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
-    )
+    ).complete = completion.DIR
     import_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/imp_url.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -11,7 +12,10 @@ class CmdImportUrl(CmdBase):
     def run(self):
         try:
             self.repo.imp_url(
-                self.args.url, out=self.args.out, fname=self.args.file
+                self.args.url,
+                out=self.args.out,
+                fname=self.args.file,
+                no_exec=self.args.no_exec,
             )
         except DvcException:
             logger.exception(
@@ -38,28 +42,29 @@ def add_parser(subparsers, parent_parser):
     )
     import_parser.add_argument(
         "url",
-        help="Supported urls:\n"
-        "/path/to/file\n"
-        "/path/to/directory\n"
-        "C:\\\\path\\to\\file\n"
-        "C:\\\\path\\to\\directory\n"
+        help="Location of the data to download. Supported URLs:\n"
+        "/absolute/path/to/file/or/dir\n"
+        "relative/path/to/file/or/dir\n"
+        "C:\\\\path\\to\\file\\or\\dir\n"
         "https://example.com/path/to/file\n"
-        "s3://bucket/path/to/file\n"
-        "s3://bucket/path/to/directory\n"
-        "gs://bucket/path/to/file\n"
-        "gs://bucket/path/to/directory\n"
+        "s3://bucket/key/path\n"
+        "gs://bucket/path/to/file/or/dir\n"
         "hdfs://example.com/path/to/file\n"
-        "ssh://example.com:/path/to/file\n"
-        "ssh://example.com:/path/to/directory\n"
-        "remote://myremote/path/to/file (see `dvc remote`)",
+        "ssh://example.com/absolute/path/to/file/or/dir\n"
+        "remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to."
-    )
+        "out", nargs="?", help="Destination path to put files to.",
+    ).complete = completion.DIR
     import_parser.add_argument(
-        "-f",
         "--file",
         help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
+    ).complete = completion.DIR
+    import_parser.add_argument(
+        "--no-exec",
+        action="store_true",
+        default=False,
+        help="Only create DVC-file without actually downloading it.",
     )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/command/init.py

@@ -8,8 +8,8 @@ logger = logging.getLogger(__name__)
 
 class CmdInit(CmdBaseNoRepo):
     def run(self):
-        from dvc.repo import Repo
         from dvc.exceptions import InitError
+        from dvc.repo import Repo
 
         try:
             self.repo = Repo.init(

dvc/command/install.py

@@ -10,7 +10,7 @@ class CmdInstall(CmdBase):
     def run(self):
         try:
             self.repo.install(self.args.use_pre_commit_tool)
-        except Exception:
+        except Exception:  # noqa, pylint:disable=broad-except
             logger.exception("failed to install DVC Git hooks")
             return 1
         return 0

dvc/command/lock.py

@@ -1,59 +0,0 @@
-import argparse
-import logging
-
-from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
-
-logger = logging.getLogger(__name__)
-
-
-class CmdLockBase(CmdBase):
-    def _run(self, unlock):
-        for target in self.args.targets:
-            try:
-                self.repo.lock_stage(target, unlock=unlock)
-            except DvcException:
-                logger.exception(
-                    "failed to {}lock '{}'".format(
-                        "un" if unlock else "", target
-                    )
-                )
-
-                return 1
-        return 0
-
-
-class CmdLock(CmdLockBase):
-    def run(self):
-        return self._run(False)
-
-
-class CmdUnlock(CmdLockBase):
-    def run(self):
-        return self._run(True)
-
-
-def add_parser(subparsers, parent_parser):
-    LOCK_HELP = "Lock DVC-files."
-    lock_parser = subparsers.add_parser(
-        "lock",
-        parents=[parent_parser],
-        description=append_doc_link(LOCK_HELP, "lock"),
-        help=LOCK_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    lock_parser.add_argument("targets", nargs="+", help="DVC-files to lock.")
-    lock_parser.set_defaults(func=CmdLock)
-
-    UNLOCK_HELP = "Unlock DVC-files."
-    unlock_parser = subparsers.add_parser(
-        "unlock",
-        parents=[parent_parser],
-        description=append_doc_link(UNLOCK_HELP, "unlock"),
-        help=UNLOCK_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    unlock_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to unlock."
-    )
-    unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/ls/__init__.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 import sys
 
+from dvc.command import completion
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.command.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
@@ -31,7 +32,7 @@ class CmdList(CmdBaseNoRepo):
                 self.args.path,
                 rev=self.args.rev,
                 recursive=self.args.recursive,
-                outs_only=self.args.outs_only,
+                dvc_only=self.args.dvc_only,
             )
             if entries:
                 entries = _prettify(entries, sys.stdout.isatty())
@@ -62,7 +63,7 @@ def add_parser(subparsers, parent_parser):
         help="Recursively list files.",
     )
     list_parser.add_argument(
-        "--outs-only", action="store_true", help="Show only DVC outputs."
+        "--dvc-only", action="store_true", help="Show only DVC outputs."
     )
     list_parser.add_argument(
         "--rev",
@@ -74,5 +75,5 @@ def add_parser(subparsers, parent_parser):
         "path",
         nargs="?",
         help="Path to directory within the repository to list outputs for",
-    )
+    ).complete = completion.DIR
     list_parser.set_defaults(func=CmdList)

dvc/command/metrics.py

@@ -1,16 +1,21 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import BadMetricError, DvcException
 
 logger = logging.getLogger(__name__)
 
 
+DEFAULT_PRECISION = 5
+
+
 def show_metrics(
     metrics, all_branches=False, all_tags=False, all_commits=False
 ):
     from flatten_json import flatten
+
     from dvc.utils.diff import format_dict
 
     # When `metrics` contains a `None` key, it means that some files
@@ -63,49 +68,41 @@ class CmdMetricsShow(CmdBase):
         return 0
 
 
-class CmdMetricsAdd(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.add(self.args.path)
-        except DvcException:
-            msg = f"failed to add metric file '{self.args.path}'"
-            logger.exception(msg)
-            return 1
-
-        return 0
-
-
-class CmdMetricsRemove(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.remove(self.args.path)
-        except DvcException:
-            msg = f"failed to remove metric file '{self.args.path}'"
-            logger.exception(msg)
-            return 1
-
-        return 0
-
-
-def _show_diff(diff, markdown=False):
+def _show_diff(diff, markdown=False, no_path=False, old=False, precision=None):
     from collections import OrderedDict
 
     from dvc.utils.diff import table
 
+    if precision is None:
+        precision = DEFAULT_PRECISION
+
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
+
+        return val
+
     rows = []
     for fname, mdiff in diff.items():
         sorted_mdiff = OrderedDict(sorted(mdiff.items()))
         for metric, change in sorted_mdiff.items():
-            rows.append(
-                [
-                    fname,
-                    metric,
-                    change["new"],
-                    change.get("diff", "diff not supported"),
-                ]
-            )
+            row = [] if no_path else [fname]
+            row.append(metric)
+            if old:
+                row.append(_round(change.get("old")))
+            row.append(_round(change["new"]))
+            row.append(_round(change.get("diff", "diff not supported")))
+            rows.append(row)
 
-    return table(["Path", "Metric", "Value", "Change"], rows, markdown)
+    header = [] if no_path else ["Path"]
+    header.append("Metric")
+    if old:
+        header.extend(["Old", "New"])
+    else:
+        header.append("Value")
+    header.append("Change")
+
+    return table(header, rows, markdown)
 
 
 class CmdMetricsDiff(CmdBase):
@@ -124,7 +121,13 @@ class CmdMetricsDiff(CmdBase):
 
                 logger.info(json.dumps(diff))
             else:
-                table = _show_diff(diff, self.args.show_md)
+                table = _show_diff(
+                    diff,
+                    self.args.show_md,
+                    self.args.no_path,
+                    self.args.old,
+                    precision=self.args.precision,
+                )
                 if table:
                     logger.info(table)
 
@@ -136,7 +139,7 @@ class CmdMetricsDiff(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    METRICS_HELP = "Commands to add, manage, collect, and display metrics."
+    METRICS_HELP = "Commands to display and compare metrics."
 
     metrics_parser = subparsers.add_parser(
         "metrics",
@@ -153,17 +156,6 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(metrics_subparsers)
 
-    METRICS_ADD_HELP = "Mark a DVC-tracked file as a metric."
-    metrics_add_parser = metrics_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
-        help=METRICS_ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_add_parser.add_argument("path", help="Path to a metric file.")
-    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
-
     METRICS_SHOW_HELP = "Print metrics, with optional formatting."
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
@@ -175,8 +167,11 @@ def add_parser(subparsers, parent_parser):
     metrics_show_parser.add_argument(
         "targets",
         nargs="*",
-        help="Metric files or directories (see -R) to show",
-    )
+        help=(
+            "Limit command scope to these metric files. Using -R, "
+            "directories to search metric files in can also be given."
+        ),
+    ).complete = completion.FILE
     metrics_show_parser.add_argument(
         "-a",
         "--all-branches",
@@ -197,6 +192,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show metrics for all commits.",
     )
+    metrics_show_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
     metrics_show_parser.add_argument(
         "-R",
         "--recursive",
@@ -207,16 +208,12 @@ def add_parser(subparsers, parent_parser):
             "metric files."
         ),
     )
-    metrics_show_parser.add_argument(
-        "--show-json",
-        action="store_true",
-        default=False,
-        help="Show output in JSON format.",
-    )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
-    METRICS_DIFF_HELP = "Show changes in metrics between commits"
-    " in the DVC repository, or between a commit and the workspace."
+    METRICS_DIFF_HELP = (
+        "Show changes in metrics between commits in the DVC repository, or "
+        "between a commit and the workspace."
+    )
     metrics_diff_parser = metrics_subparsers.add_parser(
         "diff",
         parents=[parent_parser],
@@ -230,17 +227,17 @@ def add_parser(subparsers, parent_parser):
     metrics_diff_parser.add_argument(
         "b_rev",
         nargs="?",
-        help=("New Git commit to compare (defaults to the current workspace)"),
+        help="New Git commit to compare (defaults to the current workspace)",
     )
     metrics_diff_parser.add_argument(
         "--targets",
         nargs="*",
         help=(
-            "Metric files or directories (see -R) to show diff for. "
-            "Shows diff for all metric files by default."
+            "Limit command scope to these metric files. Using -R, "
+            "directories to search metric files in can also be given."
         ),
         metavar="<paths>",
-    )
+    ).complete = completion.FILE
     metrics_diff_parser.add_argument(
         "-R",
         "--recursive",
@@ -269,15 +266,25 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show tabulated output in the Markdown format (GFM).",
     )
-    metrics_diff_parser.set_defaults(func=CmdMetricsDiff)
-
-    METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
-    metrics_remove_parser = metrics_subparsers.add_parser(
-        "remove",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
-        help=METRICS_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
+    metrics_diff_parser.add_argument(
+        "--old",
+        action="store_true",
+        default=False,
+        help="Show old metric value.",
     )
-    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
-    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
+    metrics_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show metric path.",
+    )
+    metrics_diff_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics to `n` digits precision after the decimal point. "
+            f"Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
+    metrics_diff_parser.set_defaults(func=CmdMetricsDiff)

dvc/command/move.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -36,7 +37,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src", help="Source path to a data file or directory."
-    )
-    move_parser.add_argument("dst", help="Destination path.")
+        "src", help="Source path to a data file or directory.",
+    ).complete = completion.FILE
+    move_parser.add_argument(
+        "dst", help="Destination path.",
+    ).complete = completion.FILE
     move_parser.set_defaults(func=CmdMove)

dvc/command/params.py

@@ -8,16 +8,25 @@ from dvc.exceptions import DvcException
 logger = logging.getLogger(__name__)
 
 
-def _show_diff(diff, markdown=False):
+def _show_diff(diff, markdown=False, no_path=False):
     from dvc.utils.diff import table
 
     rows = []
     for fname, pdiff in diff.items():
         sorted_pdiff = OrderedDict(sorted(pdiff.items()))
         for param, change in sorted_pdiff.items():
-            rows.append([fname, param, change["old"], change["new"]])
+            row = [] if no_path else [fname]
+            row.append(param)
+            row.append(change["old"])
+            row.append(change["new"])
+            rows.append(row)
 
-    return table(["Path", "Param", "Old", "New"], rows, markdown)
+    header = [] if no_path else ["Path"]
+    header.append("Param")
+    header.append("Old")
+    header.append("New")
+
+    return table(header, rows, markdown)
 
 
 class CmdParamsDiff(CmdBase):
@@ -34,7 +43,7 @@ class CmdParamsDiff(CmdBase):
 
                 logger.info(json.dumps(diff))
             else:
-                table = _show_diff(diff, self.args.show_md)
+                table = _show_diff(diff, self.args.show_md, self.args.no_path)
                 if table:
                     logger.info(table)
 
@@ -100,4 +109,10 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show tabulated output in the Markdown format (GFM).",
     )
+    params_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show params path.",
+    )
     params_diff_parser.set_defaults(func=CmdParamsDiff)

dvc/command/pipeline.py

@@ -1,267 +0,0 @@
-import argparse
-import logging
-
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
-from dvc.exceptions import DvcException
-
-logger = logging.getLogger(__name__)
-
-
-class CmdPipelineShow(CmdBase):
-    def _show(self, target, commands, outs, locked):
-        import networkx
-        from dvc import dvcfile
-        from dvc.utils import parse_target
-
-        path, name = parse_target(target)
-        stage = dvcfile.Dvcfile(self.repo, path).stages[name]
-        G = self.repo.graph
-        stages = networkx.dfs_postorder_nodes(G, stage)
-        if locked:
-            stages = [s for s in stages if s.locked]
-
-        for stage in stages:
-            if commands:
-                if stage.cmd is None:
-                    continue
-                logger.info(stage.cmd)
-            elif outs:
-                for out in stage.outs:
-                    logger.info(str(out))
-            else:
-                logger.info(stage.addressing)
-
-    @staticmethod
-    def _build_output_graph(G, target_stage):
-        import networkx
-        from itertools import product
-
-        nodes = {str(out) for out in target_stage.outs}
-        edges = []
-
-        for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
-            from_stage_deps = {dep.path_info.parts for dep in from_stage.deps}
-            to_outs = {
-                to_out
-                for to_out in to_stage.outs
-                if to_out.path_info.parts in from_stage_deps
-            }
-            from_outs = {
-                from_out
-                for from_out in from_stage.outs
-                if str(from_out) in nodes
-            }
-            nodes |= {str(to_out) for to_out in to_outs}
-            for from_out, to_out in product(from_outs, to_outs):
-                edges.append((str(from_out), str(to_out)))
-        return nodes, edges
-
-    def _build_graph(self, target, commands=False, outs=False):
-        import networkx
-        from dvc import dvcfile
-        from dvc.repo.graph import get_pipeline
-        from dvc.utils import parse_target
-
-        path, name = parse_target(target)
-        target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
-        G = get_pipeline(self.repo.pipelines, target_stage)
-
-        nodes = set()
-        for stage in networkx.dfs_preorder_nodes(G, target_stage):
-            if commands:
-                if stage.cmd is None:
-                    continue
-                nodes.add(stage.cmd)
-            elif not outs:
-                nodes.add(stage.addressing)
-
-        edges = []
-        for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
-            if commands:
-                if to_stage.cmd is None:
-                    continue
-                edges.append((from_stage.cmd, to_stage.cmd))
-            elif not outs:
-                edges.append((from_stage.addressing, to_stage.addressing))
-
-        if outs:
-            nodes, edges = self._build_output_graph(G, target_stage)
-
-        return list(nodes), edges, networkx.is_tree(G)
-
-    def _show_ascii(self, target, commands, outs):
-        from dvc.dagascii import draw
-
-        nodes, edges, _ = self._build_graph(target, commands, outs)
-
-        if not nodes:
-            return
-
-        draw(nodes, edges)
-
-    def _show_dependencies_tree(self, target, commands, outs):
-        from treelib import Tree
-
-        nodes, edges, is_tree = self._build_graph(target, commands, outs)
-        if not nodes:
-            return
-        if not is_tree:
-            raise DvcException(
-                "DAG is not a tree, can not print it in tree-structure way, "
-                "please use --ascii instead"
-            )
-
-        tree = Tree()
-        tree.create_node(target, target)  # Root node
-        observe_list = [target]
-        while len(observe_list) > 0:
-            current_root = observe_list[0]
-            for edge in edges:
-                if edge[0] == current_root:
-                    tree.create_node(edge[1], edge[1], parent=current_root)
-                    observe_list.append(edge[1])
-            observe_list.pop(0)
-        tree.show()
-
-    def _write_dot(self, target, commands, outs):
-        import io
-        import networkx
-        from networkx.drawing.nx_pydot import write_dot
-
-        _, edges, _ = self._build_graph(target, commands, outs)
-        edges = [edge[::-1] for edge in edges]
-
-        simple_g = networkx.DiGraph()
-        simple_g.add_edges_from(edges)
-
-        dot_file = io.StringIO()
-        write_dot(simple_g, dot_file)
-        logger.info(dot_file.getvalue())
-
-    def run(self):
-        from dvc.dvcfile import DVC_FILE
-
-        if not self.args.targets:
-            self.args.targets = [DVC_FILE]
-
-        for target in self.args.targets:
-            try:
-                if self.args.ascii:
-                    self._show_ascii(
-                        target, self.args.commands, self.args.outs
-                    )
-                elif self.args.dot:
-                    self._write_dot(target, self.args.commands, self.args.outs)
-                elif self.args.tree:
-                    self._show_dependencies_tree(
-                        target, self.args.commands, self.args.outs
-                    )
-                else:
-                    self._show(
-                        target,
-                        self.args.commands,
-                        self.args.outs,
-                        self.args.locked,
-                    )
-            except DvcException:
-                msg = f"failed to show pipeline for '{target}'"
-                logger.exception(msg)
-                return 1
-        return 0
-
-
-class CmdPipelineList(CmdBase):
-    def run(self):
-        pipelines = self.repo.pipelines
-        for pipeline in pipelines:
-            for stage in pipeline:
-                logger.info(stage.addressing)
-            if len(pipeline) != 0:
-                logger.info("=" * 80)
-        logger.info("{} pipelines total".format(len(pipelines)))
-
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    PIPELINE_HELP = "Manage pipelines."
-    pipeline_parser = subparsers.add_parser(
-        "pipeline",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_HELP, "pipeline"),
-        help=PIPELINE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-
-    pipeline_subparsers = pipeline_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc pipeline CMD --help` for command-specific help.",
-    )
-
-    fix_subparsers(pipeline_subparsers)
-
-    PIPELINE_LIST_HELP = "List connected groups of stages (pipelines)."
-    pipeline_list_parser = pipeline_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
-        help=PIPELINE_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pipeline_list_parser.set_defaults(func=CmdPipelineList)
-
-    PIPELINE_SHOW_HELP = "Show stages in a pipeline."
-    pipeline_show_parser = pipeline_subparsers.add_parser(
-        "show",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline/show"),
-        help=PIPELINE_SHOW_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
-    pipeline_show_group.add_argument(
-        "-c",
-        "--commands",
-        action="store_true",
-        default=False,
-        help="Print commands instead of paths to DVC-files.",
-    )
-    pipeline_show_group.add_argument(
-        "-o",
-        "--outs",
-        action="store_true",
-        default=False,
-        help="Print output files instead of paths to DVC-files.",
-    )
-    pipeline_show_parser.add_argument(
-        "-l",
-        "--locked",
-        action="store_true",
-        default=False,
-        help="Print locked DVC stages",
-    )
-    pipeline_show_parser.add_argument(
-        "--ascii",
-        action="store_true",
-        default=False,
-        help="Output DAG as ASCII.",
-    )
-    pipeline_show_parser.add_argument(
-        "--dot",
-        action="store_true",
-        default=False,
-        help="Print DAG with .dot format.",
-    )
-    pipeline_show_parser.add_argument(
-        "--tree",
-        action="store_true",
-        default=False,
-        help="Output DAG as Dependencies Tree.",
-    )
-    pipeline_show_parser.add_argument(
-        "targets",
-        nargs="*",
-        help="DVC-files to show pipeline for. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
-    )
-    pipeline_show_parser.set_defaults(func=CmdPipelineShow)

dvc/command/plots.py

@@ -2,8 +2,11 @@ import argparse
 import logging
 import os
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
+from dvc.schema import PLOT_PROPS
+from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
@@ -31,32 +34,28 @@ class CmdPlots(CmdBase):
     def _func(self, *args, **kwargs):
         raise NotImplementedError
 
+    def _props(self):
+        # Pass only props specified by user, to not shadow ones from plot def
+        props = {p: getattr(self.args, p) for p in PLOT_PROPS}
+        return {k: v for k, v in props.items() if v is not None}
+
     def run(self):
-        fields = None
-        jsonpath = None
-        if self.args.select:
-            if self.args.select.startswith("$"):
-                jsonpath = self.args.select
-            else:
-                fields = set(self.args.select.split(","))
+        if self.args.show_vega:
+            if not self.args.targets:
+                logger.error("please specify a target for `--show-vega`")
+                return 1
+            if len(self.args.targets) > 1:
+                logger.error(
+                    "you can only specify one target for `--show-vega`"
+                )
+                return 1
+
         try:
-            plots = self._func(
-                targets=self.args.targets,
-                template=self.args.template,
-                fields=fields,
-                x_field=self.args.x,
-                y_field=self.args.y,
-                path=jsonpath,
-                csv_header=not self.args.no_csv_header,
-                title=self.args.title,
-                x_title=self.args.xlab,
-                y_title=self.args.ylab,
-            )
+            plots = self._func(targets=self.args.targets, props=self._props())
 
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(plots))
+            if self.args.show_vega:
+                target = self.args.targets[0]
+                logger.info(plots[target])
                 return 0
 
             divs = [
@@ -90,10 +89,18 @@ class CmdPlotsDiff(CmdPlots):
         return self.repo.plots.diff(*args, revs=self.args.revisions, **kwargs)
 
 
+class CmdPlotsModify(CmdPlots):
+    def run(self):
+        self.repo.plots.modify(
+            self.args.target, props=self._props(), unset=self.args.unset,
+        )
+        return 0
+
+
 def add_parser(subparsers, parent_parser):
     PLOTS_HELP = (
-        "Generating plots for metrics stored in structured files "
-        "(JSON, CSV, TSV)."
+        "Commands to visualize and compare plot metrics in structured files "
+        "(JSON, YAML, CSV, TSV)"
     )
 
     plots_parser = subparsers.add_parser(
@@ -110,7 +117,7 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(plots_subparsers)
 
-    SHOW_HELP = "Generate a plots image file from a metrics file."
+    SHOW_HELP = "Generate plots from metric files."
     plots_show_parser = plots_subparsers.add_parser(
         "show",
         parents=[parent_parser],
@@ -118,57 +125,18 @@ def add_parser(subparsers, parent_parser):
         help=SHOW_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    plots_show_parser.add_argument(
-        "-t",
-        "--template",
-        nargs="?",
-        default=None,
-        help="File to be injected with data.",
-    )
-    plots_show_parser.add_argument(
-        "-o", "--out", default=None, help="Destination path to save plots to.",
-    )
-    plots_show_parser.add_argument(
-        "-s",
-        "--select",
-        default=None,
-        help="Choose which field(s) or JSONPath to include in the plots.",
-    )
-    plots_show_parser.add_argument(
-        "-x", default=None, help="Field name for x axis."
-    )
-    plots_show_parser.add_argument(
-        "-y", default=None, help="Field name for y axis."
-    )
-    plots_show_parser.add_argument(
-        "--no-csv-header",
-        action="store_true",
-        default=False,
-        help="Required when CSV or TSV datafile does not have a header.",
-    )
-    plots_show_parser.add_argument(
-        "--show-json",
-        action="store_true",
-        default=False,
-        help="Show output in JSON format.",
-    )
-    plots_show_parser.add_argument("--title", default=None, help="Plot title.")
-    plots_show_parser.add_argument(
-        "--xlab", default=None, help="X axis title."
-    )
-    plots_show_parser.add_argument(
-        "--ylab", default=None, help="Y axis title."
-    )
     plots_show_parser.add_argument(
         "targets",
         nargs="*",
-        help="Metrics files to visualize. Shows all plots by default.",
-    )
+        help="Plots files to visualize. Shows all plots by default.",
+    ).complete = completion.FILE
+    _add_props_arguments(plots_show_parser)
+    _add_output_arguments(plots_show_parser)
     plots_show_parser.set_defaults(func=CmdPlotsShow)
 
     PLOTS_DIFF_HELP = (
-        "Plot differences in metrics between commits in the DVC "
-        "repository, or between the last commit and the workspace."
+        "Show multiple versions of plot metrics "
+        "by plotting them in a single image."
     )
     plots_diff_parser = plots_subparsers.add_parser(
         "diff",
@@ -178,52 +146,88 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help="Plots file to visualize. Shows all plots by default.",
+        metavar="<path>",
+    ).complete = completion.FILE
+    plots_diff_parser.add_argument(
+        "revisions", nargs="*", default=None, help="Git commits to plot from",
+    )
+    _add_props_arguments(plots_diff_parser)
+    _add_output_arguments(plots_diff_parser)
+    plots_diff_parser.set_defaults(func=CmdPlotsDiff)
+
+    PLOTS_MODIFY_HELP = "Modify display properties of plot metric files."
+    plots_modify_parser = plots_subparsers.add_parser(
+        "modify",
+        parents=[parent_parser],
+        description=append_doc_link(PLOTS_MODIFY_HELP, "plots/modify"),
+        help=PLOTS_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_modify_parser.add_argument(
+        "target", help="Metric file to set properties to",
+    ).complete = completion.FILE
+    _add_props_arguments(plots_modify_parser)
+    plots_modify_parser.add_argument(
+        "--unset",
+        nargs="*",
+        metavar="<property>",
+        help="Unset one or more display properties.",
+    )
+    plots_modify_parser.set_defaults(func=CmdPlotsModify)
+
+
+def _add_props_arguments(parser):
+    parser.add_argument(
         "-t",
         "--template",
         nargs="?",
         default=None,
-        help="File to be injected with data.",
+        help=(
+            "Special JSON or HTML schema file to inject with the data. "
+            "See {}".format(
+                format_link("https://man.dvc.org/plots#plot-templates")
+            )
+        ),
+        metavar="<path>",
+    ).complete = completion.FILE
+    parser.add_argument(
+        "-x", default=None, help="Field name for X axis.", metavar="<field>"
     )
-    plots_diff_parser.add_argument(
-        "--targets",
-        nargs="*",
-        help="Metrics file to visualize. Shows all plots by default.",
+    parser.add_argument(
+        "-y", default=None, help="Field name for Y axis.", metavar="<field>"
     )
-    plots_diff_parser.add_argument(
-        "-o", "--out", default=None, help="Destination path to save plots to.",
+    parser.add_argument(
+        "--no-header",
+        action="store_false",
+        dest="header",
+        default=None,  # Use default None to distinguish when it's not used
+        help="Provided CSV or TSV datafile does not have a header.",
     )
-    plots_diff_parser.add_argument(
-        "-s",
-        "--select",
+    parser.add_argument(
+        "--title", default=None, metavar="<text>", help="Plot title."
+    )
+    parser.add_argument(
+        "--x-label", default=None, help="X axis label", metavar="<text>"
+    )
+    parser.add_argument(
+        "--y-label", default=None, help="Y axis label", metavar="<text>"
+    )
+
+
+def _add_output_arguments(parser):
+    parser.add_argument(
+        "-o",
+        "--out",
         default=None,
-        help="Choose which field(s) or JSONPath to include in the plot.",
-    )
-    plots_diff_parser.add_argument(
-        "-x", default=None, help="Field name for x axis."
-    )
-    plots_diff_parser.add_argument(
-        "-y", default=None, help="Field name for y axis."
-    )
-    plots_diff_parser.add_argument(
-        "--no-csv-header",
+        help="Destination path to save plots to",
+        metavar="<path>",
+    ).complete = completion.DIR
+    parser.add_argument(
+        "--show-vega",
         action="store_true",
         default=False,
-        help="Provided CSV ot TSV datafile does not have a header.",
+        help="Show output in Vega format.",
     )
-    plots_diff_parser.add_argument(
-        "--show-json",
-        action="store_true",
-        default=False,
-        help="Show output in JSON format.",
-    )
-    plots_diff_parser.add_argument("--title", default=None, help="Plot title.")
-    plots_diff_parser.add_argument(
-        "--xlab", default=None, help="X axis title."
-    )
-    plots_diff_parser.add_argument(
-        "--ylab", default=None, help="Y axis title."
-    )
-    plots_diff_parser.add_argument(
-        "revisions", nargs="*", default=None, help="Git commits to plot from",
-    )
-    plots_diff_parser.set_defaults(func=CmdPlotsDiff)

dvc/command/remote.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.config import ConfigError
+from dvc.config import ConfigError, merge
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
@@ -18,7 +18,7 @@ class CmdRemote(CmdConfig):
 
     def _check_exists(self, conf):
         if self.args.name not in conf["remote"]:
-            raise ConfigError(f"remote '{self.args.name}' doesn't exists.")
+            raise ConfigError(f"remote '{self.args.name}' doesn't exist.")
 
 
 class CmdRemoteAdd(CmdRemote):
@@ -61,8 +61,17 @@ class CmdRemoteRemove(CmdRemote):
 class CmdRemoteModify(CmdRemote):
     def run(self):
         with self.config.edit(self.args.level) as conf:
-            self._check_exists(conf)
-            conf["remote"][self.args.name][self.args.option] = self.args.value
+            merged = self.config.load_config_to_level(self.args.level)
+            merge(merged, conf)
+            self._check_exists(merged)
+
+            if self.args.name not in conf["remote"]:
+                conf["remote"][self.args.name] = {}
+            section = conf["remote"][self.args.name]
+            if self.args.unset:
+                section.pop(self.args.option, None)
+            else:
+                section[self.args.option] = self.args.value
         return 0
 
 
@@ -104,6 +113,35 @@ class CmdRemoteList(CmdRemote):
         return 0
 
 
+class CmdRemoteRename(CmdRemote):
+    def _rename_default(self, conf):
+        if conf["core"].get("remote") == self.args.name:
+            conf["core"]["remote"] = self.args.new
+
+    def run(self):
+        all_config = self.config.load_config_to_level(None)
+        if self.args.new in all_config.get("remote", {}):
+            raise ConfigError(
+                "Rename failed. Remote name '{}' already exists.".format(
+                    {self.args.new}
+                )
+            )
+
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            conf["remote"][self.args.new] = conf["remote"][self.args.name]
+            del conf["remote"][self.args.name]
+            self._rename_default(conf)
+
+        for level in reversed(self.config.LEVELS):
+            if level == self.args.level:
+                break
+            with self.config.edit(level) as level_conf:
+                self._rename_default(level_conf)
+
+        return 0
+
+
 def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
@@ -134,7 +172,7 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser.add_argument("name", help="Name of the remote")
     remote_add_parser.add_argument(
         "url",
-        help="URL. See full list of supported urls at {}".format(
+        help="Remote location. See full list of supported URLs at {}".format(
             format_link("https://man.dvc.org/remote")
         ),
     )
@@ -163,7 +201,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_default_parser.add_argument(
-        "name", nargs="?", help="Name of the remote."
+        "name", nargs="?", help="Name of the remote"
     )
     remote_default_parser.add_argument(
         "-u",
@@ -182,7 +220,7 @@ def add_parser(subparsers, parent_parser):
         help=REMOTE_MODIFY_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remote_modify_parser.add_argument("name", help="Name of the remote.")
+    remote_modify_parser.add_argument("name", help="Name of the remote")
     remote_modify_parser.add_argument(
         "option", help="Name of the option to modify."
     )
@@ -220,3 +258,14 @@ def add_parser(subparsers, parent_parser):
         "name", help="Name of the remote to remove."
     )
     remote_remove_parser.set_defaults(func=CmdRemoteRemove)
+    REMOTE_RENAME_HELP = "Rename a DVC remote"
+    remote_rename_parser = remote_subparsers.add_parser(
+        "rename",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_RENAME_HELP, "remote/rename"),
+        help=REMOTE_RENAME_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_rename_parser.add_argument("name", help="Remote to be renamed")
+    remote_rename_parser.add_argument("new", help="New name of the remote")
+    remote_rename_parser.set_defaults(func=CmdRemoteRename)

dvc/command/remove.py

@@ -1,7 +1,7 @@
 import argparse
 import logging
 
-import dvc.prompt as prompt
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -9,30 +9,10 @@ logger = logging.getLogger(__name__)
 
 
 class CmdRemove(CmdBase):
-    def _is_outs_only(self, target):
-        if not self.args.purge:
-            return True
-
-        if self.args.force:
-            return False
-
-        msg = "Are you sure you want to remove '{}' with its outputs?".format(
-            target
-        )
-
-        if prompt.confirm(msg):
-            return False
-
-        raise DvcException(
-            "Cannot purge without a confirmation from the user."
-            " Use `-f` to force."
-        )
-
     def run(self):
         for target in self.args.targets:
             try:
-                outs_only = self._is_outs_only(target)
-                self.repo.remove(target, outs_only=outs_only)
+                self.repo.remove(target, outs=self.args.outs)
             except DvcException:
                 logger.exception(f"failed to remove '{target}'")
                 return 1
@@ -40,7 +20,9 @@ class CmdRemove(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REMOVE_HELP = "Remove DVC-tracked files or directories."
+    REMOVE_HELP = (
+        "Remove stage entry, remove .gitignore entry and unprotect outputs"
+    )
     remove_parser = subparsers.add_parser(
         "remove",
         parents=[parent_parser],
@@ -48,29 +30,13 @@ def add_parser(subparsers, parent_parser):
         help=REMOVE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remove_parser_group = remove_parser.add_mutually_exclusive_group()
-    remove_parser_group.add_argument(
-        "-o",
+    remove_parser.add_argument(
         "--outs",
         action="store_true",
-        default=True,
-        help="Only remove DVC-file outputs. (Default)",
-    )
-    remove_parser_group.add_argument(
-        "-p",
-        "--purge",
-        action="store_true",
         default=False,
-        help="Remove DVC-file and all its outputs.",
+        help="Remove outputs as well.",
     )
     remove_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Force purge.",
-    )
-    remove_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to remove."
-    )
+        "targets", nargs="+", help="DVC-files to remove.",
+    ).complete = completion.DVC_FILE
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -2,9 +2,11 @@ import argparse
 import logging
 import os
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
+from dvc.dvcfile import PIPELINE_FILE
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -38,6 +40,11 @@ class CmdRepro(CmdBase):
                     downstream=self.args.downstream,
                     recursive=self.args.recursive,
                     force_downstream=self.args.force_downstream,
+                    experiment=self.args.experiment,
+                    queue=self.args.queue,
+                    run_all=self.args.run_all,
+                    jobs=self.args.jobs,
+                    params=self.args.params,
                 )
 
                 if len(stages) == 0:
@@ -69,8 +76,8 @@ def add_parser(subparsers, parent_parser):
     repro_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC-file to reproduce. 'Dvcfile' by default.",
-    )
+        help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
+    ).complete = completion.DVC_FILE
     repro_parser.add_argument(
         "-f",
         "--force",
@@ -164,4 +171,30 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Start from the specified stages when reproducing pipelines.",
     )
+    repro_parser.add_argument(
+        "-e",
+        "--experiment",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    repro_parser.add_argument(
+        "--params",
+        action="append",
+        default=[],
+        help="Declare parameter values for an experiment.",
+        metavar="[<filename>:]<params_list>",
+    )
+    repro_parser.add_argument(
+        "--queue", action="store_true", default=False, help=argparse.SUPPRESS
+    )
+    repro_parser.add_argument(
+        "--run-all",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    repro_parser.add_argument(
+        "-j", "--jobs", type=int, help=argparse.SUPPRESS, metavar="<number>"
+    )
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/run.py

@@ -45,7 +45,7 @@ class CmdRun(CmdBase):
                 fname=self.args.file,
                 wdir=self.args.wdir,
                 no_exec=self.args.no_exec,
-                overwrite=self.args.overwrite_dvcfile,
+                force=self.args.force,
                 run_cache=not self.args.no_run_cache,
                 no_commit=self.args.no_commit,
                 outs_persist=self.args.outs_persist,
@@ -53,6 +53,7 @@ class CmdRun(CmdBase):
                 always_changed=self.args.always_changed,
                 name=self.args.name,
                 single_stage=self.args.single_stage,
+                external=self.args.external,
             )
         except DvcException:
             logger.exception("")
@@ -132,7 +133,7 @@ def add_parser(subparsers, parent_parser):
         "--metrics",
         action="append",
         default=[],
-        help="Declare output metric file or directory.",
+        help="Declare output metric file.",
         metavar="<path>",
     )
     run_parser.add_argument(
@@ -140,8 +141,7 @@ def add_parser(subparsers, parent_parser):
         "--metrics-no-cache",
         action="append",
         default=[],
-        help="Declare output metric file or directory "
-        "(do not put into DVC cache).",
+        help="Declare output metric file (do not put into DVC cache).",
         metavar="<path>",
     )
     run_parser.add_argument(
@@ -159,10 +159,7 @@ def add_parser(subparsers, parent_parser):
         metavar="<path>",
     )
     run_parser.add_argument(
-        "-f",
-        "--file",
-        help="Specify name of the DVC-file this command will generate.",
-        metavar="<filename>",
+        "--file", metavar="<filename>", help=argparse.SUPPRESS,
     )
     run_parser.add_argument(
         "-w",
@@ -177,10 +174,11 @@ def add_parser(subparsers, parent_parser):
         help="Only create stage file without actually running it.",
     )
     run_parser.add_argument(
-        "--overwrite-dvcfile",
+        "-f",
+        "--force",
         action="store_true",
         default=False,
-        help="Overwrite existing DVC-file without asking for confirmation.",
+        help="Overwrite existing stage",
     )
     run_parser.add_argument(
         "--no-run-cache",
@@ -225,6 +223,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=argparse.SUPPRESS,
     )
+    run_parser.add_argument(
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow outputs that are outside of the DVC repository.",
+    )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."
     )

dvc/command/status.py

@@ -50,15 +50,20 @@ class CmdDataStatus(CmdDataBase):
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
             )
-            if st:
-                if self.args.quiet:
-                    return 1
-                else:
-                    self._show(st, indent)
+
+            if self.args.quiet:
+                return bool(st)
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(st))
+            elif st:
+                self._show(st, indent)
             else:
                 logger.info(self.UP_TO_DATE_MSG)
 
-        except Exception:
+        except Exception:  # noqa, pylint: disable=broad-except
             logger.exception("failed to obtain data status")
             return 1
         return 0

dvc/command/unprotect.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -32,6 +33,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets", nargs="+", help="Data files/directories to unprotect."
-    )
+        "targets", nargs="+", help="Data files/directories to unprotect.",
+    ).complete = completion.FILE
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -32,8 +33,8 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to update."
-    )
+        "targets", nargs="+", help="DVC-files to update.",
+    ).complete = completion.DVC_FILE
     update_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/version.py

@@ -10,7 +10,7 @@ from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.scm.base import SCMError
 from dvc.system import System
-from dvc.utils import is_binary, relpath
+from dvc.utils import relpath
 from dvc.utils.pkg import PKG
 from dvc.version import __version__
 
@@ -27,13 +27,18 @@ class CmdVersion(CmdBaseNoRepo):
     def run(self):
         from dvc.repo import Repo
 
+        package = PKG
+        if PKG is None:
+            package = ""
+        else:
+            package = f"({PKG})"
+
         info = [
-            f"DVC version: {__version__}",
-            f"Python version: {platform.python_version()}",
-            f"Platform: {platform.platform()}",
-            f"Binary: {is_binary()}",
-            f"Package: {PKG}",
-            f"Supported remotes: {self.get_supported_remotes()}",
+            f"DVC version: {__version__} {package}",
+            "---------------------------------",
+            f"Platform: Python {platform.python_version()} on "
+            f"{platform.platform()}",
+            f"Supports: {self.get_supported_remotes()}",
         ]
 
         try:
@@ -46,13 +51,13 @@ class CmdVersion(CmdBaseNoRepo):
             # `dvc config cache.shared group`.
             if os.path.exists(repo.cache.local.cache_dir):
                 info.append(
-                    "Cache: {}".format(self.get_linktype_support_info(repo))
+                    "Cache types: {}".format(
+                        self.get_linktype_support_info(repo)
+                    )
                 )
                 if psutil:
                     fs_type = self.get_fs_type(repo.cache.local.cache_dir)
-                    info.append(
-                        f"Filesystem type (cache directory): {fs_type}"
-                    )
+                    info.append(f"Cache directory: {fs_type}")
             else:
                 logger.warning(
                     "Unable to detect supported link types, as cache "
@@ -68,11 +73,11 @@ class CmdVersion(CmdBaseNoRepo):
             root_directory = os.getcwd()
             info.append("Repo: dvc, git (broken)")
         else:
-            info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
+            if psutil:
+                fs_root = self.get_fs_type(os.path.abspath(root_directory))
+                info.append(f"Workspace directory: {fs_root}")
 
-        if psutil:
-            fs_root = self.get_fs_type(os.path.abspath(root_directory))
-            info.append(f"Filesystem type (workspace): {fs_root}")
+            info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
 
         logger.info("\n".join(info))
         return 0
@@ -80,7 +85,7 @@ class CmdVersion(CmdBaseNoRepo):
     @staticmethod
     def get_fs_type(path):
         partition = {
-            pathlib.Path(part.mountpoint): (part.fstype, part.device)
+            pathlib.Path(part.mountpoint): (part.fstype + " on " + part.device)
             for part in psutil.disk_partitions(all=True)
         }
 
@@ -115,19 +120,27 @@ class CmdVersion(CmdBaseNoRepo):
                 os.unlink(dst)
             except DvcException:
                 status = "not supported"
-            cache.append(f"{name} - {status}")
+
+            if status == "supported":
+                cache.append(name)
         os.remove(src)
 
         return ", ".join(cache)
 
     @staticmethod
     def get_supported_remotes():
-        from dvc.remote import REMOTES
+        from dvc.tree import TREES
 
         supported_remotes = []
-        for remote in REMOTES:
-            if not remote.get_missing_deps():
-                supported_remotes.append(remote.scheme)
+        for tree_cls in TREES:
+            if not tree_cls.get_missing_deps():
+                supported_remotes.append(tree_cls.scheme)
+
+        if len(supported_remotes) == len(TREES):
+            return "All remotes"
+
+        if len(supported_remotes) == 1:
+            return supported_remotes
 
         return ", ".join(supported_remotes)
 

dvc/config.py

@@ -119,6 +119,16 @@ HTTP_COMMON = {
     "password": str,
     "ask_password": Bool,
 }
+WEBDAV_COMMON = {
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "token": str,
+    "cert_path": str,
+    "key_path": str,
+    "timeout": Coerce(int),
+}
+
 SCHEMA = {
     "core": {
         "remote": Lower,
@@ -127,6 +137,8 @@ SCHEMA = {
         Optional("analytics", default=True): Bool,
         Optional("hardlink_lock", default=False): Bool,
         Optional("no_scm", default=False): Bool,
+        Optional("experiments", default=False): Bool,
+        Optional("check_update", default=True): Bool,
     },
     "cache": {
         "local": str,
@@ -148,7 +160,9 @@ SCHEMA = {
                     "profile": str,
                     "credentialpath": str,
                     "endpointurl": str,
-                    Optional("listobjects", default=False): Bool,
+                    "access_key_id": str,
+                    "secret_access_key": str,
+                    Optional("listobjects", default=False): Bool,  # obsoleted
                     Optional("use_ssl", default=True): Bool,
                     "sse": str,
                     "sse_kms_key_id": str,
@@ -196,6 +210,8 @@ SCHEMA = {
                 },
                 "http": {**HTTP_COMMON, **REMOTE_COMMON},
                 "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
                 "remote": {str: object},  # Any of the above options are valid
             }
         )
@@ -231,7 +247,11 @@ class Config(dict):
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
-    def __init__(self, dvc_dir=None, validate=True):
+    def __init__(
+        self, dvc_dir=None, validate=True, tree=None,
+    ):  # pylint: disable=super-init-not-called
+        from dvc.tree.local import LocalTree
+
         self.dvc_dir = dvc_dir
 
         if not dvc_dir:
@@ -244,11 +264,14 @@ class Config(dict):
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
+        self.wtree = LocalTree(None, {"url": self.dvc_dir})
+        self.tree = tree or self.wtree
+
         self.load(validate=validate)
 
     @classmethod
     def get_dir(cls, level):
-        from appdirs import user_config_dir, site_config_dir
+        from appdirs import site_config_dir, user_config_dir
 
         assert level in ("global", "system")
 
@@ -302,8 +325,37 @@ class Config(dict):
         if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
+    def _get_tree(self, level):
+        # NOTE: this might be a GitTree, which doesn't see things outside of
+        # the repo.
+        return self.tree if level == "repo" else self.wtree
+
+    def _load_config(self, level):
+        filename = self.files[level]
+        tree = self._get_tree(level)
+
+        if tree.exists(filename, use_dvcignore=False):
+            with tree.open(filename) as fobj:
+                conf_obj = configobj.ConfigObj(fobj)
+        else:
+            conf_obj = configobj.ConfigObj()
+        return _parse_remotes(_lower_keys(conf_obj.dict()))
+
+    def _save_config(self, level, conf_dict):
+        filename = self.files[level]
+        tree = self._get_tree(level)
+
+        logger.debug(f"Writing '{filename}'.")
+
+        tree.makedirs(os.path.dirname(filename))
+
+        config = configobj.ConfigObj(_pack_remotes(conf_dict))
+        with tree.open(filename, "wb") as fobj:
+            config.write(fobj)
+        config.filename = filename
+
     def load_one(self, level):
-        conf = _load_config(self.files[level])
+        conf = self._load_config(level)
         conf = self._load_paths(conf, self.files[level])
 
         # Auto-verify sections
@@ -356,7 +408,7 @@ class Config(dict):
             if merge_level == level:
                 break
             if merge_level in self.files:
-                _merge(merged_conf, self.load_one(merge_level))
+                merge(merged_conf, self.load_one(merge_level))
         return merged_conf
 
     @contextmanager
@@ -370,10 +422,10 @@ class Config(dict):
         conf = self._save_paths(conf, self.files[level])
 
         merged_conf = self.load_config_to_level(level)
-        _merge(merged_conf, conf)
+        merge(merged_conf, conf)
         self.validate(merged_conf)
 
-        _save_config(self.files[level], conf)
+        self._save_config(level, conf)
         self.load()
 
     @staticmethod
@@ -384,20 +436,6 @@ class Config(dict):
             raise ConfigError(str(exc)) from None
 
 
-def _load_config(filename):
-    conf_obj = configobj.ConfigObj(filename)
-    return _parse_remotes(_lower_keys(conf_obj.dict()))
-
-
-def _save_config(filename, conf_dict):
-    logger.debug(f"Writing '{filename}'.")
-    os.makedirs(os.path.dirname(filename), exist_ok=True)
-
-    config = configobj.ConfigObj(_pack_remotes(conf_dict))
-    config.filename = filename
-    config.write()
-
-
 def _parse_remotes(conf):
     result = {"remote": {}}
 
@@ -423,11 +461,11 @@ def _pack_remotes(conf):
     return result
 
 
-def _merge(into, update):
+def merge(into, update):
     """Merges second dict into first recursively"""
     for key, val in update.items():
         if isinstance(into.get(key), dict) and isinstance(val, dict):
-            _merge(into[key], val)
+            merge(into[key], val)
         else:
             into[key] = val
 

dvc/daemon.py

@@ -3,6 +3,7 @@
 import inspect
 import logging
 import os
+import platform
 import sys
 from subprocess import Popen
 
@@ -15,25 +16,30 @@ CREATE_NEW_PROCESS_GROUP = 0x00000200
 DETACHED_PROCESS = 0x00000008
 
 
+def _popen(cmd, **kwargs):
+    prefix = [sys.executable]
+    if not is_binary():
+        prefix += [sys.argv[0]]
+
+    return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
+
+
 def _spawn_windows(cmd, env):
-    from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
+    from subprocess import STARTF_USESHOWWINDOW, STARTUPINFO
 
     creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
 
     startupinfo = STARTUPINFO()
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
-    Popen(
-        cmd,
-        env=env,
-        close_fds=True,
-        shell=False,
-        creationflags=creationflags,
-        startupinfo=startupinfo,
+    _popen(
+        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo,
     ).communicate()
 
 
 def _spawn_posix(cmd, env):
+    from dvc.main import main
+
     # NOTE: using os._exit instead of sys.exit, because dvc built
     # with PyInstaller has trouble with SystemExit exception and throws
     # errors such as "[26338] Failed to execute script __main__"
@@ -46,7 +52,6 @@ def _spawn_posix(cmd, env):
         os._exit(1)  # pylint: disable=protected-access
 
     os.setsid()
-    os.umask(0)
 
     try:
         pid = os.fork()
@@ -60,7 +65,13 @@ def _spawn_posix(cmd, env):
     sys.stdout.close()
     sys.stderr.close()
 
-    Popen(cmd, env=env, close_fds=True, shell=False).communicate()
+    if platform.system() == "Darwin":
+        # workaround for MacOS bug
+        # https://github.com/iterative/dvc/issues/4294
+        _popen(cmd, env=env).communicate()
+    else:
+        os.environ.update(env)
+        main(cmd)
 
     os._exit(0)  # pylint: disable=protected-access
 
@@ -88,10 +99,7 @@ def daemon(args):
         logger.debug("skipping launching a new daemon.")
         return
 
-    cmd = [sys.executable]
-    if not is_binary():
-        cmd += [sys.argv[0]]
-    cmd += ["daemon", "-q"] + args
+    cmd = ["daemon", "-q"] + args
 
     env = fix_env()
     file_path = os.path.abspath(inspect.stack()[0][1])

dvc/dagascii.py

@@ -96,10 +96,9 @@ class AsciiCanvas:
 
     def draw(self):
         """Draws ASCII canvas on the screen."""
-        pager = find_pager()
         lines = map("".join, self.canvas)
         joined_lines = os.linesep.join(lines)
-        pager(joined_lines)
+        return joined_lines
 
     def point(self, x, y, char):
         """Create a point on ASCII canvas.
@@ -223,7 +222,7 @@ def _build_sugiyama_layout(vertexes, edges):
         vertex.view = VertexViewer(vertex.data)
 
     # NOTE: determine min box length to create the best layout
-    minw = min([v.view.w for v in vertexes])
+    minw = min(v.view.w for v in vertexes)
 
     for edge in edges:
         edge.view = EdgeViewer()
@@ -316,4 +315,4 @@ def draw(vertexes, edges):
             int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
         )
 
-    canvas.draw()
+    return canvas.draw()

dvc/data_cloud.py

@@ -3,7 +3,7 @@
 import logging
 
 from dvc.config import NoRemoteError
-from dvc.remote import Remote
+from dvc.remote import get_remote
 
 logger = logging.getLogger(__name__)
 
@@ -45,7 +45,7 @@ class DataCloud:
         raise NoRemoteError(error_msg)
 
     def _init_remote(self, name):
-        return Remote(self.repo, name=name)
+        return get_remote(self.repo, name=name)
 
     def push(
         self, cache, jobs=None, remote=None, show_checksums=False,
@@ -85,22 +85,29 @@ class DataCloud:
             cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )
 
-        if not remote.verify:
+        if not remote.tree.verify:
             self._save_pulled_checksums(cache)
 
         return downloaded_items_num
 
     def _save_pulled_checksums(self, cache):
         for checksum in cache.scheme_keys("local"):
-            cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
-            if self.repo.cache.local.exists(cache_file):
+            cache_file = self.repo.cache.local.tree.hash_to_path_info(checksum)
+            if self.repo.cache.local.tree.exists(cache_file):
                 # We can safely save here, as existing corrupted files will
                 # be removed upon status, while files corrupted during
                 # download will not be moved from tmp_file
                 # (see `RemoteBASE.download()`)
                 self.repo.state.save(cache_file, checksum)
 
-    def status(self, cache, jobs=None, remote=None, show_checksums=False):
+    def status(
+        self,
+        cache,
+        jobs=None,
+        remote=None,
+        show_checksums=False,
+        log_missing=True,
+    ):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
@@ -111,8 +118,14 @@ class DataCloud:
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
+            log_missing (bool): log warning messages if file doesn't exist
+                neither in cache, neither in cloud.
         """
         remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
+            cache,
+            jobs=jobs,
+            remote=remote,
+            show_checksums=show_checksums,
+            log_missing=log_missing,
         )

dvc/dependency/__init__.py

@@ -2,6 +2,7 @@ from collections import defaultdict
 from urllib.parse import urlparse
 
 import dvc.output as output
+from dvc.dependency.azure import AzureDependency
 from dvc.dependency.gs import GSDependency
 from dvc.dependency.hdfs import HDFSDependency
 from dvc.dependency.http import HTTPDependency
@@ -11,12 +12,13 @@ from dvc.dependency.param import ParamsDependency
 from dvc.dependency.s3 import S3Dependency
 from dvc.dependency.ssh import SSHDependency
 from dvc.output.base import BaseOutput
-from dvc.remote import Remote
 from dvc.scheme import Schemes
 
+from ..tree import get_cloud_tree
 from .repo import RepoDependency
 
 DEPS = [
+    AzureDependency,
     GSDependency,
     HDFSDependency,
     HTTPDependency,
@@ -30,6 +32,7 @@ DEP_MAP = {
     Schemes.LOCAL: LocalDependency,
     Schemes.SSH: SSHDependency,
     Schemes.S3: S3Dependency,
+    Schemes.AZURE: AzureDependency,
     Schemes.GS: GSDependency,
     Schemes.HDFS: HDFSDependency,
     Schemes.HTTP: HTTPDependency,
@@ -51,8 +54,8 @@ SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
 def _get(stage, p, info):
     parsed = urlparse(p) if p else None
     if parsed and parsed.scheme == "remote":
-        remote = Remote(stage.repo, name=parsed.netloc)
-        return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
+        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
+        return DEP_MAP[tree.scheme](stage, p, info, tree=tree)
 
     if info and info.get(RepoDependency.PARAM_REPO):
         repo = info.pop(RepoDependency.PARAM_REPO)
@@ -77,33 +80,34 @@ def loadd_from(stage, d_list):
 
 
 def loads_from(stage, s_list, erepo=None):
-    ret = []
-    for s in s_list:
-        info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
-        ret.append(_get(stage, s, info))
-    return ret
+    assert isinstance(s_list, list)
+    info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
+    return [_get(stage, s, info.copy()) for s in s_list]
 
 
-def _parse_params(path_params):
-    path, _, params_str = path_params.rpartition(":")
-    params = params_str.split(",")
-    return path, params
+def _merge_params(s_list):
+    d = defaultdict(list)
+    default_file = ParamsDependency.DEFAULT_PARAMS_FILE
+    for key in s_list:
+        if isinstance(key, str):
+            d[default_file].append(key)
+            continue
+        if not isinstance(key, dict):
+            msg = "Only list of str/dict is supported. Got: "
+            msg += f"'{type(key).__name__}'."
+            raise ValueError(msg)
+
+        for k, params in key.items():
+            if not isinstance(params, list):
+                msg = "Expected list of params for custom params file "
+                msg += f"'{k}', got '{type(params).__name__}'."
+                raise ValueError(msg)
+            d[k].extend(params)
+    return d
 
 
 def loads_params(stage, s_list):
-    # Creates an object for each unique file that is referenced in the list
-    params_by_path = defaultdict(list)
-    for s in s_list:
-        path, params = _parse_params(s)
-        params_by_path[path].extend(params)
-
-    d_list = []
-    for path, params in params_by_path.items():
-        d_list.append(
-            {
-                BaseOutput.PARAM_PATH: path,
-                ParamsDependency.PARAM_PARAMS: params,
-            }
-        )
-
-    return loadd_from(stage, d_list)
+    d = _merge_params(s_list)
+    return [
+        ParamsDependency(stage, path, params) for path, params in d.items()
+    ]

dvc/dependency/azure.py

@@ -0,0 +1,8 @@
+from dvc.dependency.base import BaseDependency
+from dvc.output.base import BaseOutput
+
+from ..tree.azure import AzureTree
+
+
+class AzureDependency(BaseDependency, BaseOutput):
+    TREE_CLS = AzureTree

dvc/dependency/http.py

@@ -1,7 +1,8 @@
 from dvc.dependency.base import BaseDependency
 from dvc.output.base import BaseOutput
-from dvc.remote.http import HTTPRemote
+
+from ..tree.http import HTTPTree
 
 
 class HTTPDependency(BaseDependency, BaseOutput):
-    REMOTE = HTTPRemote
+    TREE_CLS = HTTPTree

dvc/dependency/https.py

@@ -1,7 +1,6 @@
-from dvc.remote.https import HTTPSRemote
-
+from ..tree.https import HTTPSTree
 from .http import HTTPDependency
 
 
 class HTTPSDependency(HTTPDependency):
-    REMOTE = HTTPSRemote
+    TREE_CLS = HTTPSTree

dvc/dependency/param.py

@@ -2,11 +2,11 @@ import os
 from collections import defaultdict
 
 import dpath.util
-import yaml
 from voluptuous import Any
 
 from dvc.dependency.local import LocalDependency
 from dvc.exceptions import DvcException
+from dvc.utils.serialize import LOADERS, ParseError
 
 
 class MissingParamsError(DvcException):
@@ -40,14 +40,13 @@ class ParamsDependency(LocalDependency):
             info=info,
         )
 
-    def _dyn_load(self, values=None):
+    def fill_values(self, values=None):
         """Load params values dynamically."""
         if not values:
             return
         for param in self.params:
-            value = values.get(param)
-            if value:
-                self.info[param] = value
+            if param in values:
+                self.info[param] = values[param]
 
     def save(self):
         super().save()
@@ -86,13 +85,14 @@ class ParamsDependency(LocalDependency):
         if not self.exists:
             return {}
 
-        with self.repo.tree.open(self.path_info, "r") as fobj:
-            try:
-                config = yaml.safe_load(fobj)
-            except yaml.YAMLError as exc:
-                raise BadParamFileError(
-                    f"Unable to read parameters from '{self}'"
-                ) from exc
+        suffix = self.path_info.suffix.lower()
+        loader = LOADERS[suffix]
+        try:
+            config = loader(self.path_info, tree=self.repo.tree)
+        except ParseError as exc:
+            raise BadParamFileError(
+                f"Unable to read parameters from '{self}'"
+            ) from exc
 
         ret = {}
         for param in self.params:

dvc/dependency/repo.py

@@ -26,7 +26,7 @@ class RepoDependency(LocalDependency):
         self.def_repo = def_repo
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, remote, path):
+    def _parse_path(self, tree, path):
         return None
 
     @property
@@ -50,15 +50,26 @@ class RepoDependency(LocalDependency):
         return external_repo(d["url"], rev=rev)
 
     def _get_checksum(self, locked=True):
+        from dvc.tree.repo import RepoTree
+
         with self._make_repo(locked=locked) as repo:
             try:
                 return repo.find_out_by_relpath(self.def_path).info["md5"]
             except OutputNotFoundError:
                 path = PathInfo(os.path.join(repo.root_dir, self.def_path))
-                # We are polluting our repo cache with some dir listing here
-                return self.repo.cache.local.get_checksum(path)
 
-    def status(self):
+                # we want stream but not fetch, so DVC out directories are
+                # walked, but dir contents is not fetched
+                tree = RepoTree(repo, stream=True)
+
+                # We are polluting our repo cache with some dir listing here
+                if tree.isdir(path):
+                    return self.repo.cache.local.tree.get_hash(
+                        path, tree=tree
+                    )[1]
+                return tree.get_file_hash(path)
+
+    def workspace_status(self):
         current_checksum = self._get_checksum(locked=True)
         updated_checksum = self._get_checksum(locked=False)
 
@@ -67,6 +78,9 @@ class RepoDependency(LocalDependency):
 
         return {}
 
+    def status(self):
+        return self.workspace_status()
+
     def save(self):
         pass
 
@@ -76,16 +90,22 @@ class RepoDependency(LocalDependency):
     def download(self, to):
         with self._make_repo() as repo:
             if self.def_repo.get(self.PARAM_REV_LOCK) is None:
-                self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+                self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
 
-            if hasattr(repo, "cache"):
-                repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
-
-            repo.pull_to(self.def_path, to.path_info)
+            cache = self.repo.cache.local
+            with repo.use_cache(cache):
+                _, _, cache_infos = repo.fetch_external([self.def_path])
+            cache.checkout(to.path_info, cache_infos[0])
 
     def update(self, rev=None):
         if rev:
             self.def_repo[self.PARAM_REV] = rev
 
         with self._make_repo(locked=False) as repo:
-            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+            self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
+
+    def changed_checksum(self):
+        # From current repo point of view what describes RepoDependency is its
+        # origin project url and rev_lock, and it makes RepoDependency
+        # immutable, hence its impossible for checksum to change.
+        return False

dvc/dvcfile.py

@@ -1,14 +1,13 @@
+import collections
 import contextlib
 import logging
 import os
 
 from voluptuous import MultipleInvalid
 
-import dvc.prompt as prompt
-from dvc import serialize
 from dvc.exceptions import DvcException
+from dvc.stage import serialize
 from dvc.stage.exceptions import (
-    StageFileAlreadyExistsError,
     StageFileBadNameError,
     StageFileDoesNotExistError,
     StageFileFormatError,
@@ -17,10 +16,12 @@ from dvc.stage.exceptions import (
 from dvc.stage.loader import SingleStageLoader, StageLoader
 from dvc.utils import relpath
 from dvc.utils.collections import apply_diff
-from dvc.utils.stage import (
-    dump_stage_file,
-    parse_stage,
-    parse_stage_for_update,
+from dvc.utils.serialize import (
+    dump_yaml,
+    load_yaml,
+    modify_yaml,
+    parse_yaml,
+    parse_yaml_for_update,
 )
 
 logger = logging.getLogger(__name__)
@@ -61,9 +62,10 @@ def check_dvc_filename(path):
 class FileMixin:
     SCHEMA = None
 
-    def __init__(self, repo, path, **kwargs):
+    def __init__(self, repo, path, verify=True, **kwargs):
         self.repo = repo
         self.path = path
+        self.verify = verify
 
     def __repr__(self):
         return "{}: {}".format(
@@ -95,41 +97,39 @@ class FileMixin:
         # 3. path doesn't represent a regular file
         if not self.exists():
             raise StageFileDoesNotExistError(self.path)
-        check_dvc_filename(self.path)
+        if self.verify:
+            check_dvc_filename(self.path)
         if not self.repo.tree.isfile(self.path):
             raise StageFileIsNotDvcFileError(self.path)
 
         with self.repo.tree.open(self.path) as fd:
             stage_text = fd.read()
-        d = parse_stage(stage_text, self.path)
+        d = parse_yaml(stage_text, self.path)
         self.validate(d, self.relpath)
         return d, stage_text
 
     @classmethod
     def validate(cls, d, fname=None):
-        assert cls.SCHEMA
+        assert isinstance(cls.SCHEMA, collections.abc.Callable)
         try:
-            cls.SCHEMA(d)
+            cls.SCHEMA(d)  # pylint: disable=not-callable
         except MultipleInvalid as exc:
             raise StageFileFormatError(f"'{fname}' format error: {exc}")
 
-    def remove_with_prompt(self, force=False):
-        raise NotImplementedError
-
-    def remove(self, force=False):
+    def remove(self, force=False):  # pylint: disable=unused-argument
         with contextlib.suppress(FileNotFoundError):
             os.unlink(self.path)
 
     def dump(self, stage, **kwargs):
         raise NotImplementedError
 
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
 
 class SingleStageFile(FileMixin):
     from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
 
-    def __init__(self, repo, path):
-        super().__init__(repo, path)
-
     @property
     def stage(self):
         data, raw = self._load()
@@ -145,26 +145,25 @@ class SingleStageFile(FileMixin):
         from dvc.stage import PipelineStage
 
         assert not isinstance(stage, PipelineStage)
-        check_dvc_filename(self.path)
+        if self.verify:
+            check_dvc_filename(self.path)
         logger.debug(
             "Saving information to '{file}'.".format(file=relpath(self.path))
         )
-        dump_stage_file(self.path, serialize.to_single_stage_file(stage))
-        self.repo.scm.track_file(relpath(self.path))
-
-    def remove_with_prompt(self, force=False):
-        if not self.exists():
-            return
-
-        msg = (
-            "'{}' already exists. Do you wish to run the command and "
-            "overwrite it?".format(relpath(self.path))
-        )
-        if not (force or prompt.confirm(msg)):
-            raise StageFileAlreadyExistsError(self.path)
+        dump_yaml(self.path, serialize.to_single_stage_file(stage))
+        self.repo.scm.track_file(self.relpath)
 
+    def remove_stage(self, stage):  # pylint: disable=unused-argument
         self.remove()
 
+    def merge(self, ancestor, other):
+        assert isinstance(ancestor, SingleStageFile)
+        assert isinstance(other, SingleStageFile)
+
+        stage = self.stage
+        stage.merge(ancestor.stage, other.stage)
+        self.dump(stage)
+
 
 class PipelineFile(FileMixin):
     """Abstraction for pipelines file, .yaml + .lock combined."""
@@ -175,37 +174,48 @@ class PipelineFile(FileMixin):
     def _lockfile(self):
         return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
 
-    def dump(self, stage, update_pipeline=False, **kwargs):
+    def dump(
+        self, stage, update_pipeline=False, no_lock=False, **kwargs
+    ):  # pylint: disable=arguments-differ
         """Dumps given stage appropriately in the dvcfile."""
         from dvc.stage import PipelineStage
 
         assert isinstance(stage, PipelineStage)
-        check_dvc_filename(self.path)
-        self._dump_lockfile(stage)
+        if self.verify:
+            check_dvc_filename(self.path)
+
         if update_pipeline and not stage.is_data_source:
             self._dump_pipeline_file(stage)
 
+        if not no_lock:
+            self._dump_lockfile(stage)
+
     def _dump_lockfile(self, stage):
         self._lockfile.dump(stage)
 
     def _dump_pipeline_file(self, stage):
-        data = {}
-        if self.exists():
-            with open(self.path) as fd:
-                data = parse_stage_for_update(fd.read(), self.path)
-        else:
-            open(self.path, "w+").close()
-
-        data["stages"] = data.get("stages", {})
         stage_data = serialize.to_pipeline_file(stage)
-        if data["stages"].get(stage.name):
-            orig_stage_data = data["stages"][stage.name]
-            apply_diff(stage_data[stage.name], orig_stage_data)
-        else:
-            data["stages"].update(stage_data)
 
-        dump_stage_file(self.path, data)
-        self.repo.scm.track_file(relpath(self.path))
+        with modify_yaml(self.path, tree=self.repo.tree) as data:
+            if not data:
+                logger.info("Creating '%s'", self.relpath)
+
+            data["stages"] = data.get("stages", {})
+            existing_entry = stage.name in data["stages"]
+            action = "Modifying" if existing_entry else "Adding"
+            logger.info(
+                "%s stage '%s' in '%s'", action, stage.name, self.relpath
+            )
+
+            if existing_entry:
+                orig_stage_data = data["stages"][stage.name]
+                if "meta" in orig_stage_data:
+                    stage_data[stage.name]["meta"] = orig_stage_data["meta"]
+                apply_diff(stage_data[stage.name], orig_stage_data)
+            else:
+                data["stages"].update(stage_data)
+
+        self.repo.scm.track_file(self.relpath)
 
     @property
     def stage(self):
@@ -227,6 +237,29 @@ class PipelineFile(FileMixin):
         super().remove()
         self._lockfile.remove()
 
+    def remove_stage(self, stage):
+        self._lockfile.remove_stage(stage)
+        if not self.exists():
+            return
+
+        with open(self.path, "r") as f:
+            d = parse_yaml_for_update(f.read(), self.path)
+
+        self.validate(d, self.path)
+        if stage.name not in d.get("stages", {}):
+            return
+
+        logger.debug("Removing '%s' from '%s'", stage.name, self.path)
+        del d["stages"][stage.name]
+
+        if d["stages"]:
+            dump_yaml(self.path, d)
+        else:
+            super().remove()
+
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
 
 class Lockfile(FileMixin):
     from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
@@ -234,8 +267,8 @@ class Lockfile(FileMixin):
     def load(self):
         if not self.exists():
             return {}
-        with self.repo.tree.open(self.path) as fd:
-            data = parse_stage(fd.read(), self.path)
+
+        data = load_yaml(self.path, tree=self.repo.tree)
         try:
             self.validate(data, fname=self.relpath)
         except StageFileFormatError:
@@ -246,16 +279,42 @@ class Lockfile(FileMixin):
 
     def dump(self, stage, **kwargs):
         stage_data = serialize.to_lockfile(stage)
-        if not self.exists():
-            data = stage_data
-            open(self.path, "w+").close()
-        else:
-            with self.repo.tree.open(self.path, "r") as fd:
-                data = parse_stage_for_update(fd.read(), self.path)
+
+        with modify_yaml(self.path, tree=self.repo.tree) as data:
+            if not data:
+                logger.info("Generating lock file '%s'", self.relpath)
+
+            modified = data.get(stage.name, {}) != stage_data.get(
+                stage.name, {}
+            )
+            if modified:
+                logger.info("Updating lock file '%s'", self.relpath)
             data.update(stage_data)
 
-        dump_stage_file(self.path, data)
-        self.repo.scm.track_file(relpath(self.path))
+        if modified:
+            self.repo.scm.track_file(self.relpath)
+
+    def remove_stage(self, stage):
+        if not self.exists():
+            return
+
+        with open(self.path) as f:
+            d = parse_yaml_for_update(f.read(), self.path)
+        self.validate(d, self.path)
+
+        if stage.name not in d:
+            return
+
+        logger.debug("Removing '%s' from '%s'", stage.name, self.path)
+        del d[stage.name]
+
+        if d:
+            dump_yaml(self.path, d)
+        else:
+            self.remove()
+
+    def merge(self, ancestor, other):
+        raise NotImplementedError
 
 
 class Dvcfile:

dvc/exceptions.py

@@ -1,7 +1,7 @@
 """Exceptions raised by the dvc."""
 from funcy import first
 
-from dvc.utils import format_link, relpath
+from dvc.utils import error_link, format_link, relpath
 
 
 class DvcException(Exception):
@@ -171,19 +171,14 @@ class BadMetricError(DvcException):
 
 
 class NoMetricsError(DvcException):
+    pass
+
+
+class NoPlotsError(DvcException):
     def __init__(self):
         super().__init__(
-            "no metric files in this repository. "
-            "Use `dvc metrics add` to add a metric file to track."
-        )
-
-
-class StageFileCorruptedError(DvcException):
-    def __init__(self, path):
-        path = relpath(path)
-        super().__init__(
-            "unable to read DVC-file: {} "
-            "YAML file structure is corrupted".format(path)
+            "no plots in this repository. Use `--plots/--plots-no-cache` "
+            "options for `dvc run` to mark stage outputs as plots."
         )
 
 
@@ -215,9 +210,12 @@ class ETagMismatchError(DvcException):
 
 
 class FileMissingError(DvcException):
-    def __init__(self, path):
+    def __init__(self, path, hint=None):
         self.path = path
-        super().__init__(f"Can't find '{path}' neither locally nor on remote")
+        hint = "" if hint is None else f". {hint}"
+        super().__init__(
+            f"Can't find '{path}' neither locally nor on remote{hint}"
+        )
 
 
 class DvcIgnoreInCollectedDirError(DvcException):
@@ -258,8 +256,10 @@ class CheckoutError(DvcException):
         self.stats = stats
         targets = [str(t) for t in target_infos]
         m = (
-            "Checkout failed for following targets:\n{}\nDid you "
-            "forget to fetch?".format("\n".join(targets))
+            "Checkout failed for following targets:\n{}\nIs your "
+            "cache up to date?\n{}".format(
+                "\n".join(targets), error_link("missing-files"),
+            )
         )
         super().__init__(m)
 
@@ -294,19 +294,15 @@ class PathMissingError(DvcException):
         "The path '{}' does not exist in the target repository '{}'"
         " neither as a DVC output nor as a Git-tracked file."
     )
-    default_msg_output_only = (
+    default_msg_dvc_only = (
         "The path '{}' does not exist in the target repository '{}'"
         " as an DVC output."
     )
 
-    def __init__(self, path, repo, output_only=False):
-        msg = (
-            self.default_msg
-            if not output_only
-            else self.default_msg_output_only
-        )
+    def __init__(self, path, repo, dvc_only=False):
+        msg = self.default_msg if not dvc_only else self.default_msg_dvc_only
         super().__init__(msg.format(path, repo))
-        self.output_only = output_only
+        self.dvc_only = dvc_only
 
 
 class RemoteCacheRequiredError(DvcException):
@@ -324,5 +320,21 @@ class RemoteCacheRequiredError(DvcException):
         )
 
 
-class IsADirectoryError(DvcException):
+class IsADirectoryError(DvcException):  # noqa,pylint:disable=redefined-builtin
     """Raised when a file operation is requested on a directory."""
+
+
+class NoOutputOrStageError(DvcException):
+    """
+    Raised when the target is neither an output nor a stage name in dvc.yaml
+    """
+
+    def __init__(self, target, file):
+        super().__init__(
+            f"'{target}' "
+            f"does not exist as an output or a stage name in '{file}'"
+        )
+
+
+class MergeError(DvcException):
+    pass

dvc/external_repo.py

@@ -4,12 +4,12 @@ import tempfile
 import threading
 from contextlib import contextmanager
 from distutils.dir_util import copy_tree
+from typing import Iterable
 
-from funcy import cached_property, retry, suppress, wrap_with
+from funcy import cached_property, retry, wrap_with
 
 from dvc.config import NoRemoteError, NotDvcRepoError
 from dvc.exceptions import (
-    CheckoutError,
     FileMissingError,
     NoOutputInExternalRepoError,
     NoRemoteInExternalRepoError,
@@ -18,9 +18,12 @@ from dvc.exceptions import (
 )
 from dvc.path_info import PathInfo
 from dvc.repo import Repo
+from dvc.scm.base import CloneError
 from dvc.scm.git import Git
-from dvc.utils import tmp_fname
-from dvc.utils.fs import fs_copy, move, remove
+from dvc.tree.local import LocalTree
+from dvc.tree.repo import RepoTree
+from dvc.utils import relpath
+from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
@@ -29,10 +32,15 @@ logger = logging.getLogger(__name__)
 def external_repo(url, rev=None, for_write=False):
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
+    if not rev:
+        # Local HEAD points to the tip of whatever branch we first cloned from
+        # (which may not be the default branch), use origin/HEAD here to get
+        # the tip of the default branch
+        rev = "refs/remotes/origin/HEAD"
     try:
-        repo = ExternalRepo(path, url)
+        repo = ExternalRepo(path, url, rev, for_write=for_write)
     except NotDvcRepoError:
-        repo = ExternalGitRepo(path, url)
+        repo = ExternalGitRepo(path, url, rev)
 
     try:
         yield repo
@@ -56,7 +64,7 @@ CACHE_DIRS = {}
 
 def clean_repos():
     # Outside code should not see cache while we are removing
-    paths = list(CLONES.values()) + list(CACHE_DIRS.values())
+    paths = [path for path, _ in CLONES.values()] + list(CACHE_DIRS.values())
     CLONES.clear()
     CACHE_DIRS.clear()
 
@@ -64,59 +72,116 @@ def clean_repos():
         _remove(path)
 
 
-class ExternalRepo(Repo):
-    def __init__(self, root_dir, url):
-        super().__init__(root_dir)
+class BaseExternalRepo:
+    # pylint: disable=no-member
+
+    _local_cache = None
+
+    def __str__(self):
+        return self.url
+
+    @property
+    def local_cache(self):
+        if hasattr(self, "cache"):
+            return self.cache.local
+        return self._local_cache
+
+    @contextmanager
+    def use_cache(self, cache):
+        """Use the specified cache in place of default tmpdir cache for
+        download operations.
+        """
+        has_cache = hasattr(self, "cache")
+
+        if has_cache:
+            save_cache = self.cache.local  # pylint: disable=E0203
+            self.cache.local = cache  # pylint: disable=E0203
+        else:
+            from collections import namedtuple
+
+            mock_cache = namedtuple("MockCache", ["local"])(local=cache)
+            self.cache = mock_cache  # pylint: disable=W0201
+
+        self._local_cache = cache
+
+        yield
+
+        if has_cache:
+            self.cache.local = save_cache
+        else:
+            del self.cache
+
+        self._local_cache = None
+
+    @cached_property
+    def repo_tree(self):
+        return RepoTree(self, fetch=True)
+
+    def get_rev(self):
+        if isinstance(self.tree, LocalTree):
+            return self.scm.get_rev()
+        return self.tree.rev
+
+    def fetch_external(self, paths: Iterable, **kwargs):
+        """Fetch specified external repo paths into cache.
+
+        Returns 3-tuple in the form
+            (downloaded, failed, list(cache_infos))
+        where cache_infos can be used as checkout targets for the
+        fetched paths.
+        """
+        download_results = []
+        failed = 0
+
+        paths = [PathInfo(self.root_dir) / path for path in paths]
+
+        def download_update(result):
+            download_results.append(result)
+
+        save_infos = []
+        for path in paths:
+            if not self.repo_tree.exists(path):
+                raise PathMissingError(path, self.url)
+            hash_info = self.repo_tree.save_info(
+                path, download_callback=download_update
+            )
+            self.local_cache.save(
+                path,
+                self.repo_tree,
+                hash_info,
+                save_link=False,
+                download_callback=download_update,
+            )
+            save_infos.append(hash_info)
+
+        return sum(download_results), failed, save_infos
+
+    def get_external(self, path, dest):
+        """Convenience wrapper for fetch_external and checkout."""
+        if self.local_cache:
+            # fetch DVC and git files to tmpdir cache, then checkout
+            _, _, save_infos = self.fetch_external([path])
+            self.local_cache.checkout(PathInfo(dest), save_infos[0])
+        else:
+            # git-only erepo with no cache, just copy files directly
+            # to dest
+            path = PathInfo(self.root_dir) / path
+            if not self.repo_tree.exists(path):
+                raise PathMissingError(path, self.url)
+            self.repo_tree.copytree(path, dest)
+
+
+class ExternalRepo(Repo, BaseExternalRepo):
+    def __init__(self, root_dir, url, rev, for_write=False):
+        if for_write:
+            super().__init__(root_dir)
+        else:
+            root_dir = os.path.realpath(root_dir)
+            super().__init__(root_dir, scm=Git(root_dir), rev=rev)
         self.url = url
         self._set_cache_dir()
         self._fix_upstream()
 
-    def pull_to(self, path, to_info):
-        """
-        Pull the corresponding file or directory specified by `path` and
-        checkout it into `to_info`.
-
-        It works with files tracked by Git and DVC, and also local files
-        outside the repository.
-        """
-        out = None
-        path_info = PathInfo(self.root_dir) / path
-
-        with suppress(OutputNotFoundError):
-            (out,) = self.find_outs_by_path(path_info, strict=False)
-
-        try:
-            if out and out.use_cache:
-                self._pull_cached(out, path_info, to_info)
-                return
-
-            # Check if it is handled by Git (it can't have an absolute path)
-            if os.path.isabs(path):
-                raise FileNotFoundError
-
-            fs_copy(path_info, to_info)
-        except FileNotFoundError:
-            raise PathMissingError(path, self.url)
-
-    def _pull_cached(self, out, path_info, dest):
-        with self.state:
-            tmp = PathInfo(tmp_fname(dest))
-            src = tmp / path_info.relative_to(out.path_info)
-
-            out.path_info = tmp
-
-            # Only pull unless all needed cache is present
-            if out.changed_cache(filter_info=src):
-                self.cloud.pull(out.get_used_cache(filter_info=src))
-
-            try:
-                out.checkout(filter_info=src)
-            except CheckoutError:
-                raise FileNotFoundError
-
-            move(src, dest)
-            remove(tmp)
-
     @wrap_with(threading.Lock())
     def _set_cache_dir(self):
         try:
@@ -125,6 +190,7 @@ class ExternalRepo(Repo):
             cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
 
         self.cache.local.cache_dir = cache_dir
+        self._local_cache = self.cache.local
 
     def _fix_upstream(self):
         if not os.path.isdir(self.url):
@@ -165,10 +231,11 @@ class ExternalRepo(Repo):
         self.config["core"]["remote"] = "auto-generated-upstream"
 
 
-class ExternalGitRepo:
-    def __init__(self, root_dir, url):
-        self.root_dir = root_dir
+class ExternalGitRepo(BaseExternalRepo):
+    def __init__(self, root_dir, url, rev):
+        self.root_dir = os.path.realpath(root_dir)
         self.url = url
+        self.tree = self.scm.get_tree(rev)
 
     @cached_property
     def scm(self):
@@ -181,23 +248,15 @@ class ExternalGitRepo:
     def find_out_by_relpath(self, path):
         raise OutputNotFoundError(path, self)
 
-    def pull_to(self, path, to_info):
-        try:
-            # Git handled files can't have absolute path
-            if os.path.isabs(path):
-                raise FileNotFoundError
-
-            fs_copy(os.path.join(self.root_dir, path), to_info)
-        except FileNotFoundError:
-            raise PathMissingError(path, self.url)
-
     @contextmanager
     def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
         """Opens a specified resource as a file object."""
+        tree = RepoTree(self)
         try:
-            abs_path = os.path.join(self.root_dir, path)
-            with open(abs_path, mode, encoding=encoding) as fd:
-                yield fd
+            with tree.open(
+                path, mode=mode, encoding=encoding, **kwargs
+            ) as fobj:
+                yield fobj
         except FileNotFoundError:
             raise PathMissingError(path, self.url)
 
@@ -209,14 +268,12 @@ def _cached_clone(url, rev, for_write=False):
     revision checked out. If for_write is set prevents reusing this dir via
     cache.
     """
-    if not for_write and Git.is_sha(rev) and (url, rev) in CLONES:
-        return CLONES[url, rev]
+    # even if we have already cloned this repo, we may need to
+    # fetch/fast-forward to get specified rev
+    clone_path, shallow = _clone_default_branch(url, rev, for_write=for_write)
 
-    clone_path = _clone_default_branch(url, rev)
-    rev_sha = Git(clone_path).resolve_rev(rev or "HEAD")
-
-    if not for_write and (url, rev_sha) in CLONES:
-        return CLONES[url, rev_sha]
+    if not for_write and (url) in CLONES:
+        return CLONES[url][0]
 
     # Copy to a new dir to keep the clone clean
     repo_path = tempfile.mkdtemp("dvc-erepo")
@@ -224,21 +281,20 @@ def _cached_clone(url, rev, for_write=False):
     copy_tree(clone_path, repo_path)
 
     # Check out the specified revision
-    if rev is not None:
+    if for_write:
         _git_checkout(repo_path, rev)
-
-    if not for_write:
-        CLONES[url, rev_sha] = repo_path
+    else:
+        CLONES[url] = (repo_path, shallow)
     return repo_path
 
 
 @wrap_with(threading.Lock())
-def _clone_default_branch(url, rev):
+def _clone_default_branch(url, rev, for_write=False):
     """Get or create a clean clone of the url.
 
     The cloned is reactualized with git pull unless rev is a known sha.
     """
-    clone_path = CLONES.get(url)
+    clone_path, shallow = CLONES.get(url, (None, False))
 
     git = None
     try:
@@ -246,18 +302,55 @@ def _clone_default_branch(url, rev):
             git = Git(clone_path)
             # Do not pull for known shas, branches and tags might move
             if not Git.is_sha(rev) or not git.has_rev(rev):
-                logger.debug("erepo: git pull %s", url)
-                git.pull()
+                if shallow:
+                    # If we are missing a rev in a shallow clone, fallback to
+                    # a full (unshallowed) clone. Since fetching specific rev
+                    # SHAs is only available in certain git versions, if we
+                    # have need to reference multiple specific revs for a
+                    # given repo URL it is easier/safer for us to work with
+                    # full clones in this case.
+                    logger.debug("erepo: unshallowing clone for '%s'", url)
+                    _unshallow(git)
+                    shallow = False
+                    CLONES[url] = (clone_path, shallow)
+                else:
+                    logger.debug("erepo: git pull '%s'", url)
+                    git.pull()
         else:
-            logger.debug("erepo: git clone %s to a temporary dir", url)
+            logger.debug("erepo: git clone '%s' to a temporary dir", url)
             clone_path = tempfile.mkdtemp("dvc-clone")
-            git = Git.clone(url, clone_path)
-            CLONES[url] = clone_path
+            if not for_write and rev and not Git.is_sha(rev):
+                # If rev is a tag or branch name try shallow clone first
+                try:
+                    git = Git.clone(url, clone_path, shallow_branch=rev)
+                    shallow = True
+                    logger.debug(
+                        "erepo: using shallow clone for branch '%s'", rev
+                    )
+                except CloneError:
+                    pass
+            if not git:
+                git = Git.clone(url, clone_path)
+                shallow = False
+            CLONES[url] = (clone_path, shallow)
     finally:
         if git:
             git.close()
 
-    return clone_path
+    return clone_path, shallow
+
+
+def _unshallow(git):
+    if git.repo.head.is_detached:
+        # If this is a detached head (i.e. we shallow cloned a tag) switch to
+        # the default branch
+        origin_refs = git.repo.remotes["origin"].refs
+        ref = origin_refs["HEAD"].reference
+        branch_name = ref.name.split("/")[-1]
+        branch = git.repo.create_head(branch_name, ref)
+        branch.set_tracking_branch(ref)
+        branch.checkout()
+    git.pull(unshallow=True)
 
 
 def _git_checkout(repo_path, rev):
@@ -273,6 +366,11 @@ def _remove(path):
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
         os_retry = retry(5, errors=OSError, timeout=0.1)
-        os_retry(remove)(path)
+        try:
+            os_retry(remove)(path)
+        except PermissionError:
+            logger.warning(
+                "Failed to remove '%s'", relpath(path), exc_info=True
+            )
     else:
         remove(path)

dvc/ignore.py

@@ -1,12 +1,17 @@
 import logging
 import os
+import re
+from collections import namedtuple
+from itertools import groupby, takewhile
 
-from funcy import cached_property
-from pathspec import PathSpec
 from pathspec.patterns import GitWildMatchPattern
+from pathspec.util import normalize_file
 
-from dvc.scm.tree import BaseTree
+from dvc.path_info import PathInfo
+from dvc.pathspec_math import PatternInfo, merge_patterns
+from dvc.system import System
 from dvc.utils import relpath
+from dvc.utils.collections import PathStringTrie
 
 logger = logging.getLogger(__name__)
 
@@ -19,125 +24,324 @@ class DvcIgnore:
 
 
 class DvcIgnorePatterns(DvcIgnore):
-    def __init__(self, ignore_file_path, tree):
+    def __init__(self, pattern_list, dirname):
+        if pattern_list:
+            if isinstance(pattern_list[0], str):
+                pattern_list = [
+                    PatternInfo(pattern, "") for pattern in pattern_list
+                ]
+
+        self.pattern_list = pattern_list
+        self.dirname = dirname
+        self.prefix = self.dirname + os.sep
+
+        self.regex_pattern_list = [
+            GitWildMatchPattern.pattern_to_regex(pattern_info.patterns)
+            for pattern_info in pattern_list
+        ]
+
+        self.ignore_spec = [
+            (ignore, re.compile("|".join(item[0] for item in group)))
+            for ignore, group in groupby(
+                self.regex_pattern_list, lambda x: x[1]
+            )
+            if ignore is not None
+        ]
+
+    @classmethod
+    def from_files(cls, ignore_file_path, tree):
         assert os.path.isabs(ignore_file_path)
-
-        self.ignore_file_path = ignore_file_path
-        self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
-
+        dirname = os.path.normpath(os.path.dirname(ignore_file_path))
+        ignore_file_rel_path = os.path.relpath(
+            ignore_file_path, tree.tree_root
+        )
         with tree.open(ignore_file_path, encoding="utf-8") as fobj:
-            self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
+            path_spec_lines = [
+                PatternInfo(
+                    line,
+                    "{}:{}:{}".format(ignore_file_rel_path, line_no + 1, line),
+                )
+                for line_no, line in enumerate(
+                    map(str.strip, fobj.readlines())
+                )
+                if line and not (line.strip().startswith("#"))
+            ]
+
+        return cls(path_spec_lines, dirname)
 
     def __call__(self, root, dirs, files):
         files = [f for f in files if not self.matches(root, f)]
-        dirs = [d for d in dirs if not self.matches(root, d)]
+        dirs = [d for d in dirs if not self.matches(root, d, True)]
 
         return dirs, files
 
-    def matches(self, dirname, basename):
-        abs_path = os.path.join(dirname, basename)
-        rel_path = relpath(abs_path, self.dirname)
-
-        if os.pardir + os.sep in rel_path:
+    def _get_normalize_path(self, dirname, basename):
+        # NOTE: `relpath` is too slow, so we have to assume that both
+        # `dirname` and `self.dirname` are relative or absolute together.
+        if dirname == self.dirname:
+            path = basename
+        elif dirname.startswith(self.prefix):
+            rel = dirname[len(self.prefix) :]
+            # NOTE: `os.path.join` is ~x5.5 slower
+            path = f"{rel}{os.sep}{basename}"
+        else:
             return False
-        return self.ignore_spec.match_file(rel_path)
+
+        if not System.is_unix():
+            path = normalize_file(path)
+        return path
+
+    def matches(self, dirname, basename, is_dir=False):
+        path = self._get_normalize_path(dirname, basename)
+        if not path:
+            return False
+        return self.ignore(path, is_dir)
+
+    def ignore(self, path, is_dir):
+        result = False
+        if is_dir:
+            path_dir = f"{path}/"
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path) or pattern.match(path_dir):
+                    result = ignore
+        else:
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path):
+                    result = ignore
+        return result
+
+    def match_details(self, dirname, basename, is_dir=False):
+        path = self._get_normalize_path(dirname, basename)
+        if not path:
+            return False
+        return self._ignore_details(path, is_dir)
+
+    def _ignore_details(self, path, is_dir):
+        result = []
+        for ignore, pattern in zip(self.regex_pattern_list, self.pattern_list):
+            regex = re.compile(ignore[0])
+            # skip system pattern
+            if not pattern.file_info:
+                continue
+            if is_dir:
+                path_dir = f"{path}/"
+                if regex.match(path) or regex.match(path_dir):
+                    result.append(pattern.file_info)
+            else:
+                if regex.match(path):
+                    result.append(pattern.file_info)
+        return result
 
     def __hash__(self):
-        return hash(self.ignore_file_path)
+        return hash(self.dirname + ":" + str(self.pattern_list))
 
     def __eq__(self, other):
         if not isinstance(other, DvcIgnorePatterns):
             return NotImplemented
+        return (self.dirname == other.dirname) & (
+            [pattern.patterns for pattern in self.pattern_list]
+            == [pattern.patterns for pattern in other.pattern_list]
+        )
 
-        return self.ignore_file_path == other.ignore_file_path
+    def __bool__(self):
+        return bool(self.pattern_list)
 
 
-class DvcIgnoreDirs(DvcIgnore):
-    def __init__(self, basenames):
-        self.basenames = set(basenames)
+CheckIgnoreResult = namedtuple(
+    "CheckIgnoreResult", ["file", "match", "patterns"]
+)
 
-    def __call__(self, root, dirs, files):
-        dirs = [d for d in dirs if d not in self.basenames]
 
+def _no_match(path):
+    return CheckIgnoreResult(path, False, ["::"])
+
+
+class DvcIgnoreFilterNoop:
+    def __init__(self, tree, root_dir):
+        pass
+
+    def __call__(self, root, dirs, files, **kwargs):
         return dirs, files
 
-    def __hash__(self):
-        return hash(tuple(self.basenames))
+    def is_ignored_dir(self, _):
+        return False
 
-    def __eq__(self, other):
-        if not isinstance(other, DvcIgnoreDirs):
-            return NotImplemented
+    def is_ignored_file(self, _):
+        return False
 
-        return self.basenames == other.basenames
-
-
-class DvcIgnoreRepo(DvcIgnore):
-    def __call__(self, root, dirs, files):
-        def is_dvc_repo(directory):
-            from dvc.repo import Repo
-
-            return os.path.isdir(os.path.join(directory, Repo.DVC_DIR))
-
-        dirs = [d for d in dirs if not is_dvc_repo(d)]
-
-        return dirs, files
+    def check_ignore(self, path):
+        return _no_match(path)
 
 
 class DvcIgnoreFilter:
-    def __init__(self, tree):
+    @staticmethod
+    def _is_dvc_repo(root, directory):
+        from dvc.repo import Repo
+
+        return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
+
+    def __init__(self, tree, root_dir):
+        from dvc.repo import Repo
+
+        default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
+
         self.tree = tree
-        self.ignores = {
-            DvcIgnoreDirs([".git", ".hg", ".dvc"]),
-            DvcIgnoreRepo(),
-        }
-        for root, dirs, files in self.tree.walk(self.tree.tree_root):
-            self._update(root)
-            dirs[:], files[:] = self(root, dirs, files)
+        self.root_dir = root_dir
+        self.ignores_trie_tree = PathStringTrie()
+        self.ignores_trie_tree[root_dir] = DvcIgnorePatterns(
+            default_ignore_patterns, root_dir
+        )
+        self._ignored_subrepos = PathStringTrie()
+        self._update(self.root_dir)
 
     def _update(self, dirname):
+        old_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
+        matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
+
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if self.tree.exists(ignore_file_path):
-            self.ignores.add(DvcIgnorePatterns(ignore_file_path, self.tree))
+        if not matches and self.tree.exists(
+            ignore_file_path, use_dvcignore=False
+        ):
+            new_pattern = DvcIgnorePatterns.from_files(
+                ignore_file_path, self.tree
+            )
+            if old_pattern:
+                self.ignores_trie_tree[dirname] = DvcIgnorePatterns(
+                    *merge_patterns(
+                        old_pattern.pattern_list,
+                        old_pattern.dirname,
+                        new_pattern.pattern_list,
+                        new_pattern.dirname,
+                    )
+                )
+            else:
+                self.ignores_trie_tree[dirname] = new_pattern
+        elif old_pattern:
+            self.ignores_trie_tree[dirname] = old_pattern
 
-    def __call__(self, root, dirs, files):
-        for ignore in self.ignores:
-            dirs, files = ignore(root, dirs, files)
+        # NOTE: using `walk` + `break` because tree doesn't have `listdir()`
+        for root, dirs, _ in self.tree.walk(dirname, use_dvcignore=False):
+            self._update_sub_repo(root, dirs)
+            break
 
+    def _update_sub_repo(self, root, dirs):
+        for d in dirs:
+            if self._is_dvc_repo(root, d):
+                self._ignored_subrepos[root] = self._ignored_subrepos.get(
+                    root, set()
+                ) | {d}
+                new_pattern = DvcIgnorePatterns([f"/{d}/"], root)
+                old_pattern = self.ignores_trie_tree.longest_prefix(root).value
+                if old_pattern:
+                    self.ignores_trie_tree[root] = DvcIgnorePatterns(
+                        *merge_patterns(
+                            old_pattern.pattern_list,
+                            old_pattern.dirname,
+                            new_pattern.pattern_list,
+                            new_pattern.dirname,
+                        )
+                    )
+                else:
+                    self.ignores_trie_tree[root] = new_pattern
+
+    def __call__(self, root, dirs, files, ignore_subrepos=True):
+        ignore_pattern = self._get_trie_pattern(root)
+        if ignore_pattern:
+            dirs, files = ignore_pattern(root, dirs, files)
+            if not ignore_subrepos:
+                dirs.extend(self._ignored_subrepos.get(root, []))
         return dirs, files
 
+    def _get_trie_pattern(self, dirname):
+        ignore_pattern = self.ignores_trie_tree.get(dirname)
+        if ignore_pattern:
+            return ignore_pattern
 
-class CleanTree(BaseTree):
-    def __init__(self, tree):
-        self.tree = tree
+        prefix = self.ignores_trie_tree.longest_prefix(dirname).key
+        if not prefix:
+            # outside of the repo
+            return None
 
-    @cached_property
-    def dvcignore(self):
-        return DvcIgnoreFilter(self.tree)
+        dirs = list(
+            takewhile(
+                lambda path: path != prefix,
+                (parent.fspath for parent in PathInfo(dirname).parents),
+            )
+        )
+        dirs.reverse()
+        dirs.append(dirname)
 
-    @property
-    def tree_root(self):
-        return self.tree.tree_root
+        for parent in dirs:
+            self._update(parent)
 
-    def open(self, path, mode="r", encoding="utf-8"):
-        return self.tree.open(path, mode, encoding)
+        return self.ignores_trie_tree.get(dirname)
 
-    def exists(self, path):
-        return self.tree.exists(path)
+    def _is_ignored(self, path, is_dir=False):
+        if self._outside_repo(path):
+            return False
+        dirname, basename = os.path.split(os.path.normpath(path))
+        ignore_pattern = self._get_trie_pattern(dirname)
+        if ignore_pattern:
+            return ignore_pattern.matches(dirname, basename, is_dir)
+        else:
+            return False
 
-    def isdir(self, path):
-        return self.tree.isdir(path)
+    def _is_subrepo(self, path):
+        dirname, basename = os.path.split(os.path.normpath(path))
+        return basename in self._ignored_subrepos.get(dirname, set())
 
-    def isfile(self, path):
-        return self.tree.isfile(path)
+    def is_ignored_dir(self, path, ignore_subrepos=True):
+        path = os.path.abspath(path)
+        if not ignore_subrepos:
+            return not self._is_subrepo(path)
+        if path == self.root_dir:
+            return False
 
-    def isexec(self, path):
-        return self.tree.isexec(path)
+        return self._is_ignored(path, True)
 
-    def walk(self, top, topdown=True):
-        for root, dirs, files in self.tree.walk(top, topdown):
-            dirs[:], files[:] = self.dvcignore(root, dirs, files)
+    def is_ignored_file(self, path):
+        path = os.path.abspath(path)
+        return self._is_ignored(path, False)
 
-            yield root, dirs, files
+    def _outside_repo(self, path):
+        path = PathInfo(path)
 
-    def stat(self, path):
-        return self.tree.stat(path)
+        # paths outside of the repo should be ignored
+        path = relpath(path, self.root_dir)
+        if path.startswith("..") or (
+            os.name == "nt"
+            and not os.path.commonprefix(
+                [os.path.abspath(path), self.root_dir]
+            )
+        ):
+            return True
+        return False
+
+    def check_ignore(self, target):
+        full_target = os.path.abspath(target)
+        if not self._outside_repo(full_target):
+            dirname, basename = os.path.split(os.path.normpath(full_target))
+            pattern = self._get_trie_pattern(dirname)
+            if pattern:
+                matches = pattern.match_details(
+                    dirname, basename, os.path.isdir(full_target)
+                )
+
+                if matches:
+                    return CheckIgnoreResult(target, True, matches)
+        return _no_match(target)
+
+
+def init(path):
+    dvcignore = os.path.join(path, DvcIgnore.DVCIGNORE_FILE)
+    if os.path.exists(dvcignore):
+        return dvcignore
+
+    with open(dvcignore, "w") as fobj:
+        fobj.write(
+            "# Add patterns of files dvc should ignore, which could improve\n"
+            "# the performance. Learn more at\n"
+            "# https://dvc.org/doc/user-guide/dvcignore\n"
+        )
+
+    return dvcignore

dvc/istextfile.py

@@ -7,13 +7,17 @@
 TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
 
 
-def istextfile(fname, blocksize=512):
+def istextfile(fname, blocksize=512, tree=None):
     """ Uses heuristics to guess whether the given file is text or binary,
         by reading a single block of bytes from the file.
         If more than 30% of the chars in the block are non-text, or there
         are NUL ('\x00') bytes in the block, assume this is a binary file.
     """
-    with open(fname, "rb") as fobj:
+    if tree:
+        open_func = tree.open
+    else:
+        open_func = open
+    with open_func(fname, "rb") as fobj:
         block = fobj.read(blocksize)
 
     if not block:

dvc/lock.py

@@ -47,7 +47,7 @@ class Lock:
                 disable=not self._friendly,
                 desc=(
                     "If DVC froze, see `hardlink_lock` in {}".format(
-                        format_link("man.dvc.org/config#core")
+                        format_link("https://man.dvc.org/config#core")
                     )
                 ),
             ):
@@ -88,7 +88,9 @@ class HardlinkLock(flufl.lock.Lock):
         tmp_dir (str): a directory to store claim files.
     """
 
-    def __init__(self, lockfile, tmp_dir=None, **kwargs):
+    def __init__(
+        self, lockfile, tmp_dir=None, **kwargs
+    ):  # pylint: disable=super-init-not-called
         import socket
 
         self._tmp_dir = tmp_dir
@@ -117,7 +119,7 @@ class HardlinkLock(flufl.lock.Lock):
     def lockfile(self):
         return self._lockfile
 
-    def lock(self):
+    def lock(self):  # pylint: disable=arguments-differ
         try:
             super().lock(timedelta(seconds=DEFAULT_TIMEOUT))
         except flufl.lock.TimeOutError:

dvc/logger.py

@@ -18,20 +18,46 @@ FOOTER = (
 )
 
 
+def addLoggingLevel(levelName, levelNum, methodName=None):
+    """
+    Adds a new logging level to the `logging` module and the
+    currently configured logging class.
+
+    Based on https://stackoverflow.com/questions/2183233
+    """
+    if methodName is None:
+        methodName = levelName.lower()
+
+    assert not hasattr(logging, levelName)
+    assert not hasattr(logging, methodName)
+    assert not hasattr(logging.getLoggerClass(), methodName)
+
+    def logForLevel(self, message, *args, **kwargs):
+        if self.isEnabledFor(levelNum):
+            # pylint: disable=protected-access
+            self._log(levelNum, message, args, **kwargs)
+
+    def logToRoot(message, *args, **kwargs):
+        logging.log(levelNum, message, *args, **kwargs)
+
+    logging.addLevelName(levelNum, levelName)
+    setattr(logging, levelName, levelNum)
+    setattr(logging.getLoggerClass(), methodName, logForLevel)
+    setattr(logging, methodName, logToRoot)
+
+
 class LoggingException(Exception):
     def __init__(self, record):
         msg = "failed to log {}".format(str(record))
         super().__init__(msg)
 
 
-class ExcludeErrorsFilter(logging.Filter):
-    def filter(self, record):
-        return record.levelno < logging.WARNING
+def excludeFilter(level):
+    class ExcludeLevelFilter(logging.Filter):
+        def filter(self, record):
+            return record.levelno < level
 
-
-class ExcludeInfoFilter(logging.Filter):
-    def filter(self, record):
-        return record.levelno < logging.INFO
+    return ExcludeLevelFilter
 
 
 class ColorFormatter(logging.Formatter):
@@ -47,6 +73,7 @@ class ColorFormatter(logging.Formatter):
     """
 
     color_code = {
+        "TRACE": colorama.Fore.GREEN,
         "DEBUG": colorama.Fore.BLUE,
         "WARNING": colorama.Fore.YELLOW,
         "ERROR": colorama.Fore.RED,
@@ -111,12 +138,16 @@ class LoggerHandler(logging.StreamHandler):
             self.flush()
         except RecursionError:
             raise
-        except Exception:
+        except Exception:  # noqa, pylint: disable=broad-except
             self.handleError(record)
 
 
 def _is_verbose():
-    return logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG
+    return (
+        logging.NOTSET
+        < logging.getLogger("dvc").getEffectiveLevel()
+        <= logging.DEBUG
+    )
 
 
 def _iter_causes(exc):
@@ -152,12 +183,14 @@ def disable_other_loggers():
 def setup(level=logging.INFO):
     colorama.init()
 
+    addLoggingLevel("TRACE", logging.DEBUG - 5)
     logging.config.dictConfig(
         {
             "version": 1,
             "filters": {
-                "exclude_errors": {"()": ExcludeErrorsFilter},
-                "exclude_info": {"()": ExcludeInfoFilter},
+                "exclude_errors": {"()": excludeFilter(logging.WARNING)},
+                "exclude_info": {"()": excludeFilter(logging.INFO)},
+                "exclude_debug": {"()": excludeFilter(logging.DEBUG)},
             },
             "formatters": {"color": {"()": ColorFormatter}},
             "handlers": {
@@ -175,6 +208,13 @@ def setup(level=logging.INFO):
                     "stream": "ext://sys.stdout",
                     "filters": ["exclude_info"],
                 },
+                "console_trace": {
+                    "class": "dvc.logger.LoggerHandler",
+                    "level": "TRACE",
+                    "formatter": "color",
+                    "stream": "ext://sys.stdout",
+                    "filters": ["exclude_debug"],
+                },
                 "console_errors": {
                     "class": "dvc.logger.LoggerHandler",
                     "level": "WARNING",
@@ -188,6 +228,7 @@ def setup(level=logging.INFO):
                     "handlers": [
                         "console_info",
                         "console_debug",
+                        "console_trace",
                         "console_errors",
                     ],
                 },

dvc/main.py

@@ -2,6 +2,7 @@
 
 import errno
 import logging
+from contextlib import contextmanager
 
 from dvc import analytics
 from dvc.cli import parse_args
@@ -9,8 +10,8 @@ from dvc.config import ConfigError
 from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
 from dvc.external_repo import clean_repos
 from dvc.logger import FOOTER, disable_other_loggers
-from dvc.remote.pool import close_pools
-from dvc.utils import format_link
+from dvc.tree.pool import close_pools
+from dvc.utils import error_link
 
 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
@@ -18,11 +19,29 @@ from dvc.utils import format_link
 
 "".encode("idna")
 
-
 logger = logging.getLogger("dvc")
 
 
-def main(argv=None):
+@contextmanager
+def profile(enable, dump):
+    if not enable:
+        yield
+        return
+
+    import cProfile
+
+    prof = cProfile.Profile()
+    prof.enable()
+
+    yield
+
+    if not dump:
+        prof.print_stats(sort="cumtime")
+        return
+    prof.dump_stats(dump)
+
+
+def main(argv=None):  # noqa: C901
     """Run dvc CLI command.
 
     Args:
@@ -38,14 +57,22 @@ def main(argv=None):
     try:
         args = parse_args(argv)
 
+        level = None
         if args.quiet:
-            logger.setLevel(logging.CRITICAL)
+            level = logging.CRITICAL
+        elif args.verbose == 1:
+            level = logging.DEBUG
+        elif args.verbose > 1:
+            level = logging.TRACE
 
-        elif args.verbose:
-            logger.setLevel(logging.DEBUG)
+        if level is not None:
+            logger.setLevel(level)
 
-        cmd = args.func(args)
-        ret = cmd.run()
+        logger.trace(args)
+
+        with profile(enable=args.cprofile, dump=args.cprofile_dump):
+            cmd = args.func(args)
+            ret = cmd.run()
     except ConfigError:
         logger.exception("configuration error")
         ret = 251
@@ -60,23 +87,35 @@ def main(argv=None):
     except DvcException:
         ret = 255
         logger.exception("")
-    except Exception as exc:  # pylint: disable=broad-except
-        if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+    except OSError as exc:
+        if exc.errno == errno.EMFILE:
             logger.exception(
                 "too many open files, please visit "
                 "{} to see how to handle this "
-                "problem".format(
-                    format_link("https://error.dvc.org/many-files")
-                ),
+                "problem".format(error_link("many-files")),
                 extra={"tb_only": True},
             )
         else:
             logger.exception("unexpected error")
         ret = 255
+    except Exception:  # noqa, pylint: disable=broad-except
+        logger.exception("unexpected error")
+        ret = 255
+
+    try:
+        if ret != 0 and (
+            ret != 1 or getattr(args, "cmd", "") != "check-ignore"
+        ):
+            logger.info(FOOTER)
+
+        if analytics.is_enabled():
+            analytics.collect_and_send_report(args, ret)
+
+        return ret
     finally:
         logger.setLevel(outerLogLevel)
 
-        # Closing pools by-hand to prevent wierd messages when closing SSH
+        # Closing pools by-hand to prevent weird messages when closing SSH
         # connections. See https://github.com/iterative/dvc/issues/3248 for
         # more info.
         close_pools()
@@ -84,11 +123,3 @@ def main(argv=None):
         # Remove cached repos in the end of the call, these are anonymous
         # so won't be reused by any other subsequent run anyway.
         clean_repos()
-
-    if ret != 0:
-        logger.info(FOOTER)
-
-    if analytics.is_enabled():
-        analytics.collect_and_send_report(args, ret)
-
-    return ret

dvc/output/__init__.py

@@ -1,5 +1,7 @@
+from collections import defaultdict
 from urllib.parse import urlparse
 
+from funcy import collecting, project
 from voluptuous import And, Any, Coerce, Length, Lower, Required, SetTo
 
 from dvc.output.base import BaseOutput
@@ -8,12 +10,13 @@ from dvc.output.hdfs import HDFSOutput
 from dvc.output.local import LocalOutput
 from dvc.output.s3 import S3Output
 from dvc.output.ssh import SSHOutput
-from dvc.remote import Remote
-from dvc.remote.hdfs import HDFSRemote
-from dvc.remote.local import LocalRemote
-from dvc.remote.s3 import S3Remote
 from dvc.scheme import Schemes
 
+from ..tree import get_cloud_tree
+from ..tree.hdfs import HDFSTree
+from ..tree.local import LocalTree
+from ..tree.s3 import S3Tree
+
 OUTS = [
     HDFSOutput,
     S3Output,
@@ -45,9 +48,9 @@ CHECKSUM_SCHEMA = Any(
 # so when a few types of outputs share the same name, we only need
 # specify it once.
 CHECKSUMS_SCHEMA = {
-    LocalRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    S3Remote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    HDFSRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    LocalTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    S3Tree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    HDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
 }
 
 SCHEMA = CHECKSUMS_SCHEMA.copy()
@@ -58,17 +61,19 @@ SCHEMA[BaseOutput.PARAM_PLOT] = bool
 SCHEMA[BaseOutput.PARAM_PERSIST] = bool
 
 
-def _get(stage, p, info, cache, metric, plot=False, persist=False):
+def _get(
+    stage, p, info=None, cache=True, metric=False, plot=False, persist=False
+):
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":
-        remote = Remote(stage.repo, name=parsed.netloc)
-        return OUTS_MAP[remote.scheme](
+        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
+        return OUTS_MAP[tree.scheme](
             stage,
             p,
             info,
             cache=cache,
-            remote=remote,
+            tree=tree,
             metric=metric,
             plot=plot,
             persist=persist,
@@ -81,7 +86,7 @@ def _get(stage, p, info, cache, metric, plot=False, persist=False):
                 p,
                 info,
                 cache=cache,
-                remote=None,
+                tree=None,
                 metric=metric,
                 plot=plot,
                 persist=persist,
@@ -91,7 +96,7 @@ def _get(stage, p, info, cache, metric, plot=False, persist=False):
         p,
         info,
         cache=cache,
-        remote=None,
+        tree=None,
         metric=metric,
         plot=plot,
         persist=persist,
@@ -135,3 +140,45 @@ def loads_from(
         )
         for s in s_list
     ]
+
+
+def _split_dict(d, keys):
+    return project(d, keys), project(d, d.keys() - keys)
+
+
+def _merge_data(s_list):
+    d = defaultdict(dict)
+    for key in s_list:
+        if isinstance(key, str):
+            d[key].update({})
+            continue
+        if not isinstance(key, dict):
+            raise ValueError(f"'{type(key).__name__}' not supported.")
+
+        for k, flags in key.items():
+            if not isinstance(flags, dict):
+                raise ValueError(
+                    f"Expected dict for '{k}', got: '{type(flags).__name__}'"
+                )
+            d[k].update(flags)
+    return d
+
+
+@collecting
+def load_from_pipeline(stage, s_list, typ="outs"):
+    if typ not in (stage.PARAM_OUTS, stage.PARAM_METRICS, stage.PARAM_PLOTS):
+        raise ValueError(f"'{typ}' key is not allowed for pipeline files.")
+
+    metric = typ == stage.PARAM_METRICS
+    plot = typ == stage.PARAM_PLOTS
+
+    d = _merge_data(s_list)
+
+    for path, flags in d.items():
+        plt_d = {}
+        if plot:
+            from dvc.schema import PLOT_PROPS
+
+            plt_d, flags = _split_dict(flags, keys=PLOT_PROPS.keys())
+        extra = project(flags, ["cache", "persist"])
+        yield _get(stage, path, {}, plot=plt_d or plot, metric=metric, **extra)

dvc/output/base.py

@@ -10,9 +10,11 @@ from dvc.cache import NamedCache
 from dvc.exceptions import (
     CollectCacheError,
     DvcException,
+    MergeError,
     RemoteCacheRequiredError,
 )
-from dvc.remote.base import BaseRemote
+
+from ..tree.base import BaseTree
 
 logger = logging.getLogger(__name__)
 
@@ -31,7 +33,11 @@ class OutputIsNotFileOrDirError(DvcException):
 
 class OutputAlreadyTrackedError(DvcException):
     def __init__(self, path):
-        msg = f"output '{path}' is already tracked by SCM (e.g. Git)"
+        msg = f""" output '{path}' is already tracked by SCM (e.g. Git).
+    You can remove it from Git, then add to DVC.
+        To stop tracking from Git:
+            git rm -r --cached '{path}'
+            git commit -m "stop tracking {path}" """
         super().__init__(msg)
 
 
@@ -40,10 +46,16 @@ class OutputIsStageFileError(DvcException):
         super().__init__(f"Stage file '{path}' cannot be an output.")
 
 
+class OutputIsIgnoredError(DvcException):
+    def __init__(self, match):
+        lines = "\n".join(match.patterns)
+        super().__init__(f"Path '{match.file}' is ignored by\n{lines}")
+
+
 class BaseOutput:
     IS_DEPENDENCY = False
 
-    REMOTE = BaseRemote
+    TREE_CLS = BaseTree
 
     PARAM_PATH = "path"
     PARAM_CACHE = "cache"
@@ -52,6 +64,12 @@ class BaseOutput:
     PARAM_METRIC_XPATH = "xpath"
     PARAM_PLOT = "plot"
     PARAM_PLOT_TEMPLATE = "template"
+    PARAM_PLOT_X = "x"
+    PARAM_PLOT_Y = "y"
+    PARAM_PLOT_X_LABEL = "x_label"
+    PARAM_PLOT_Y_LABEL = "y_label"
+    PARAM_PLOT_TITLE = "title"
+    PARAM_PLOT_HEADER = "header"
     PARAM_PERSIST = "persist"
 
     METRIC_SCHEMA = Any(
@@ -66,6 +84,7 @@ class BaseOutput:
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
     IsStageFileError = OutputIsStageFileError
+    IsIgnoredError = OutputIsIgnoredError
 
     sep = "/"
 
@@ -74,13 +93,13 @@ class BaseOutput:
         stage,
         path,
         info=None,
-        remote=None,
+        tree=None,
         cache=True,
         metric=False,
         plot=False,
         persist=False,
     ):
-        self._validate_output_path(path)
+        self._validate_output_path(path, stage)
         # This output (and dependency) objects have too many paths/urls
         # here is a list and comments:
         #
@@ -95,21 +114,24 @@ class BaseOutput:
         self.repo = stage.repo if stage else None
         self.def_path = path
         self.info = info
-        self.remote = remote or self.REMOTE(self.repo, {})
+        if tree:
+            self.tree = tree
+        else:
+            self.tree = self.TREE_CLS(self.repo, {})
         self.use_cache = False if self.IS_DEPENDENCY else cache
         self.metric = False if self.IS_DEPENDENCY else metric
         self.plot = False if self.IS_DEPENDENCY else plot
         self.persist = persist
 
-        self.path_info = self._parse_path(remote, path)
+        self.path_info = self._parse_path(tree, path)
         if self.use_cache and self.cache is None:
             raise RemoteCacheRequiredError(self.path_info)
 
-    def _parse_path(self, remote, path):
-        if remote:
+    def _parse_path(self, tree, path):
+        if tree:
             parsed = urlparse(path)
-            return remote.path_info / parsed.path.lstrip("/")
-        return self.REMOTE.path_cls(path)
+            return tree.path_info / parsed.path.lstrip("/")
+        return self.TREE_CLS.PATH_CLS(path)
 
     def __repr__(self):
         return "{class_name}: '{def_path}'".format(
@@ -121,7 +143,7 @@ class BaseOutput:
 
     @property
     def scheme(self):
-        return self.REMOTE.scheme
+        return self.TREE_CLS.scheme
 
     @property
     def is_in_repo(self):
@@ -144,37 +166,29 @@ class BaseOutput:
 
     @classmethod
     def supported(cls, url):
-        return cls.REMOTE.supported(url)
+        return cls.TREE_CLS.supported(url)
 
     @property
     def cache_path(self):
-        return self.cache.checksum_to_path_info(self.checksum).url
-
-    @property
-    def checksum_type(self):
-        return self.remote.PARAM_CHECKSUM
+        return self.cache.tree.hash_to_path_info(self.checksum).url
 
     @property
     def checksum(self):
-        return self.info.get(self.remote.PARAM_CHECKSUM)
-
-    @checksum.setter
-    def checksum(self, checksum):
-        self.info[self.remote.PARAM_CHECKSUM] = checksum
+        return self.info.get(self.tree.PARAM_CHECKSUM)
 
     def get_checksum(self):
-        return self.remote.get_checksum(self.path_info)
+        return self.tree.get_hash(self.path_info)[1]
 
     @property
     def is_dir_checksum(self):
-        return self.remote.is_dir_checksum(self.checksum)
+        return self.tree.is_dir_hash(self.checksum)
 
     @property
     def exists(self):
-        return self.remote.exists(self.path_info)
+        return self.tree.exists(self.path_info)
 
     def save_info(self):
-        return self.remote.save_info(self.path_info)
+        return self.tree.save_info(self.path_info)
 
     def changed_checksum(self):
         return self.checksum != self.get_checksum()
@@ -185,10 +199,7 @@ class BaseOutput:
 
         return self.cache.changed_cache(self.checksum, filter_info=filter_info)
 
-    def status(self):
-        if self.checksum and self.use_cache and self.changed_cache():
-            return {str(self): "not in cache"}
-
+    def workspace_status(self):
         if not self.exists:
             return {str(self): "deleted"}
 
@@ -200,6 +211,12 @@ class BaseOutput:
 
         return {}
 
+    def status(self):
+        if self.checksum and self.use_cache and self.changed_cache():
+            return {str(self): "not in cache"}
+
+        return self.workspace_status()
+
     def changed(self):
         status = self.status()
         logger.debug(str(status))
@@ -207,13 +224,15 @@ class BaseOutput:
 
     @property
     def is_empty(self):
-        return self.remote.is_empty(self.path_info)
+        return self.tree.is_empty(self.path_info)
 
     def isdir(self):
-        return self.remote.isdir(self.path_info)
+        return self.tree.isdir(self.path_info)
 
     def isfile(self):
-        return self.remote.isfile(self.path_info)
+        return self.tree.isfile(self.path_info)
+
+    # pylint: disable=no-member
 
     def ignore(self):
         if not self.use_scm_ignore:
@@ -224,6 +243,14 @@ class BaseOutput:
 
         self.repo.scm.ignore(self.fspath)
 
+    def ignore_remove(self):
+        if not self.use_scm_ignore:
+            return
+
+        self.repo.scm.ignore_remove(self.fspath)
+
+    # pylint: enable=no-member
+
     def save(self):
         if not self.exists:
             raise self.DoesNotExistError(self)
@@ -236,29 +263,29 @@ class BaseOutput:
 
         self.ignore()
 
+        if self.metric or self.plot:
+            self.verify_metric()
+
         if not self.use_cache:
             self.info = self.save_info()
-            if self.metric or self.plot:
-                self.verify_metric()
             if not self.IS_DEPENDENCY:
-                logger.info(
-                    "Output '{}' doesn't use cache. Skipping saving.".format(
-                        self
-                    )
+                logger.debug(
+                    "Output '%s' doesn't use cache. Skipping saving.", self
                 )
             return
 
         assert not self.IS_DEPENDENCY
 
         if not self.changed():
-            logger.info(f"Output '{self}' didn't change. Skipping saving.")
+            logger.debug("Output '%s' didn't change. Skipping saving.", self)
             return
 
         self.info = self.save_info()
 
     def commit(self):
+        assert self.info
         if self.use_cache:
-            self.cache.save(self.path_info, self.info)
+            self.cache.save(self.path_info, self.cache.tree, self.info)
 
     def dumpd(self):
         ret = copy(self.info)
@@ -267,7 +294,8 @@ class BaseOutput:
         if self.IS_DEPENDENCY:
             return ret
 
-        ret[self.PARAM_CACHE] = self.use_cache
+        if not self.use_cache:
+            ret[self.PARAM_CACHE] = self.use_cache
 
         if isinstance(self.metric, dict):
             if (
@@ -276,12 +304,14 @@ class BaseOutput:
             ):
                 del self.metric[self.PARAM_METRIC_XPATH]
 
-        ret[self.PARAM_METRIC] = self.metric
+        if self.metric:
+            ret[self.PARAM_METRIC] = self.metric
 
         if self.plot:
             ret[self.PARAM_PLOT] = self.plot
 
-        ret[self.PARAM_PERSIST] = self.persist
+        if self.persist:
+            ret[self.PARAM_PERSIST] = self.persist
 
         return ret
 
@@ -289,7 +319,7 @@ class BaseOutput:
         raise DvcException(f"verify metric is not supported for {self.scheme}")
 
     def download(self, to):
-        self.remote.download(self.path_info, to.path_info)
+        self.tree.download(self.path_info, to.path_info)
 
     def checkout(
         self,
@@ -315,18 +345,19 @@ class BaseOutput:
         )
 
     def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+        self.tree.remove(self.path_info)
         if self.scheme != "local":
             return
 
-        if ignore_remove and self.use_scm_ignore:
-            self.repo.scm.ignore_remove(self.fspath)
+        if ignore_remove:
+            self.ignore_remove()
 
     def move(self, out):
+        # pylint: disable=no-member
         if self.scheme == "local" and self.use_scm_ignore:
             self.repo.scm.ignore_remove(self.fspath)
 
-        self.remote.move(self.path_info, out.path_info)
+        self.tree.move(self.path_info, out.path_info)
         self.def_path = out.def_path
         self.path_info = out.path_info
         self.save()
@@ -345,7 +376,18 @@ class BaseOutput:
 
     def unprotect(self):
         if self.exists:
-            self.remote.unprotect(self.path_info)
+            self.tree.unprotect(self.path_info)
+
+    def get_dir_cache(self, **kwargs):
+        if not self.is_dir_checksum:
+            raise DvcException("cannot get dir cache for file checksum")
+        if self.cache.changed_cache_file(self.checksum):
+            self.repo.cloud.pull(
+                NamedCache.make("local", self.checksum, str(self)),
+                show_checksums=False,
+                **kwargs,
+            )
+        return self.dir_cache
 
     def collect_used_dir_cache(
         self, remote=None, force=False, jobs=None, filter_info=None
@@ -371,16 +413,10 @@ class BaseOutput:
 
         cache = NamedCache()
 
-        if self.cache.changed_cache_file(self.checksum):
-            try:
-                self.repo.cloud.pull(
-                    NamedCache.make("local", self.checksum, str(self)),
-                    jobs=jobs,
-                    remote=remote,
-                    show_checksums=False,
-                )
-            except DvcException:
-                logger.debug(f"failed to pull cache for '{self}'")
+        try:
+            self.get_dir_cache(jobs=jobs, remote=remote)
+        except DvcException:
+            logger.debug(f"failed to pull cache for '{self}'")
 
         if self.cache.changed_cache_file(self.checksum):
             msg = (
@@ -400,8 +436,8 @@ class BaseOutput:
         filter_path = str(filter_info) if filter_info else None
         is_win = os.name == "nt"
         for entry in self.dir_cache:
-            checksum = entry[self.remote.PARAM_CHECKSUM]
-            entry_relpath = entry[self.remote.PARAM_RELPATH]
+            checksum = entry[self.tree.PARAM_CHECKSUM]
+            entry_relpath = entry[self.tree.PARAM_RELPATH]
             if is_win:
                 entry_relpath = entry_relpath.replace("/", os.sep)
             entry_path = os.path.join(path, entry_relpath)
@@ -463,8 +499,47 @@ class BaseOutput:
         return ret
 
     @classmethod
-    def _validate_output_path(cls, path):
+    def _validate_output_path(cls, path, stage=None):
         from dvc.dvcfile import is_valid_filename
 
         if is_valid_filename(path):
             raise cls.IsStageFileError(path)
+
+        if stage:
+            check = stage.repo.tree.dvcignore.check_ignore(path)
+            if check.match:
+                raise cls.IsIgnoredError(check)
+
+    def _check_can_merge(self, out):
+        if self.scheme != out.scheme:
+            raise MergeError("unable to auto-merge outputs of different types")
+
+        my = self.dumpd()
+        other = out.dumpd()
+
+        my.pop(self.tree.PARAM_CHECKSUM)
+        other.pop(self.tree.PARAM_CHECKSUM)
+
+        if my != other:
+            raise MergeError(
+                "unable to auto-merge outputs with different options"
+            )
+
+        if not out.is_dir_checksum:
+            raise MergeError(
+                "unable to auto-merge outputs that are not directories"
+            )
+
+    def merge(self, ancestor, other):
+        assert other
+
+        if ancestor:
+            self._check_can_merge(ancestor)
+            ancestor_info = ancestor.info
+        else:
+            ancestor_info = None
+
+        self._check_can_merge(self)
+        self._check_can_merge(other)
+
+        self.info = self.cache.merge(ancestor_info, self.info, other.info)

dvc/output/gs.py

@@ -1,6 +1,7 @@
 from dvc.output.s3 import S3Output
-from dvc.remote.gs import GSRemote
+
+from ..tree.gs import GSTree
 
 
 class GSOutput(S3Output):
-    REMOTE = GSRemote
+    TREE_CLS = GSTree

dvc/output/hdfs.py

@@ -1,6 +1,7 @@
 from dvc.output.base import BaseOutput
-from dvc.remote.hdfs import HDFSRemote
+
+from ..tree.hdfs import HDFSTree
 
 
 class HDFSOutput(BaseOutput):
-    REMOTE = HDFSRemote
+    TREE_CLS = HDFSTree

dvc/output/local.py

@@ -5,15 +5,16 @@ from urllib.parse import urlparse
 from dvc.exceptions import DvcException
 from dvc.istextfile import istextfile
 from dvc.output.base import BaseOutput
-from dvc.remote.local import LocalRemote
 from dvc.utils import relpath
 from dvc.utils.fs import path_isin
 
+from ..tree.local import LocalTree
+
 logger = logging.getLogger(__name__)
 
 
 class LocalOutput(BaseOutput):
-    REMOTE = LocalRemote
+    TREE_CLS = LocalTree
     sep = os.sep
 
     def __init__(self, stage, path, *args, **kwargs):
@@ -21,11 +22,17 @@ class LocalOutput(BaseOutput):
             path = relpath(path, stage.wdir)
 
         super().__init__(stage, path, *args, **kwargs)
+        if (
+            self.is_in_repo
+            and self.repo
+            and isinstance(self.repo.tree, LocalTree)
+        ):
+            self.tree = self.repo.tree
 
-    def _parse_path(self, remote, path):
+    def _parse_path(self, tree, path):
         parsed = urlparse(path)
         if parsed.scheme == "remote":
-            p = remote.path_info / parsed.path.lstrip("/")
+            p = tree.path_info / parsed.path.lstrip("/")
         else:
             # NOTE: we can path either from command line or .dvc file,
             # so we should expect both posix and windows style paths.
@@ -33,15 +40,15 @@ class LocalOutput(BaseOutput):
             #
             # FIXME: if we have Windows path containing / or posix one with \
             # then we have #2059 bug and can't really handle that.
-            p = self.REMOTE.path_cls(path)
-            if not p.is_absolute():
+            p = self.TREE_CLS.PATH_CLS(path)
+            if self.stage and not p.is_absolute():
                 p = self.stage.wdir / p
 
         abs_p = os.path.abspath(os.path.normpath(p))
-        return self.REMOTE.path_cls(abs_p)
+        return self.TREE_CLS.PATH_CLS(abs_p)
 
     def __str__(self):
-        if not self.is_in_repo:
+        if not self.repo or not self.is_in_repo:
             return str(self.def_path)
 
         cur_dir = os.getcwd()
@@ -80,8 +87,9 @@ class LocalOutput(BaseOutput):
 
         name = "metrics" if self.metric else "plot"
         if os.path.isdir(path):
-            msg = "directory '{}' cannot be used as {}."
-            raise DvcException(msg.format(self.path_info, name))
+            msg = "directory '%s' cannot be used as %s."
+            logger.debug(msg, str(self.path_info), name)
+            return
 
         if not istextfile(path):
             msg = "binary file '{}' cannot be used as {}."

dvc/output/s3.py

@@ -1,6 +1,7 @@
 from dvc.output.base import BaseOutput
-from dvc.remote.s3 import S3Remote
+
+from ..tree.s3 import S3Tree
 
 
 class S3Output(BaseOutput):
-    REMOTE = S3Remote
+    TREE_CLS = S3Tree

dvc/output/ssh.py

@@ -1,6 +1,7 @@
 from dvc.output.base import BaseOutput
-from dvc.remote.ssh import SSHRemote
+
+from ..tree.ssh import SSHTree
 
 
 class SSHOutput(BaseOutput):
-    REMOTE = SSHRemote
+    TREE_CLS = SSHTree

dvc/path_info.py

@@ -1,3 +1,4 @@
+# pylint: disable=protected-access
 import os
 import pathlib
 import posixpath
@@ -17,7 +18,7 @@ class _BasePath:
         return self.isin_or_eq(other) or other.isin(self)
 
     def isin_or_eq(self, other):
-        return self == other or self.isin(other)
+        return self == other or self.isin(other)  # pylint: disable=no-member
 
 
 class PathInfo(pathlib.PurePath, _BasePath):
@@ -30,11 +31,14 @@ class PathInfo(pathlib.PurePath, _BasePath):
     def __new__(cls, *args):
         # Construct a proper subclass depending on current os
         if cls is PathInfo:
-            cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo
+            cls = (  # pylint: disable=self-cls-assignment
+                WindowsPathInfo if os.name == "nt" else PosixPathInfo
+            )
+
         return cls._from_parts(args)
 
     def as_posix(self):
-        f = self._flavour
+        f = self._flavour  # pylint: disable=no-member
         # Unlike original implementation [1] that uses `str()` we actually need
         # to use `fspath`, because we've overridden `__str__` method to return
         # relative paths, which will break original `as_posix`.
@@ -261,7 +265,7 @@ class HTTPURLInfo(URLInfo):
         params=None,
         query=None,
         fragment=None,
-    ):
+    ):  # pylint: disable=arguments-differ
         assert bool(host) ^ bool(netloc)
 
         if netloc is not None:
@@ -311,3 +315,11 @@ class HTTPURLInfo(URLInfo):
             and self._path == other._path
             and self._extra_parts == other._extra_parts
         )
+
+
+class WebDAVURLInfo(URLInfo):
+    @cached_property
+    def url(self):
+        return "{}://{}{}".format(
+            self.scheme.replace("webdav", "http"), self.netloc, self._spath
+        )

dvc/pathspec_math.py

@@ -0,0 +1,93 @@
+# Path Specification Pattern Math
+# Including changing base dir of path specification patterns and merging
+# of two path specification patterns with different base
+# All the operations follow the documents of `gitignore`
+import os
+from collections import namedtuple
+
+from pathspec.util import normalize_file
+
+from dvc.utils import relpath
+
+PatternInfo = namedtuple("PatternInfo", ["patterns", "file_info"])
+
+
+def _not_ignore(rule):
+    return (True, rule[1:]) if rule.startswith("!") else (False, rule)
+
+
+def _is_comment(rule):
+    return rule.startswith("#")
+
+
+def _remove_slash(rule):
+    if rule.startswith("\\"):
+        return rule[1:]
+    return rule
+
+
+def _match_all_level(rule):
+    if rule[:-1].find("/") >= 0 and not rule.startswith("**/"):
+        if rule.startswith("/"):
+            rule = rule[1:]
+        return False, rule
+    if rule.startswith("**/"):
+        rule = rule[3:]
+    return True, rule
+
+
+def change_rule(rule, rel):
+    rule = rule.strip()
+    if _is_comment(rule):
+        return rule
+    not_ignore, rule = _not_ignore(rule)
+    match_all, rule = _match_all_level(rule)
+    rule = _remove_slash(rule)
+    if not match_all:
+        rule = f"/{rule}"
+    else:
+        rule = f"/**/{rule}"
+    if not_ignore:
+        rule = f"!/{rel}{rule}"
+    else:
+        rule = f"/{rel}{rule}"
+    rule = normalize_file(rule)
+    return rule
+
+
+def _change_dirname(dirname, pattern_list, new_dirname):
+    if new_dirname == dirname:
+        return pattern_list
+    rel = relpath(dirname, new_dirname)
+    if rel.startswith(".."):
+        raise ValueError("change dirname can only change to parent path")
+
+    return [
+        PatternInfo(change_rule(rule.patterns, rel), rule.file_info)
+        for rule in pattern_list
+    ]
+
+
+def merge_patterns(pattern_a, prefix_a, pattern_b, prefix_b):
+    """
+    Merge two path specification patterns.
+
+    This implementation merge two path specification patterns on different
+    bases. It returns the longest common parent directory, and the patterns
+    based on this new base directory.
+    """
+    if not pattern_a:
+        return pattern_b, prefix_b
+    elif not pattern_b:
+        return pattern_a, prefix_a
+
+    longest_common_dir = os.path.commonpath([prefix_a, prefix_b])
+    new_pattern_a = _change_dirname(prefix_a, pattern_a, longest_common_dir)
+    new_pattern_b = _change_dirname(prefix_b, pattern_b, longest_common_dir)
+
+    if len(prefix_a) <= len(prefix_b):
+        merged_pattern = new_pattern_a + new_pattern_b
+    else:
+        merged_pattern = new_pattern_b + new_pattern_a
+
+    return merged_pattern, longest_common_dir

dvc/prompt.py

@@ -7,7 +7,7 @@ from getpass import getpass
 logger = logging.getLogger(__name__)
 
 
-def _ask(prompt, limited_to=None):
+def ask(prompt, limited_to=None):
     if not sys.stdout.isatty():
         return None
 
@@ -39,7 +39,7 @@ def confirm(statement):
         bool: whether or not specified statement was confirmed.
     """
     prompt = f"{statement} [y/n]"
-    answer = _ask(prompt, limited_to=["yes", "no", "y", "n"])
+    answer = ask(prompt, limited_to=["yes", "no", "y", "n"])
     return answer and answer.startswith("y")
 
 

dvc/remote/__init__.py

@@ -1,74 +1,13 @@
-import posixpath
-from urllib.parse import urlparse
-
-from dvc.remote.azure import AzureRemote
-from dvc.remote.gdrive import GDriveRemote
-from dvc.remote.gs import GSRemote
-from dvc.remote.hdfs import HDFSRemote
-from dvc.remote.http import HTTPRemote
-from dvc.remote.https import HTTPSRemote
-from dvc.remote.local import LocalRemote
-from dvc.remote.oss import OSSRemote
-from dvc.remote.s3 import S3Remote
-from dvc.remote.ssh import SSHRemote
-
-REMOTES = [
-    AzureRemote,
-    GDriveRemote,
-    GSRemote,
-    HDFSRemote,
-    HTTPRemote,
-    HTTPSRemote,
-    S3Remote,
-    SSHRemote,
-    OSSRemote,
-    # NOTE: LocalRemote is the default
-]
+from ..tree import get_cloud_tree
+from .base import Remote
+from .local import LocalRemote
+from .ssh import SSHRemote
 
 
-def _get(remote_conf):
-    for remote in REMOTES:
-        if remote.supported(remote_conf):
-            return remote
-    return LocalRemote
-
-
-def Remote(repo, **kwargs):
-    name = kwargs.get("name")
-    if name:
-        remote_conf = repo.config["remote"][name.lower()]
-    else:
-        remote_conf = kwargs
-    remote_conf = _resolve_remote_refs(repo.config, remote_conf)
-    return _get(remote_conf)(repo, remote_conf)
-
-
-def _resolve_remote_refs(config, remote_conf):
-    # Support for cross referenced remotes.
-    # This will merge the settings, shadowing base ref with remote_conf.
-    # For example, having:
-    #
-    #       dvc remote add server ssh://localhost
-    #       dvc remote modify server user root
-    #       dvc remote modify server ask_password true
-    #
-    #       dvc remote add images remote://server/tmp/pictures
-    #       dvc remote modify images user alice
-    #       dvc remote modify images ask_password false
-    #       dvc remote modify images password asdf1234
-    #
-    # Results on a config dictionary like:
-    #
-    #       {
-    #           "url": "ssh://localhost/tmp/pictures",
-    #           "user": "alice",
-    #           "password": "asdf1234",
-    #           "ask_password": False,
-    #       }
-    parsed = urlparse(remote_conf["url"])
-    if parsed.scheme != "remote":
-        return remote_conf
-
-    base = config["remote"][parsed.netloc]
-    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
-    return {**base, **remote_conf, "url": url}
+def get_remote(repo, **kwargs):
+    tree = get_cloud_tree(repo, **kwargs)
+    if tree.scheme == "local":
+        return LocalRemote(tree)
+    if tree.scheme == "ssh":
+        return SSHRemote(tree)
+    return Remote(tree)

dvc/remote/azure.py

@@ -1,160 +0,0 @@
-import logging
-import os
-import posixpath
-import re
-import threading
-from datetime import datetime, timedelta
-from urllib.parse import urlparse
-
-from funcy import cached_property, wrap_prop
-
-from dvc.path_info import CloudURLInfo
-from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
-from dvc.scheme import Schemes
-
-logger = logging.getLogger(__name__)
-
-
-class AzureRemote(BaseRemote):
-    scheme = Schemes.AZURE
-    path_cls = CloudURLInfo
-    REGEX = (
-        r"azure://((?P<path>[^=;]*)?|("
-        # backward compatibility
-        r"(ContainerName=(?P<container_name>[^;]+);?)?"
-        r"(?P<connection_string>.+)?)?)$"
-    )
-    REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
-    PARAM_CHECKSUM = "etag"
-    COPY_POLL_SECONDS = 5
-    LIST_OBJECT_PAGE_SIZE = 5000
-
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-
-        url = config.get("url", "azure://")
-
-        match = re.match(self.REGEX, url)  # backward compatibility
-        path = match.group("path")
-        bucket = (
-            urlparse(url if path else "").netloc
-            or match.group("container_name")  # backward compatibility
-            or os.getenv("AZURE_STORAGE_CONTAINER_NAME")
-        )
-
-        self.connection_string = (
-            config.get("connection_string")
-            or match.group("connection_string")  # backward compatibility
-            or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
-        )
-
-        if not bucket:
-            raise ValueError("azure storage container name missing")
-
-        if not self.connection_string:
-            raise ValueError("azure storage connection string missing")
-
-        self.path_info = (
-            self.path_cls(url)
-            if path
-            else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
-        )
-
-    @wrap_prop(threading.Lock())
-    @cached_property
-    def blob_service(self):
-        from azure.storage.blob import BlockBlobService
-        from azure.common import AzureMissingResourceHttpError
-
-        logger.debug(f"URL {self.path_info}")
-        logger.debug(f"Connection string {self.connection_string}")
-        blob_service = BlockBlobService(
-            connection_string=self.connection_string
-        )
-        logger.debug(f"Container name {self.path_info.bucket}")
-        try:  # verify that container exists
-            blob_service.list_blobs(
-                self.path_info.bucket, delimiter="/", num_results=1
-            )
-        except AzureMissingResourceHttpError:
-            blob_service.create_container(self.path_info.bucket)
-        return blob_service
-
-    def remove(self, path_info):
-        if path_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        logger.debug(f"Removing {path_info}")
-        self.blob_service.delete_blob(path_info.bucket, path_info.path)
-
-    def _list_paths(self, bucket, prefix, progress_callback=None):
-        blob_service = self.blob_service
-        next_marker = None
-        while True:
-            blobs = blob_service.list_blobs(
-                bucket, prefix=prefix, marker=next_marker
-            )
-
-            for blob in blobs:
-                if progress_callback:
-                    progress_callback()
-                yield blob.name
-
-            if not blobs.next_marker:
-                break
-
-            next_marker = blobs.next_marker
-
-    def list_cache_paths(self, prefix=None, progress_callback=None):
-        if prefix:
-            prefix = posixpath.join(
-                self.path_info.path, prefix[:2], prefix[2:]
-            )
-        else:
-            prefix = self.path_info.path
-        return self._list_paths(
-            self.path_info.bucket, prefix, progress_callback
-        )
-
-    def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
-    ):
-        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
-            self.blob_service.create_blob_from_path(
-                to_info.bucket,
-                to_info.path,
-                from_file,
-                progress_callback=pbar.update_to,
-            )
-
-    def _download(
-        self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
-    ):
-        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
-            self.blob_service.get_blob_to_path(
-                from_info.bucket,
-                from_info.path,
-                to_file,
-                progress_callback=pbar.update_to,
-            )
-
-    def exists(self, path_info):
-        paths = self._list_paths(path_info.bucket, path_info.path)
-        return any(path_info.path == path for path in paths)
-
-    def _generate_download_url(self, path_info, expires=3600):
-        from azure.storage.blob import BlobPermissions
-
-        expires_at = datetime.utcnow() + timedelta(seconds=expires)
-
-        sas_token = self.blob_service.generate_blob_shared_access_signature(
-            path_info.bucket,
-            path_info.path,
-            permission=BlobPermissions.READ,
-            expiry=expires_at,
-        )
-        download_url = self.blob_service.make_blob_url(
-            path_info.bucket, path_info.path, sas_token=sas_token
-        )
-        return download_url

dvc/remote/https.py

@@ -1,7 +0,0 @@
-from dvc.scheme import Schemes
-
-from .http import HTTPRemote
-
-
-class HTTPSRemote(HTTPRemote):
-    scheme = Schemes.HTTPS

dvc/remote/index.py

@@ -25,15 +25,15 @@ class RemoteIndexNoop:
     def __iter__(self):
         return iter([])
 
-    def __contains__(self, checksum):
+    def __contains__(self, hash_):
         return False
 
     @staticmethod
-    def checksums():
+    def hashes():
         return []
 
     @staticmethod
-    def dir_checksums():
+    def dir_hashes():
         return []
 
     def load(self):
@@ -54,19 +54,19 @@ class RemoteIndexNoop:
 
 
 class RemoteIndex:
-    """Class for indexing remote checksums in a sqlite3 database.
+    """Class for indexing remote hashes in a sqlite3 database.
 
     Args:
         repo: repo for this remote index.
         name: name for this index. Index db will be loaded from and saved to
             ``.dvc/tmp/index/{name}.idx``.
-        dir_suffix: suffix used for naming directory checksums
+        dir_suffix: suffix used for naming directory hashes
     """
 
     INDEX_SUFFIX = ".idx"
     VERSION = 1
     INDEX_TABLE = "remote_index"
-    INDEX_TABLE_LAYOUT = "checksum TEXT PRIMARY KEY, " "dir INTEGER NOT NULL"
+    INDEX_TABLE_LAYOUT = "checksum TEXT PRIMARY KEY, dir INTEGER NOT NULL"
 
     def __init__(self, repo, name, dir_suffix=".dir"):
         self.path = os.path.join(repo.index_dir, f"{name}{self.INDEX_SUFFIX}")
@@ -79,8 +79,8 @@ class RemoteIndex:
 
     def __iter__(self):
         cmd = f"SELECT checksum FROM {self.INDEX_TABLE}"
-        for (checksum,) in self._execute(cmd):
-            yield checksum
+        for (hash_,) in self._execute(cmd):
+            yield hash_
 
     def __enter__(self):
         self.lock.acquire()
@@ -90,25 +90,25 @@ class RemoteIndex:
         self.dump()
         self.lock.release()
 
-    def __contains__(self, checksum):
+    def __contains__(self, hash_):
         cmd = "SELECT checksum FROM {} WHERE checksum = (?)".format(
             self.INDEX_TABLE
         )
-        self._execute(cmd, (checksum,))
+        self._execute(cmd, (hash_,))
         return self.cursor.fetchone() is not None
 
-    def checksums(self):
-        """Iterate over checksums stored in the index."""
+    def hashes(self):
+        """Iterate over hashes stored in the index."""
         return iter(self)
 
-    def dir_checksums(self):
-        """Iterate over .dir checksums stored in the index."""
+    def dir_hashes(self):
+        """Iterate over .dir hashes stored in the index."""
         cmd = f"SELECT checksum FROM {self.INDEX_TABLE} WHERE dir = 1"
-        for (checksum,) in self._execute(cmd):
-            yield checksum
+        for (hash_,) in self._execute(cmd):
+            yield hash_
 
-    def is_dir_checksum(self, checksum):
-        return checksum.endswith(self.dir_suffix)
+    def is_dir_hash(self, hash_):
+        return hash_.endswith(self.dir_suffix)
 
     def _execute(self, cmd, parameters=()):
         return self.cursor.execute(cmd, parameters)
@@ -185,28 +185,24 @@ class RemoteIndex:
         cmd = f"DELETE FROM {self.INDEX_TABLE}"
         self._execute(cmd)
 
-    def update(self, dir_checksums, file_checksums):
-        """Update this index, adding the specified checksums.
+    def update(self, dir_hashes, file_hashes):
+        """Update this index, adding the specified hashes.
 
         Changes to the index will not committed until dump() is called.
         """
         cmd = "INSERT OR IGNORE INTO {} (checksum, dir) VALUES (?, ?)".format(
             self.INDEX_TABLE
         )
-        self._executemany(
-            cmd, ((checksum, True) for checksum in dir_checksums)
-        )
-        self._executemany(
-            cmd, ((checksum, False) for checksum in file_checksums)
-        )
+        self._executemany(cmd, ((hash_, True) for hash_ in dir_hashes))
+        self._executemany(cmd, ((hash_, False) for hash_ in file_hashes))
 
-    def intersection(self, checksums):
-        """Iterate over values from `checksums` which exist in the index."""
+    def intersection(self, hashes):
+        """Iterate over values from `hashes` which exist in the index."""
         # sqlite has a compile time limit of 999, see:
         # https://www.sqlite.org/c3ref/c_limit_attached.html#sqlitelimitvariablenumber
-        for chunk in lchunks(999, checksums):
+        for chunk in lchunks(999, hashes):
             cmd = "SELECT checksum FROM {} WHERE checksum IN ({})".format(
-                self.INDEX_TABLE, ",".join("?" for checksum in chunk)
+                self.INDEX_TABLE, ",".join("?" for hash_ in chunk)
             )
-            for (checksum,) in self._execute(cmd, chunk):
-                yield checksum
+            for (hash_,) in self._execute(cmd, chunk):
+                yield hash_

dvc/remote/local.py

@@ -1,716 +1,6 @@
-import errno
-import logging
-import os
-import stat
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial
-
-from funcy import cached_property, concat
-from shortuuid import uuid
-
-from dvc.exceptions import DownloadError, DvcException, UploadError
-from dvc.path_info import PathInfo
-from dvc.progress import Tqdm
-from dvc.remote.base import (
-    STATUS_DELETED,
-    STATUS_MAP,
-    STATUS_MISSING,
-    STATUS_NEW,
-    BaseRemote,
-    index_locked,
-)
-from dvc.remote.index import RemoteIndexNoop
-from dvc.scheme import Schemes
-from dvc.scm.tree import is_working_tree
-from dvc.system import System
-from dvc.utils import file_md5, relpath, tmp_fname
-from dvc.utils.fs import copyfile, makedirs, move, remove, walk_files
-
-logger = logging.getLogger(__name__)
+from .base import Remote
+from .index import RemoteIndexNoop
 
 
-class LocalRemote(BaseRemote):
-    scheme = Schemes.LOCAL
-    path_cls = PathInfo
-    PARAM_CHECKSUM = "md5"
-    PARAM_PATH = "path"
-    TRAVERSE_PREFIX_LEN = 2
+class LocalRemote(Remote):
     INDEX_CLS = RemoteIndexNoop
-
-    UNPACKED_DIR_SUFFIX = ".unpacked"
-
-    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
-
-    CACHE_MODE = 0o444
-    SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
-
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-        self.cache_dir = config.get("url")
-        self._dir_info = {}
-
-    @property
-    def state(self):
-        return self.repo.state
-
-    @property
-    def cache_dir(self):
-        return self.path_info.fspath if self.path_info else None
-
-    @cache_dir.setter
-    def cache_dir(self, value):
-        self.path_info = PathInfo(value) if value else None
-
-    @classmethod
-    def supported(cls, config):
-        return True
-
-    @cached_property
-    def cache_path(self):
-        return os.path.abspath(self.cache_dir)
-
-    def checksum_to_path(self, checksum):
-        return os.path.join(self.cache_path, checksum[0:2], checksum[2:])
-
-    def list_cache_paths(self, prefix=None, progress_callback=None):
-        assert self.path_info is not None
-        if prefix:
-            path_info = self.path_info / prefix[:2]
-        else:
-            path_info = self.path_info
-        if progress_callback:
-            for path in walk_files(path_info):
-                progress_callback()
-                yield path
-        else:
-            yield from walk_files(path_info)
-
-    def get(self, md5):
-        if not md5:
-            return None
-
-        return self.checksum_to_path_info(md5).url
-
-    def exists(self, path_info):
-        assert is_working_tree(self.repo.tree)
-        assert isinstance(path_info, str) or path_info.scheme == "local"
-        return self.repo.tree.exists(path_info)
-
-    def makedirs(self, path_info):
-        makedirs(path_info, exist_ok=True, mode=self._dir_mode)
-
-    def already_cached(self, path_info):
-        assert path_info.scheme in ["", "local"]
-
-        current_md5 = self.get_checksum(path_info)
-
-        if not current_md5:
-            return False
-
-        return not self.changed_cache(current_md5)
-
-    def _verify_link(self, path_info, link_type):
-        if link_type == "hardlink" and self.getsize(path_info) == 0:
-            return
-
-        super()._verify_link(path_info, link_type)
-
-    def is_empty(self, path_info):
-        path = path_info.fspath
-
-        if self.isfile(path_info) and os.path.getsize(path) == 0:
-            return True
-
-        if self.isdir(path_info) and len(os.listdir(path)) == 0:
-            return True
-
-        return False
-
-    @staticmethod
-    def isfile(path_info):
-        return os.path.isfile(path_info)
-
-    @staticmethod
-    def isdir(path_info):
-        return os.path.isdir(path_info)
-
-    def iscopy(self, path_info):
-        return not (
-            System.is_symlink(path_info) or System.is_hardlink(path_info)
-        )
-
-    @staticmethod
-    def getsize(path_info):
-        return os.path.getsize(path_info)
-
-    def walk_files(self, path_info):
-        assert is_working_tree(self.repo.tree)
-
-        for fname in self.repo.tree.walk_files(path_info):
-            yield PathInfo(fname)
-
-    def get_file_checksum(self, path_info):
-        return file_md5(path_info)[0]
-
-    def remove(self, path_info):
-        if isinstance(path_info, PathInfo):
-            if path_info.scheme != "local":
-                raise NotImplementedError
-            path = path_info.fspath
-        else:
-            path = path_info
-
-        if self.exists(path):
-            remove(path)
-
-    def move(self, from_info, to_info, mode=None):
-        if from_info.scheme != "local" or to_info.scheme != "local":
-            raise NotImplementedError
-
-        self.makedirs(to_info.parent)
-
-        if mode is None:
-            if self.isfile(from_info):
-                mode = self._file_mode
-            else:
-                mode = self._dir_mode
-
-        move(from_info, to_info, mode=mode)
-
-    def copy(self, from_info, to_info):
-        tmp_info = to_info.parent / tmp_fname(to_info.name)
-        try:
-            System.copy(from_info, tmp_info)
-            os.chmod(tmp_info, self._file_mode)
-            os.rename(tmp_info, to_info)
-        except Exception:
-            self.remove(tmp_info)
-            raise
-
-    @staticmethod
-    def symlink(from_info, to_info):
-        System.symlink(from_info, to_info)
-
-    @staticmethod
-    def is_symlink(path_info):
-        return System.is_symlink(path_info)
-
-    def hardlink(self, from_info, to_info):
-        # If there are a lot of empty files (which happens a lot in datasets),
-        # and the cache type is `hardlink`, we might reach link limits and
-        # will get something like: `too many links error`
-        #
-        # This is because all those empty files will have the same checksum
-        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
-        # linked to the same file in the cache.
-        #
-        # From https://en.wikipedia.org/wiki/Hard_link
-        #   * ext4 limits the number of hard links on a file to 65,000
-        #   * Windows with NTFS has a limit of 1024 hard links on a file
-        #
-        # That's why we simply create an empty file rather than a link.
-        if self.getsize(from_info) == 0:
-            self.open(to_info, "w").close()
-
-            logger.debug(
-                "Created empty file: {src} -> {dest}".format(
-                    src=str(from_info), dest=str(to_info)
-                )
-            )
-            return
-
-        System.hardlink(from_info, to_info)
-
-    @staticmethod
-    def is_hardlink(path_info):
-        return System.is_hardlink(path_info)
-
-    def reflink(self, from_info, to_info):
-        tmp_info = to_info.parent / tmp_fname(to_info.name)
-        System.reflink(from_info, tmp_info)
-        # NOTE: reflink has its own separate inode, so you can set permissions
-        # that are different from the source.
-        os.chmod(tmp_info, self._file_mode)
-        os.rename(tmp_info, to_info)
-
-    def cache_exists(self, checksums, jobs=None, name=None):
-        return [
-            checksum
-            for checksum in Tqdm(
-                checksums,
-                unit="file",
-                desc="Querying "
-                + ("cache in " + name if name else "local cache"),
-            )
-            if not self.changed_cache_file(checksum)
-        ]
-
-    def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
-    ):
-        makedirs(to_info.parent, exist_ok=True)
-
-        tmp_file = tmp_fname(to_info)
-        copyfile(
-            from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
-        )
-        os.rename(tmp_file, to_info)
-
-    def _download(
-        self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
-    ):
-        copyfile(
-            from_info, to_file, no_progress_bar=no_progress_bar, name=name
-        )
-
-    @staticmethod
-    def open(path_info, mode="r", encoding=None):
-        return open(path_info, mode=mode, encoding=encoding)
-
-    @index_locked
-    def status(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        # Return flattened dict containing all status info
-        dir_status, file_status, _ = self._status(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=download,
-        )
-        return dict(dir_status, **file_status)
-
-    def _status(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        """Return a tuple of (dir_status_info, file_status_info, dir_contents).
-
-        dir_status_info contains status for .dir files, file_status_info
-        contains status for all other files, and dir_contents is a dict of
-        {dir_checksum: set(file_checksum, ...)} which can be used to map
-        a .dir file to its file contents.
-        """
-        logger.debug(f"Preparing to collect status from {remote.path_info}")
-        md5s = set(named_cache.scheme_keys(self.scheme))
-
-        logger.debug("Collecting information from local cache...")
-        local_exists = frozenset(
-            self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
-        )
-
-        # This is a performance optimization. We can safely assume that,
-        # if the resources that we want to fetch are already cached,
-        # there's no need to check the remote storage for the existence of
-        # those files.
-        if download and local_exists == md5s:
-            remote_exists = local_exists
-        else:
-            logger.debug("Collecting information from remote cache...")
-            remote_exists = set()
-            dir_md5s = set(named_cache.dir_keys(self.scheme))
-            if dir_md5s:
-                remote_exists.update(
-                    self._indexed_dir_checksums(named_cache, remote, dir_md5s)
-                )
-                md5s.difference_update(remote_exists)
-            if md5s:
-                remote_exists.update(
-                    remote.cache_exists(
-                        md5s, jobs=jobs, name=str(remote.path_info)
-                    )
-                )
-        return self._make_status(
-            named_cache, show_checksums, local_exists, remote_exists
-        )
-
-    def _make_status(
-        self, named_cache, show_checksums, local_exists, remote_exists
-    ):
-        def make_names(checksum, names):
-            return {"name": checksum if show_checksums else " ".join(names)}
-
-        dir_status = {}
-        file_status = {}
-        dir_contents = {}
-        for checksum, item in named_cache[self.scheme].items():
-            if item.children:
-                dir_status[checksum] = make_names(checksum, item.names)
-                dir_contents[checksum] = set()
-                for child_checksum, child in item.children.items():
-                    file_status[child_checksum] = make_names(
-                        child_checksum, child.names
-                    )
-                    dir_contents[checksum].add(child_checksum)
-            else:
-                file_status[checksum] = make_names(checksum, item.names)
-
-        self._fill_statuses(dir_status, local_exists, remote_exists)
-        self._fill_statuses(file_status, local_exists, remote_exists)
-
-        self._log_missing_caches(dict(dir_status, **file_status))
-
-        return dir_status, file_status, dir_contents
-
-    def _indexed_dir_checksums(self, named_cache, remote, dir_md5s):
-        # Validate our index by verifying all indexed .dir checksums
-        # still exist on the remote
-        indexed_dirs = set(remote.index.dir_checksums())
-        indexed_dir_exists = set()
-        if indexed_dirs:
-            indexed_dir_exists.update(
-                remote._cache_object_exists(indexed_dirs)
-            )
-            missing_dirs = indexed_dirs.difference(indexed_dir_exists)
-            if missing_dirs:
-                logger.debug(
-                    "Remote cache missing indexed .dir checksums '{}', "
-                    "clearing remote index".format(", ".join(missing_dirs))
-                )
-                remote.index.clear()
-
-        # Check if non-indexed (new) dir checksums exist on remote
-        dir_exists = dir_md5s.intersection(indexed_dir_exists)
-        dir_exists.update(remote._cache_object_exists(dir_md5s - dir_exists))
-
-        # If .dir checksum exists on the remote, assume directory contents
-        # still exists on the remote
-        for dir_checksum in dir_exists:
-            file_checksums = list(
-                named_cache.child_keys(self.scheme, dir_checksum)
-            )
-            if dir_checksum not in remote.index:
-                logger.debug(
-                    "Indexing new .dir '{}' with '{}' nested files".format(
-                        dir_checksum, len(file_checksums)
-                    )
-                )
-                remote.index.update([dir_checksum], file_checksums)
-            yield dir_checksum
-            yield from file_checksums
-
-    @staticmethod
-    def _fill_statuses(checksum_info_dir, local_exists, remote_exists):
-        # Using sets because they are way faster for lookups
-        local = set(local_exists)
-        remote = set(remote_exists)
-
-        for md5, info in checksum_info_dir.items():
-            status = STATUS_MAP[(md5 in local, md5 in remote)]
-            info["status"] = status
-
-    def _get_plans(self, download, remote, status_info, status):
-        cache = []
-        path_infos = []
-        names = []
-        checksums = []
-        for md5, info in Tqdm(
-            status_info.items(), desc="Analysing status", unit="file"
-        ):
-            if info["status"] == status:
-                cache.append(self.checksum_to_path_info(md5))
-                path_infos.append(remote.checksum_to_path_info(md5))
-                names.append(info["name"])
-                checksums.append(md5)
-
-        if download:
-            to_infos = cache
-            from_infos = path_infos
-        else:
-            to_infos = path_infos
-            from_infos = cache
-
-        return from_infos, to_infos, names, checksums
-
-    def _process(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        logger.debug(
-            "Preparing to {} '{}'".format(
-                "download data from" if download else "upload data to",
-                remote.path_info,
-            )
-        )
-
-        if download:
-            func = partial(
-                remote.download,
-                dir_mode=self._dir_mode,
-                file_mode=self._file_mode,
-            )
-            status = STATUS_DELETED
-            desc = "Downloading"
-        else:
-            func = remote.upload
-            status = STATUS_NEW
-            desc = "Uploading"
-
-        if jobs is None:
-            jobs = remote.JOBS
-
-        dir_status, file_status, dir_contents = self._status(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=download,
-        )
-
-        dir_plans = self._get_plans(download, remote, dir_status, status)
-        file_plans = self._get_plans(download, remote, file_status, status)
-
-        total = len(dir_plans[0]) + len(file_plans[0])
-        if total == 0:
-            return 0
-
-        with Tqdm(total=total, unit="file", desc=desc) as pbar:
-            func = pbar.wrap_fn(func)
-            with ThreadPoolExecutor(max_workers=jobs) as executor:
-                if download:
-                    fails = sum(executor.map(func, *dir_plans))
-                    fails += sum(executor.map(func, *file_plans))
-                else:
-                    # for uploads, push files first, and any .dir files last
-
-                    file_futures = {}
-                    for from_info, to_info, name, checksum in zip(*file_plans):
-                        file_futures[checksum] = executor.submit(
-                            func, from_info, to_info, name
-                        )
-                    dir_futures = {}
-                    for from_info, to_info, name, dir_checksum in zip(
-                        *dir_plans
-                    ):
-                        wait_futures = {
-                            future
-                            for file_checksum, future in file_futures.items()
-                            if file_checksum in dir_contents[dir_checksum]
-                        }
-                        dir_futures[dir_checksum] = executor.submit(
-                            self._dir_upload,
-                            func,
-                            wait_futures,
-                            from_info,
-                            to_info,
-                            name,
-                        )
-                    fails = sum(
-                        future.result()
-                        for future in concat(
-                            file_futures.values(), dir_futures.values()
-                        )
-                    )
-
-        if fails:
-            if download:
-                remote.index.clear()
-                raise DownloadError(fails)
-            raise UploadError(fails)
-
-        if not download:
-            # index successfully pushed dirs
-            for dir_checksum, future in dir_futures.items():
-                if future.result() == 0:
-                    file_checksums = dir_contents[dir_checksum]
-                    logger.debug(
-                        "Indexing pushed dir '{}' with "
-                        "'{}' nested files".format(
-                            dir_checksum, len(file_checksums)
-                        )
-                    )
-                    remote.index.update([dir_checksum], file_checksums)
-
-        return len(dir_plans[0]) + len(file_plans[0])
-
-    @staticmethod
-    def _dir_upload(func, futures, from_info, to_info, name):
-        for future in as_completed(futures):
-            if future.result():
-                # do not upload this .dir file if any file in this
-                # directory failed to upload
-                logger.debug(
-                    "failed to upload full contents of '{}', "
-                    "aborting .dir file upload".format(name)
-                )
-                logger.error(f"failed to upload '{from_info}' to '{to_info}'")
-                return 1
-        return func(from_info, to_info, name)
-
-    @index_locked
-    def push(self, named_cache, remote, jobs=None, show_checksums=False):
-        return self._process(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=False,
-        )
-
-    @index_locked
-    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
-        return self._process(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=True,
-        )
-
-    @staticmethod
-    def _log_missing_caches(checksum_info_dict):
-        missing_caches = [
-            (md5, info)
-            for md5, info in checksum_info_dict.items()
-            if info["status"] == STATUS_MISSING
-        ]
-        if missing_caches:
-            missing_desc = "".join(
-                "\nname: {}, md5: {}".format(info["name"], md5)
-                for md5, info in missing_caches
-            )
-            msg = (
-                "Some of the cache files do not exist neither locally "
-                "nor on remote. Missing cache files: {}".format(missing_desc)
-            )
-            logger.warning(msg)
-
-    def _unprotect_file(self, path):
-        if System.is_symlink(path) or System.is_hardlink(path):
-            logger.debug(f"Unprotecting '{path}'")
-            tmp = os.path.join(os.path.dirname(path), "." + uuid())
-
-            # The operations order is important here - if some application
-            # would access the file during the process of copyfile then it
-            # would get only the part of file. So, at first, the file should be
-            # copied with the temporary name, and then original file should be
-            # replaced by new.
-            copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
-            remove(path)
-            os.rename(tmp, path)
-
-        else:
-            logger.debug(
-                "Skipping copying for '{}', since it is not "
-                "a symlink or a hardlink.".format(path)
-            )
-
-        os.chmod(path, self._file_mode)
-
-    def _unprotect_dir(self, path):
-        assert is_working_tree(self.repo.tree)
-
-        for fname in self.repo.tree.walk_files(path):
-            self._unprotect_file(fname)
-
-    def unprotect(self, path_info):
-        path = path_info.fspath
-        if not os.path.exists(path):
-            raise DvcException(f"can't unprotect non-existing data '{path}'")
-
-        if os.path.isdir(path):
-            self._unprotect_dir(path)
-        else:
-            self._unprotect_file(path)
-
-    def protect(self, path_info):
-        path = os.fspath(path_info)
-        mode = self.CACHE_MODE
-
-        try:
-            os.chmod(path, mode)
-        except OSError as exc:
-            # There is nothing we need to do in case of a read-only file system
-            if exc.errno == errno.EROFS:
-                return
-
-            # In shared cache scenario, we might not own the cache file, so we
-            # need to check if cache file is already protected.
-            if exc.errno not in [errno.EPERM, errno.EACCES]:
-                raise
-
-            actual = stat.S_IMODE(os.stat(path).st_mode)
-            if actual != mode:
-                raise
-
-    def _get_unpacked_dir_path_info(self, checksum):
-        info = self.checksum_to_path_info(checksum)
-        return info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
-
-    def _remove_unpacked_dir(self, checksum):
-        path_info = self._get_unpacked_dir_path_info(checksum)
-        self.remove(path_info)
-
-    def _path_info_changed(self, path_info):
-        if self.exists(path_info) and self.state.get(path_info):
-            return False
-        return True
-
-    def _update_unpacked_dir(self, checksum):
-        unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
-
-        if not self._path_info_changed(unpacked_dir_info):
-            return
-
-        self.remove(unpacked_dir_info)
-
-        try:
-            dir_info = self.get_dir_cache(checksum)
-            self._create_unpacked_dir(checksum, dir_info, unpacked_dir_info)
-        except DvcException:
-            logger.warning(f"Could not create '{unpacked_dir_info}'")
-
-            self.remove(unpacked_dir_info)
-
-    def _create_unpacked_dir(self, checksum, dir_info, unpacked_dir_info):
-        self.makedirs(unpacked_dir_info)
-
-        for entry in Tqdm(dir_info, desc="Creating unpacked dir", unit="file"):
-            entry_cache_info = self.checksum_to_path_info(
-                entry[self.PARAM_CHECKSUM]
-            )
-            relative_path = entry[self.PARAM_RELPATH]
-            # In shared cache mode some cache files might not be owned by the
-            # user, so we need to use symlinks because, unless
-            # /proc/sys/fs/protected_hardlinks is disabled, the user is not
-            # allowed to create hardlinks to files that he doesn't own.
-            link_types = ["hardlink", "symlink"]
-            self._link(
-                entry_cache_info, unpacked_dir_info / relative_path, link_types
-            )
-
-        self.state.save(unpacked_dir_info, checksum)
-
-    def _changed_unpacked_dir(self, checksum):
-        status_unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
-
-        return not self.state.get(status_unpacked_dir_info)
-
-    def _get_unpacked_dir_names(self, checksums):
-        unpacked = set()
-        for c in checksums:
-            if self.is_dir_checksum(c):
-                unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
-        return unpacked
-
-    def is_protected(self, path_info):
-        if not self.exists(path_info):
-            return False
-
-        mode = os.stat(path_info).st_mode
-
-        return stat.S_IMODE(mode) == self.CACHE_MODE

dvc/remote/ssh.py

@@ -0,0 +1,73 @@
+import errno
+import itertools
+import logging
+from concurrent.futures import ThreadPoolExecutor
+
+from dvc.progress import Tqdm
+from dvc.utils import to_chunks
+
+from .base import Remote
+
+logger = logging.getLogger(__name__)
+
+
+class SSHRemote(Remote):
+    def batch_exists(self, path_infos, callback):
+        def _exists(chunk_and_channel):
+            chunk, channel = chunk_and_channel
+            ret = []
+            for path in chunk:
+                try:
+                    channel.stat(path)
+                    ret.append(True)
+                except OSError as exc:
+                    if exc.errno != errno.ENOENT:
+                        raise
+                    ret.append(False)
+                callback(path)
+            return ret
+
+        with self.tree.ssh(path_infos[0]) as ssh:
+            channels = ssh.open_max_sftp_channels()
+            max_workers = len(channels)
+
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                paths = [path_info.path for path_info in path_infos]
+                chunks = to_chunks(paths, num_chunks=max_workers)
+                chunks_and_channels = zip(chunks, channels)
+                outcome = executor.map(_exists, chunks_and_channels)
+                results = list(itertools.chain.from_iterable(outcome))
+
+            return results
+
+    def hashes_exist(self, hashes, jobs=None, name=None):
+        """This is older implementation used in remote/base.py
+        We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
+        faster than current approach (relying on exists(path_info)) applied in
+        remote/base.
+        """
+        if not self.tree.CAN_TRAVERSE:
+            return list(set(hashes) & set(self.tree.all()))
+
+        # possibly prompt for credentials before "Querying" progress output
+        self.tree.ensure_credentials()
+
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(hashes),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(chunks):
+                return self.batch_exists(chunks, callback=pbar.update_msg)
+
+            with ThreadPoolExecutor(
+                max_workers=jobs or self.tree.JOBS
+            ) as executor:
+                path_infos = [self.tree.hash_to_path_info(x) for x in hashes]
+                chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
+                results = executor.map(exists_with_progress, chunks)
+                in_remote = itertools.chain.from_iterable(results)
+                ret = list(itertools.compress(hashes, in_remote))
+                return ret

dvc/repo/__init__.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from contextlib import contextmanager
 from functools import wraps
@@ -5,22 +6,30 @@ from functools import wraps
 from funcy import cached_property, cat, first
 
 from dvc.config import Config
+from dvc.dvcfile import PIPELINE_FILE, Dvcfile, is_valid_filename
 from dvc.exceptions import FileMissingError
 from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
-from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
-from dvc.ignore import CleanTree
+from dvc.exceptions import (
+    NoOutputOrStageError,
+    NotDvcRepoError,
+    OutputNotFoundError,
+)
 from dvc.path_info import PathInfo
-from dvc.repo.tree import RepoTree
+from dvc.tree.repo import RepoTree
 from dvc.utils.fs import path_isin
 
+from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
 from ..utils import parse_target
 from .graph import check_acyclic, get_pipeline, get_pipelines
 
+logger = logging.getLogger(__name__)
+
 
 def locked(f):
     @wraps(f)
     def wrapper(repo, *args, **kwargs):
         with repo.lock, repo.state:
+            # pylint: disable=protected-access
             repo._reset()
             ret = f(repo, *args, **kwargs)
             # Our graph cache is no longer valid after we release the repo.lock
@@ -33,54 +42,68 @@ def locked(f):
 class Repo:
     DVC_DIR = ".dvc"
 
-    from dvc.repo.destroy import destroy
-    from dvc.repo.install import install
     from dvc.repo.add import add
-    from dvc.repo.remove import remove
-    from dvc.repo.ls import ls
-    from dvc.repo.lock import lock as lock_stage
-    from dvc.repo.move import move
-    from dvc.repo.run import run
-    from dvc.repo.imp import imp
-    from dvc.repo.imp_url import imp_url
-    from dvc.repo.reproduce import reproduce
-    from dvc.repo.checkout import _checkout
-    from dvc.repo.push import push
-    from dvc.repo.fetch import _fetch
-    from dvc.repo.pull import pull
-    from dvc.repo.status import status
-    from dvc.repo.gc import gc
-    from dvc.repo.commit import commit
-    from dvc.repo.diff import diff
     from dvc.repo.brancher import brancher
+    from dvc.repo.checkout import _checkout
+    from dvc.repo.commit import commit
+    from dvc.repo.destroy import destroy
+    from dvc.repo.diff import diff
+    from dvc.repo.fetch import _fetch
+    from dvc.repo.freeze import freeze, unfreeze
+    from dvc.repo.gc import gc
     from dvc.repo.get import get
     from dvc.repo.get_url import get_url
+    from dvc.repo.imp import imp
+    from dvc.repo.imp_url import imp_url
+    from dvc.repo.install import install
+    from dvc.repo.ls import ls
+    from dvc.repo.move import move
+    from dvc.repo.pull import pull
+    from dvc.repo.push import push
+    from dvc.repo.remove import remove
+    from dvc.repo.reproduce import reproduce
+    from dvc.repo.run import run
+    from dvc.repo.status import status
     from dvc.repo.update import update
 
-    def __init__(self, root_dir=None):
-        from dvc.state import State
-        from dvc.lock import make_lock
-        from dvc.scm import SCM
+    def __init__(self, root_dir=None, scm=None, rev=None):
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
+        from dvc.lock import make_lock
+        from dvc.repo.experiments import Experiments
         from dvc.repo.metrics import Metrics
-        from dvc.repo.plots import Plots
         from dvc.repo.params import Params
-        from dvc.scm.tree import WorkingTree
-        from dvc.utils.fs import makedirs
+        from dvc.repo.plots import Plots
+        from dvc.scm import SCM
         from dvc.stage.cache import StageCache
+        from dvc.state import State, StateNoop
+        from dvc.tree.local import LocalTree
+        from dvc.utils.fs import makedirs
 
-        root_dir = self.find_root(root_dir)
+        if scm:
+            tree = scm.get_tree(rev)
+            self.root_dir = self.find_root(root_dir, tree)
+            self.scm = scm
+            self.tree = scm.get_tree(
+                rev, use_dvcignore=True, dvcignore_root=self.root_dir
+            )
+            self.state = StateNoop()
+        else:
+            root_dir = self.find_root(root_dir)
+            self.root_dir = os.path.abspath(os.path.realpath(root_dir))
+            self.tree = LocalTree(
+                self,
+                {"url": self.root_dir},
+                use_dvcignore=True,
+                dvcignore_root=self.root_dir,
+            )
 
-        self.root_dir = os.path.abspath(os.path.realpath(root_dir))
         self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
+        self.config = Config(self.dvc_dir, tree=self.tree)
 
-        self.config = Config(self.dvc_dir)
-
-        no_scm = self.config["core"].get("no_scm", False)
-        self.scm = SCM(self.root_dir, no_scm=no_scm)
-
-        self.tree = WorkingTree(self.root_dir)
+        if not scm:
+            no_scm = self.config["core"].get("no_scm", False)
+            self.scm = SCM(self.root_dir, no_scm=no_scm)
 
         self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
         self.index_dir = os.path.join(self.tmp_dir, "index")
@@ -94,19 +117,26 @@ class Repo:
             friendly=True,
         )
 
-        # NOTE: storing state and link_state in the repository itself to avoid
-        # any possible state corruption in 'shared cache dir' scenario.
-        self.state = State(self)
-
         self.cache = Cache(self)
         self.cloud = DataCloud(self)
 
+        if not scm:
+            # NOTE: storing state and link_state in the repository itself to
+            # avoid any possible state corruption in 'shared cache dir'
+            # scenario.
+            self.state = State(self.cache.local)
+
         self.stage_cache = StageCache(self)
 
         self.metrics = Metrics(self)
         self.plots = Plots(self)
         self.params = Params(self)
 
+        try:
+            self.experiments = Experiments(self)
+        except NotImplementedError:
+            self.experiments = None
+
         self._ignore()
 
     @property
@@ -115,7 +145,7 @@ class Repo:
 
     @tree.setter
     def tree(self, tree):
-        self._tree = tree if isinstance(tree, CleanTree) else CleanTree(tree)
+        self._tree = tree
         # Our graph cache is no longer valid, as it was based on the previous
         # tree.
         self._reset()
@@ -124,9 +154,14 @@ class Repo:
         return f"{self.__class__.__name__}: '{self.root_dir}'"
 
     @classmethod
-    def find_root(cls, root=None):
+    def find_root(cls, root=None, tree=None):
         root_dir = os.path.realpath(root or os.curdir)
 
+        if tree:
+            if tree.isdir(os.path.join(root_dir, cls.DVC_DIR)):
+                return root_dir
+            raise NotDvcRepoError(f"'{root}' does not contain DVC directory")
+
         if not os.path.isdir(root_dir):
             raise NotDvcRepoError(f"directory '{root}' does not exist")
 
@@ -157,16 +192,40 @@ class Repo:
         return Repo(root_dir)
 
     def unprotect(self, target):
-        return self.cache.local.unprotect(PathInfo(target))
+        return self.cache.local.tree.unprotect(PathInfo(target))
 
     def _ignore(self):
-        flist = [self.config.files["local"], self.tmp_dir]
+        flist = [
+            self.config.files["local"],
+            self.tmp_dir,
+        ]
+        if self.experiments:
+            flist.append(self.experiments.exp_dir)
 
         if path_isin(self.cache.local.cache_dir, self.root_dir):
             flist += [self.cache.local.cache_dir]
 
         self.scm.ignore_list(flist)
 
+    def get_stage(self, path=None, name=None):
+        if not path:
+            path = PIPELINE_FILE
+            logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
+
+        dvcfile = Dvcfile(self, path)
+        return dvcfile.stages[name]
+
+    def get_stages(self, path=None, name=None):
+        if not path:
+            path = PIPELINE_FILE
+            logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
+
+        if name:
+            return [self.get_stage(path, name)]
+
+        dvcfile = Dvcfile(self, path)
+        return list(dvcfile.stages.values())
+
     def check_modified_graph(self, new_stages):
         """Generate graph including the new stage to check for errors"""
         # Building graph might be costly for the ones with many DVC-files,
@@ -190,10 +249,9 @@ class Repo:
         stages = nx.dfs_postorder_nodes(graph)
         return [stage for stage in stages if path_isin(stage.path, path)]
 
-    def collect(self, target, with_deps=False, recursive=False, graph=None):
-        import networkx as nx
-        from ..dvcfile import Dvcfile
-
+    def collect(
+        self, target=None, with_deps=False, recursive=False, graph=None
+    ):
         if not target:
             return list(graph) if graph else self.stages
 
@@ -203,36 +261,81 @@ class Repo:
             )
 
         path, name = parse_target(target)
-        dvcfile = Dvcfile(self, path)
-        stages = list(dvcfile.stages.filter(name).values())
+        stages = self.get_stages(path, name)
         if not with_deps:
             return stages
 
         res = set()
         for stage in stages:
-            pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
-            res.update(nx.dfs_postorder_nodes(pipeline, stage))
+            res.update(self._collect_pipeline(stage, graph=graph))
         return res
 
-    def collect_granular(self, target, *args, **kwargs):
-        from ..dvcfile import Dvcfile, is_valid_filename
+    def _collect_pipeline(self, stage, graph=None):
+        import networkx as nx
 
+        pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
+        return nx.dfs_postorder_nodes(pipeline, stage)
+
+    def _collect_from_default_dvcfile(self, target):
+        dvcfile = Dvcfile(self, PIPELINE_FILE)
+        if dvcfile.exists():
+            return dvcfile.stages.get(target)
+
+    def collect_granular(
+        self, target=None, with_deps=False, recursive=False, graph=None
+    ):
+        """
+        Priority is in the order of following in case of ambiguity:
+            - .dvc file or .yaml file
+            - dir if recursive and directory exists
+            - stage_name
+            - output file
+        """
         if not target:
             return [(stage, None) for stage in self.stages]
 
         file, name = parse_target(target)
-        if is_valid_filename(file) and not kwargs.get("with_deps"):
-            # Optimization: do not collect the graph for a specific .dvc target
-            stages = Dvcfile(self, file).stages.filter(name)
-            return [(stage, None) for stage in stages.values()]
+        stages = []
 
-        try:
-            (out,) = self.find_outs_by_path(file, strict=False)
-            filter_info = PathInfo(os.path.abspath(file))
-            return [(out.stage, filter_info)]
-        except OutputNotFoundError:
-            stages = self.collect(target, *args, **kwargs)
-            return [(stage, None) for stage in stages]
+        # Optimization: do not collect the graph for a specific target
+        if not file:
+            # parsing is ambiguous when it does not have a colon
+            # or if it's not a dvcfile, as it can be a stage name
+            # in `dvc.yaml` or, an output in a stage.
+            logger.debug(
+                "Checking if stage '%s' is in '%s'", target, PIPELINE_FILE
+            )
+            if not (recursive and os.path.isdir(target)):
+                stage = self._collect_from_default_dvcfile(target)
+                if stage:
+                    stages = (
+                        self._collect_pipeline(stage) if with_deps else [stage]
+                    )
+        elif not with_deps and is_valid_filename(file):
+            stages = self.get_stages(file, name)
+
+        if not stages:
+            if not (recursive and os.path.isdir(target)):
+                try:
+                    (out,) = self.find_outs_by_path(target, strict=False)
+                    filter_info = PathInfo(os.path.abspath(target))
+                    return [(out.stage, filter_info)]
+                except OutputNotFoundError:
+                    pass
+
+            try:
+                stages = self.collect(target, with_deps, recursive, graph)
+            except StageFileDoesNotExistError as exc:
+                # collect() might try to use `target` as a stage name
+                # and throw error that dvc.yaml does not exist, whereas it
+                # should say that both stage name and file does not exist.
+                if file and is_valid_filename(file):
+                    raise
+                raise NoOutputOrStageError(target, exc.file) from exc
+            except StageNotFound as exc:
+                raise NoOutputOrStageError(target, exc.file) from exc
+
+        return [(stage, None) for stage in stages]
 
     def used_cache(
         self,
@@ -338,17 +441,18 @@ class Repo:
         """
         import networkx as nx
         from pygtrie import Trie
+
         from dvc.exceptions import (
             OutputDuplicationError,
-            StagePathAsOutputError,
             OverlappingOutputPathsError,
+            StagePathAsOutputError,
         )
 
         G = nx.DiGraph()
         stages = stages or self.stages
         outs = Trie()  # Use trie to efficiently find overlapping outs and deps
 
-        for stage in filter(bool, stages):
+        for stage in filter(bool, stages):  # bug? not using it later
             for out in stage.outs:
                 out_key = out.path_info.parts
 
@@ -392,7 +496,7 @@ class Repo:
                     continue
 
                 dep_key = dep.path_info.parts
-                overlapping = list(n.value for n in outs.prefixes(dep_key))
+                overlapping = [n.value for n in outs.prefixes(dep_key)]
                 if outs.has_subtrie(dep_key):
                     overlapping.extend(outs.values(prefix=dep_key))
 
@@ -429,22 +533,18 @@ class Repo:
         return PlotTemplates(self.dvc_dir)
 
     def _collect_stages(self):
-        from dvc.dvcfile import Dvcfile, is_valid_filename
-
         stages = []
         outs = set()
 
         for root, dirs, files in self.tree.walk(self.root_dir):
             for file_name in filter(is_valid_filename, files):
-                path = os.path.join(root, file_name)
-                stage_loader = Dvcfile(self, path).stages
-                stages.extend(stage_loader.values())
+                new_stages = self.get_stages(os.path.join(root, file_name))
+                stages.extend(new_stages)
                 outs.update(
                     out.fspath
-                    for stage in stages
-                    for out in (
-                        out for out in stage.outs if out.scheme == "local"
-                    )
+                    for stage in new_stages
+                    for out in stage.outs
+                    if out.scheme == "local"
                 )
             dirs[:] = [d for d in dirs if os.path.join(root, d) not in outs]
         return stages
@@ -488,17 +588,18 @@ class Repo:
         tree = RepoTree(self, stream=True)
         path = os.path.join(self.root_dir, path)
         try:
-            with tree.open(
-                os.path.join(self.root_dir, path),
-                mode=mode,
-                encoding=encoding,
-                remote=remote,
-            ) as fobj:
-                yield fobj
+            with self.state:
+                with tree.open(
+                    os.path.join(self.root_dir, path),
+                    mode=mode,
+                    encoding=encoding,
+                    remote=remote,
+                ) as fobj:
+                    yield fobj
         except FileNotFoundError as exc:
             raise FileMissingError(path) from exc
         except IsADirectoryError as exc:
-            raise DvcIsADirectoryError from exc
+            raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
 
     def close(self):
         self.scm.close()
@@ -515,4 +616,3 @@ class Repo:
         self.__dict__.pop("graph", None)
         self.__dict__.pop("stages", None)
         self.__dict__.pop("pipelines", None)
-        self.__dict__.pop("dvcignore", None)

dvc/repo/add.py

@@ -21,7 +21,9 @@ logger = logging.getLogger(__name__)
 
 @locked
 @scm_context
-def add(repo, targets, recursive=False, no_commit=False, fname=None):
+def add(
+    repo, targets, recursive=False, no_commit=False, fname=None, external=False
+):
     if recursive and fname:
         raise RecursiveAddingWhileUsingFilename()
 
@@ -52,7 +54,9 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
                     )
                 )
 
-            stages = _create_stages(repo, sub_targets, fname, pbar=pbar)
+            stages = _create_stages(
+                repo, sub_targets, fname, pbar=pbar, external=external
+            )
 
             try:
                 repo.check_modified_graph(stages)
@@ -105,22 +109,22 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
 def _find_all_targets(repo, target, recursive):
     if os.path.isdir(target) and recursive:
         return [
-            fname
-            for fname in Tqdm(
+            os.fspath(path)
+            for path in Tqdm(
                 repo.tree.walk_files(target),
                 desc="Searching " + target,
                 bar_format=Tqdm.BAR_FMT_NOTOTAL,
                 unit="file",
             )
-            if not repo.is_dvc_internal(fname)
-            if not is_dvc_file(fname)
-            if not repo.scm.belongs_to_scm(fname)
-            if not repo.scm.is_tracked(fname)
+            if not repo.is_dvc_internal(os.fspath(path))
+            if not is_dvc_file(os.fspath(path))
+            if not repo.scm.belongs_to_scm(os.fspath(path))
+            if not repo.scm.is_tracked(os.fspath(path))
         ]
     return [target]
 
 
-def _create_stages(repo, targets, fname, pbar=None):
+def _create_stages(repo, targets, fname, pbar=None, external=False):
     from dvc.stage import Stage, create_stage
 
     stages = []
@@ -132,11 +136,18 @@ def _create_stages(repo, targets, fname, pbar=None):
         unit="file",
     ):
         path, wdir, out = resolve_paths(repo, out)
-        stage = create_stage(Stage, repo, fname or path, wdir=wdir, outs=[out])
+        stage = create_stage(
+            Stage,
+            repo,
+            fname or path,
+            wdir=wdir,
+            outs=[out],
+            external=external,
+        )
         if stage:
-            Dvcfile(repo, stage.path).remove_with_prompt(force=True)
+            Dvcfile(repo, stage.path).remove()
 
-        repo._reset()
+        repo._reset()  # pylint: disable=protected-access
 
         if not stage:
             if pbar is not None:

dvc/repo/brancher.py

@@ -1,6 +1,6 @@
 from funcy import group_by
 
-from dvc.scm.tree import WorkingTree
+from dvc.tree.local import LocalTree
 
 
 def brancher(  # noqa: E302
@@ -18,7 +18,7 @@ def brancher(  # noqa: E302
         str: the display name for the currently selected tree, it could be:
             - a git revision identifier
             - empty string it there is no branches to iterate over
-            - "Working Tree" if there are uncommitted changes in the SCM repo
+            - "workspace" if there are uncommitted changes in the SCM repo
     """
     if not any([revs, all_branches, all_tags, all_commits]):
         yield ""
@@ -29,11 +29,11 @@ def brancher(  # noqa: E302
 
     scm = self.scm
 
-    self.tree = WorkingTree(self.root_dir)
-    yield "working tree"
+    self.tree = LocalTree(self, {"url": self.root_dir}, use_dvcignore=True)
+    yield "workspace"
 
-    if revs and "working tree" in revs:
-        revs.remove("working tree")
+    if revs and "workspace" in revs:
+        revs.remove("workspace")
 
     if all_commits:
         revs = scm.list_all_commits()
@@ -47,7 +47,12 @@ def brancher(  # noqa: E302
     try:
         if revs:
             for sha, names in group_by(scm.resolve_rev, revs).items():
-                self.tree = scm.get_tree(sha)
-                yield ", ".join(names)
+                self.tree = scm.get_tree(
+                    sha, use_dvcignore=True, dvcignore_root=self.root_dir
+                )
+                # ignore revs that don't contain repo root
+                # (i.e. revs from before a subdir=True repo was init'ed)
+                if self.tree.exists(self.root_dir):
+                    yield ", ".join(names)
     finally:
         self.tree = saved_tree

dvc/repo/checkout.py

@@ -1,7 +1,11 @@
 import logging
 import os
 
-from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit
+from dvc.exceptions import (
+    CheckoutError,
+    CheckoutErrorSuggestGit,
+    NoOutputOrStageError,
+)
 from dvc.progress import Tqdm
 from dvc.utils import relpath
 
@@ -18,11 +22,11 @@ def _get_unused_links(repo):
     return repo.state.get_unused_links(used)
 
 
-def _fspath_dir(path, root):
+def _fspath_dir(path):
     if not os.path.exists(str(path)):
         return str(path)
 
-    path = relpath(path, root)
+    path = relpath(path)
     return os.path.join(path, "") if os.path.isdir(path) else path
 
 
@@ -56,7 +60,7 @@ def _checkout(
         targets = [None]
         unused = _get_unused_links(self)
 
-    stats["deleted"] = [_fspath_dir(u, self.root_dir) for u in unused]
+    stats["deleted"] = [_fspath_dir(u) for u in unused]
     self.state.remove_links(unused)
 
     if isinstance(targets, str):
@@ -70,7 +74,11 @@ def _checkout(
                     target, with_deps=with_deps, recursive=recursive
                 )
             )
-        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+        except (
+            StageFileDoesNotExistError,
+            StageFileBadNameError,
+            NoOutputOrStageError,
+        ) as exc:
             if not target:
                 raise
             raise CheckoutErrorSuggestGit(target) from exc
@@ -87,9 +95,7 @@ def _checkout(
                 filter_info=filter_info,
             )
             for key, items in result.items():
-                stats[key].extend(
-                    _fspath_dir(path, self.root_dir) for path in items
-                )
+                stats[key].extend(_fspath_dir(path) for path in items)
 
     if stats.get("failed"):
         raise CheckoutError(stats["failed"], stats)

dvc/repo/commit.py

@@ -46,3 +46,4 @@ def commit(self, target, with_deps=False, recursive=False, force=False):
         stage.commit()
 
         Dvcfile(self, stage.path).dump(stage)
+    return stages

dvc/repo/diff.py

@@ -1,13 +1,13 @@
 import os
 
 from dvc.repo import locked
-from dvc.scm.tree import is_working_tree
+from dvc.tree.local import LocalTree
 
 
 @locked
 def diff(self, a_rev="HEAD", b_rev=None):
     """
-    By default, it compares the working tree with the last commit's tree.
+    By default, it compares the workspace with the last commit's tree.
 
     This implementation differs from `git diff` since DVC doesn't have
     the concept of `index`, but it keeps the same interface, thus,
@@ -33,11 +33,11 @@ def diff(self, a_rev="HEAD", b_rev=None):
                 else os.path.join(str(output), "")
             )
 
-        on_working_tree = is_working_tree(self.tree)
+        on_working_tree = isinstance(self.tree, LocalTree)
 
         def _to_checksum(output):
             if on_working_tree:
-                return self.cache.local.get_checksum(output.path_info)
+                return self.cache.local.tree.get_hash(output.path_info)[1]
             return output.checksum
 
         def _exists(output):

dvc/repo/experiments/__init__.py

@@ -0,0 +1,591 @@
+import logging
+import os
+import re
+import tempfile
+from collections.abc import Mapping
+from concurrent.futures import (
+    ProcessPoolExecutor,
+    ThreadPoolExecutor,
+    as_completed,
+)
+from contextlib import contextmanager
+from functools import partial, wraps
+from typing import Iterable, Optional
+
+from funcy import cached_property, first
+
+from dvc.exceptions import DownloadError, DvcException, UploadError
+from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
+from dvc.repo.experiments.executor import ExperimentExecutor, LocalExecutor
+from dvc.scm.git import Git
+from dvc.stage.serialize import to_lockfile
+from dvc.tree.repo import RepoTree
+from dvc.utils import dict_sha256, env2bool, relpath
+from dvc.utils.fs import remove
+
+logger = logging.getLogger(__name__)
+
+
+def scm_locked(f):
+    # Lock the experiments workspace so that we don't try to perform two
+    # different sequences of git operations at once
+    @wraps(f)
+    def wrapper(exp, *args, **kwargs):
+        with exp.scm_lock:
+            return f(exp, *args, **kwargs)
+
+    return wrapper
+
+
+def hash_exp(stages):
+    exp_data = {}
+    for stage in stages:
+        exp_data.update(to_lockfile(stage))
+    return dict_sha256(exp_data)
+
+
+class UnchangedExperimentError(DvcException):
+    def __init__(self, rev):
+        super().__init__(f"Experiment identical to baseline '{rev[:7]}'.")
+        self.rev = rev
+
+
+class BaselineMismatchError(DvcException):
+    def __init__(self, rev, expected):
+        if hasattr(rev, "hexsha"):
+            rev = rev.hexsha
+        rev_str = f"{rev[:7]}" if rev is not None else "dangling commit"
+        super().__init__(
+            f"Experiment derived from '{rev_str}', expected '{expected[:7]}'."
+        )
+        self.rev = rev
+        self.expected_rev = expected
+
+
+class Experiments:
+    """Class that manages experiments in a DVC repo.
+
+    Args:
+        repo (dvc.repo.Repo): repo instance that these experiments belong to.
+    """
+
+    EXPERIMENTS_DIR = "experiments"
+    PACKED_ARGS_FILE = "repro.dat"
+    STASH_MSG_PREFIX = "dvc-exp-"
+    STASH_EXPERIMENT_RE = re.compile(
+        r"(?:On \(.*\): )dvc-exp-(?P<baseline_rev>[0-9a-f]+)$"
+    )
+
+    def __init__(self, repo):
+        from dvc.lock import make_lock
+
+        if not (
+            env2bool("DVC_TEST")
+            or repo.config["core"].get("experiments", False)
+        ):
+            raise NotImplementedError
+
+        self.repo = repo
+        self.scm_lock = make_lock(
+            os.path.join(self.repo.tmp_dir, "exp_scm_lock"),
+            tmp_dir=self.repo.tmp_dir,
+        )
+
+    @cached_property
+    def exp_dir(self):
+        return os.path.join(self.repo.dvc_dir, self.EXPERIMENTS_DIR)
+
+    @cached_property
+    def scm(self):
+        """Experiments clone scm instance."""
+        if os.path.exists(self.exp_dir):
+            return Git(self.exp_dir)
+        return self._init_clone()
+
+    @cached_property
+    def dvc_dir(self):
+        return relpath(self.repo.dvc_dir, self.repo.scm.root_dir)
+
+    @cached_property
+    def exp_dvc_dir(self):
+        return os.path.join(self.exp_dir, self.dvc_dir)
+
+    @cached_property
+    def exp_dvc(self):
+        """Return clone dvc Repo instance."""
+        from dvc.repo import Repo
+
+        return Repo(self.exp_dvc_dir)
+
+    @contextmanager
+    def chdir(self):
+        cwd = os.getcwd()
+        os.chdir(self.exp_dvc.root_dir)
+        yield self.exp_dvc.root_dir
+        os.chdir(cwd)
+
+    @cached_property
+    def args_file(self):
+        return os.path.join(self.exp_dvc.tmp_dir, self.PACKED_ARGS_FILE)
+
+    @property
+    def stash_reflog(self):
+        if "refs/stash" in self.scm.repo.refs:
+            return self.scm.repo.refs["refs/stash"].log()
+        return []
+
+    @property
+    def stash_revs(self):
+        revs = {}
+        for i, entry in enumerate(self.stash_reflog):
+            m = self.STASH_EXPERIMENT_RE.match(entry.message)
+            if m:
+                revs[entry.newhexsha] = (i, m.group("baseline_rev"))
+        return revs
+
+    def _init_clone(self):
+        src_dir = self.repo.scm.root_dir
+        logger.debug("Initializing experiments clone")
+        git = Git.clone(src_dir, self.exp_dir)
+        self._config_clone()
+        return git
+
+    def _config_clone(self):
+        dvc_dir = relpath(self.repo.dvc_dir, self.repo.scm.root_dir)
+        local_config = os.path.join(self.exp_dir, dvc_dir, "config.local")
+        cache_dir = self.repo.cache.local.cache_dir
+        logger.debug("Writing experiments local config '%s'", local_config)
+        with open(local_config, "w") as fobj:
+            fobj.write(f"[cache]\n    dir = {cache_dir}")
+
+    def _scm_checkout(self, rev):
+        self.scm.repo.git.reset(hard=True)
+        if self.scm.repo.head.is_detached:
+            self._checkout_default_branch()
+        if not Git.is_sha(rev) or not self.scm.has_rev(rev):
+            self.scm.pull()
+        logger.debug("Checking out experiment commit '%s'", rev)
+        self.scm.checkout(rev)
+
+    def _checkout_default_branch(self):
+        # switch to default branch
+        git_repo = self.scm.repo
+        origin_refs = git_repo.remotes["origin"].refs
+        ref = origin_refs["HEAD"].reference
+        branch_name = ref.name.split("/")[-1]
+        if branch_name in git_repo.heads:
+            branch = git_repo.heads[branch_name]
+        else:
+            branch = git_repo.create_head(branch_name, ref)
+            branch.set_tracking_branch(ref)
+        branch.checkout()
+
+    def _stash_exp(self, *args, params: Optional[dict] = None, **kwargs):
+        """Stash changes from the current (parent) workspace as an experiment.
+
+        Args:
+            params: Optional dictionary of parameter values to be used.
+                Values take priority over any parameters specified in the
+                user's workspace.
+        """
+        rev = self.scm.get_rev()
+
+        # patch user's workspace into experiments clone
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        try:
+            self.repo.scm.repo.git.diff(
+                patch=True, full_index=True, binary=True, output=tmp
+            )
+            if os.path.getsize(tmp):
+                logger.debug("Patching experiment workspace")
+                self.scm.repo.git.apply(tmp)
+            elif not params:
+                # experiment matches original baseline
+                raise UnchangedExperimentError(rev)
+        finally:
+            remove(tmp)
+
+        # update experiment params from command line
+        if params:
+            self._update_params(params)
+
+        # save additional repro command line arguments
+        self._pack_args(*args, **kwargs)
+
+        # save experiment as a stash commit w/message containing baseline rev
+        # (stash commits are merge commits and do not contain a parent commit
+        # SHA)
+        msg = f"{self.STASH_MSG_PREFIX}{rev}"
+        self.scm.repo.git.stash("push", "-m", msg)
+        return self.scm.resolve_rev("stash@{0}")
+
+    def _pack_args(self, *args, **kwargs):
+        ExperimentExecutor.pack_repro_args(self.args_file, *args, **kwargs)
+        self.scm.add(self.args_file)
+
+    def _unpack_args(self, tree=None):
+        return ExperimentExecutor.unpack_repro_args(self.args_file, tree=tree)
+
+    def _update_params(self, params: dict):
+        """Update experiment params files with the specified values."""
+        from dvc.utils.serialize import MODIFIERS
+
+        logger.debug("Using experiment params '%s'", params)
+
+        # recursive dict update
+        def _update(dict_, other):
+            for key, value in other.items():
+                if isinstance(value, Mapping):
+                    dict_[key] = _update(dict_.get(key, {}), value)
+                else:
+                    dict_[key] = value
+            return dict_
+
+        for params_fname in params:
+            path = PathInfo(self.exp_dvc.root_dir) / params_fname
+            suffix = path.suffix.lower()
+            modify_data = MODIFIERS[suffix]
+            with modify_data(path, tree=self.exp_dvc.tree) as data:
+                _update(data, params[params_fname])
+
+    def _commit(self, exp_hash, check_exists=True, branch=True):
+        """Commit stages as an experiment and return the commit SHA."""
+        if not self.scm.is_dirty():
+            raise UnchangedExperimentError(self.scm.get_rev())
+
+        rev = self.scm.get_rev()
+        exp_name = f"{rev[:7]}-{exp_hash}"
+        if branch:
+            if check_exists and exp_name in self.scm.list_branches():
+                logger.debug("Using existing experiment branch '%s'", exp_name)
+                return self.scm.resolve_rev(exp_name)
+            self.scm.checkout(exp_name, create_new=True)
+        logger.debug("Commit new experiment branch '%s'", exp_name)
+        self.scm.repo.git.add(A=True)
+        self.scm.commit(f"Add experiment {exp_name}")
+        return self.scm.get_rev()
+
+    def reproduce_one(self, queue=False, **kwargs):
+        """Reproduce and checkout a single experiment."""
+        stash_rev = self.new(**kwargs)
+        if queue:
+            logger.info(
+                "Queued experiment '%s' for future execution.", stash_rev[:7]
+            )
+            return [stash_rev]
+        results = self.reproduce([stash_rev], keep_stash=False)
+        exp_rev = first(results)
+        if exp_rev is not None:
+            self.checkout_exp(exp_rev)
+        return results
+
+    def reproduce_queued(self, **kwargs):
+        results = self.reproduce(**kwargs)
+        if results:
+            revs = [f"{rev[:7]}" for rev in results]
+            logger.info(
+                "Successfully reproduced experiment(s) '%s'.\n"
+                "Use `dvc exp checkout <exp_rev>` to apply the results of "
+                "a specific experiment to your workspace.",
+                ", ".join(revs),
+            )
+        return results
+
+    @scm_locked
+    def new(self, *args, **kwargs):
+        """Create a new experiment.
+
+        Experiment will be reproduced and checked out into the user's
+        workspace.
+        """
+        rev = self.repo.scm.get_rev()
+        self._scm_checkout(rev)
+        try:
+            stash_rev = self._stash_exp(*args, **kwargs)
+        except UnchangedExperimentError as exc:
+            logger.info("Reproducing existing experiment '%s'.", rev[:7])
+            raise exc
+        logger.debug(
+            "Stashed experiment '%s' for future execution.", stash_rev[:7]
+        )
+        return stash_rev
+
+    @scm_locked
+    def reproduce(
+        self,
+        revs: Optional[Iterable] = None,
+        keep_stash: Optional[bool] = True,
+        **kwargs,
+    ):
+        """Reproduce the specified experiments.
+
+        Args:
+            revs: If revs is not specified, all stashed experiments will be
+                reproduced.
+            keep_stash: If True, stashed experiments will be preserved if they
+                fail to reproduce successfully.
+        """
+        stash_revs = self.stash_revs
+
+        # to_run contains mapping of:
+        #   input_rev: (stash_index, baseline_rev)
+        # where input_rev contains the changes to execute (usually a stash
+        # commit) and baseline_rev is the baseline to compare output against.
+        # The final experiment commit will be branched from baseline_rev.
+        if revs is None:
+            to_run = {
+                rev: baseline_rev
+                for rev, (_, baseline_rev) in stash_revs.items()
+            }
+        else:
+            to_run = {
+                rev: stash_revs[rev][1] if rev in stash_revs else rev
+                for rev in revs
+            }
+
+        logger.debug(
+            "Reproducing experiment revs '%s'",
+            ", ".join((rev[:7] for rev in to_run)),
+        )
+
+        # setup executors - unstash experiment, generate executor, upload
+        # contents of (unstashed) exp workspace to the executor tree
+        executors = {}
+        for rev, baseline_rev in to_run.items():
+            self._scm_checkout(baseline_rev)
+            self.scm.repo.git.stash("apply", rev)
+            repro_args, repro_kwargs = self._unpack_args()
+            executor = LocalExecutor(
+                baseline_rev,
+                repro_args=repro_args,
+                repro_kwargs=repro_kwargs,
+                dvc_dir=self.dvc_dir,
+                cache_dir=self.repo.cache.local.cache_dir,
+            )
+            self._collect_input(executor)
+            executors[rev] = executor
+
+        exec_results = self._reproduce(executors, **kwargs)
+
+        if keep_stash:
+            # only drop successfully run stashed experiments
+            to_drop = sorted(
+                (
+                    stash_revs[rev][0]
+                    for rev in exec_results
+                    if rev in stash_revs
+                ),
+                reverse=True,
+            )
+        else:
+            # drop all stashed experiments
+            to_drop = sorted(
+                (stash_revs[rev][0] for rev in to_run if rev in stash_revs),
+                reverse=True,
+            )
+        for index in to_drop:
+            self.scm.repo.git.stash("drop", index)
+
+        result = {}
+        for _, exp_result in exec_results.items():
+            result.update(exp_result)
+        return result
+
+    def _reproduce(self, executors: dict, jobs: Optional[int] = 1) -> dict:
+        """Run dvc repro for the specified ExperimentExecutors in parallel.
+
+        Returns dict containing successfully executed experiments.
+        """
+        result = {}
+
+        with ProcessPoolExecutor(max_workers=jobs) as workers:
+            futures = {}
+            for rev, executor in executors.items():
+                future = workers.submit(
+                    executor.reproduce,
+                    executor.dvc_dir,
+                    cwd=executor.dvc.root_dir,
+                    **executor.repro_kwargs,
+                )
+                futures[future] = (rev, executor)
+            for future in as_completed(futures):
+                rev, executor = futures[future]
+                exc = future.exception()
+                if exc is None:
+                    exp_hash = future.result()
+                    self._scm_checkout(executor.baseline_rev)
+                    try:
+                        self._collect_output(executor)
+                    except DownloadError:
+                        logger.error(
+                            "Failed to collect output for experiment '%s'",
+                            rev,
+                        )
+                        continue
+                    finally:
+                        if os.path.exists(self.args_file):
+                            remove(self.args_file)
+
+                    try:
+                        exp_rev = self._commit(exp_hash)
+                    except UnchangedExperimentError:
+                        logger.debug(
+                            "Experiment '%s' identical to baseline '%s'",
+                            rev,
+                            executor.baseline_rev,
+                        )
+                        exp_rev = executor.baseline_rev
+                    logger.info("Reproduced experiment '%s'.", exp_rev[:7])
+                    result[rev] = {exp_rev: exp_hash}
+                else:
+                    logger.exception(
+                        "Failed to reproduce experiment '%s'", rev
+                    )
+                executor.cleanup()
+
+        return result
+
+    def _collect_input(self, executor: ExperimentExecutor):
+        """Copy (upload) input from the experiments workspace to the executor
+        tree.
+        """
+        logger.debug("Collecting input for '%s'", executor.tmp_dir)
+        repo_tree = RepoTree(self.exp_dvc)
+        self._process(
+            executor.tree,
+            self.exp_dvc.tree,
+            executor.collect_files(self.exp_dvc.tree, repo_tree),
+        )
+
+    def _collect_output(self, executor: ExperimentExecutor):
+        """Copy (download) output from the executor tree into experiments
+        workspace.
+        """
+        logger.debug("Collecting output from '%s'", executor.tmp_dir)
+        self._process(
+            self.exp_dvc.tree,
+            executor.tree,
+            executor.collect_output(),
+            download=True,
+        )
+
+    @staticmethod
+    def _process(dest_tree, src_tree, collected_files, download=False):
+        from dvc.cache.local import _log_exceptions
+
+        from_infos = []
+        to_infos = []
+        names = []
+        for from_info in collected_files:
+            from_infos.append(from_info)
+            fname = from_info.relative_to(src_tree.path_info)
+            names.append(str(fname))
+            to_infos.append(dest_tree.path_info / fname)
+        total = len(from_infos)
+
+        if download:
+            func = partial(
+                _log_exceptions(src_tree.download, "download"),
+                dir_mode=dest_tree.dir_mode,
+                file_mode=dest_tree.file_mode,
+            )
+            desc = "Downloading"
+        else:
+            func = partial(_log_exceptions(dest_tree.upload, "upload"))
+            desc = "Uploading"
+
+        with Tqdm(total=total, unit="file", desc=desc) as pbar:
+            func = pbar.wrap_fn(func)
+            # TODO: parallelize this, currently --jobs for repro applies to
+            # number of repro executors not download threads
+            with ThreadPoolExecutor(max_workers=1) as dl_executor:
+                fails = sum(dl_executor.map(func, from_infos, to_infos, names))
+
+        if fails:
+            if download:
+                raise DownloadError(fails)
+            raise UploadError(fails)
+
+    @scm_locked
+    def checkout_exp(self, rev):
+        """Checkout an experiment to the user's workspace."""
+        from git.exc import GitCommandError
+
+        from dvc.repo.checkout import _checkout as dvc_checkout
+
+        self._check_baseline(rev)
+        self._scm_checkout(rev)
+
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        self.scm.repo.head.commit.diff(
+            "HEAD~1", patch=True, full_index=True, binary=True, output=tmp
+        )
+
+        dirty = self.repo.scm.is_dirty()
+        if dirty:
+            logger.debug("Stashing workspace changes.")
+            self.repo.scm.repo.git.stash("push", "--include-untracked")
+
+        try:
+            if os.path.getsize(tmp):
+                logger.debug("Patching local workspace")
+                self.repo.scm.repo.git.apply(tmp, reverse=True)
+                need_checkout = True
+            else:
+                need_checkout = False
+        except GitCommandError:
+            raise DvcException("failed to apply experiment changes.")
+        finally:
+            remove(tmp)
+            if dirty:
+                self._unstash_workspace()
+
+        if need_checkout:
+            dvc_checkout(self.repo)
+
+    def _check_baseline(self, exp_rev):
+        baseline_sha = self.repo.scm.get_rev()
+        exp_commit = self.scm.repo.rev_parse(exp_rev)
+        parent = first(exp_commit.parents)
+        if parent is not None and parent.hexsha == baseline_sha:
+            return
+        raise BaselineMismatchError(parent, baseline_sha)
+
+    def _unstash_workspace(self):
+        # Essentially we want `git stash pop` with `-X ours` merge strategy
+        # to prefer the applied experiment changes over stashed workspace
+        # changes. git stash doesn't support merge strategy parameters, but we
+        # can do it ourselves with checkout/reset.
+        from git.exc import GitCommandError
+
+        logger.debug("Unstashing workspace changes.")
+        git_repo = self.repo.scm.repo.git
+
+        # stage workspace changes, then apply stashed changes on top
+        git_repo.add(A=True)
+        try:
+            git_repo.stash("apply", "stash@{0}")
+        except GitCommandError:
+            # stash apply will return error code on merge conflicts,
+            # prefer workspace changes over stash changes
+            git_repo.checkout("--ours", "--", ".")
+
+        # unstage changes and drop the stash entry
+        git_repo.reset("HEAD")
+        git_repo.stash("drop", "stash@{0}")
+
+    def checkout(self, *args, **kwargs):
+        from dvc.repo.experiments.checkout import checkout
+
+        return checkout(self.repo, *args, **kwargs)
+
+    def diff(self, *args, **kwargs):
+        from dvc.repo.experiments.diff import diff
+
+        return diff(self.repo, *args, **kwargs)
+
+    def show(self, *args, **kwargs):
+        from dvc.repo.experiments.show import show
+
+        return show(self.repo, *args, **kwargs)