dvc: bump to 1.0.0a3

remote: is_dir_checksum: handle empty/None checksum (#3836 )
Fixed #3434
2026-11-04 07:18:27 +01:00 · 2020-05-20 18:01:24 +03:00 · 2020-05-20 18:00:41 +03:00 · 2020-05-20 17:53:25 +03:00 · 2020-05-20 16:05:09 +03:00 · 2020-05-20 15:21:07 +03:00
312 changed files with 17793 additions and 7747 deletions
@@ -1,7 +0,0 @@
-[flake8]
-ignore =
-    E203, # Whitespace before ':'
-    E266, # Too many leading '#' for block comment
-    W503, # Line break occurred before a binary operator
-max-line-length = 79
-select = B,C,E,F,W,T4,B9
@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+patreon: DVCorg
@@ -1,7 +1,7 @@
-* [ ] ❗ Have you followed the guidelines in the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) list?
+* [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.

-* [ ] 📖 Check this box if this PR **does not** require [documentation](https://dvc.org/doc) updates, or if it does **and** you have created a separate PR in [dvc.org](https://github.com/iterative/dvc.org) with such updates (or at least opened an issue about it in that repo). Please link below to your PR (or issue) in the [dvc.org](https://github.com/iterative/dvc.org) repo.
+* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here. If the CLI API is changed, I have updated [tab completion scripts](https://github.com/iterative/dvc/tree/master/scripts/completion).

-* [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect everything to be addressed. Please review them carefully and fix those that actually improve code or fix bugs.
+* [ ] ❌ I will check DeepSource, CodeClimate, and other sanity checks below. (We consider them recommendatory and don't expect everything to be addressed. Please fix things that actually improve code or fix bugs.)

 Thank you for the contribution - we'll try to review it as soon as possible. 🙏
@@ -26,7 +26,7 @@ innosetup/config.ini
 .coverage
 .coverage.*

-*.swp
+*.sw?

 pip-wheel-metadata/
 .vscode/
@@ -1,17 +1,44 @@
 repos:
-   repo: https://github.com/ambv/black
-    rev: 19.3b0
+- hooks:
+  - id: black
+    language_version: python3
+  repo: https://github.com/ambv/black
+  rev: 19.10b0
+-   repo: https://github.com/asottile/seed-isort-config
+    rev: v2.1.1
    hooks:
-    - id: black
-      language_version: python3
-   repo: https://gitlab.com/pycqa/flake8
-    rev: master
-    hooks:
-    - id: flake8
-      language_version: python3
-   repo: https://github.com/lovesegfault/beautysh
-    rev: master
-    hooks:
-    - id: beautysh
-      language_version: python3
-      args: [-i, '2']  # 2-space indentaion
+    -   id: seed-isort-config
+- hooks:
+  - id: isort
+    language_version: python3
+  repo: https://github.com/timothycrosley/isort
+  rev: 4.3.21
+- hooks:
+  - id: flake8
+    language_version: python3
+  repo: https://gitlab.com/pycqa/flake8
+  rev: 3.7.9
+- hooks:
+  - args:
+    - -i
+    - '2'
+    id: beautysh
+    language_version: python3
+  repo: https://github.com/lovesegfault/beautysh
+  rev: master
+- hooks:
+  - id: dvc-pre-commit
+    language_version: python3
+    stages:
+    - commit
+  - id: dvc-pre-push
+    language_version: python3
+    stages:
+    - push
+  - always_run: true
+    id: dvc-post-checkout
+    language_version: python3
+    stages:
+    - post-checkout
+  repo: https://github.com/iterative/dvc
+  rev: master
@@ -0,0 +1,33 @@
+- args:
+  - git-hook
+  - pre-commit
+  entry: dvc
+  id: dvc-pre-commit
+  language: python
+  language_version: python3
+  name: DVC pre-commit
+  stages:
+  - commit
+  verbose: true
+- args:
+  - git-hook
+  - pre-push
+  entry: dvc
+  id: dvc-pre-push
+  language: python
+  language_version: python3
+  name: DVC pre-push
+  stages:
+  - push
+- always_run: true
+  args:
+  - git-hook
+  - post-checkout
+  entry: dvc
+  id: dvc-post-checkout
+  language: python
+  language_version: python3
+  minimum_pre_commit_version: 2.2.0
+  name: DVC post-checkout
+  stages:
+  - post-checkout
@@ -1,7 +1,7 @@
 ---
 restylers:
  - name: black
-    image: restyled/restyler-black:v19.3b0
+    image: restyled/restyler-black:v19.10b0
    command:
    - black
    arguments: []
@@ -9,3 +9,12 @@ restylers:
    - "**/*.py"
    interpreters:
    - python
+  - name: isort
+    image: restyled/restyler-isort:v4.3.21
+    command:
+    - isort
+    arguments: []
+    include:
+    - "**/*.py"
+    interpreters:
+    - python
@@ -37,15 +37,24 @@ jobs:
    stage: test
    os: windows           # Windows 10.0.17134 N/A Build 17134
    language: shell       # 'language: python' is an error on Travis CI Windows
-  - name: "3.5 on Linux"
-    language: python
-    python: 3.5
+    env:
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
+  - name: "3.8 on Windows"
+    os: windows           # Windows 10.0.17134 N/A Build 17134
+    language: shell       # 'language: python' is an error on Travis CI Windows
+    env:
+      - PYTHON_VERSION=3.8.0
+      - PATH=/c/Python38:/c/Python38/Scripts:$PATH
  - name: "3.6 on Linux"
    language: python
    python: 3.6
  - name: "3.7 on Linux"
    language: python
    python: 3.7
+  - name: "3.8 on Linux"
+    language: python
+    python: 3.8
  - name: "3.7 on Mac"
    os: osx
    osx_image: xcode11.3
@@ -63,6 +72,9 @@ jobs:
    language: shell
    install:
    script: ./scripts/build_windows.cmd
+    env:
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
  - name: "Linux pkgs"
    language: python
    python: 3.7
@@ -71,16 +83,18 @@ jobs:
    script: ./scripts/build_posix.sh
  - name: "PyPI pkgs"
    language: python
-    python: 3.7
+    python: 3.8
    script: ./scripts/build_package.sh
  - name: Snapcraft snap
    language: python
-    python: 3.7
+    python: 3.8
    addons:
     snaps:
     - name: snapcraft
       channel: stable
       confinement: classic
+     - name: review-tools
+       channel: stable
     - name: lxd
       channel: stable
    env:
@@ -89,6 +103,7 @@ jobs:
    - sudo /snap/bin/lxd.migrate -yes
    - sudo /snap/bin/lxd waitready
    - sudo /snap/bin/lxd init --auto
+    - git fetch --tags
    script:
    - ./scripts/build_snap.sh
    after_failure:
@@ -134,10 +149,9 @@ deploy:
      tags: true
      repo: iterative/dvc
      stage: build
-  - provider: snap
-    snap: dvc_*.snap
-    channel: $SNAP_CHANNEL
+  - provider: script
    skip_cleanup: true
+    script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
    on:
      all_branches: true
      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG)"
@@ -0,0 +1,8 @@
+{
+  "title": "DVC: Data Version Control - Git for Data & Models",
+  "keywords": [
+    "data-science", "data-version-control", "machine-learning", "git",
+    "developer-tools", "reproducibility", "collaboration", "ai", "python"],
+  "contributors": [
+    {"name": "DVC team", "type": "Other", "affiliation": "Iterative"}]
+}
@@ -1,6 +1,4 @@
-.. image:: https://dvc.org/static/img/logo-github-readme.png
-  :target: https://dvc.org
-  :alt: DVC logo
+|Banner|

 `Website <https://dvc.org>`_
 • `Docs <https://dvc.org/doc>`_
@@ -10,33 +8,9 @@
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_

-.. image:: https://img.shields.io/badge/release-ok-brightgreen
-  :target: https://travis-ci.com/iterative/dvc
-  :alt: Release
+|Release| |CI| |Maintainability| |Coverage| |Donate| |DOI|

-.. image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev
-  :target: https://travis-ci.com/iterative/dvc
-  :alt: Travis dev branch
-
-.. image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
-  :target: https://codeclimate.com/github/iterative/dvc
-  :alt: Code Climate
-
-.. image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
-  :target: https://codecov.io/gh/iterative/dvc
-  :alt: Codecov
-
-.. image:: https://img.shields.io/badge/patreon-donate-green.svg
-  :target: https://www.patreon.com/DVCorg/overview
-  :alt: Donate
-
-.. image:: https://anaconda.org/conda-forge/dvc/badges/version.svg
-  :target: https://anaconda.org/conda-forge/dvc
-  :alt: Conda-forge
-
-.. image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
-  :target: https://snapcraft.io/dvc
-  :alt: Snapcraft
+|Snap| |Choco| |Brew| |Conda| |PyPI| |Packages|

 |

@@ -79,9 +53,7 @@ to store data and model files seamlessly out of Git, while preserving almost the
 were stored in Git itself. To store and share the data cache, DVC supports multiple remotes - any cloud (S3, Azure,
 Google Cloud, etc) or any on-premise network storage (via SSH, for example).

-.. image:: https://dvc.org/static/img/flow.gif
-   :target: https://dvc.org/static/img/flow.gif
-   :alt: how_dvc_works
+|Flowchart|

 The DVC pipelines (computational graph) feature connects code and data together. It is possible to explicitly
 specify all steps required to produce a model: input dependencies including data, commands to run,
@@ -119,9 +91,52 @@ Installation
 There are four options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package.
 Full instructions are `available here <https://dvc.org/doc/get-started/install>`_.

+Snap (Snapcraft/Linux)
+----------------------
+
+|Snap|
+
+.. code-block:: bash
+
+   snap install dvc --classic
+
+This corresponds to the latest tagged release.
+Add ``--edge`` for the latest ``master`` version.
+
+Choco (Chocolatey/Windows)
+--------------------------
+
+|Choco|
+
+.. code-block:: bash
+
+   choco install dvc
+
+Brew (Homebrew/Mac OS)
+----------------------
+
+|Brew|
+
+.. code-block:: bash
+
+   brew install dvc
+
+Conda (Anaconda)
+----------------
+
+|Conda|
+
+.. code-block:: bash
+
+   conda install -c conda-forge dvc
+
+Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
+
 pip (PyPI)
 ----------

+|PyPI|
+
 .. code-block:: bash

   pip install dvc
@@ -137,36 +152,11 @@ To install the development version, run:

   pip install git+git://github.com/iterative/dvc

-Homebrew
--------
-
-.. code-block:: bash
-
-   brew install dvc
-
-
-Conda (Anaconda)
----------------
-
-.. code-block:: bash
-
-   conda install -c conda-forge dvc
-
-Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
-
-Snap (Snapcraft)
----------------
-
-.. code-block:: bash
-
-   snap install dvc --classic
-
-This corresponds to the latest tagged release.
-Add ``--edge`` for the latest ``master`` version.
-
 Package
 -------

+|Packages|
+
 Self-contained packages for Linux, Windows, and Mac are available. The latest version of the packages
 can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.

@@ -206,40 +196,43 @@ Comparison to related technologies

 Contributing
 ============
+
+|Maintainability| |Donate|
+
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
-  :alt: 0
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+   :alt: 0

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
-  :alt: 1
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+   :alt: 1

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
-  :alt: 2
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+   :alt: 2

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
-  :alt: 3
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
+   :alt: 3

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
-  :alt: 4
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+   :alt: 4

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
-  :alt: 5
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+   :alt: 5

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
-  :alt: 6
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+   :alt: 6

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
-  :alt: 7
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+   :alt: 7

 Mailing List
 ============
@@ -253,3 +246,67 @@ This project is distributed under the Apache license version 2.0 (see the LICENS

 By submitting a pull request to this project, you agree to license your contribution under the Apache license version
 2.0 to this project.
+
+Citation
+========
+
+|DOI|
+
+Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
+`DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
+
+.. |Banner| image:: https://dvc.org/img/logo-github-readme.png
+   :target: https://dvc.org
+   :alt: DVC logo
+
+.. |Release| image:: https://img.shields.io/badge/release-ok-brightgreen
+   :target: https://travis-ci.com/iterative/dvc/branches
+   :alt: Release
+
+.. |CI| image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev&logo=travis
+   :target: https://travis-ci.com/iterative/dvc/builds
+   :alt: Travis dev branch
+
+.. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
+   :target: https://codeclimate.com/github/iterative/dvc
+   :alt: Code Climate
+
+.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/iterative/dvc
+   :alt: Codecov
+
+.. |Donate| image:: https://img.shields.io/badge/patreon-donate-green.svg?logo=patreon
+   :target: https://www.patreon.com/DVCorg/overview
+   :alt: Donate
+
+.. |Snap| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
+   :target: https://snapcraft.io/dvc
+   :alt: Snapcraft
+
+.. |Choco| image:: https://img.shields.io/chocolatey/v/dvc?label=choco
+   :target: https://chocolatey.org/packages/dvc
+   :alt: Chocolatey
+
+.. |Brew| image:: https://img.shields.io/homebrew/v/dvc?label=brew
+   :target: https://formulae.brew.sh/formula/dvc
+   :alt: Homebrew
+
+.. |Conda| image:: https://img.shields.io/conda/v/conda-forge/dvc.svg?label=conda&logo=conda-forge
+   :target: https://anaconda.org/conda-forge/dvc
+   :alt: Conda-forge
+
+.. |PyPI| image:: https://img.shields.io/pypi/v/dvc.svg?label=pip&logo=PyPI&logoColor=white
+   :target: https://pypi.org/project/dvc
+   :alt: PyPI
+
+.. |Packages| image:: https://img.shields.io/github/v/release/iterative/dvc?label=deb|pkg|rpm|exe&logo=GitHub
+   :target: https://github.com/iterative/dvc/releases/latest
+   :alt: deb|pkg|rpm|exe
+
+.. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.3677553-blue.svg
+   :target: https://doi.org/10.5281/zenodo.3677553
+   :alt: DOI
+
+.. |Flowchart| image:: https://dvc.org/img/flow.gif
+   :target: https://dvc.org/img/flow.gif
+   :alt: how_dvc_works
@@ -3,17 +3,7 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-import warnings
-
 import dvc.logger
 from dvc.version import __version__  # noqa: F401

-
 dvc.logger.setup()
-
-# Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
-# We don't directly import numpy, but our dependency networkx does, causing
-# these warnings in some environments. Luckily these warnings are benign and
-# we can simply ignore them so that they don't show up when you are using dvc.
-warnings.filterwarnings("ignore", message="numpy.dtype size changed")
-warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
@@ -2,12 +2,12 @@ import json
 import logging
 import os
 import platform
-import requests
 import sys
 import tempfile
 import uuid

 import distro
+import requests

 from dvc import __version__
 from dvc.config import Config, to_bool
@@ -15,11 +15,11 @@ from dvc.daemon import daemon
 from dvc.exceptions import NotDvcRepoError
 from dvc.lock import Lock, LockError
 from dvc.repo import Repo
-from dvc.scm import SCM
+from dvc.scm import SCM, NoSCM
+from dvc.scm.base import SCMError
 from dvc.utils import env2bool, is_binary
 from dvc.utils.fs import makedirs

-
 logger = logging.getLogger(__name__)


@@ -35,7 +35,7 @@ def collect_and_send_report(args=None, return_code=None):
    report as a JSON, where the _collector_ generates it and the _sender_
    removes it after sending it.
    """
-    report = _runtime_info()
+    report = {}

    # Include command execution information on the report only when available.
    if args and hasattr(args, "func"):
@@ -54,17 +54,14 @@ def is_enabled():
        return False

    enabled = to_bool(
-        Config(validate=False)
-        .config.get(Config.SECTION_CORE, {})
-        .get(Config.SECTION_CORE_ANALYTICS, "true")
+        Config(validate=False).get("core", {}).get("analytics", "true")
    )
-
    logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))

    return enabled


-def send(report):
+def send(path):
    """
    Side effect: Removes the report after sending it.

@@ -75,19 +72,25 @@ def send(report):
    url = "https://analytics.dvc.org"
    headers = {"content-type": "application/json"}

-    with open(report, "rb") as fobj:
-        try:
-            requests.post(url, data=fobj, headers=headers, timeout=5)
-        except requests.exceptions.RequestException:
-            logger.debug("failed to send analytics report", exc_info=True)
+    with open(path) as fobj:
+        report = json.load(fobj)

-    os.remove(report)
+    report.update(_runtime_info())
+
+    try:
+        requests.post(url, json=report, headers=headers, timeout=5)
+    except requests.exceptions.RequestException:
+        logger.debug("failed to send analytics report", exc_info=True)
+
+    os.remove(path)


 def _scm_in_use():
    try:
        scm = SCM(root_dir=Repo.find_root())
        return type(scm).__name__
+    except SCMError:
+        return NoSCM.__name__
    except NotDvcRepoError:
        pass

@@ -144,7 +147,7 @@ def _find_or_create_user_id():

    IDs are generated randomly with UUID.
    """
-    config_dir = Config.get_global_config_dir()
+    config_dir = Config.get_dir("global")
    fname = os.path.join(config_dir, "user_id")
    lockfile = os.path.join(config_dir, "user_id.lock")

@@ -155,7 +158,7 @@ def _find_or_create_user_id():
    try:
        with Lock(lockfile):
            try:
-                with open(fname, "r") as fobj:
+                with open(fname) as fobj:
                    user_id = json.load(fobj)["user_id"]

            except (FileNotFoundError, ValueError, KeyError):
@@ -167,6 +170,4 @@ def _find_or_create_user_id():
            return user_id

    except LockError:
-        logger.debug(
-            "Failed to acquire '{lockfile}'".format(lockfile=lockfile)
-        )
+        logger.debug(f"Failed to acquire '{lockfile}'")
@@ -1,37 +1,51 @@
 import os
-from contextlib import contextmanager, _GeneratorContextManager as GCM
+from contextlib import _GeneratorContextManager as GCM
+from contextlib import contextmanager

-from dvc.repo import Repo
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.external_repo import external_repo
+from dvc.repo import Repo


 class UrlNotDvcRepoError(DvcException):
-    """Thrown if given url is not a DVC repository.
-
-    Args:
-        url (str): URL to the repository
-    """
+    """Thrown if the given URL is not a DVC repository."""

    def __init__(self, url):
-        super().__init__("'{}' is not a DVC repository.".format(url))
+        super().__init__(f"'{url}' is not a DVC repository.")


 def get_url(path, repo=None, rev=None, remote=None):
    """
-    Returns the full URL to the data artifact specified by its `path` in a
-    `repo`.
-    NOTE: There is no guarantee that the file actually exists in that location.
+    Returns the URL to the storage location of a data file or directory tracked
+    in a DVC repo. For Git repos, HEAD is used unless a rev argument is
+    supplied. The default remote is tried unless a remote argument is supplied.
+
+    Raises UrlNotDvcRepoError if repo is not a DVC project.
+
+    NOTE: This function does not check for the actual existence of the file or
+    directory in the remote storage.
    """
    with _make_repo(repo, rev=rev) as _repo:
-        _require_dvc(_repo)
+        if not isinstance(_repo, Repo):
+            raise UrlNotDvcRepoError(_repo.url)
        out = _repo.find_out_by_relpath(path)
        remote_obj = _repo.cloud.get_remote(remote)
        return str(remote_obj.checksum_to_path_info(out.checksum))


 def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Context manager to open a file artifact as a file object."""
+    """
+    Open file in the supplied path tracked in a repo (both DVC projects and
+    plain Git repos are supported). For Git repos, HEAD is used unless a rev
+    argument is supplied. The default remote is tried unless a remote argument
+    is supplied. It may only be used as a context manager:
+
+        with dvc.api.open(
+                'path/to/file',
+                repo='https://example.com/url/to/repo'
+                ) as fd:
+            # ... Handle file object fd
+    """
    args = (path,)
    kwargs = {
        "repo": repo,
@@ -63,7 +77,11 @@ def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):


 def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Returns the contents of a file artifact."""
+    """
+    Returns the contents of a tracked file (by DVC or Git). For Git repos, HEAD
+    is used unless a rev argument is supplied. The default remote is tried
+    unless a remote argument is supplied.
+    """
    with open(
        path, repo=repo, rev=rev, remote=remote, mode=mode, encoding=encoding
    ) as fd:
@@ -81,8 +99,3 @@ def _make_repo(repo_url=None, rev=None):
            pass  # fallthrough to external_repo
    with external_repo(url=repo_url, rev=rev) as repo:
        yield repo
-
-
-def _require_dvc(repo):
-    if not isinstance(repo, Repo):
-        raise UrlNotDvcRepoError(repo.url)
@@ -1,25 +1,8 @@
 """Manages cache of a DVC repo."""
-import os
 from collections import defaultdict

 from funcy import cached_property

-from dvc.config import Config
-
-
-class CacheConfig(object):
-    def __init__(self, config):
-        self.config = config
-
-    def set_dir(self, dname, level=None):
-        from dvc.remote.config import RemoteConfig
-
-        configobj = self.config.get_configobj(level)
-        path = RemoteConfig.resolve_path(dname, configobj.filename)
-        self.config.set(
-            Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level
-        )
-

 def _make_remote_property(name):
    """
@@ -57,7 +40,7 @@ def _make_remote_property(name):
    return cached_property(getter)


-class Cache(object):
+class Cache:
    """Class that manages cache locations of a DVC repo.

    Args:
@@ -70,42 +53,57 @@ class Cache(object):
        from dvc.remote import Remote

        self.repo = repo
+        self.config = config = repo.config["cache"]

-        self.config = config = repo.config.config[Config.SECTION_CACHE]
-        local = config.get(Config.SECTION_CACHE_LOCAL)
+        local = config.get("local")

        if local:
-            name = Config.SECTION_REMOTE_FMT.format(local)
-            settings = repo.config.config[name]
+            settings = {"name": local}
        else:
-            default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
-            cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
-            cache_type = config.get(Config.SECTION_CACHE_TYPE)
-            protected = config.get(Config.SECTION_CACHE_PROTECTED)
-            shared = config.get(Config.SECTION_CACHE_SHARED)
-
-            settings = {
-                Config.PRIVATE_CWD: config.get(
-                    Config.PRIVATE_CWD, repo.dvc_dir
-                ),
-                Config.SECTION_REMOTE_URL: cache_dir,
-                Config.SECTION_CACHE_TYPE: cache_type,
-                Config.SECTION_CACHE_PROTECTED: protected,
-                Config.SECTION_CACHE_SHARED: shared,
-            }
+            settings = {**config, "url": config["dir"]}

        self.local = Remote(repo, **settings)

-    s3 = _make_remote_property(Config.SECTION_CACHE_S3)
-    gs = _make_remote_property(Config.SECTION_CACHE_GS)
-    ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
-    hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
-    azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
+    s3 = _make_remote_property("s3")
+    gs = _make_remote_property("gs")
+    ssh = _make_remote_property("ssh")
+    hdfs = _make_remote_property("hdfs")
+    azure = _make_remote_property("azure")


-class NamedCache(object):
+class NamedCacheItem:
    def __init__(self):
-        self._items = defaultdict(lambda: defaultdict(set))
+        self.names = set()
+        self.children = defaultdict(NamedCacheItem)
+
+    def __eq__(self, other):
+        return self.names == other.names and self.children == other.children
+
+    def child_keys(self):
+        for key, child in self.children.items():
+            yield key
+            yield from child.child_keys()
+
+    def child_names(self):
+        for key, child in self.children.items():
+            yield key, child.names
+            yield from child.child_names()
+
+    def add(self, checksum, item):
+        self.children[checksum].update(item)
+
+    def update(self, item, suffix=""):
+        if suffix:
+            self.names.update(n + suffix for n in item.names)
+        else:
+            self.names.update(item.names)
+        for checksum, child_item in item.children.items():
+            self.children[checksum].update(child_item)
+
+
+class NamedCache:
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(NamedCacheItem))
        self.external = defaultdict(set)

    @classmethod
@@ -118,7 +116,18 @@ class NamedCache(object):
        return self._items[key]

    def add(self, scheme, checksum, name):
-        self._items[scheme][checksum].add(name)
+        """Add a mapped name for the specified checksum."""
+        self._items[scheme][checksum].names.add(name)
+
+    def add_child_cache(self, checksum, cache, suffix=""):
+        """Add/update child cache for the specified checksum."""
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme][checksum].children
+            for child_checksum, item in src.items():
+                dst[child_checksum].update(item, suffix=suffix)
+
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)

    def add_external(self, url, rev, path):
        self.external[url, rev].add(path)
@@ -126,11 +135,32 @@ class NamedCache(object):
    def update(self, cache, suffix=""):
        for scheme, src in cache._items.items():
            dst = self._items[scheme]
-            for checksum, names in src.items():
-                if suffix:
-                    dst[checksum].update(n + suffix for n in names)
-                else:
-                    dst[checksum].update(names)
+            for checksum, item in src.items():
+                dst[checksum].update(item, suffix=suffix)

        for repo_pair, files in cache.external.items():
            self.external[repo_pair].update(files)
+
+    def scheme_keys(self, scheme):
+        """Iterate over a flat list of all keys for the specified scheme,
+        including children.
+        """
+        for key, item in self._items[scheme].items():
+            yield key
+            yield from item.child_keys()
+
+    def scheme_names(self, scheme):
+        """Iterate over a flat list of checksum, names items for the specified
+        scheme, including children.
+        """
+        for key, item in self._items[scheme].items():
+            yield key, item.names
+            yield from item.child_names()
+
+    def dir_keys(self, scheme):
+        return (
+            key for key, item in self._items[scheme].items() if item.children
+        )
+
+    def child_keys(self, scheme, checksum):
+        return self._items[scheme][checksum].child_keys()
@@ -16,20 +16,23 @@ from .command import (
    gc,
    get,
    get_url,
+    git_hook,
    imp,
    imp_url,
    init,
    install,
    lock,
+    ls,
    metrics,
    move,
+    params,
    pipeline,
+    plots,
    remote,
    remove,
    repro,
    root,
    run,
-    tag,
    unprotect,
    update,
    version,
@@ -37,7 +40,6 @@ from .command import (
 from .command.base import fix_subparsers
 from .exceptions import DvcParserError

-
 logger = logging.getLogger(__name__)

 COMMANDS = [
@@ -60,16 +62,19 @@ COMMANDS = [
    remote,
    cache,
    metrics,
+    params,
    install,
    root,
+    ls,
    lock,
    pipeline,
    daemon,
    commit,
-    tag,
    diff,
    version,
    update,
+    git_hook,
+    plots,
 ]


@@ -1,11 +1,8 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.exceptions import DvcException
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
-
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename

 logger = logging.getLogger(__name__)

@@ -56,6 +53,7 @@ def add_parser(subparsers, parent_parser):
        "-f",
        "--file",
        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
    )
    parser.add_argument(
        "targets", nargs="+", help="Input files/directories to add."
@@ -1,6 +1,5 @@
 import logging

-
 logger = logging.getLogger(__name__)


@@ -27,7 +26,7 @@ def append_doc_link(help_message, path):
    )


-class CmdBase(object):
+class CmdBase:
    def __init__(self, args):
        from dvc.repo import Repo
        from dvc.updater import Updater
@@ -35,18 +34,17 @@ class CmdBase(object):
        self.repo = Repo()
        self.config = self.repo.config
        self.args = args
-        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
-        updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
+        hardlink_lock = self.config["core"].get("hardlink_lock", False)
+        updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
        updater.check()

    @property
    def default_targets(self):
-        """Default targets for `dvc repro` and `dvc pipeline`."""
-        from dvc.stage import Stage
+        """Default targets for `dvc repro`."""
+        from dvc.dvcfile import PIPELINE_FILE

-        msg = "assuming default target '{}'.".format(Stage.STAGE_FILE)
-        logger.warning(msg)
-        return [Stage.STAGE_FILE]
+        logger.debug(f"assuming default target '{PIPELINE_FILE}'.")
+        return [PIPELINE_FILE]

    # Abstract methods that have to be implemented by any inheritance class
    def run(self):
@@ -1,18 +1,13 @@
 import argparse

-from dvc.cache import CacheConfig
-from dvc.command.base import append_doc_link
-from dvc.command.base import fix_subparsers
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig


 class CmdCacheDir(CmdConfig):
-    def __init__(self, args):
-        super().__init__(args)
-        self.cache_config = CacheConfig(self.config)
-
    def run(self):
-        self.cache_config.set_dir(self.args.value, level=self.args.level)
+        with self.config.edit(level=self.args.level) as edit:
+            edit["cache"]["dir"] = self.args.value
        return 0


@@ -1,18 +1,67 @@
 import argparse
+import logging
+import operator

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+import colorama
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import CheckoutError
+from dvc.utils.humanize import get_summary
+
+logger = logging.getLogger(__name__)
+
+
+def log_changes(stats):
+    colors = [
+        ("modified", colorama.Fore.YELLOW,),
+        ("added", colorama.Fore.GREEN),
+        ("deleted", colorama.Fore.RED,),
+    ]
+
+    for state, color in colors:
+        entries = stats.get(state)
+
+        if not entries:
+            continue
+
+        for entry in entries:
+            logger.info(
+                "{color}{state}{nc}{spacing}{entry}".format(
+                    color=color,
+                    state=state[0].upper(),
+                    nc=colorama.Fore.RESET,
+                    spacing="\t",
+                    entry=entry,
+                )
+            )


 class CmdCheckout(CmdBase):
    def run(self):
-        self.repo.checkout(
-            targets=self.args.targets,
-            with_deps=self.args.with_deps,
-            force=self.args.force,
-            relink=self.args.relink,
-            recursive=self.args.recursive,
-        )
+        stats, exc = None, None
+        try:
+            stats = self.repo.checkout(
+                targets=self.args.targets,
+                with_deps=self.args.with_deps,
+                force=self.args.force,
+                relink=self.args.relink,
+                recursive=self.args.recursive,
+            )
+        except CheckoutError as _exc:
+            exc = _exc
+            stats = exc.stats
+
+        if self.args.summary:
+            default_message = "No changes."
+            msg = get_summary(
+                sorted(stats.items(), key=operator.itemgetter(0))
+            )
+            logger.info(msg or default_message)
+        else:
+            log_changes(stats)
+
+        if exc:
+            raise exc
        return 0


@@ -26,6 +75,12 @@ def add_parser(subparsers, parent_parser):
        help=CHECKOUT_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
+    checkout_parser.add_argument(
+        "--summary",
+        action="store_true",
+        default=False,
+        help="Show summary of the changes.",
+    )
    checkout_parser.add_argument(
        "-d",
        "--with-deps",
@@ -1,11 +1,9 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException

-
 logger = logging.getLogger(__name__)


@@ -1,10 +1,8 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
-from dvc.config import Config
-
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.config import Config, ConfigError

 logger = logging.getLogger(__name__)

@@ -18,17 +16,21 @@ class CmdConfig(CmdBaseNoRepo):
    def run(self):
        section, opt = self.args.name.lower().strip().split(".", 1)

-        if self.args.unset:
-            self.config.unset(section, opt, level=self.args.level)
-        elif self.args.value is None:
-            logger.info(self.config.get(section, opt, level=self.args.level))
-        else:
-            self.config.set(
-                section, opt, self.args.value, level=self.args.level
-            )
+        if self.args.value is None and not self.args.unset:
+            conf = self.config.load_one(self.args.level)
+            self._check(conf, section, opt)
+            logger.info(conf[section][opt])
+            return 0

-        is_write = self.args.unset or self.args.value is not None
-        if is_write and self.args.name == "cache.type":
+        with self.config.edit(self.args.level) as conf:
+            if self.args.unset:
+                self._check(conf, section, opt)
+                del conf[section][opt]
+            else:
+                self._check(conf, section)
+                conf[section][opt] = self.args.value
+
+        if self.args.name == "cache.type":
            logger.warning(
                "You have changed the 'cache.type' option. This doesn't update"
                " any existing workspace file links, but it can be done with:"
@@ -37,30 +39,39 @@ class CmdConfig(CmdBaseNoRepo):

        return 0

+    def _check(self, conf, section, opt=None):
+        if section not in conf:
+            msg = "section {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+
+        if opt and opt not in conf[section]:
+            msg = "option {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+

 parent_config_parser = argparse.ArgumentParser(add_help=False)
 parent_config_parser.add_argument(
    "--global",
    dest="level",
    action="store_const",
-    const=Config.LEVEL_GLOBAL,
+    const="global",
    help="Use global config.",
 )
 parent_config_parser.add_argument(
    "--system",
    dest="level",
    action="store_const",
-    const=Config.LEVEL_SYSTEM,
+    const="system",
    help="Use system config.",
 )
 parent_config_parser.add_argument(
    "--local",
    dest="level",
    action="store_const",
-    const=Config.LEVEL_LOCAL,
+    const="local",
    help="Use local config.",
 )
-parent_config_parser.set_defaults(level=Config.LEVEL_REPO)
+parent_config_parser.set_defaults(level="repo")


 def add_parser(subparsers, parent_parser):
@@ -1,5 +1,4 @@
-from dvc.command.base import CmdBaseNoRepo
-from dvc.command.base import fix_subparsers
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers


 class CmdDaemonBase(CmdBaseNoRepo):
@@ -15,11 +14,10 @@ class CmdDaemonUpdater(CmdDaemonBase):

        root_dir = Repo.find_root()
        dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
+        tmp_dir = os.path.join(dvc_dir, "tmp")
        config = Config(dvc_dir, validate=False)
-        hardlink_lock = config.config.get("core", {}).get(
-            "hardlink_lock", False
-        )
-        updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
+        hardlink_lock = config.get("core", {}).get("hardlink_lock", False)
+        updater = Updater(tmp_dir, hardlink_lock=hardlink_lock)
        updater.fetch(detach=False)

        return 0
@@ -1,38 +1,45 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.exceptions import DvcException
-
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.checkout import log_changes
+from dvc.exceptions import CheckoutError, DvcException
+from dvc.utils.humanize import get_summary

 logger = logging.getLogger(__name__)


 class CmdDataBase(CmdBase):
-    @classmethod
-    def check_up_to_date(cls, processed_files_count):
-        if processed_files_count == 0:
-            logger.info("Everything is up to date.")
+    def log_summary(self, stats):
+        default_msg = "Everything is up to date."
+        logger.info(get_summary(stats.items()) or default_msg)


 class CmdDataPull(CmdDataBase):
+    def log_summary(self, stats):
+        log_changes(stats)
+        super().log_summary(stats)
+
    def run(self):
        try:
-            processed_files_count = self.repo.pull(
+            stats = self.repo.pull(
                targets=self.args.targets,
                jobs=self.args.jobs,
                remote=self.args.remote,
                all_branches=self.args.all_branches,
                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                with_deps=self.args.with_deps,
                force=self.args.force,
                recursive=self.args.recursive,
+                run_cache=self.args.run_cache,
            )
-        except DvcException:
+            self.log_summary(stats)
+        except (CheckoutError, DvcException) as exc:
+            self.log_summary(getattr(exc, "stats", {}))
            logger.exception("failed to pull data from the cloud")
            return 1
-        self.check_up_to_date(processed_files_count)
+
        return 0


@@ -45,13 +52,15 @@ class CmdDataPush(CmdDataBase):
                remote=self.args.remote,
                all_branches=self.args.all_branches,
                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                with_deps=self.args.with_deps,
                recursive=self.args.recursive,
+                run_cache=self.args.run_cache,
            )
+            self.log_summary({"pushed": processed_files_count})
        except DvcException:
            logger.exception("failed to push data to the cloud")
            return 1
-        self.check_up_to_date(processed_files_count)
        return 0


@@ -64,13 +73,15 @@ class CmdDataFetch(CmdDataBase):
                remote=self.args.remote,
                all_branches=self.args.all_branches,
                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                with_deps=self.args.with_deps,
                recursive=self.args.recursive,
+                run_cache=self.args.run_cache,
            )
+            self.log_summary({"fetched": processed_files_count})
        except DvcException:
            logger.exception("failed to fetch data from the cloud")
            return 1
-        self.check_up_to_date(processed_files_count)
        return 0


@@ -78,27 +89,31 @@ def shared_parent_parser():
    from dvc.cli import get_parent_parser

    # Parent parser used in pull/push/status
-    shared_parent_parser = argparse.ArgumentParser(
+    parent_parser = argparse.ArgumentParser(
        add_help=False, parents=[get_parent_parser()]
    )
-    shared_parent_parser.add_argument(
-        "-j", "--jobs", type=int, help="Number of jobs to run simultaneously."
+    parent_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help="Number of jobs to run simultaneously.",
+        metavar="<number>",
    )
-    shared_parent_parser.add_argument(
+    parent_parser.add_argument(
        "targets",
        nargs="*",
        help="Limit command scope to these DVC-files. "
        "Using -R, directories to search DVC-files in can also be given.",
    )

-    return shared_parent_parser
+    return parent_parser


 def add_parser(subparsers, _parent_parser):
    from dvc.command.status import CmdDataStatus

    # Pull
-    PULL_HELP = "Pull data files from a DVC remote storage."
+    PULL_HELP = "Download tracked files or directories from remote storage."

    pull_parser = subparsers.add_parser(
        "pull",
@@ -108,7 +123,7 @@ def add_parser(subparsers, _parent_parser):
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    pull_parser.add_argument(
-        "-r", "--remote", help="Remote repository to pull from."
+        "-r", "--remote", help="Remote storage to pull from", metavar="<name>",
    )
    pull_parser.add_argument(
        "-a",
@@ -124,6 +139,12 @@ def add_parser(subparsers, _parent_parser):
        default=False,
        help="Fetch cache for all tags.",
    )
+    pull_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all commits.",
+    )
    pull_parser.add_argument(
        "-f",
        "--force",
@@ -145,10 +166,16 @@ def add_parser(subparsers, _parent_parser):
        default=False,
        help="Pull cache for subdirectories of the specified directory.",
    )
+    pull_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
    pull_parser.set_defaults(func=CmdDataPull)

    # Push
-    PUSH_HELP = "Push data files to a DVC remote storage."
+    PUSH_HELP = "Upload tracked files or directories to remote storage."

    push_parser = subparsers.add_parser(
        "push",
@@ -158,7 +185,7 @@ def add_parser(subparsers, _parent_parser):
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    push_parser.add_argument(
-        "-r", "--remote", help="Remote repository to push to."
+        "-r", "--remote", help="Remote storage to push to", metavar="<name>",
    )
    push_parser.add_argument(
        "-a",
@@ -174,6 +201,12 @@ def add_parser(subparsers, _parent_parser):
        default=False,
        help="Push cache for all tags.",
    )
+    push_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Push cache for all commits.",
+    )
    push_parser.add_argument(
        "-d",
        "--with-deps",
@@ -186,12 +219,21 @@ def add_parser(subparsers, _parent_parser):
        "--recursive",
        action="store_true",
        default=False,
-        help="Push cache from subdirectories of specified directory.",
+        help="Push cache for subdirectories of specified directory.",
+    )
+    push_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
    )
    push_parser.set_defaults(func=CmdDataPush)

    # Fetch
-    FETCH_HELP = "Fetch data files from a DVC remote storage."
+    FETCH_HELP = (
+        "Get tracked files or directories from remote storage"
+        " into the cache."
+    )

    fetch_parser = subparsers.add_parser(
        "fetch",
@@ -201,7 +243,10 @@ def add_parser(subparsers, _parent_parser):
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    fetch_parser.add_argument(
-        "-r", "--remote", help="Remote repository to fetch from."
+        "-r",
+        "--remote",
+        help="Remote storage to fetch from",
+        metavar="<name>",
    )
    fetch_parser.add_argument(
        "-a",
@@ -217,6 +262,12 @@ def add_parser(subparsers, _parent_parser):
        default=False,
        help="Fetch cache for all tags.",
    )
+    fetch_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all commits.",
+    )
    fetch_parser.add_argument(
        "-d",
        "--with-deps",
@@ -231,6 +282,12 @@ def add_parser(subparsers, _parent_parser):
        default=False,
        help="Fetch cache for subdirectories of specified directory.",
    )
+    fetch_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
    fetch_parser.set_defaults(func=CmdDataFetch)

    # Status
@@ -264,7 +321,10 @@ def add_parser(subparsers, _parent_parser):
        help="Show status of a local cache compared to a remote repository.",
    )
    status_parser.add_argument(
-        "-r", "--remote", help="Remote repository to compare local cache to."
+        "-r",
+        "--remote",
+        help="Remote storage to compare local cache to",
+        metavar="<name>",
    )
    status_parser.add_argument(
        "-a",
@@ -282,6 +342,13 @@ def add_parser(subparsers, _parent_parser):
        help="Show status of a local cache compared to a remote repository "
        "for all tags.",
    )
+    status_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show status of a local cache compared to a remote repository "
+        "for all commits.",
+    )
    status_parser.add_argument(
        "-d",
        "--with-deps",
@@ -289,4 +356,12 @@ def add_parser(subparsers, _parent_parser):
        default=False,
        help="Show status for all dependencies of the specified target.",
    )
+    status_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Show status of all stages in the specified directory.",
+    )
+
    status_parser.set_defaults(func=CmdDataStatus)
@@ -2,11 +2,9 @@ import argparse
 import logging

 import dvc.prompt as prompt
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException

-
 logger = logging.getLogger(__name__)


@@ -8,10 +8,24 @@ import colorama
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException

-
 logger = logging.getLogger(__name__)


+def _show_md(diff):
+    from dvc.utils.diff import table
+
+    rows = []
+    for status in ["added", "deleted", "modified"]:
+        entries = diff.get(status, [])
+        if not entries:
+            continue
+        paths = sorted([entry["path"] for entry in entries])
+        for path in paths:
+            rows.append([status, path])
+
+    return table(["Status", "Path"], rows, True)
+
+
 class CmdDiff(CmdBase):
    @staticmethod
    def _format(diff):
@@ -28,7 +42,7 @@ class CmdDiff(CmdBase):
                dir/
                dir/1

-        An example of a diff formatted when entries contain checksums:
+        An example of a diff formatted when entries contain hash:

            Added:
                d3b07384 foo
@@ -66,7 +80,7 @@ class CmdDiff(CmdBase):

            for entry in entries:
                path = entry["path"]
-                checksum = entry.get("checksum")
+                checksum = entry.get("hash")
                summary[state] += 1 if not path.endswith(os.sep) else 0
                content.append(
                    "{space}{checksum}{separator}{path}".format(
@@ -97,20 +111,17 @@ class CmdDiff(CmdBase):
        try:
            diff = self.repo.diff(self.args.a_rev, self.args.b_rev)

-            if not any(diff.values()):
-                return 0
-
-            if not self.args.checksums:
+            if not self.args.show_hash:
                for _, entries in diff.items():
                    for entry in entries:
-                        del entry["checksum"]
+                        del entry["hash"]

            if self.args.show_json:
-                res = json.dumps(diff)
-            else:
-                res = self._format(diff)
-
-            logger.info(res)
+                logger.info(json.dumps(diff))
+            elif self.args.show_md:
+                logger.info(_show_md(diff))
+            elif diff:
+                logger.info(self._format(diff))

        except DvcException:
            logger.exception("failed to get diff")
@@ -120,8 +131,8 @@ class CmdDiff(CmdBase):

 def add_parser(subparsers, parent_parser):
    DIFF_DESCRIPTION = (
-        "Compare two versions of the DVC repository.\n"
-        "Shows the list of paths added, modified, or deleted"
+        "Show added, modified, or deleted data between commits in the DVC"
+        " repository, or between a commit and the workspace."
    )
    diff_parser = subparsers.add_parser(
        "diff",
@@ -148,9 +159,15 @@ def add_parser(subparsers, parent_parser):
        default=False,
    )
    diff_parser.add_argument(
-        "--checksums",
+        "--show-hash",
        help="Display hash value for each entry",
        action="store_true",
        default=False,
    )
+    diff_parser.add_argument(
+        "--show-md",
+        help="Show tabulated output in the Markdown format (GFM).",
+        action="store_true",
+        default=False,
+    )
    diff_parser.set_defaults(func=CmdDiff)
@@ -3,15 +3,22 @@ import logging
 import os

 import dvc.prompt as prompt
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-
+from dvc.command.base import CmdBase, append_doc_link

 logger = logging.getLogger(__name__)


 class CmdGC(CmdBase):
    def run(self):
+        from dvc.repo.gc import _raise_error_if_all_disabled
+
+        _raise_error_if_all_disabled(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+        )
+
        msg = "This will remove all cache except items used in "

        msg += "the working tree"
@@ -47,6 +54,7 @@ class CmdGC(CmdBase):
            force=self.args.force,
            jobs=self.args.jobs,
            repos=self.args.repos,
+            workspace=self.args.workspace,
        )
        return 0

@@ -64,25 +72,32 @@ def add_parser(subparsers, parent_parser):
        help=GC_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
+    gc_parser.add_argument(
+        "-w",
+        "--workspace",
+        action="store_true",
+        default=False,
+        help="Keep data files used in the current workspace (working tree).",
+    )
    gc_parser.add_argument(
        "-a",
        "--all-branches",
        action="store_true",
        default=False,
-        help="Keep data files for the tips of all git branches.",
+        help="Keep data files for the tips of all Git branches.",
    )
    gc_parser.add_argument(
        "-T",
        "--all-tags",
        action="store_true",
        default=False,
-        help="Keep data files for all git tags.",
+        help="Keep data files for all Git tags.",
    )
    gc_parser.add_argument(
        "--all-commits",
        action="store_true",
        default=False,
-        help=argparse.SUPPRESS,
+        help="Keep data files for all Git commits.",
    )
    gc_parser.add_argument(
        "-c",
@@ -92,7 +107,10 @@ def add_parser(subparsers, parent_parser):
        help="Collect garbage in remote repository.",
    )
    gc_parser.add_argument(
-        "-r", "--remote", help="Remote storage to collect garbage in."
+        "-r",
+        "--remote",
+        help="Remote storage to collect garbage in",
+        metavar="<name>",
    )
    gc_parser.add_argument(
        "-f",
@@ -102,7 +120,11 @@ def add_parser(subparsers, parent_parser):
        help="Force garbage collection - automatically agree to all prompts.",
    )
    gc_parser.add_argument(
-        "-j", "--jobs", type=int, help="Number of jobs to run simultaneously."
+        "-j",
+        "--jobs",
+        type=int,
+        help="Number of jobs to run simultaneously.",
+        metavar="<number>",
    )
    gc_parser.add_argument(
        "-p",
@@ -113,5 +135,6 @@ def add_parser(subparsers, parent_parser):
        help="Keep data files required by these projects "
        "in addition to the current one. "
        "Useful if you share a single cache across repos.",
+        metavar="<paths>",
    )
    gc_parser.set_defaults(func=CmdGC)
@@ -1,10 +1,9 @@
 import argparse
 import logging

-from .base import append_doc_link
-from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException

+from .base import CmdBaseNoRepo, append_doc_link

 logger = logging.getLogger(__name__)

@@ -66,10 +65,17 @@ def add_parser(subparsers, parent_parser):
        "path", help="Path to a file or directory within the repository"
    )
    get_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to download files to"
+        "-o",
+        "--out",
+        nargs="?",
+        help="Destination path to download files to",
+        metavar="<path>",
    )
    get_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
    )
    get_parser.add_argument(
        "--show-url",
@@ -1,10 +1,9 @@
 import argparse
 import logging

-from .base import append_doc_link
-from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException

+from .base import CmdBaseNoRepo, append_doc_link

 logger = logging.getLogger(__name__)

@@ -17,7 +16,7 @@ class CmdGetUrl(CmdBaseNoRepo):
            Repo.get_url(self.args.url, out=self.args.out)
            return 0
        except DvcException:
-            logger.exception("failed to get '{}'".format(self.args.url))
+            logger.exception(f"failed to get '{self.args.url}'")
            return 1


@@ -0,0 +1,112 @@
+import logging
+import os
+
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.exceptions import NotDvcRepoError
+
+logger = logging.getLogger(__name__)
+
+
+class CmdHookBase(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            repo = Repo()
+            repo.close()
+        except NotDvcRepoError:
+            return 0
+
+        return self._run()
+
+
+class CmdPreCommit(CmdHookBase):
+    def _run(self):
+        from dvc.main import main
+
+        return main(["status"])
+
+
+class CmdPostCheckout(CmdHookBase):
+    def run(self):
+        # when we are running from pre-commit tool, it doesn't provide CLI
+        # flags, but instead provides respective env vars that we could use.
+        flag = os.environ.get("PRE_COMMIT_CHECKOUT_TYPE")
+        if flag is None and len(self.args.args) >= 3:
+            # see https://git-scm.com/docs/githooks#_post_checkout
+            flag = self.args.args[2]
+
+        # checking out some reference and not specific file.
+        if flag != "1":
+            return 0
+
+        # make sure we are not in the middle of a rebase/merge, so we
+        # don't accidentally break it with an unsuccessful checkout.
+        # Note that git hooks are always running in repo root.
+        if os.path.isdir(os.path.join(".git", "rebase-merge")):
+            return 0
+
+        from dvc.main import main
+
+        return main(["checkout"])
+
+
+class CmdPrePush(CmdHookBase):
+    def run(self):
+        from dvc.main import main
+
+        return main(["push"])
+
+
+def add_parser(subparsers, parent_parser):
+    GIT_HOOK_HELP = "Run GIT hook."
+
+    git_hook_parser = subparsers.add_parser(
+        "git-hook",
+        parents=[parent_parser],
+        description=GIT_HOOK_HELP,
+        add_help=False,
+    )
+
+    git_hook_subparsers = git_hook_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc daemon CMD --help` for command-specific help.",
+    )
+
+    fix_subparsers(git_hook_subparsers)
+
+    PRE_COMMIT_HELP = "Run pre-commit GIT hook."
+    pre_commit_parser = git_hook_subparsers.add_parser(
+        "pre-commit",
+        parents=[parent_parser],
+        description=PRE_COMMIT_HELP,
+        help=PRE_COMMIT_HELP,
+    )
+    pre_commit_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    pre_commit_parser.set_defaults(func=CmdPreCommit)
+
+    POST_CHECKOUT_HELP = "Run post-checkout GIT hook."
+    post_checkout_parser = git_hook_subparsers.add_parser(
+        "post-checkout",
+        parents=[parent_parser],
+        description=POST_CHECKOUT_HELP,
+        help=POST_CHECKOUT_HELP,
+    )
+    post_checkout_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    post_checkout_parser.set_defaults(func=CmdPostCheckout)
+
+    PRE_PUSH_HELP = "Run pre-push GIT hook."
+    pre_push_parser = git_hook_subparsers.add_parser(
+        "pre-push",
+        parents=[parent_parser],
+        description=PRE_PUSH_HELP,
+        help=PRE_PUSH_HELP,
+    )
+    pre_push_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    pre_push_parser.set_defaults(func=CmdPrePush)
@@ -1,11 +1,9 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException

-
 logger = logging.getLogger(__name__)


@@ -48,9 +46,16 @@ def add_parser(subparsers, parent_parser):
        "path", help="Path to a file or directory within the repository"
    )
    import_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to download files to"
+        "-o",
+        "--out",
+        nargs="?",
+        help="Destination path to download files to",
+        metavar="<path>",
    )
    import_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
    )
    import_parser.set_defaults(func=CmdImport)
@@ -1,11 +1,9 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException

-
 logger = logging.getLogger(__name__)


@@ -59,6 +57,9 @@ def add_parser(subparsers, parent_parser):
        "out", nargs="?", help="Destination path to put files to."
    )
    import_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC-file it generates."
+        "-f",
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
    )
    import_parser.set_defaults(func=CmdImportUrl)
@@ -1,9 +1,7 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
-
+from dvc.command.base import CmdBaseNoRepo, append_doc_link

 logger = logging.getLogger(__name__)

@@ -15,7 +13,10 @@ class CmdInit(CmdBaseNoRepo):

        try:
            self.repo = Repo.init(
-                ".", no_scm=self.args.no_scm, force=self.args.force
+                ".",
+                no_scm=self.args.no_scm,
+                force=self.args.force,
+                subdir=self.args.subdir,
            )
            self.config = self.repo.config
        except InitError:
@@ -44,7 +45,7 @@ def add_parser(subparsers, parent_parser):
        action="store_true",
        default=False,
        help="Initiate DVC in directory that is "
-        "not tracked by any scm tool (e.g. git).",
+        "not tracked by any SCM tool (e.g. Git).",
    )
    init_parser.add_argument(
        "-f",
@@ -56,4 +57,13 @@ def add_parser(subparsers, parent_parser):
            "This operation removes local cache."
        ),
    )
+    init_parser.add_argument(
+        "--subdir",
+        action="store_true",
+        default=False,
+        help=(
+            "Necessary for running this command inside a subdirectory of a "
+            "parent SCM repository."
+        ),
+    )
    init_parser.set_defaults(func=CmdInit)
@@ -1,9 +1,7 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-
+from dvc.command.base import CmdBase, append_doc_link

 logger = logging.getLogger(__name__)

@@ -11,7 +9,7 @@ logger = logging.getLogger(__name__)
 class CmdInstall(CmdBase):
    def run(self):
        try:
-            self.repo.install()
+            self.repo.install(self.args.use_pre_commit_tool)
        except Exception:
            logger.exception("failed to install DVC Git hooks")
            return 1
@@ -27,4 +25,11 @@ def add_parser(subparsers, parent_parser):
        help=INSTALL_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
+    install_parser.add_argument(
+        "--use-pre-commit-tool",
+        action="store_true",
+        default=False,
+        help="Install DVC hooks using pre-commit "
+        "(https://pre-commit.com) if it is installed.",
+    )
    install_parser.set_defaults(func=CmdInstall)
@@ -1,11 +1,9 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException

-
 logger = logging.getLogger(__name__)


@@ -0,0 +1,78 @@
+import argparse
+import logging
+import sys
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.ls.ls_colors import LsColors
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _prettify(entries, with_color=False):
+    if with_color:
+        ls_colors = LsColors()
+        fmt = ls_colors.format
+    else:
+
+        def fmt(entry):
+            return entry["path"]
+
+    return [fmt(entry) for entry in entries]
+
+
+class CmdList(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            entries = Repo.ls(
+                self.args.url,
+                self.args.path,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+                outs_only=self.args.outs_only,
+            )
+            if entries:
+                entries = _prettify(entries, sys.stdout.isatty())
+                logger.info("\n".join(entries))
+            return 0
+        except DvcException:
+            logger.exception(f"failed to list '{self.args.url}'")
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    LIST_HELP = (
+        "List repository contents, including files"
+        " and directories tracked by DVC and by Git."
+    )
+    list_parser = subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(LIST_HELP, "list"),
+        help=LIST_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    list_parser.add_argument("url", help="Location of DVC repository to list")
+    list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        help="Recursively list files.",
+    )
+    list_parser.add_argument(
+        "--outs-only", action="store_true", help="Show only DVC outputs."
+    )
+    list_parser.add_argument(
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    list_parser.add_argument(
+        "path",
+        nargs="?",
+        help="Path to directory within the repository to list outputs for",
+    )
+    list_parser.set_defaults(func=CmdList)
@@ -0,0 +1,48 @@
+import os
+
+
+class LsColors:
+    default = "rs=0:di=01;34:ex=01;32"
+
+    def __init__(self, lscolors=None):
+        self._extensions = {}
+        self._codes = {}
+        self._load(lscolors or os.environ.get("LS_COLORS") or LsColors.default)
+
+    def _load(self, lscolors):
+        for item in lscolors.split(":"):
+            try:
+                code, color = item.split("=", 1)
+            except ValueError:
+                continue
+            if code.startswith("*."):
+                self._extensions[code[1:]] = color
+            else:
+                self._codes[code] = color
+
+    def format(self, entry):
+        text = entry["path"]
+
+        if entry.get("isout", False) and "out" in self._codes:
+            return self._format(text, code="out")
+
+        if entry.get("isdir", False):
+            return self._format(text, code="di")
+
+        if entry.get("isexec", False):
+            return self._format(text, code="ex")
+
+        _, ext = os.path.splitext(text)
+        return self._format(text, ext=ext)
+
+    def _format(self, text, code=None, ext=None):
+        val = None
+        if ext:
+            val = self._extensions.get(ext, None)
+        if code:
+            val = self._codes.get(code, None)
+
+        if not val:
+            return text
+        rs = self._codes.get("rs", 0)
+        return f"\033[{val}m{text}\033[{rs}m"
@@ -1,46 +1,34 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import BadMetricError
-from dvc.exceptions import DvcException
-
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import BadMetricError, DvcException

 logger = logging.getLogger(__name__)


-def show_metrics(metrics, all_branches=False, all_tags=False):
-    """
-    Args:
-        metrics (list): Where each element is either a `list`
-            if an xpath was specified, otherwise a `str`
-    """
+def show_metrics(
+    metrics, all_branches=False, all_tags=False, all_commits=False
+):
+    from flatten_json import flatten
+    from dvc.utils.diff import format_dict
+
    # When `metrics` contains a `None` key, it means that some files
    # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
    missing = metrics.pop(None, None)

    for branch, val in metrics.items():
-        if all_branches or all_tags:
-            logger.info("{branch}:".format(branch=branch))
+        if all_branches or all_tags or all_commits:
+            logger.info(f"{branch}:")

        for fname, metric in val.items():
-            if isinstance(metric, dict):
-                lines = list(metric.values())
-            elif isinstance(metric, list):
-                lines = metric
-            else:
-                lines = metric.splitlines()
+            if not isinstance(metric, dict):
+                logger.info("\t{}: {}".format(fname, str(metric)))
+                continue

-            if len(lines) > 1:
-                logger.info("\t{fname}:".format(fname=fname))
-
-                for line in lines:
-                    logger.info("\t\t{content}".format(content=line))
-
-            else:
-                logger.info("\t{}: {}".format(fname, metric))
+            logger.info(f"\t{fname}:")
+            for key, value in flatten(format_dict(metric), ".").items():
+                logger.info(f"\t\t{key}: {value}")

    if missing:
        raise BadMetricError(missing)
@@ -51,14 +39,23 @@ class CmdMetricsShow(CmdBase):
        try:
            metrics = self.repo.metrics.show(
                self.args.targets,
-                typ=self.args.type,
-                xpath=self.args.xpath,
                all_branches=self.args.all_branches,
                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                recursive=self.args.recursive,
            )

-            show_metrics(metrics, self.args.all_branches, self.args.all_tags)
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(metrics))
+            else:
+                show_metrics(
+                    metrics,
+                    self.args.all_branches,
+                    self.args.all_tags,
+                    self.args.all_commits,
+                )
        except DvcException:
            logger.exception("failed to show metrics")
            return 1
@@ -66,27 +63,12 @@ class CmdMetricsShow(CmdBase):
        return 0


-class CmdMetricsModify(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.modify(
-                self.args.path, typ=self.args.type, xpath=self.args.xpath
-            )
-        except DvcException:
-            logger.exception("failed to modify metric file settings")
-            return 1
-
-        return 0
-
-
 class CmdMetricsAdd(CmdBase):
    def run(self):
        try:
-            self.repo.metrics.add(
-                self.args.path, self.args.type, self.args.xpath
-            )
+            self.repo.metrics.add(self.args.path)
        except DvcException:
-            msg = "failed to add metric file '{}'".format(self.args.path)
+            msg = f"failed to add metric file '{self.args.path}'"
            logger.exception(msg)
            return 1

@@ -98,28 +80,22 @@ class CmdMetricsRemove(CmdBase):
        try:
            self.repo.metrics.remove(self.args.path)
        except DvcException:
-            msg = "failed to remove metric file '{}'".format(self.args.path)
+            msg = f"failed to remove metric file '{self.args.path}'"
            logger.exception(msg)
            return 1

        return 0


-def _show_diff(diff):
-    from texttable import Texttable
+def _show_diff(diff, markdown=False):
+    from collections import OrderedDict

-    if not diff:
-        return "No changes."
+    from dvc.utils.diff import table

-    table = Texttable()
-
-    # remove borders to make it easier for users to copy stuff
-    table.set_chars(("", "", "", ""))
-    table.set_deco(0)
-
-    rows = [["Path", "Metric", "Value", "Change"]]
+    rows = []
    for fname, mdiff in diff.items():
-        for metric, change in mdiff.items():
+        sorted_mdiff = OrderedDict(sorted(mdiff.items()))
+        for metric, change in sorted_mdiff.items():
            rows.append(
                [
                    fname,
@@ -128,8 +104,8 @@ def _show_diff(diff):
                    change.get("diff", "diff not supported"),
                ]
            )
-    table.add_rows(rows)
-    return table.draw()
+
+    return table(["Path", "Metric", "Value", "Change"], rows, markdown)


 class CmdMetricsDiff(CmdBase):
@@ -139,9 +115,8 @@ class CmdMetricsDiff(CmdBase):
                a_rev=self.args.a_rev,
                b_rev=self.args.b_rev,
                targets=self.args.targets,
-                typ=self.args.type,
-                xpath=self.args.xpath,
                recursive=self.args.recursive,
+                all=self.args.all,
            )

            if self.args.show_json:
@@ -149,7 +124,9 @@ class CmdMetricsDiff(CmdBase):

                logger.info(json.dumps(diff))
            else:
-                logger.info(_show_diff(diff))
+                table = _show_diff(diff, self.args.show_md)
+                if table:
+                    logger.info(table)

        except DvcException:
            logger.exception("failed to show metrics diff")
@@ -159,7 +136,7 @@ class CmdMetricsDiff(CmdBase):


 def add_parser(subparsers, parent_parser):
-    METRICS_HELP = "Commands to add, manage, collect and display metrics."
+    METRICS_HELP = "Commands to add, manage, collect, and display metrics."

    metrics_parser = subparsers.add_parser(
        "metrics",
@@ -176,7 +153,18 @@ def add_parser(subparsers, parent_parser):

    fix_subparsers(metrics_subparsers)

-    METRICS_SHOW_HELP = "Output metric values."
+    METRICS_ADD_HELP = "Mark a DVC-tracked file as a metric."
+    metrics_add_parser = metrics_subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
+        help=METRICS_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_add_parser.add_argument("path", help="Path to a metric file.")
+    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
+
+    METRICS_SHOW_HELP = "Print metrics, with optional formatting."
    metrics_show_parser = metrics_subparsers.add_parser(
        "show",
        parents=[parent_parser],
@@ -187,20 +175,7 @@ def add_parser(subparsers, parent_parser):
    metrics_show_parser.add_argument(
        "targets",
        nargs="*",
-        help="Metric files or directories (see -R) to show "
-        "(leave empty to display all)",
-    )
-    metrics_show_parser.add_argument(
-        "-t",
-        "--type",
-        help=(
-            "Type of metrics (json/tsv/htsv/csv/hcsv). "
-            "It can be detected by the file extension automatically. "
-            "Unsupported types will be treated as raw."
-        ),
-    )
-    metrics_show_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
+        help="Metric files or directories (see -R) to show",
    )
    metrics_show_parser.add_argument(
        "-a",
@@ -216,6 +191,12 @@ def add_parser(subparsers, parent_parser):
        default=False,
        help="Show metrics for all tags.",
    )
+    metrics_show_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show metrics for all commits.",
+    )
    metrics_show_parser.add_argument(
        "-R",
        "--recursive",
@@ -226,55 +207,16 @@ def add_parser(subparsers, parent_parser):
            "metric files."
        ),
    )
+    metrics_show_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
    metrics_show_parser.set_defaults(func=CmdMetricsShow)

-    METRICS_ADD_HELP = "Tag file as a metric file."
-    metrics_add_parser = metrics_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
-        help=METRICS_ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_add_parser.add_argument(
-        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
-    )
-    metrics_add_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
-    )
-    metrics_add_parser.add_argument("path", help="Path to a metric file.")
-    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
-
-    METRICS_MODIFY_HELP = "Modify metric file options."
-    metrics_modify_parser = metrics_subparsers.add_parser(
-        "modify",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_MODIFY_HELP, "metrics/modify"),
-        help=METRICS_MODIFY_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_modify_parser.add_argument(
-        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
-    )
-    metrics_modify_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
-    )
-    metrics_modify_parser.add_argument("path", help="Path to a metric file.")
-    metrics_modify_parser.set_defaults(func=CmdMetricsModify)
-
-    METRICS_REMOVE_HELP = "Remove files's metric tag."
-    metrics_remove_parser = metrics_subparsers.add_parser(
-        "remove",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
-        help=METRICS_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
-    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
-
-    METRICS_DIFF_HELP = "Show a table of changes between metrics among "
-    "versions of the DVC repository."
+    METRICS_DIFF_HELP = "Show changes in metrics between commits"
+    " in the DVC repository, or between a commit and the workspace."
    metrics_diff_parser = metrics_subparsers.add_parser(
        "diff",
        parents=[parent_parser],
@@ -297,18 +239,7 @@ def add_parser(subparsers, parent_parser):
            "Metric files or directories (see -R) to show diff for. "
            "Shows diff for all metric files by default."
        ),
-    )
-    metrics_diff_parser.add_argument(
-        "-t",
-        "--type",
-        help=(
-            "Type of metrics (json/tsv/htsv/csv/hcsv). "
-            "It can be detected by the file extension automatically. "
-            "Unsupported types will be treated as raw."
-        ),
-    )
-    metrics_diff_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
+        metavar="<paths>",
    )
    metrics_diff_parser.add_argument(
        "-R",
@@ -320,10 +251,33 @@ def add_parser(subparsers, parent_parser):
            "metric files."
        ),
    )
+    metrics_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged metrics as well.",
+    )
    metrics_diff_parser.add_argument(
        "--show-json",
        action="store_true",
        default=False,
        help="Show output in JSON format.",
    )
+    metrics_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
    metrics_diff_parser.set_defaults(func=CmdMetricsDiff)
+
+    METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
+    metrics_remove_parser = metrics_subparsers.add_parser(
+        "remove",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
+        help=METRICS_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
+    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
@@ -1,11 +1,9 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException

-
 logger = logging.getLogger(__name__)


@@ -0,0 +1,103 @@
+import argparse
+import logging
+from collections import OrderedDict
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _show_diff(diff, markdown=False):
+    from dvc.utils.diff import table
+
+    rows = []
+    for fname, pdiff in diff.items():
+        sorted_pdiff = OrderedDict(sorted(pdiff.items()))
+        for param, change in sorted_pdiff.items():
+            rows.append([fname, param, change["old"], change["new"]])
+
+    return table(["Path", "Param", "Old", "New"], rows, markdown)
+
+
+class CmdParamsDiff(CmdBase):
+    def run(self):
+        try:
+            diff = self.repo.params.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                all=self.args.all,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                table = _show_diff(diff, self.args.show_md)
+                if table:
+                    logger.info(table)
+
+        except DvcException:
+            logger.exception("failed to show params diff")
+            return 1
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    PARAMS_HELP = "Commands to display params."
+
+    params_parser = subparsers.add_parser(
+        "params",
+        parents=[parent_parser],
+        description=append_doc_link(PARAMS_HELP, "params"),
+        help=PARAMS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    params_subparsers = params_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc params CMD --help` to display command-specific help.",
+    )
+
+    fix_subparsers(params_subparsers)
+
+    PARAMS_DIFF_HELP = (
+        "Show changes in params between commits in the DVC repository, or "
+        "between a commit and the workspace."
+    )
+    params_diff_parser = params_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(PARAMS_DIFF_HELP, "params/diff"),
+        help=PARAMS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    params_diff_parser.add_argument(
+        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
+    )
+    params_diff_parser.add_argument(
+        "b_rev",
+        nargs="?",
+        help=("New Git commit to compare (defaults to the current workspace)"),
+    )
+    params_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged params as well.",
+    )
+    params_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    params_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    params_diff_parser.set_defaults(func=CmdParamsDiff)
@@ -4,19 +4,19 @@ import logging
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException

-
 logger = logging.getLogger(__name__)


 class CmdPipelineShow(CmdBase):
    def _show(self, target, commands, outs, locked):
        import networkx
-        from dvc.stage import Stage
+        from dvc import dvcfile
+        from dvc.utils import parse_target

-        stage = Stage.load(self.repo, target)
+        path, name = parse_target(target)
+        stage = dvcfile.Dvcfile(self.repo, path).stages[name]
        G = self.repo.graph
        stages = networkx.dfs_postorder_nodes(G, stage)
-
        if locked:
            stages = [s for s in stages if s.locked]

@@ -29,14 +29,41 @@ class CmdPipelineShow(CmdBase):
                for out in stage.outs:
                    logger.info(str(out))
            else:
-                logger.info(stage.path_in_repo)
+                logger.info(stage.addressing)

-    def _build_graph(self, target, commands, outs):
+    @staticmethod
+    def _build_output_graph(G, target_stage):
        import networkx
-        from dvc.stage import Stage
-        from dvc.repo.graph import get_pipeline
+        from itertools import product

-        target_stage = Stage.load(self.repo, target)
+        nodes = {str(out) for out in target_stage.outs}
+        edges = []
+
+        for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
+            from_stage_deps = {dep.path_info.parts for dep in from_stage.deps}
+            to_outs = {
+                to_out
+                for to_out in to_stage.outs
+                if to_out.path_info.parts in from_stage_deps
+            }
+            from_outs = {
+                from_out
+                for from_out in from_stage.outs
+                if str(from_out) in nodes
+            }
+            nodes |= {str(to_out) for to_out in to_outs}
+            for from_out, to_out in product(from_outs, to_outs):
+                edges.append((str(from_out), str(to_out)))
+        return nodes, edges
+
+    def _build_graph(self, target, commands=False, outs=False):
+        import networkx
+        from dvc import dvcfile
+        from dvc.repo.graph import get_pipeline
+        from dvc.utils import parse_target
+
+        path, name = parse_target(target)
+        target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
        G = get_pipeline(self.repo.pipelines, target_stage)

        nodes = set()
@@ -45,29 +72,20 @@ class CmdPipelineShow(CmdBase):
                if stage.cmd is None:
                    continue
                nodes.add(stage.cmd)
-            elif outs:
-                for out in stage.outs:
-                    nodes.add(str(out))
-                for dep in stage.deps:
-                    nodes.add(str(dep))
-            else:
-                nodes.add(stage.relpath)
+            elif not outs:
+                nodes.add(stage.addressing)

        edges = []
+        for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
+            if commands:
+                if to_stage.cmd is None:
+                    continue
+                edges.append((from_stage.cmd, to_stage.cmd))
+            elif not outs:
+                edges.append((from_stage.addressing, to_stage.addressing))

        if outs:
-            for stage in networkx.dfs_preorder_nodes(G, target_stage):
-                for dep in stage.deps:
-                    for out in stage.outs:
-                        edges.append((str(out), str(dep)))
-        else:
-            for from_stage, to_stage in networkx.dfs_edges(G, target_stage):
-                if commands:
-                    if to_stage.cmd is None:
-                        continue
-                    edges.append((from_stage.cmd, to_stage.cmd))
-                else:
-                    edges.append((from_stage.relpath, to_stage.relpath))
+            nodes, edges = self._build_output_graph(G, target_stage)

        return list(nodes), edges, networkx.is_tree(G)

@@ -121,8 +139,10 @@ class CmdPipelineShow(CmdBase):
        logger.info(dot_file.getvalue())

    def run(self):
+        from dvc.dvcfile import DVC_FILE
+
        if not self.args.targets:
-            self.args.targets = self.default_targets
+            self.args.targets = [DVC_FILE]

        for target in self.args.targets:
            try:
@@ -144,7 +164,7 @@ class CmdPipelineShow(CmdBase):
                        self.args.locked,
                    )
            except DvcException:
-                msg = "failed to show pipeline for '{}'".format(target)
+                msg = f"failed to show pipeline for '{target}'"
                logger.exception(msg)
                return 1
        return 0
@@ -155,7 +175,7 @@ class CmdPipelineList(CmdBase):
        pipelines = self.repo.pipelines
        for pipeline in pipelines:
            for stage in pipeline:
-                logger.info(stage.relpath)
+                logger.info(stage.addressing)
            if len(pipeline) != 0:
                logger.info("=" * 80)
        logger.info("{} pipelines total".format(len(pipelines)))
@@ -180,7 +200,17 @@ def add_parser(subparsers, parent_parser):

    fix_subparsers(pipeline_subparsers)

-    PIPELINE_SHOW_HELP = "Show pipelines."
+    PIPELINE_LIST_HELP = "List connected groups of stages (pipelines)."
+    pipeline_list_parser = pipeline_subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
+        help=PIPELINE_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    pipeline_list_parser.set_defaults(func=CmdPipelineList)
+
+    PIPELINE_SHOW_HELP = "Show stages in a pipeline."
    pipeline_show_parser = pipeline_subparsers.add_parser(
        "show",
        parents=[parent_parser],
@@ -235,13 +265,3 @@ def add_parser(subparsers, parent_parser):
        "(Finds all DVC-files in the workspace by default.)",
    )
    pipeline_show_parser.set_defaults(func=CmdPipelineShow)
-
-    PIPELINE_LIST_HELP = "List pipelines."
-    pipeline_list_parser = pipeline_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
-        help=PIPELINE_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pipeline_list_parser.set_defaults(func=CmdPipelineList)
@@ -0,0 +1,229 @@
+import argparse
+import logging
+import os
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+PAGE_HTML = """<!DOCTYPE html>
+<html>
+<head>
+    <title>DVC Plot</title>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+</head>
+<body>
+    {divs}
+</body>
+</html>"""
+
+DIV_HTML = """<div id = "{id}"></div>
+<script type = "text/javascript">
+    var spec = {vega_json};
+    vegaEmbed('#{id}', spec);
+</script>"""
+
+
+class CmdPlots(CmdBase):
+    def _func(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def run(self):
+        fields = None
+        jsonpath = None
+        if self.args.select:
+            if self.args.select.startswith("$"):
+                jsonpath = self.args.select
+            else:
+                fields = set(self.args.select.split(","))
+        try:
+            plots = self._func(
+                targets=self.args.targets,
+                template=self.args.template,
+                fields=fields,
+                x_field=self.args.x,
+                y_field=self.args.y,
+                path=jsonpath,
+                csv_header=not self.args.no_csv_header,
+                title=self.args.title,
+                x_title=self.args.xlab,
+                y_title=self.args.ylab,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(plots))
+                return 0
+
+            divs = [
+                DIV_HTML.format(id=f"plot{i}", vega_json=plot)
+                for i, plot in enumerate(plots.values())
+            ]
+            html = PAGE_HTML.format(divs="\n".join(divs))
+            path = self.args.out or "plots.html"
+
+            with open(path, "w") as fobj:
+                fobj.write(html)
+
+            logger.info(
+                "file://{}".format(os.path.join(self.repo.root_dir, path))
+            )
+
+        except DvcException:
+            logger.exception("")
+            return 1
+
+        return 0
+
+
+class CmdPlotsShow(CmdPlots):
+    def _func(self, *args, **kwargs):
+        return self.repo.plots.show(*args, **kwargs)
+
+
+class CmdPlotsDiff(CmdPlots):
+    def _func(self, *args, **kwargs):
+        return self.repo.plots.diff(*args, revs=self.args.revisions, **kwargs)
+
+
+def add_parser(subparsers, parent_parser):
+    PLOTS_HELP = (
+        "Generating plots for metrics stored in structured files "
+        "(JSON, CSV, TSV)."
+    )
+
+    plots_parser = subparsers.add_parser(
+        "plots",
+        parents=[parent_parser],
+        description=append_doc_link(PLOTS_HELP, "plots"),
+        help=PLOTS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_subparsers = plots_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc plots CMD --help` to display command-specific help.",
+    )
+
+    fix_subparsers(plots_subparsers)
+
+    SHOW_HELP = "Generate a plots image file from a metrics file."
+    plots_show_parser = plots_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(SHOW_HELP, "plots/show"),
+        help=SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_show_parser.add_argument(
+        "-t",
+        "--template",
+        nargs="?",
+        default=None,
+        help="File to be injected with data.",
+    )
+    plots_show_parser.add_argument(
+        "-o", "--out", default=None, help="Destination path to save plots to.",
+    )
+    plots_show_parser.add_argument(
+        "-s",
+        "--select",
+        default=None,
+        help="Choose which field(s) or JSONPath to include in the plots.",
+    )
+    plots_show_parser.add_argument(
+        "-x", default=None, help="Field name for x axis."
+    )
+    plots_show_parser.add_argument(
+        "-y", default=None, help="Field name for y axis."
+    )
+    plots_show_parser.add_argument(
+        "--no-csv-header",
+        action="store_true",
+        default=False,
+        help="Required when CSV or TSV datafile does not have a header.",
+    )
+    plots_show_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    plots_show_parser.add_argument("--title", default=None, help="Plot title.")
+    plots_show_parser.add_argument(
+        "--xlab", default=None, help="X axis title."
+    )
+    plots_show_parser.add_argument(
+        "--ylab", default=None, help="Y axis title."
+    )
+    plots_show_parser.add_argument(
+        "targets",
+        nargs="*",
+        help="Metrics files to visualize. Shows all plots by default.",
+    )
+    plots_show_parser.set_defaults(func=CmdPlotsShow)
+
+    PLOTS_DIFF_HELP = (
+        "Plot differences in metrics between commits in the DVC "
+        "repository, or between the last commit and the workspace."
+    )
+    plots_diff_parser = plots_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(PLOTS_DIFF_HELP, "plots/diff"),
+        help=PLOTS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_diff_parser.add_argument(
+        "-t",
+        "--template",
+        nargs="?",
+        default=None,
+        help="File to be injected with data.",
+    )
+    plots_diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help="Metrics file to visualize. Shows all plots by default.",
+    )
+    plots_diff_parser.add_argument(
+        "-o", "--out", default=None, help="Destination path to save plots to.",
+    )
+    plots_diff_parser.add_argument(
+        "-s",
+        "--select",
+        default=None,
+        help="Choose which field(s) or JSONPath to include in the plot.",
+    )
+    plots_diff_parser.add_argument(
+        "-x", default=None, help="Field name for x axis."
+    )
+    plots_diff_parser.add_argument(
+        "-y", default=None, help="Field name for y axis."
+    )
+    plots_diff_parser.add_argument(
+        "--no-csv-header",
+        action="store_true",
+        default=False,
+        help="Provided CSV ot TSV datafile does not have a header.",
+    )
+    plots_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    plots_diff_parser.add_argument("--title", default=None, help="Plot title.")
+    plots_diff_parser.add_argument(
+        "--xlab", default=None, help="X axis title."
+    )
+    plots_diff_parser.add_argument(
+        "--ylab", default=None, help="Y axis title."
+    )
+    plots_diff_parser.add_argument(
+        "revisions", nargs="*", default=None, help="Git commits to plot from",
+    )
+    plots_diff_parser.set_defaults(func=CmdPlotsDiff)
@@ -1,79 +1,142 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import fix_subparsers
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.remote.config import RemoteConfig
+from dvc.config import ConfigError
 from dvc.utils import format_link

 logger = logging.getLogger(__name__)


-class CmdRemoteConfig(CmdConfig):
+class CmdRemote(CmdConfig):
    def __init__(self, args):
        super().__init__(args)
-        self.remote_config = RemoteConfig(self.config)
+
+        if getattr(self.args, "name", None):
+            self.args.name = self.args.name.lower()
+
+    def _check_exists(self, conf):
+        if self.args.name not in conf["remote"]:
+            raise ConfigError(f"remote '{self.args.name}' doesn't exists.")


-class CmdRemoteAdd(CmdRemoteConfig):
+class CmdRemoteAdd(CmdRemote):
    def run(self):
        if self.args.default:
-            logger.info(
-                "Setting '{}' as a default remote.".format(self.args.name)
-            )
-        self.remote_config.add(
-            self.args.name,
-            self.args.url,
-            force=self.args.force,
-            default=self.args.default,
-            level=self.args.level,
-        )
+            logger.info(f"Setting '{self.args.name}' as a default remote.")
+
+        with self.config.edit(self.args.level) as conf:
+            if self.args.name in conf["remote"] and not self.args.force:
+                raise ConfigError(
+                    "remote '{}' already exists. Use `-f|--force` to "
+                    "overwrite it.".format(self.args.name)
+                )
+
+            conf["remote"][self.args.name] = {"url": self.args.url}
+            if self.args.default:
+                conf["core"]["remote"] = self.args.name
+
        return 0


-class CmdRemoteRemove(CmdRemoteConfig):
+class CmdRemoteRemove(CmdRemote):
    def run(self):
-        self.remote_config.remove(self.args.name, level=self.args.level)
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            del conf["remote"][self.args.name]
+
+        # Remove core.remote refs to this remote in any shadowing configs
+        for level in reversed(self.config.LEVELS):
+            with self.config.edit(level) as conf:
+                if conf["core"].get("remote") == self.args.name:
+                    del conf["core"]["remote"]
+
+            if level == self.args.level:
+                break
+
        return 0


-class CmdRemoteModify(CmdRemoteConfig):
+class CmdRemoteModify(CmdRemote):
    def run(self):
-        self.remote_config.modify(
-            self.args.name,
-            self.args.option,
-            self.args.value,
-            level=self.args.level,
-        )
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            conf["remote"][self.args.name][self.args.option] = self.args.value
        return 0


-class CmdRemoteDefault(CmdRemoteConfig):
+class CmdRemoteDefault(CmdRemote):
    def run(self):
+
        if self.args.name is None and not self.args.unset:
-            name = self.remote_config.get_default(level=self.args.level)
-            print(name)
+            conf = self.config.load_one(self.args.level)
+            try:
+                print(conf["core"]["remote"])
+            except KeyError:
+                logger.info("No default remote set")
+                return 1
        else:
-            self.remote_config.set_default(
-                self.args.name, unset=self.args.unset, level=self.args.level
-            )
+            with self.config.edit(self.args.level) as conf:
+                if self.args.unset:
+                    conf["core"].pop("remote", None)
+                else:
+                    merged_conf = self.config.load_config_to_level(
+                        self.args.level
+                    )
+                    if (
+                        self.args.name in conf["remote"]
+                        or self.args.name in merged_conf["remote"]
+                    ):
+                        conf["core"]["remote"] = self.args.name
+                    else:
+                        raise ConfigError(
+                            "default remote must be present in remote list."
+                        )
        return 0


-class CmdRemoteList(CmdRemoteConfig):
+class CmdRemoteList(CmdRemote):
    def run(self):
-        for name, url in self.remote_config.list(
-            level=self.args.level
-        ).items():
-            logger.info("{}\t{}".format(name, url))
+        conf = self.config.load_one(self.args.level)
+        for name, conf in conf["remote"].items():
+            logger.info("{}\t{}".format(name, conf["url"]))
+        return 0
+
+
+class CmdRemoteRename(CmdRemote):
+    def _rename_default(self, conf):
+        if conf["core"].get("remote") == self.args.name:
+            conf["core"]["remote"] = self.args.new
+
+    def run(self):
+        all_config = self.config.load_config_to_level(None)
+        if self.args.new in all_config.get("remote", {}):
+            raise ConfigError(
+                "Rename failed. Remote name '{}' already exists.".format(
+                    {self.args.new}
+                )
+            )
+
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            conf["remote"][self.args.new] = conf["remote"][self.args.name]
+            del conf["remote"][self.args.name]
+            self._rename_default(conf)
+
+        for level in reversed(self.config.LEVELS):
+            if level == self.args.level:
+                break
+            with self.config.edit(level) as level_conf:
+                self._rename_default(level_conf)
+
        return 0


 def add_parser(subparsers, parent_parser):
    from dvc.command.config import parent_config_parser

-    REMOTE_HELP = "Manage remote storage configuration."
+    REMOTE_HELP = "Set up and manage data remotes."
    remote_parser = subparsers.add_parser(
        "remote",
        parents=[parent_parser],
@@ -89,7 +152,7 @@ def add_parser(subparsers, parent_parser):

    fix_subparsers(remote_subparsers)

-    REMOTE_ADD_HELP = "Add remote."
+    REMOTE_ADD_HELP = "Add a new data remote."
    remote_add_parser = remote_subparsers.add_parser(
        "add",
        parents=[parent_config_parser, parent_parser],
@@ -120,7 +183,7 @@ def add_parser(subparsers, parent_parser):
    )
    remote_add_parser.set_defaults(func=CmdRemoteAdd)

-    REMOTE_DEFAULT_HELP = "Set/unset default remote."
+    REMOTE_DEFAULT_HELP = "Set/unset the default data remote."
    remote_default_parser = remote_subparsers.add_parser(
        "default",
        parents=[parent_config_parser, parent_parser],
@@ -140,20 +203,7 @@ def add_parser(subparsers, parent_parser):
    )
    remote_default_parser.set_defaults(func=CmdRemoteDefault)

-    REMOTE_REMOVE_HELP = "Remove remote."
-    remote_remove_parser = remote_subparsers.add_parser(
-        "remove",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
-        help=REMOTE_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    remote_remove_parser.add_argument(
-        "name", help="Name of the remote to remove."
-    )
-    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
-
-    REMOTE_MODIFY_HELP = "Modify remote."
+    REMOTE_MODIFY_HELP = "Modify the configuration of a data remote."
    remote_modify_parser = remote_subparsers.add_parser(
        "modify",
        parents=[parent_config_parser, parent_parser],
@@ -177,7 +227,7 @@ def add_parser(subparsers, parent_parser):
    )
    remote_modify_parser.set_defaults(func=CmdRemoteModify)

-    REMOTE_LIST_HELP = "List available remotes."
+    REMOTE_LIST_HELP = "List all available data remotes."
    remote_list_parser = remote_subparsers.add_parser(
        "list",
        parents=[parent_config_parser, parent_parser],
@@ -186,3 +236,27 @@ def add_parser(subparsers, parent_parser):
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    remote_list_parser.set_defaults(func=CmdRemoteList)
+
+    REMOTE_REMOVE_HELP = "Remove a data remote."
+    remote_remove_parser = remote_subparsers.add_parser(
+        "remove",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
+        help=REMOTE_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_remove_parser.add_argument(
+        "name", help="Name of the remote to remove."
+    )
+    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
+    REMOTE_RENAME_HELP = "Rename a DVC remote"
+    remote_rename_parser = remote_subparsers.add_parser(
+        "rename",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_RENAME_HELP, "remote/rename"),
+        help=REMOTE_RENAME_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_rename_parser.add_argument("name", help="Remote to be renamed")
+    remote_rename_parser.add_argument("new", help="New name of the remote")
+    remote_rename_parser.set_defaults(func=CmdRemoteRename)
@@ -2,11 +2,9 @@ import argparse
 import logging

 import dvc.prompt as prompt
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException

-
 logger = logging.getLogger(__name__)


@@ -36,13 +34,13 @@ class CmdRemove(CmdBase):
                outs_only = self._is_outs_only(target)
                self.repo.remove(target, outs_only=outs_only)
            except DvcException:
-                logger.exception("failed to remove '{}'".format(target))
+                logger.exception(f"failed to remove '{target}'")
                return 1
        return 0


 def add_parser(subparsers, parent_parser):
-    REMOVE_HELP = "Remove DVC-file outputs."
+    REMOVE_HELP = "Remove DVC-tracked files or directories."
    remove_parser = subparsers.add_parser(
        "remove",
        parents=[parent_parser],
@@ -2,13 +2,11 @@ import argparse
 import logging
 import os

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
 from dvc.exceptions import DvcException

-
 logger = logging.getLogger(__name__)


@@ -35,10 +33,11 @@ class CmdRepro(CmdBase):
                    interactive=self.args.interactive,
                    pipeline=self.args.pipeline,
                    all_pipelines=self.args.all_pipelines,
-                    ignore_build_cache=self.args.ignore_build_cache,
+                    run_cache=not self.args.no_run_cache,
                    no_commit=self.args.no_commit,
                    downstream=self.args.downstream,
                    recursive=self.args.recursive,
+                    force_downstream=self.args.force_downstream,
                )

                if len(stages) == 0:
@@ -57,7 +56,9 @@ class CmdRepro(CmdBase):


 def add_parser(subparsers, parent_parser):
-    REPRO_HELP = "Check for changes and reproduce stages and dependencies."
+    REPRO_HELP = (
+        "Reproduce complete or partial pipelines by executing their stages."
+    )
    repro_parser = subparsers.add_parser(
        "repro",
        parents=[parent_parser],
@@ -90,6 +91,7 @@ def add_parser(subparsers, parent_parser):
        "--cwd",
        default=os.path.curdir,
        help="Directory within your repo to reproduce from.",
+        metavar="<path>",
    )
    repro_parser.add_argument(
        "-m",
@@ -135,7 +137,16 @@ def add_parser(subparsers, parent_parser):
        help="Reproduce all stages in the specified directory.",
    )
    repro_parser.add_argument(
-        "--ignore-build-cache",
+        "--no-run-cache",
+        action="store_true",
+        default=False,
+        help=(
+            "Execute stage commands even if they have already been run with "
+            "the same command/dependencies/outputs/etc before."
+        ),
+    )
+    repro_parser.add_argument(
+        "--force-downstream",
        action="store_true",
        default=False,
        help="Reproduce all descendants of a changed stage even if their "
@@ -1,8 +1,7 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.utils import relpath

 logger = logging.getLogger(__name__)
@@ -17,7 +16,7 @@ class CmdRoot(CmdBaseNoRepo):


 def add_parser(subparsers, parent_parser):
-    ROOT_HELP = "Relative path to the repository's directory."
+    ROOT_HELP = "Return the relative path to the root of the DVC project."
    root_parser = subparsers.add_parser(
        "root",
        parents=[parent_parser],
@@ -1,18 +1,14 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException

-
 logger = logging.getLogger(__name__)


 class CmdRun(CmdBase):
    def run(self):
-        overwrite = self.args.yes or self.args.overwrite_dvcfile
-
        if not any(
            [
                self.args.deps,
@@ -20,15 +16,18 @@ class CmdRun(CmdBase):
                self.args.outs_no_cache,
                self.args.metrics,
                self.args.metrics_no_cache,
+                self.args.plots,
+                self.args.plots_no_cache,
                self.args.outs_persist,
                self.args.outs_persist_no_cache,
+                self.args.params,
                self.args.command,
            ]
        ):  # pragma: no cover
            logger.error(
                "too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
-                "`-m`, `-M`, `--outs-persist`, `--outs-persist-no-cache`, "
-                "`command`."
+                "`-m`, `-M`, `-p`, `--plots`, `--plots-no-cache`, "
+                "`--outs-persist`, `--outs-persist-no-cache`, `command`."
            )
            return 1

@@ -39,21 +38,24 @@ class CmdRun(CmdBase):
                outs_no_cache=self.args.outs_no_cache,
                metrics=self.args.metrics,
                metrics_no_cache=self.args.metrics_no_cache,
+                plots=self.args.plots,
+                plots_no_cache=self.args.plots_no_cache,
                deps=self.args.deps,
+                params=self.args.params,
                fname=self.args.file,
-                cwd=self.args.cwd,
                wdir=self.args.wdir,
                no_exec=self.args.no_exec,
-                overwrite=overwrite,
-                ignore_build_cache=self.args.ignore_build_cache,
-                remove_outs=self.args.remove_outs,
+                overwrite=self.args.overwrite_dvcfile,
+                run_cache=not self.args.no_run_cache,
                no_commit=self.args.no_commit,
                outs_persist=self.args.outs_persist,
                outs_persist_no_cache=self.args.outs_persist_no_cache,
                always_changed=self.args.always_changed,
+                name=self.args.name,
+                single_stage=self.args.single_stage,
            )
        except DvcException:
-            logger.exception("failed to run command")
+            logger.exception("")
            return 1

        return 0
@@ -77,7 +79,7 @@ class CmdRun(CmdBase):
        if " " not in argument or '"' in argument:
            return argument

-        return '"{}"'.format(argument)
+        return f'"{argument}"'


 def add_parser(subparsers, parent_parser):
@@ -95,6 +97,10 @@ def add_parser(subparsers, parent_parser):
        action="append",
        default=[],
        help="Declare dependencies for reproducible cmd.",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "-n", "--name", help="Stage name.",
    )
    run_parser.add_argument(
        "-o",
@@ -102,6 +108,7 @@ def add_parser(subparsers, parent_parser):
        action="append",
        default=[],
        help="Declare output file or directory.",
+        metavar="<filename>",
    )
    run_parser.add_argument(
        "-O",
@@ -110,6 +117,15 @@ def add_parser(subparsers, parent_parser):
        default=[],
        help="Declare output file or directory "
        "(do not put into DVC cache).",
+        metavar="<filename>",
+    )
+    run_parser.add_argument(
+        "-p",
+        "--params",
+        action="append",
+        default=[],
+        help="Declare parameter to use as additional dependency.",
+        metavar="[<filename>:]<params_list>",
    )
    run_parser.add_argument(
        "-m",
@@ -117,6 +133,7 @@ def add_parser(subparsers, parent_parser):
        action="append",
        default=[],
        help="Declare output metric file or directory.",
+        metavar="<path>",
    )
    run_parser.add_argument(
        "-M",
@@ -125,17 +142,33 @@ def add_parser(subparsers, parent_parser):
        default=[],
        help="Declare output metric file or directory "
        "(do not put into DVC cache).",
+        metavar="<path>",
    )
    run_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC-file it generates."
+        "--plots",
+        action="append",
+        default=[],
+        help="Declare output plot file.",
+        metavar="<path>",
    )
    run_parser.add_argument(
-        "-c", "--cwd", help="Deprecated, use -w and -f instead."
+        "--plots-no-cache",
+        action="append",
+        default=[],
+        help="Declare output plot file (do not put into DVC cache).",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
    )
    run_parser.add_argument(
        "-w",
        "--wdir",
        help="Directory within your repo to run your command in.",
+        metavar="<path>",
    )
    run_parser.add_argument(
        "--no-exec",
@@ -143,13 +176,6 @@ def add_parser(subparsers, parent_parser):
        default=False,
        help="Only create stage file without actually running it.",
    )
-    run_parser.add_argument(
-        "-y",
-        "--yes",
-        action="store_true",
-        default=False,
-        help="Deprecated, use --overwrite-dvcfile instead",
-    )
    run_parser.add_argument(
        "--overwrite-dvcfile",
        action="store_true",
@@ -157,17 +183,13 @@ def add_parser(subparsers, parent_parser):
        help="Overwrite existing DVC-file without asking for confirmation.",
    )
    run_parser.add_argument(
-        "--ignore-build-cache",
+        "--no-run-cache",
        action="store_true",
        default=False,
-        help="Run this stage even if it has been already ran with the same "
-        "command/dependencies/outputs/etc before.",
-    )
-    run_parser.add_argument(
-        "--remove-outs",
-        action="store_true",
-        default=False,
-        help="Deprecated, this is now the default behavior",
+        help=(
+            "Execute the command even if this stage has already been run "
+            "with the same command/dependencies/outputs/etc before."
+        ),
    )
    run_parser.add_argument(
        "--no-commit",
@@ -181,6 +203,7 @@ def add_parser(subparsers, parent_parser):
        default=[],
        help="Declare output file or directory that will not be "
        "removed upon repro.",
+        metavar="<filename>",
    )
    run_parser.add_argument(
        "--outs-persist-no-cache",
@@ -188,6 +211,7 @@ def add_parser(subparsers, parent_parser):
        default=[],
        help="Declare output file or directory that will not be "
        "removed upon repro (do not put into DVC cache).",
+        metavar="<filename>",
    )
    run_parser.add_argument(
        "--always-changed",
@@ -195,6 +219,12 @@ def add_parser(subparsers, parent_parser):
        default=False,
        help="Always consider this DVC-file as changed.",
    )
+    run_parser.add_argument(
+        "--single-stage",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
    run_parser.add_argument(
        "command", nargs=argparse.REMAINDER, help="Command to execute."
    )
@@ -2,7 +2,6 @@ import logging

 from dvc.command.data_sync import CmdDataBase

-
 logger = logging.getLogger(__name__)


@@ -20,7 +19,7 @@ class CmdDataStatus(CmdDataBase):
        ind = indent * self.STATUS_INDENT

        if isinstance(status, str):
-            logger.info("{}{}".format(ind, status))
+            logger.info(f"{ind}{status}")
            return

        if isinstance(status, list):
@@ -34,7 +33,7 @@ class CmdDataStatus(CmdDataBase):
            if isinstance(value, str):
                logger.info("{}{}{}".format(ind, self._normalize(value), key))
            elif value:
-                logger.info("{}{}:".format(ind, key))
+                logger.info(f"{ind}{key}:")
                self._show(value, indent + 1)

    def run(self):
@@ -47,7 +46,9 @@ class CmdDataStatus(CmdDataBase):
                remote=self.args.remote,
                all_branches=self.args.all_branches,
                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                with_deps=self.args.with_deps,
+                recursive=self.args.recursive,
            )
            if st:
                if self.args.quiet:
@@ -1,156 +0,0 @@
-import logging
-
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import DvcException
-from dvc.utils import to_yaml_string
-
-
-logger = logging.getLogger(__name__)
-
-
-class CmdTagAdd(CmdBase):
-    def run(self):
-        for target in self.args.targets:
-            try:
-                self.repo.tag.add(
-                    self.args.tag,
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    recursive=self.args.recursive,
-                )
-            except DvcException:
-                logger.exception("failed to add tag")
-                return 1
-        return 0
-
-
-class CmdTagRemove(CmdBase):
-    def run(self):
-        for target in self.args.targets:
-            try:
-                self.repo.tag.remove(
-                    self.args.tag,
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    recursive=self.args.recursive,
-                )
-            except DvcException:
-                logger.exception("failed to remove tag")
-                return 1
-        return 0
-
-
-class CmdTagList(CmdBase):
-    def run(self):
-        for target in self.args.targets:
-            try:
-                tags = self.repo.tag.list(
-                    target,
-                    with_deps=self.args.with_deps,
-                    recursive=self.args.recursive,
-                )
-                if tags:
-                    logger.info(to_yaml_string(tags))
-            except DvcException:
-                logger.exception("failed list tags")
-                return 1
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    TAG_HELP = "A set of commands to manage DVC tags."
-    tag_parser = subparsers.add_parser(
-        "tag",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_HELP, "tag"),
-        add_help=False,
-    )
-
-    tag_subparsers = tag_parser.add_subparsers(
-        dest="cmd",
-        help="Use DVC tag CMD --help to display command-specific help.",
-    )
-
-    fix_subparsers(tag_subparsers)
-
-    TAG_ADD_HELP = "Add DVC tag."
-    tag_add_parser = tag_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_ADD_HELP, "tag-add"),
-        help=TAG_ADD_HELP,
-    )
-    tag_add_parser.add_argument("tag", help="Dvc tag.")
-    tag_add_parser.add_argument(
-        "targets", nargs="*", default=[None], help="DVC-files."
-    )
-    tag_add_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="Add tag for all dependencies of the specified DVC-file.",
-    )
-    tag_add_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="Add tag for subdirectories of the specified directory.",
-    )
-    tag_add_parser.set_defaults(func=CmdTagAdd)
-
-    TAG_REMOVE_HELP = "Remove DVC tag."
-    tag_remove_parser = tag_subparsers.add_parser(
-        "remove",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_REMOVE_HELP, "tag-remove"),
-        help=TAG_REMOVE_HELP,
-    )
-    tag_remove_parser.add_argument("tag", help="Dvc tag.")
-    tag_remove_parser.add_argument(
-        "targets", nargs="*", default=[None], help="DVC-files."
-    )
-    tag_remove_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="Remove tag for all dependencies of the specified DVC-file.",
-    )
-    tag_remove_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="Remove tag for subdirectories of the specified directory.",
-    )
-    tag_remove_parser.set_defaults(func=CmdTagRemove)
-
-    TAG_LIST_HELP = "List DVC tags."
-    tag_list_parser = tag_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_LIST_HELP, "tag-list"),
-        help=TAG_LIST_HELP,
-    )
-    tag_list_parser.add_argument(
-        "targets", nargs="*", default=[None], help="DVC-files."
-    )
-    tag_list_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="List tags for all dependencies of the specified DVC-file.",
-    )
-    tag_list_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="List tags for subdirectories of the specified directory.",
-    )
-    tag_list_parser.set_defaults(func=CmdTagList)
@@ -1,11 +1,9 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException

-
 logger = logging.getLogger(__name__)


@@ -15,14 +13,17 @@ class CmdUnprotect(CmdBase):
            try:
                self.repo.unprotect(target)
            except DvcException:
-                msg = "failed to unprotect '{}'".format(target)
+                msg = f"failed to unprotect '{target}'"
                logger.exception(msg)
                return 1
        return 0


 def add_parser(subparsers, parent_parser):
-    UNPROTECT_HELP = "Unprotect data files or directories."
+    UNPROTECT_HELP = (
+        "Unprotect tracked files or directories (when hardlinks or symlinks "
+        "have been enabled with `dvc config cache.type`)"
+    )
    unprotect_parser = subparsers.add_parser(
        "unprotect",
        parents=[parent_parser],
@@ -1,23 +1,24 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException

-
 logger = logging.getLogger(__name__)


 class CmdUpdate(CmdBase):
    def run(self):
        ret = 0
-        for target in self.args.targets:
-            try:
-                self.repo.update(target)
-            except DvcException:
-                logger.exception("failed to update '{}'.".format(target))
-                ret = 1
+        try:
+            self.repo.update(
+                targets=self.args.targets,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+            )
+        except DvcException:
+            logger.exception("failed update data")
+            ret = 1
        return ret


@@ -33,4 +34,17 @@ def add_parser(subparsers, parent_parser):
    update_parser.add_argument(
        "targets", nargs="+", help="DVC-files to update."
    )
+    update_parser.add_argument(
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    update_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Update all stages in the specified directory.",
+    )
    update_parser.set_defaults(func=CmdUpdate)
@@ -2,21 +2,23 @@ import argparse
 import itertools
 import logging
 import os
+import pathlib
 import platform
 import uuid
-import pathlib
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.scm.base import SCMError
+from dvc.system import System
+from dvc.utils import is_binary, relpath
+from dvc.utils.pkg import PKG
+from dvc.version import __version__

 try:
    import psutil
 except ImportError:
    psutil = None

-from dvc.utils import is_binary, relpath
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.version import __version__
-from dvc.exceptions import DvcException, NotDvcRepoError
-from dvc.system import System
-from dvc.utils.pkg import PKG

 logger = logging.getLogger(__name__)

@@ -26,11 +28,12 @@ class CmdVersion(CmdBaseNoRepo):
        from dvc.repo import Repo

        info = [
-            "DVC version: {}".format(__version__),
-            "Python version: {}".format(platform.python_version()),
-            "Platform: {}".format(platform.platform()),
-            "Binary: {}".format(is_binary()),
-            "Package: {}".format(PKG),
+            f"DVC version: {__version__}",
+            f"Python version: {platform.python_version()}",
+            f"Platform: {platform.platform()}",
+            f"Binary: {is_binary()}",
+            f"Package: {PKG}",
+            f"Supported remotes: {self.get_supported_remotes()}",
        ]

        try:
@@ -48,7 +51,7 @@ class CmdVersion(CmdBaseNoRepo):
                if psutil:
                    fs_type = self.get_fs_type(repo.cache.local.cache_dir)
                    info.append(
-                        "Filesystem type (cache directory): {}".format(fs_type)
+                        f"Filesystem type (cache directory): {fs_type}"
                    )
            else:
                logger.warning(
@@ -61,10 +64,15 @@ class CmdVersion(CmdBaseNoRepo):

        except NotDvcRepoError:
            root_directory = os.getcwd()
+        except SCMError:
+            root_directory = os.getcwd()
+            info.append("Repo: dvc, git (broken)")
+        else:
+            info.append("Repo: {}".format(_get_dvc_repo_info(repo)))

        if psutil:
            fs_root = self.get_fs_type(os.path.abspath(root_directory))
-            info.append("Filesystem type (workspace): {}".format(fs_root))
+            info.append(f"Filesystem type (workspace): {fs_root}")

        logger.info("\n".join(info))
        return 0
@@ -107,15 +115,37 @@ class CmdVersion(CmdBaseNoRepo):
                os.unlink(dst)
            except DvcException:
                status = "not supported"
-            cache.append("{name} - {status}".format(name=name, status=status))
+            cache.append(f"{name} - {status}")
        os.remove(src)

        return ", ".join(cache)

+    @staticmethod
+    def get_supported_remotes():
+        from dvc.remote import REMOTES
+
+        supported_remotes = []
+        for remote in REMOTES:
+            if not remote.get_missing_deps():
+                supported_remotes.append(remote.scheme)
+
+        return ", ".join(supported_remotes)
+
+
+def _get_dvc_repo_info(repo):
+    if repo.config.get("core", {}).get("no_scm", False):
+        return "dvc (no_scm)"
+
+    if repo.root_dir != repo.scm.root_dir:
+        return "dvc (subdir), git"
+
+    return "dvc, git"
+

 def add_parser(subparsers, parent_parser):
-    VERSION_HELP = "Show DVC version and system/environment information."
-
+    VERSION_HELP = (
+        "Display the DVC version and system/environment information."
+    )
    version_parser = subparsers.add_parser(
        "version",
        parents=[parent_parser],
@@ -1,42 +0,0 @@
-"""Python version compatibility code"""
-
-# Backport os.fspath() from Python 3.6
-try:
-    from os import fspath  # noqa: F821
-
-    fspath_py35 = lambda s: s  # noqa: E731
-except ImportError:
-
-    def fspath(path):
-        """Return the path representation of a path-like object.
-
-        If str or bytes is passed in, it is returned unchanged. Otherwise the
-        os.PathLike interface is used to get the path representation. If the
-        path representation is not str or bytes, TypeError is raised. If the
-        provided path is not str, bytes, or os.PathLike, TypeError is raised.
-        """
-        if isinstance(path, (str, bytes)):
-            return path
-
-        # Work from the object's type to match method resolution of other magic
-        # methods.
-        path_type = type(path)
-        try:
-            path_repr = path_type.__fspath__(path)
-        except AttributeError:
-            if hasattr(path_type, "__fspath__"):
-                raise
-            else:
-                raise TypeError(
-                    "expected str, bytes or os.PathLike object, "
-                    "not " + path_type.__name__
-                )
-        if isinstance(path_repr, (str, bytes)):
-            return path_repr
-        else:
-            raise TypeError(
-                "expected {}.__fspath__() to return str or bytes, "
-                "not {}".format(path_type.__name__, type(path_repr).__name__)
-            )
-
-    fspath_py35 = fspath
@@ -1,17 +1,28 @@
 """DVC config objects."""
-
-import copy
-import errno
 import logging
 import os
 import re
+from contextlib import contextmanager
+from functools import partial
+from urllib.parse import urlparse

 import configobj
-from voluptuous import Schema, Required, Optional, Invalid
-from voluptuous import All, Any, Lower, Range, Coerce, Match
+from funcy import cached_property, compact, re_find, walk_values
+from voluptuous import (
+    ALLOW_EXTRA,
+    All,
+    Any,
+    Coerce,
+    Invalid,
+    Lower,
+    Optional,
+    Range,
+    Schema,
+)

-from dvc.exceptions import DvcException
-from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.path_info import PathInfo
+from dvc.utils import relpath

 logger = logging.getLogger(__name__)

@@ -20,7 +31,7 @@ class ConfigError(DvcException):
    """DVC config exception."""

    def __init__(self, msg):
-        super().__init__("config file error: {}".format(msg))
+        super().__init__(f"config file error: {msg}")


 class NoRemoteError(ConfigError):
@@ -48,13 +59,12 @@ def supported_cache_type(types):


 # Checks that value is either true or false and converts it to bool
-Bool = All(
+to_bool = Bool = All(
    Lower,
    Any("true", "false"),
    lambda v: v == "true",
    msg="expected true or false",
 )
-to_bool = Schema(Bool)


 def Choices(*choices):
@@ -67,7 +77,138 @@ def Choices(*choices):
    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))


-class Config(object):  # pylint: disable=too-many-instance-attributes
+def ByUrl(mapping):
+    schemas = walk_values(Schema, mapping)
+
+    def validate(data):
+        if "url" not in data:
+            raise Invalid("expected 'url'")
+
+        parsed = urlparse(data["url"])
+        # Windows absolute paths should really have scheme == "" (local)
+        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+            return schemas[""](data)
+        if parsed.scheme not in schemas:
+            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
+
+        return schemas[parsed.scheme](data)
+
+    return validate
+
+
+class RelPath(str):
+    pass
+
+
+REMOTE_COMMON = {
+    "url": str,
+    "checksum_jobs": All(Coerce(int), Range(1)),
+    Optional("no_traverse"): Bool,  # obsoleted
+    "verify": Bool,
+}
+LOCAL_COMMON = {
+    "type": supported_cache_type,
+    Optional("protected", default=False): Bool,  # obsoleted
+    "shared": All(Lower, Choices("group")),
+    Optional("slow_link_warning", default=True): Bool,
+}
+HTTP_COMMON = {
+    "auth": All(Lower, Choices("basic", "digest", "custom")),
+    "custom_auth_header": str,
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+}
+SCHEMA = {
+    "core": {
+        "remote": Lower,
+        "checksum_jobs": All(Coerce(int), Range(1)),
+        Optional("interactive", default=False): Bool,
+        Optional("analytics", default=True): Bool,
+        Optional("hardlink_lock", default=False): Bool,
+        Optional("no_scm", default=False): Bool,
+    },
+    "cache": {
+        "local": str,
+        "s3": str,
+        "gs": str,
+        "hdfs": str,
+        "ssh": str,
+        "azure": str,
+        # This is for default local cache
+        "dir": str,
+        **LOCAL_COMMON,
+    },
+    "remote": {
+        str: ByUrl(
+            {
+                "": {**LOCAL_COMMON, **REMOTE_COMMON},
+                "s3": {
+                    "region": str,
+                    "profile": str,
+                    "credentialpath": str,
+                    "endpointurl": str,
+                    Optional("listobjects", default=False): Bool,
+                    Optional("use_ssl", default=True): Bool,
+                    "sse": str,
+                    "sse_kms_key_id": str,
+                    "acl": str,
+                    "grant_read": str,
+                    "grant_read_acp": str,
+                    "grant_write_acp": str,
+                    "grant_full_control": str,
+                    **REMOTE_COMMON,
+                },
+                "gs": {
+                    "projectname": str,
+                    "credentialpath": str,
+                    **REMOTE_COMMON,
+                },
+                "ssh": {
+                    "type": supported_cache_type,
+                    "port": Coerce(int),
+                    "user": str,
+                    "password": str,
+                    "ask_password": Bool,
+                    "keyfile": str,
+                    "timeout": Coerce(int),
+                    "gss_auth": Bool,
+                    **REMOTE_COMMON,
+                },
+                "hdfs": {"user": str, **REMOTE_COMMON},
+                "azure": {"connection_string": str, **REMOTE_COMMON},
+                "oss": {
+                    "oss_key_id": str,
+                    "oss_key_secret": str,
+                    "oss_endpoint": str,
+                    **REMOTE_COMMON,
+                },
+                "gdrive": {
+                    "gdrive_use_service_account": Bool,
+                    "gdrive_client_id": str,
+                    "gdrive_client_secret": str,
+                    "gdrive_user_credentials_file": str,
+                    "gdrive_service_account_email": str,
+                    "gdrive_service_account_user_email": str,
+                    "gdrive_service_account_p12_file_path": str,
+                    Optional("gdrive_trash_only", default=False): Bool,
+                    **REMOTE_COMMON,
+                },
+                "http": {**HTTP_COMMON, **REMOTE_COMMON},
+                "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "remote": {str: object},  # Any of the above options are valid
+            }
+        )
+    },
+    "state": {
+        "row_limit": All(Coerce(int), Range(1)),
+        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
+    },
+}
+COMPILED_SCHEMA = Schema(SCHEMA)
+
+
+class Config(dict):
    """Class that manages configuration files for a DVC repo.

    Args:
@@ -77,176 +218,21 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
        validate (bool): optional flag to tell dvc if it should validate the
            config or just load it as is. 'True' by default.

-
    Raises:
-        ConfigError: thrown when config has an invalid format.
+        ConfigError: thrown if config has an invalid format.
    """

    APPNAME = "dvc"
    APPAUTHOR = "iterative"

-    # NOTE: used internally in RemoteLOCAL to know config
-    # location, that url should resolved relative to.
-    PRIVATE_CWD = "_cwd"
+    # In the order they shadow each other
+    LEVELS = ("system", "global", "repo", "local")

    CONFIG = "config"
    CONFIG_LOCAL = "config.local"

-    CREDENTIALPATH = "credentialpath"
-
-    LEVEL_LOCAL = 0
-    LEVEL_REPO = 1
-    LEVEL_GLOBAL = 2
-    LEVEL_SYSTEM = 3
-
-    SECTION_CORE = "core"
-    SECTION_CORE_LOGLEVEL = "loglevel"
-    SECTION_CORE_LOGLEVEL_SCHEMA = All(
-        Lower, Choices("info", "debug", "warning", "error")
-    )
-    SECTION_CORE_REMOTE = "remote"
-    SECTION_CORE_INTERACTIVE = "interactive"
-    SECTION_CORE_ANALYTICS = "analytics"
-    SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_CORE_HARDLINK_LOCK = "hardlink_lock"
-
-    SECTION_CACHE = "cache"
-    SECTION_CACHE_DIR = "dir"
-    SECTION_CACHE_TYPE = "type"
-    SECTION_CACHE_PROTECTED = "protected"
-    SECTION_CACHE_SHARED = "shared"
-    SECTION_CACHE_SHARED_SCHEMA = All(Lower, Choices("group"))
-    SECTION_CACHE_LOCAL = "local"
-    SECTION_CACHE_S3 = "s3"
-    SECTION_CACHE_GS = "gs"
-    SECTION_CACHE_SSH = "ssh"
-    SECTION_CACHE_HDFS = "hdfs"
-    SECTION_CACHE_AZURE = "azure"
-    SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
-    SECTION_CACHE_SCHEMA = {
-        SECTION_CACHE_LOCAL: str,
-        SECTION_CACHE_S3: str,
-        SECTION_CACHE_GS: str,
-        SECTION_CACHE_HDFS: str,
-        SECTION_CACHE_SSH: str,
-        SECTION_CACHE_AZURE: str,
-        SECTION_CACHE_DIR: str,
-        SECTION_CACHE_TYPE: supported_cache_type,
-        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
-        SECTION_CACHE_SHARED: SECTION_CACHE_SHARED_SCHEMA,
-        PRIVATE_CWD: str,
-        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): Bool,
-    }
-
-    SECTION_CORE_SCHEMA = {
-        SECTION_CORE_LOGLEVEL: SECTION_CORE_LOGLEVEL_SCHEMA,
-        SECTION_CORE_REMOTE: Lower,
-        Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
-        Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
-        SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
-        Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool,
-    }
-
-    # aws specific options
-    SECTION_AWS_CREDENTIALPATH = CREDENTIALPATH
-    SECTION_AWS_ENDPOINT_URL = "endpointurl"
-    SECTION_AWS_LIST_OBJECTS = "listobjects"
-    SECTION_AWS_REGION = "region"
-    SECTION_AWS_PROFILE = "profile"
-    SECTION_AWS_USE_SSL = "use_ssl"
-    SECTION_AWS_SSE = "sse"
-    SECTION_AWS_ACL = "acl"
-    SECTION_AWS_GRANT_READ = "grant_read"
-    SECTION_AWS_GRANT_READ_ACP = "grant_read_acp"
-    SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp"
-    SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control"
-
-    # gcp specific options
-    SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH
-    SECTION_GCP_PROJECTNAME = "projectname"
-
-    # azure specific option
-    SECTION_AZURE_CONNECTION_STRING = "connection_string"
-
-    # Alibabacloud oss options
-    SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
-    SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
-    SECTION_OSS_ENDPOINT = "oss_endpoint"
-
-    # GDrive options
-    SECTION_GDRIVE_CLIENT_ID = "gdrive_client_id"
-    SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret"
-    SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file"
-
-    SECTION_REMOTE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
-    SECTION_REMOTE_FMT = 'remote "{}"'
-    SECTION_REMOTE_URL = "url"
-    SECTION_REMOTE_USER = "user"
-    SECTION_REMOTE_PORT = "port"
-    SECTION_REMOTE_KEY_FILE = "keyfile"
-    SECTION_REMOTE_TIMEOUT = "timeout"
-    SECTION_REMOTE_PASSWORD = "password"
-    SECTION_REMOTE_ASK_PASSWORD = "ask_password"
-    SECTION_REMOTE_GSS_AUTH = "gss_auth"
-    SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
-    SECTION_REMOTE_VERIFY = "verify"
-    SECTION_REMOTE_SCHEMA = {
-        Required(SECTION_REMOTE_URL): str,
-        SECTION_AWS_REGION: str,
-        SECTION_AWS_PROFILE: str,
-        SECTION_AWS_CREDENTIALPATH: str,
-        SECTION_AWS_ENDPOINT_URL: str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): Bool,
-        Optional(SECTION_AWS_USE_SSL, default=True): Bool,
-        SECTION_AWS_SSE: str,
-        SECTION_AWS_ACL: str,
-        SECTION_AWS_GRANT_READ: str,
-        SECTION_AWS_GRANT_READ_ACP: str,
-        SECTION_AWS_GRANT_WRITE_ACP: str,
-        SECTION_AWS_GRANT_FULL_CONTROL: str,
-        SECTION_GCP_PROJECTNAME: str,
-        SECTION_CACHE_TYPE: supported_cache_type,
-        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
-        SECTION_REMOTE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
-        SECTION_REMOTE_USER: str,
-        SECTION_REMOTE_PORT: Coerce(int),
-        SECTION_REMOTE_KEY_FILE: str,
-        SECTION_REMOTE_TIMEOUT: Coerce(int),
-        SECTION_REMOTE_PASSWORD: str,
-        SECTION_REMOTE_ASK_PASSWORD: Bool,
-        SECTION_REMOTE_GSS_AUTH: Bool,
-        SECTION_AZURE_CONNECTION_STRING: str,
-        SECTION_OSS_ACCESS_KEY_ID: str,
-        SECTION_OSS_ACCESS_KEY_SECRET: str,
-        SECTION_OSS_ENDPOINT: str,
-        SECTION_GDRIVE_CLIENT_ID: str,
-        SECTION_GDRIVE_CLIENT_SECRET: str,
-        SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
-        PRIVATE_CWD: str,
-        SECTION_REMOTE_NO_TRAVERSE: Bool,
-        SECTION_REMOTE_VERIFY: Bool,
-    }
-
-    SECTION_STATE = "state"
-    SECTION_STATE_ROW_LIMIT = "row_limit"
-    SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
-    SECTION_STATE_SCHEMA = {
-        SECTION_STATE_ROW_LIMIT: All(Coerce(int), Range(1)),
-        SECTION_STATE_ROW_CLEANUP_QUOTA: All(Coerce(int), Range(0, 100)),
-    }
-
-    SCHEMA = {
-        Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
-        Match(SECTION_REMOTE_REGEX): SECTION_REMOTE_SCHEMA,
-        Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
-        Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
-    }
-    COMPILED_SCHEMA = Schema(SCHEMA)
-
    def __init__(self, dvc_dir=None, validate=True):
        self.dvc_dir = dvc_dir
-        self.should_validate = validate

        if not dvc_dir:
            try:
@@ -258,33 +244,31 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
        else:
            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))

-        self.load()
+        self.load(validate=validate)

-    @staticmethod
-    def get_global_config_dir():
-        """Returns global config location. E.g. ~/.config/dvc/config.
+    @classmethod
+    def get_dir(cls, level):
+        from appdirs import user_config_dir, site_config_dir

-        Returns:
-            str: path to the global config directory.
-        """
-        from appdirs import user_config_dir
+        assert level in ("global", "system")

-        return user_config_dir(
-            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
-        )
+        if level == "global":
+            return user_config_dir(cls.APPNAME, cls.APPAUTHOR)
+        if level == "system":
+            return site_config_dir(cls.APPNAME, cls.APPAUTHOR)

-    @staticmethod
-    def get_system_config_dir():
-        """Returns system config location. E.g. /etc/dvc.conf.
+    @cached_property
+    def files(self):
+        files = {
+            level: os.path.join(self.get_dir(level), self.CONFIG)
+            for level in ("system", "global")
+        }

-        Returns:
-            str: path to the system config directory.
-        """
-        from appdirs import site_config_dir
+        if self.dvc_dir is not None:
+            files["repo"] = os.path.join(self.dvc_dir, self.CONFIG)
+            files["local"] = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)

-        return site_config_dir(
-            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
-        )
+        return files

    @staticmethod
    def init(dvc_dir):
@@ -300,250 +284,156 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
        open(config_file, "w+").close()
        return Config(dvc_dir)

-    def _resolve_cache_path(self, config):
-        cache = config.get(self.SECTION_CACHE)
-        if cache is None:
-            return
-
-        cache_dir = cache.get(self.SECTION_CACHE_DIR)
-        if cache_dir is None:
-            return
-
-        cache[self.PRIVATE_CWD] = os.path.dirname(config.filename)
-
-    def _resolve_paths(self, config):
-        if config.filename is None:
-            return config
-
-        ret = copy.deepcopy(config)
-        self._resolve_cache_path(ret)
-
-        for section in ret.values():
-            if self.SECTION_REMOTE_URL not in section.keys():
-                continue
-
-            section[self.PRIVATE_CWD] = os.path.dirname(ret.filename)
-
-        return ret
-
-    def _load_configs(self):
-        system_config_file = os.path.join(
-            self.get_system_config_dir(), self.CONFIG
-        )
-
-        global_config_file = os.path.join(
-            self.get_global_config_dir(), self.CONFIG
-        )
-
-        self._system_config = configobj.ConfigObj(system_config_file)
-        self._global_config = configobj.ConfigObj(global_config_file)
-        self._repo_config = configobj.ConfigObj()
-        self._local_config = configobj.ConfigObj()
-
-        if not self.dvc_dir:
-            return
-
-        config_file = os.path.join(self.dvc_dir, self.CONFIG)
-        config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
-
-        self._repo_config = configobj.ConfigObj(config_file)
-        self._local_config = configobj.ConfigObj(config_local_file)
-
-    @property
-    def config_local_file(self):
-        return self._local_config.filename
-
-    @property
-    def config_file(self):
-        return self._repo_config.filename
-
-    def load(self):
+    def load(self, validate=True):
        """Loads config from all the config files.

        Raises:
-            dvc.config.ConfigError: thrown if config has invalid format.
+            ConfigError: thrown if config has an invalid format.
        """
-        self._load_configs()
+        conf = self.load_config_to_level()

-        self.config = configobj.ConfigObj()
-        for c in [
-            self._system_config,
-            self._global_config,
-            self._repo_config,
-            self._local_config,
-        ]:
-            c = self._resolve_paths(c)
-            c = self._lower(c)
-            self.config.merge(c)
+        if validate:
+            conf = self.validate(conf)

-        if not self.should_validate:
-            return
+        self.clear()
+        self.update(conf)

-        d = self.validate(self.config)
-        self.config = configobj.ConfigObj(d, write_empty_values=True)
+        # Add resolved default cache.dir
+        if not self["cache"].get("dir") and self.dvc_dir:
+            self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")

-    def save(self, config=None):
-        """Saves config to config files.
+    def load_one(self, level):
+        conf = _load_config(self.files[level])
+        conf = self._load_paths(conf, self.files[level])

-        Raises:
-            dvc.config.ConfigError: thrown if failed to write config file.
-        """
-        if config is not None:
-            clist = [config]
-        else:
-            clist = [
-                self._system_config,
-                self._global_config,
-                self._repo_config,
-                self._local_config,
-            ]
+        # Auto-verify sections
+        for key in COMPILED_SCHEMA.schema:
+            conf.setdefault(key, {})

-        for conf in clist:
-            self._save(conf)
+        return conf

+    @staticmethod
+    def _load_paths(conf, filename):
+        abs_conf_dir = os.path.abspath(os.path.dirname(filename))
+
+        def resolve(path):
+            if os.path.isabs(path) or re.match(r"\w+://", path):
+                return path
+            return RelPath(os.path.join(abs_conf_dir, path))
+
+        return Config._map_dirs(conf, resolve)
+
+    @staticmethod
+    def _to_relpath(conf_dir, path):
+        if re.match(r"\w+://", path):
+            return path
+
+        if isinstance(path, RelPath) or not os.path.isabs(path):
+            path = relpath(path, conf_dir)
+
+        return PathInfo(path).as_posix()
+
+    @staticmethod
+    def _save_paths(conf, filename):
+        conf_dir = os.path.dirname(filename)
+        rel = partial(Config._to_relpath, conf_dir)
+
+        return Config._map_dirs(conf, rel)
+
+    @staticmethod
+    def _map_dirs(conf, func):
+        dirs_schema = {
+            "cache": {"dir": func},
+            "remote": {
+                str: {"url": func, "gdrive_user_credentials_file": func}
+            },
+        }
+        return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
+
+    def load_config_to_level(self, level=None):
+        merged_conf = {}
+        for merge_level in self.LEVELS:
+            if merge_level == level:
+                break
+            if merge_level in self.files:
+                _merge(merged_conf, self.load_one(merge_level))
+        return merged_conf
+
+    @contextmanager
+    def edit(self, level="repo"):
+        if level in {"repo", "local"} and self.dvc_dir is None:
+            raise ConfigError("Not inside a DVC repo")
+
+        conf = self.load_one(level)
+        yield conf
+
+        conf = self._save_paths(conf, self.files[level])
+
+        merged_conf = self.load_config_to_level(level)
+        _merge(merged_conf, conf)
+        self.validate(merged_conf)
+
+        _save_config(self.files[level], conf)
        self.load()

    @staticmethod
-    def _save(config):
-        if config.filename is None:
-            return
-
-        logger.debug("Writing '{}'.".format(config.filename))
-        dname = os.path.dirname(os.path.abspath(config.filename))
+    def validate(data):
        try:
-            os.makedirs(dname)
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                raise
-        config.write()
-
-    def validate(self, config):
-        try:
-            return self.COMPILED_SCHEMA(config.dict())
+            return COMPILED_SCHEMA(data)
        except Invalid as exc:
-            raise ConfigError(str(exc)) from exc
+            raise ConfigError(str(exc)) from None

-    def unset(self, section, opt=None, level=None, force=False):
-        """Unsets specified option and/or section in the config.

-        Args:
-            section (str): section name.
-            opt (str): optional option name.
-            level (int): config level to use.
-            force (bool): don't error-out even if section doesn't exist. False
-                by default.
+def _load_config(filename):
+    conf_obj = configobj.ConfigObj(filename)
+    return _parse_remotes(_lower_keys(conf_obj.dict()))

-        Raises:
-            dvc.config.ConfigError: thrown if section doesn't exist and
-                `force != True`.
-        """
-        config = self.get_configobj(level)

-        if section not in config.keys():
-            if force:
-                return
-            raise ConfigError("section '{}' doesn't exist".format(section))
+def _save_config(filename, conf_dict):
+    logger.debug(f"Writing '{filename}'.")
+    os.makedirs(os.path.dirname(filename), exist_ok=True)

-        if opt:
-            if opt not in config[section].keys():
-                if force:
-                    return
-                raise ConfigError(
-                    "option '{}.{}' doesn't exist".format(section, opt)
-                )
-            del config[section][opt]
+    config = configobj.ConfigObj(_pack_remotes(conf_dict))
+    config.filename = filename
+    config.write()

-            if not config[section]:
-                del config[section]
+
+def _parse_remotes(conf):
+    result = {"remote": {}}
+
+    for section, val in conf.items():
+        name = re_find(r'^\s*remote\s*"(.*)"\s*$', section)
+        if name:
+            result["remote"][name] = val
        else:
-            del config[section]
+            result[section] = val

-        self.save(config)
+    return result

-    def set(self, section, opt, value, level=None, force=True):
-        """Sets specified option in the config.

-        Args:
-            section (str): section name.
-            opt (str): option name.
-            value: value to set option to.
-            level (int): config level to use.
-            force (bool): set option even if section already exists. True by
-                default.
+def _pack_remotes(conf):
+    # Drop empty sections
+    result = compact(conf)

-        Raises:
-            dvc.config.ConfigError: thrown if section already exists and
-                `force != True`.
+    # Transform remote.name -> 'remote "name"'
+    for name, val in conf["remote"].items():
+        result[f'remote "{name}"'] = val
+    result.pop("remote", None)

-        """
-        config = self.get_configobj(level)
+    return result

-        if section not in config.keys():
-            config[section] = {}
-        elif not force:
-            raise ConfigError(
-                "Section '{}' already exists. Use `-f|--force` to overwrite "
-                "section with new value.".format(section)
-            )

-        config[section][opt] = value
+def _merge(into, update):
+    """Merges second dict into first recursively"""
+    for key, val in update.items():
+        if isinstance(into.get(key), dict) and isinstance(val, dict):
+            _merge(into[key], val)
+        else:
+            into[key] = val

-        result = copy.deepcopy(self.config)
-        result.merge(config)
-        self.validate(result)

-        self.save(config)
-
-    def get(self, section, opt=None, level=None):
-        """Return option value from the config.
-
-        Args:
-            section (str): section name.
-            opt (str): option name.
-            level (int): config level to use.
-
-        Returns:
-            value (str, int): option value.
-        """
-        config = self.get_configobj(level)
-
-        if section not in config.keys():
-            raise ConfigError("section '{}' doesn't exist".format(section))
-
-        if opt not in config[section].keys():
-            raise ConfigError(
-                "option '{}.{}' doesn't exist".format(section, opt)
-            )
-
-        return config[section][opt]
-
-    @staticmethod
-    def _lower(config):
-        new_config = configobj.ConfigObj()
-        for s_key, s_value in config.items():
-            new_s = {}
-            for key, value in s_value.items():
-                new_s[key.lower()] = str(value)
-            new_config[s_key.lower()] = new_s
-        return new_config
-
-    def get_configobj(self, level):
-        configs = {
-            self.LEVEL_LOCAL: self._local_config,
-            self.LEVEL_REPO: self._repo_config,
-            self.LEVEL_GLOBAL: self._global_config,
-            self.LEVEL_SYSTEM: self._system_config,
-        }
-
-        return configs.get(level, self._repo_config)
-
-    def list_options(self, section_regex, option, level=None):
-        ret = {}
-        config = self.get_configobj(level)
-        for section in config.keys():
-            r = re.match(section_regex, section)
-            if r:
-                name = r.group("name")
-                value = config[section].get(option, "")
-                ret[name] = value
-        return ret
+def _lower_keys(data):
+    return {
+        k.lower(): _lower_keys(v) if isinstance(v, dict) else v
+        for k, v in data.items()
+    }
@@ -7,9 +7,7 @@ import sys
 from subprocess import Popen

 from dvc.env import DVC_DAEMON
-from dvc.utils import fix_env
-from dvc.utils import is_binary
-
+from dvc.utils import fix_env, is_binary

 logger = logging.getLogger(__name__)

@@ -68,7 +66,7 @@ def _spawn_posix(cmd, env):


 def _spawn(cmd, env):
-    logger.debug("Trying to spawn '{}'".format(cmd))
+    logger.debug(f"Trying to spawn '{cmd}'")

    if os.name == "nt":
        _spawn_windows(cmd, env)
@@ -77,7 +75,7 @@ def _spawn(cmd, env):
    else:
        raise NotImplementedError

-    logger.debug("Spawned '{}'".format(cmd))
+    logger.debug(f"Spawned '{cmd}'")


 def daemon(args):
@@ -6,12 +6,9 @@ import os
 import pydoc
 import sys

-from grandalf.graphs import Edge
-from grandalf.graphs import Graph
-from grandalf.graphs import Vertex
+from grandalf.graphs import Edge, Graph, Vertex
 from grandalf.layouts import SugiyamaLayout
-from grandalf.routing import EdgeViewer
-from grandalf.routing import route_with_lines
+from grandalf.routing import EdgeViewer, route_with_lines

 from dvc.env import DVC_PAGER
 from dvc.utils import format_link
@@ -40,7 +37,7 @@ def find_pager():
    if env_pager:
        return make_pager(env_pager)

-    if os.system("({}) 2>{}".format(DEFAULT_PAGER, os.devnull)) == 0:
+    if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
        return make_pager(DEFAULT_PAGER_FORMATTED)

    logger.warning(
@@ -52,7 +49,7 @@ def find_pager():
    return pydoc.plainpager


-class VertexViewer(object):
+class VertexViewer:
    """Class to define vertex box boundaries that will be accounted for during
    graph building by grandalf.

@@ -78,7 +75,7 @@ class VertexViewer(object):
        return self._w


-class AsciiCanvas(object):
+class AsciiCanvas:
    """Class for drawing in ASCII.

    Args:
@@ -95,7 +92,7 @@ class AsciiCanvas(object):
        self.cols = cols
        self.lines = lines

-        self.canvas = [[" "] * cols for l in range(lines)]
+        self.canvas = [[" "] * cols for line in range(lines)]

    def draw(self):
        """Draws ASCII canvas on the screen."""
@@ -148,21 +145,21 @@ class AsciiCanvas(object):
                if dx == 0:
                    y = y0
                else:
-                    y = y0 + int(round((x - x0) * dy / float((dx))))
+                    y = y0 + int(round((x - x0) * dy / float(dx)))
                self.point(x, y, char)
        elif y0 < y1:
            for y in range(y0, y1 + 1):
                if dy == 0:
                    x = x0
                else:
-                    x = x0 + int(round((y - y0) * dx / float((dy))))
+                    x = x0 + int(round((y - y0) * dx / float(dy)))
                self.point(x, y, char)
        else:
            for y in range(y1, y0 + 1):
                if dy == 0:
                    x = x0
                else:
-                    x = x1 + int(round((y - y1) * dx / float((dy))))
+                    x = x1 + int(round((y - y1) * dx / float(dy)))
                self.point(x, y, char)

    def text(self, x, y, text):
@@ -216,7 +213,7 @@ def _build_sugiyama_layout(vertexes, edges):
    # Y
    #

-    vertexes = {v: Vertex(" {} ".format(v)) for v in vertexes}
+    vertexes = {v: Vertex(f" {v} ") for v in vertexes}
    # NOTE: reverting edges to correctly orientate the graph
    edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
    vertexes = vertexes.values()
@@ -2,15 +2,13 @@

 import logging

-from dvc.config import Config
 from dvc.config import NoRemoteError
 from dvc.remote import Remote

-
 logger = logging.getLogger(__name__)


-class DataCloud(object):
+class DataCloud:
    """Class that manages dvc remotes.

    Args:
@@ -24,28 +22,14 @@ class DataCloud(object):
    def __init__(self, repo):
        self.repo = repo

-    @property
-    def _config(self):
-        return self.repo.config.config
+    def get_remote(self, name=None, command="<command>"):
+        if not name:
+            name = self.repo.config["core"].get("remote")

-    @property
-    def _core(self):
-        return self._config.get(Config.SECTION_CORE, {})
+        if name:
+            return self._init_remote(name)

-    def get_remote(self, remote=None, command="<command>"):
-        if not remote:
-            remote = self._core.get(Config.SECTION_CORE_REMOTE)
-
-        if remote:
-            return self._init_remote(remote)
-
-        has_non_default_remote = bool(
-            self.repo.config.list_options(
-                Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL
-            )
-        )
-
-        if has_non_default_remote:
+        if bool(self.repo.config["remote"]):
            error_msg = (
                "no remote specified. Setup default remote with\n"
                "    dvc remote default <remote name>\n"
@@ -60,39 +44,43 @@ class DataCloud(object):

        raise NoRemoteError(error_msg)

-    def _init_remote(self, remote):
-        return Remote(self.repo, name=remote)
+    def _init_remote(self, name):
+        return Remote(self.repo, name=name)

-    def push(self, cache, jobs=None, remote=None, show_checksums=False):
+    def push(
+        self, cache, jobs=None, remote=None, show_checksums=False,
+    ):
        """Push data items in a cloud-agnostic way.

        Args:
            cache (NamedCache): named checksums to push to the cloud.
            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.RemoteBASE): optional remote to push to.
+            remote (dvc.remote.base.BaseRemote): optional remote to push to.
                By default remote from core.remote config option is used.
            show_checksums (bool): show checksums instead of file names in
                information messages.
        """
+        remote = self.get_remote(remote, "push")
+
        return self.repo.cache.local.push(
-            cache,
-            jobs=jobs,
-            remote=self.get_remote(remote, "push"),
-            show_checksums=show_checksums,
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums,
        )

-    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
+    def pull(
+        self, cache, jobs=None, remote=None, show_checksums=False,
+    ):
        """Pull data items in a cloud-agnostic way.

        Args:
            cache (NamedCache): named checksums to pull from the cloud.
            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
+            remote (dvc.remote.base.BaseRemote): optional remote to pull from.
                By default remote from core.remote config option is used.
            show_checksums (bool): show checksums instead of file names in
                information messages.
        """
        remote = self.get_remote(remote, "pull")
+
        downloaded_items_num = self.repo.cache.local.pull(
            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
        )
@@ -103,12 +91,13 @@ class DataCloud(object):
        return downloaded_items_num

    def _save_pulled_checksums(self, cache):
-        for checksum in cache["local"].keys():
+        for checksum in cache.scheme_keys("local"):
            cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
            if self.repo.cache.local.exists(cache_file):
-                # We can safely save here, as existing corrupted files will be
-                # removed upon status, while files corrupted during download
-                # will not be moved from tmp_file (see `RemoteBASE.download()`)
+                # We can safely save here, as existing corrupted files will
+                # be removed upon status, while files corrupted during
+                # download will not be moved from tmp_file
+                # (see `RemoteBASE.download()`)
                self.repo.state.save(cache_file, checksum)

    def status(self, cache, jobs=None, remote=None, show_checksums=False):
@@ -117,7 +106,7 @@ class DataCloud(object):
        Args:
            cache (NamedCache): named checksums to check status for.
            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.RemoteBASE): optional remote to compare
+            remote (dvc.remote.base.BaseRemote): optional remote to compare
                cache to. By default remote from core.remote config option
                is used.
            show_checksums (bool): show checksums instead of file names in
@@ -1,37 +1,39 @@
+from collections import defaultdict
 from urllib.parse import urlparse

 import dvc.output as output
-from dvc.dependency.gs import DependencyGS
-from dvc.dependency.hdfs import DependencyHDFS
-from dvc.dependency.http import DependencyHTTP
-from dvc.dependency.https import DependencyHTTPS
-from dvc.dependency.local import DependencyLOCAL
-from dvc.dependency.s3 import DependencyS3
-from dvc.dependency.ssh import DependencySSH
-from dvc.output.base import OutputBase
+from dvc.dependency.gs import GSDependency
+from dvc.dependency.hdfs import HDFSDependency
+from dvc.dependency.http import HTTPDependency
+from dvc.dependency.https import HTTPSDependency
+from dvc.dependency.local import LocalDependency
+from dvc.dependency.param import ParamsDependency
+from dvc.dependency.s3 import S3Dependency
+from dvc.dependency.ssh import SSHDependency
+from dvc.output.base import BaseOutput
 from dvc.remote import Remote
 from dvc.scheme import Schemes
-from .repo import DependencyREPO

+from .repo import RepoDependency

 DEPS = [
-    DependencyGS,
-    DependencyHDFS,
-    DependencyHTTP,
-    DependencyHTTPS,
-    DependencyS3,
-    DependencySSH,
-    # NOTE: DependencyLOCAL is the default choice
+    GSDependency,
+    HDFSDependency,
+    HTTPDependency,
+    HTTPSDependency,
+    S3Dependency,
+    SSHDependency,
+    # NOTE: LocalDependency is the default choice
 ]

 DEP_MAP = {
-    Schemes.LOCAL: DependencyLOCAL,
-    Schemes.SSH: DependencySSH,
-    Schemes.S3: DependencyS3,
-    Schemes.GS: DependencyGS,
-    Schemes.HDFS: DependencyHDFS,
-    Schemes.HTTP: DependencyHTTP,
-    Schemes.HTTPS: DependencyHTTPS,
+    Schemes.LOCAL: LocalDependency,
+    Schemes.SSH: SSHDependency,
+    Schemes.S3: S3Dependency,
+    Schemes.GS: GSDependency,
+    Schemes.HDFS: HDFSDependency,
+    Schemes.HTTP: HTTPDependency,
+    Schemes.HTTPS: HTTPSDependency,
 }


@@ -40,32 +42,36 @@ DEP_MAP = {
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
 # output is a metric file and how to parse it, see `-M` flag for `dvc run`).
 SCHEMA = output.SCHEMA.copy()
-del SCHEMA[OutputBase.PARAM_CACHE]
-del SCHEMA[OutputBase.PARAM_METRIC]
-SCHEMA[DependencyREPO.PARAM_REPO] = DependencyREPO.REPO_SCHEMA
+del SCHEMA[BaseOutput.PARAM_CACHE]
+del SCHEMA[BaseOutput.PARAM_METRIC]
+SCHEMA.update(RepoDependency.REPO_SCHEMA)
+SCHEMA.update(ParamsDependency.PARAM_SCHEMA)


 def _get(stage, p, info):
-    parsed = urlparse(p)
-
-    if parsed.scheme == "remote":
+    parsed = urlparse(p) if p else None
+    if parsed and parsed.scheme == "remote":
        remote = Remote(stage.repo, name=parsed.netloc)
        return DEP_MAP[remote.scheme](stage, p, info, remote=remote)

-    if info and info.get(DependencyREPO.PARAM_REPO):
-        repo = info.pop(DependencyREPO.PARAM_REPO)
-        return DependencyREPO(repo, stage, p, info)
+    if info and info.get(RepoDependency.PARAM_REPO):
+        repo = info.pop(RepoDependency.PARAM_REPO)
+        return RepoDependency(repo, stage, p, info)
+
+    if info and info.get(ParamsDependency.PARAM_PARAMS):
+        params = info.pop(ParamsDependency.PARAM_PARAMS)
+        return ParamsDependency(stage, p, params)

    for d in DEPS:
        if d.supported(p):
            return d(stage, p, info)
-    return DependencyLOCAL(stage, p, info)
+    return LocalDependency(stage, p, info)


 def loadd_from(stage, d_list):
    ret = []
    for d in d_list:
-        p = d.pop(OutputBase.PARAM_PATH)
+        p = d.pop(BaseOutput.PARAM_PATH, None)
        ret.append(_get(stage, p, d))
    return ret

@@ -73,6 +79,31 @@ def loadd_from(stage, d_list):
 def loads_from(stage, s_list, erepo=None):
    ret = []
    for s in s_list:
-        info = {DependencyREPO.PARAM_REPO: erepo} if erepo else {}
+        info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
        ret.append(_get(stage, s, info))
    return ret
+
+
+def _parse_params(path_params):
+    path, _, params_str = path_params.rpartition(":")
+    params = params_str.split(",")
+    return path, params
+
+
+def loads_params(stage, s_list):
+    # Creates an object for each unique file that is referenced in the list
+    params_by_path = defaultdict(list)
+    for s in s_list:
+        path, params = _parse_params(s)
+        params_by_path[path].extend(params)
+
+    d_list = []
+    for path, params in params_by_path.items():
+        d_list.append(
+            {
+                BaseOutput.PARAM_PATH: path,
+                ParamsDependency.PARAM_PARAMS: params,
+            }
+        )
+
+    return loadd_from(stage, d_list)
@@ -3,29 +3,27 @@ from dvc.exceptions import DvcException

 class DependencyDoesNotExistError(DvcException):
    def __init__(self, path):
-        msg = "dependency '{}' does not exist".format(path)
+        msg = f"dependency '{path}' does not exist"
        super().__init__(msg)


 class DependencyIsNotFileOrDirError(DvcException):
    def __init__(self, path):
-        msg = "dependency '{}' is not a file or directory".format(path)
+        msg = f"dependency '{path}' is not a file or directory"
        super().__init__(msg)


 class DependencyIsStageFileError(DvcException):
    def __init__(self, path):
-        super().__init__(
-            "Stage file '{}' cannot be a dependency.".format(path)
-        )
+        super().__init__(f"Stage file '{path}' cannot be a dependency.")


-class DependencyBase(object):
+class BaseDependency:
    IS_DEPENDENCY = True

    DoesNotExistError = DependencyDoesNotExistError
    IsNotFileOrDirError = DependencyIsNotFileOrDirError
    IsStageFileError = DependencyIsStageFileError

-    def update(self):
+    def update(self, rev=None):
        pass
@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.gs import OutputGS
+from dvc.dependency.base import BaseDependency
+from dvc.output.gs import GSOutput


-class DependencyGS(DependencyBase, OutputGS):
+class GSDependency(BaseDependency, GSOutput):
    pass
@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.hdfs import OutputHDFS
+from dvc.dependency.base import BaseDependency
+from dvc.output.hdfs import HDFSOutput


-class DependencyHDFS(DependencyBase, OutputHDFS):
+class HDFSDependency(BaseDependency, HDFSOutput):
    pass
@@ -1,7 +1,7 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.base import OutputBase
-from dvc.remote.http import RemoteHTTP
+from dvc.dependency.base import BaseDependency
+from dvc.output.base import BaseOutput
+from dvc.remote.http import HTTPRemote


-class DependencyHTTP(DependencyBase, OutputBase):
-    REMOTE = RemoteHTTP
+class HTTPDependency(BaseDependency, BaseOutput):
+    REMOTE = HTTPRemote
@@ -1,6 +1,7 @@
-from .http import DependencyHTTP
-from dvc.remote.https import RemoteHTTPS
+from dvc.remote.https import HTTPSRemote
+
+from .http import HTTPDependency


-class DependencyHTTPS(DependencyHTTP):
-    REMOTE = RemoteHTTPS
+class HTTPSDependency(HTTPDependency):
+    REMOTE = HTTPSRemote
@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.local import OutputLOCAL
+from dvc.dependency.base import BaseDependency
+from dvc.output.local import LocalOutput


-class DependencyLOCAL(DependencyBase, OutputLOCAL):
+class LocalDependency(BaseDependency, LocalOutput):
    pass
@@ -0,0 +1,116 @@
+import os
+from collections import defaultdict
+
+import dpath.util
+import yaml
+from voluptuous import Any
+
+from dvc.dependency.local import LocalDependency
+from dvc.exceptions import DvcException
+
+
+class MissingParamsError(DvcException):
+    pass
+
+
+class BadParamFileError(DvcException):
+    pass
+
+
+class ParamsDependency(LocalDependency):
+    PARAM_PARAMS = "params"
+    PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
+    DEFAULT_PARAMS_FILE = "params.yaml"
+
+    def __init__(self, stage, path, params):
+        info = {}
+        self.params = []
+        if params:
+            if isinstance(params, list):
+                self.params = params
+            else:
+                assert isinstance(params, dict)
+                self.params = list(params.keys())
+                info = params
+
+        super().__init__(
+            stage,
+            path
+            or os.path.join(stage.repo.root_dir, self.DEFAULT_PARAMS_FILE),
+            info=info,
+        )
+
+    def _dyn_load(self, values=None):
+        """Load params values dynamically."""
+        if not values:
+            return
+        for param in self.params:
+            value = values.get(param)
+            if value:
+                self.info[param] = value
+
+    def save(self):
+        super().save()
+        self.info = self.save_info()
+
+    def status(self):
+        status = super().status()
+
+        if status[str(self)] == "deleted":
+            return status
+
+        status = defaultdict(dict)
+        info = self.read_params()
+        for param in self.params:
+            if param not in info.keys():
+                st = "deleted"
+            elif param not in self.info:
+                st = "new"
+            elif info[param] != self.info[param]:
+                st = "modified"
+            else:
+                assert info[param] == self.info[param]
+                continue
+
+            status[str(self)][param] = st
+
+        return status
+
+    def dumpd(self):
+        return {
+            self.PARAM_PATH: self.def_path,
+            self.PARAM_PARAMS: self.info or self.params,
+        }
+
+    def read_params(self):
+        if not self.exists:
+            return {}
+
+        with self.repo.tree.open(self.path_info, "r") as fobj:
+            try:
+                config = yaml.safe_load(fobj)
+            except yaml.YAMLError as exc:
+                raise BadParamFileError(
+                    f"Unable to read parameters from '{self}'"
+                ) from exc
+
+        ret = {}
+        for param in self.params:
+            try:
+                ret[param] = dpath.util.get(config, param, separator=".")
+            except KeyError:
+                pass
+        return ret
+
+    def save_info(self):
+        info = self.read_params()
+
+        missing_params = set(self.params) - set(info.keys())
+        if missing_params:
+            raise MissingParamsError(
+                "Parameters '{}' are missing from '{}'.".format(
+                    ", ".join(missing_params), self,
+                )
+            )
+
+        return info
@@ -2,21 +2,24 @@ import os

 from voluptuous import Required

-from .local import DependencyLOCAL
 from dvc.exceptions import OutputNotFoundError
 from dvc.path_info import PathInfo

+from .local import LocalDependency

-class DependencyREPO(DependencyLOCAL):
+
+class RepoDependency(LocalDependency):
    PARAM_REPO = "repo"
    PARAM_URL = "url"
    PARAM_REV = "rev"
    PARAM_REV_LOCK = "rev_lock"

    REPO_SCHEMA = {
-        Required(PARAM_URL): str,
-        PARAM_REV: str,
-        PARAM_REV_LOCK: str,
+        PARAM_REPO: {
+            Required(PARAM_URL): str,
+            PARAM_REV: str,
+            PARAM_REV_LOCK: str,
+        }
    }

    def __init__(self, def_repo, stage, *args, **kwargs):
@@ -80,6 +83,9 @@ class DependencyREPO(DependencyLOCAL):

            repo.pull_to(self.def_path, to.path_info)

-    def update(self):
+    def update(self, rev=None):
+        if rev:
+            self.def_repo[self.PARAM_REV] = rev
+
        with self._make_repo(locked=False) as repo:
            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.s3 import OutputS3
+from dvc.dependency.base import BaseDependency
+from dvc.output.s3 import S3Output


-class DependencyS3(DependencyBase, OutputS3):
+class S3Dependency(BaseDependency, S3Output):
    pass
@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.ssh import OutputSSH
+from dvc.dependency.base import BaseDependency
+from dvc.output.ssh import SSHOutput


-class DependencySSH(DependencyBase, OutputSSH):
+class SSHDependency(BaseDependency, SSHOutput):
    pass
@@ -0,0 +1,270 @@
+import contextlib
+import logging
+import os
+
+from voluptuous import MultipleInvalid
+
+import dvc.prompt as prompt
+from dvc import serialize
+from dvc.exceptions import DvcException
+from dvc.stage.exceptions import (
+    StageFileAlreadyExistsError,
+    StageFileBadNameError,
+    StageFileDoesNotExistError,
+    StageFileFormatError,
+    StageFileIsNotDvcFileError,
+)
+from dvc.stage.loader import SingleStageLoader, StageLoader
+from dvc.utils import relpath
+from dvc.utils.collections import apply_diff
+from dvc.utils.stage import (
+    dump_stage_file,
+    parse_stage,
+    parse_stage_for_update,
+)
+
+logger = logging.getLogger(__name__)
+
+DVC_FILE = "Dvcfile"
+DVC_FILE_SUFFIX = ".dvc"
+PIPELINE_FILE = "dvc.yaml"
+PIPELINE_LOCK = "dvc.lock"
+
+
+class LockfileCorruptedError(DvcException):
+    pass
+
+
+def is_valid_filename(path):
+    return path.endswith(DVC_FILE_SUFFIX) or os.path.basename(path) in [
+        DVC_FILE,
+        PIPELINE_FILE,
+    ]
+
+
+def is_dvc_file(path):
+    return os.path.isfile(path) and (
+        is_valid_filename(path) or os.path.basename(path) == PIPELINE_LOCK
+    )
+
+
+def check_dvc_filename(path):
+    if not is_valid_filename(path):
+        raise StageFileBadNameError(
+            "bad DVC-file name '{}'. DVC-files should be named "
+            "'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
+                relpath(path), os.path.basename(path)
+            )
+        )
+
+
+class FileMixin:
+    SCHEMA = None
+
+    def __init__(self, repo, path, **kwargs):
+        self.repo = repo
+        self.path = path
+
+    def __repr__(self):
+        return "{}: {}".format(
+            self.__class__.__name__, relpath(self.path, self.repo.root_dir)
+        )
+
+    def __hash__(self):
+        return hash(self.path)
+
+    def __eq__(self, other):
+        return self.repo == other.repo and os.path.abspath(
+            self.path
+        ) == os.path.abspath(other.path)
+
+    def __str__(self):
+        return f"{self.__class__.__name__}: {self.relpath}"
+
+    @property
+    def relpath(self):
+        return relpath(self.path)
+
+    def exists(self):
+        return self.repo.tree.exists(self.path)
+
+    def _load(self):
+        # it raises the proper exceptions by priority:
+        # 1. when the file doesn't exists
+        # 2. filename is not a DVC-file
+        # 3. path doesn't represent a regular file
+        if not self.exists():
+            raise StageFileDoesNotExistError(self.path)
+        check_dvc_filename(self.path)
+        if not self.repo.tree.isfile(self.path):
+            raise StageFileIsNotDvcFileError(self.path)
+
+        with self.repo.tree.open(self.path) as fd:
+            stage_text = fd.read()
+        d = parse_stage(stage_text, self.path)
+        self.validate(d, self.relpath)
+        return d, stage_text
+
+    @classmethod
+    def validate(cls, d, fname=None):
+        assert cls.SCHEMA
+        try:
+            cls.SCHEMA(d)
+        except MultipleInvalid as exc:
+            raise StageFileFormatError(f"'{fname}' format error: {exc}")
+
+    def remove_with_prompt(self, force=False):
+        raise NotImplementedError
+
+    def remove(self, force=False):
+        with contextlib.suppress(FileNotFoundError):
+            os.unlink(self.path)
+
+    def dump(self, stage, **kwargs):
+        raise NotImplementedError
+
+
+class SingleStageFile(FileMixin):
+    from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
+
+    def __init__(self, repo, path):
+        super().__init__(repo, path)
+
+    @property
+    def stage(self):
+        data, raw = self._load()
+        return SingleStageLoader.load_stage(self, data, raw)
+
+    @property
+    def stages(self):
+        data, raw = self._load()
+        return SingleStageLoader(self, data, raw)
+
+    def dump(self, stage, **kwargs):
+        """Dumps given stage appropriately in the dvcfile."""
+        from dvc.stage import PipelineStage
+
+        assert not isinstance(stage, PipelineStage)
+        check_dvc_filename(self.path)
+        logger.debug(
+            "Saving information to '{file}'.".format(file=relpath(self.path))
+        )
+        dump_stage_file(self.path, serialize.to_single_stage_file(stage))
+        self.repo.scm.track_file(relpath(self.path))
+
+    def remove_with_prompt(self, force=False):
+        if not self.exists():
+            return
+
+        msg = (
+            "'{}' already exists. Do you wish to run the command and "
+            "overwrite it?".format(relpath(self.path))
+        )
+        if not (force or prompt.confirm(msg)):
+            raise StageFileAlreadyExistsError(self.path)
+
+        self.remove()
+
+
+class PipelineFile(FileMixin):
+    """Abstraction for pipelines file, .yaml + .lock combined."""
+
+    from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA
+
+    @property
+    def _lockfile(self):
+        return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
+
+    def dump(self, stage, update_pipeline=False, **kwargs):
+        """Dumps given stage appropriately in the dvcfile."""
+        from dvc.stage import PipelineStage
+
+        assert isinstance(stage, PipelineStage)
+        check_dvc_filename(self.path)
+        self._dump_lockfile(stage)
+        if update_pipeline and not stage.is_data_source:
+            self._dump_pipeline_file(stage)
+
+    def _dump_lockfile(self, stage):
+        self._lockfile.dump(stage)
+
+    def _dump_pipeline_file(self, stage):
+        data = {}
+        if self.exists():
+            with open(self.path) as fd:
+                data = parse_stage_for_update(fd.read(), self.path)
+        else:
+            open(self.path, "w+").close()
+
+        data["stages"] = data.get("stages", {})
+        stage_data = serialize.to_pipeline_file(stage)
+        if data["stages"].get(stage.name):
+            orig_stage_data = data["stages"][stage.name]
+            apply_diff(stage_data[stage.name], orig_stage_data)
+        else:
+            data["stages"].update(stage_data)
+
+        dump_stage_file(self.path, data)
+        self.repo.scm.track_file(relpath(self.path))
+
+    @property
+    def stage(self):
+        raise DvcException(
+            "PipelineFile has multiple stages. Please specify it's name."
+        )
+
+    @property
+    def stages(self):
+        data, _ = self._load()
+        lockfile_data = self._lockfile.load()
+        return StageLoader(self, data.get("stages", {}), lockfile_data)
+
+    def remove(self, force=False):
+        if not force:
+            logger.warning("Cannot remove pipeline file.")
+            return
+
+        super().remove()
+        self._lockfile.remove()
+
+
+class Lockfile(FileMixin):
+    from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
+
+    def load(self):
+        if not self.exists():
+            return {}
+        with self.repo.tree.open(self.path) as fd:
+            data = parse_stage(fd.read(), self.path)
+        try:
+            self.validate(data, fname=self.relpath)
+        except StageFileFormatError:
+            raise LockfileCorruptedError(
+                f"Lockfile '{self.relpath}' is corrupted."
+            )
+        return data
+
+    def dump(self, stage, **kwargs):
+        stage_data = serialize.to_lockfile(stage)
+        if not self.exists():
+            data = stage_data
+            open(self.path, "w+").close()
+        else:
+            with self.repo.tree.open(self.path, "r") as fd:
+                data = parse_stage_for_update(fd.read(), self.path)
+            data.update(stage_data)
+
+        dump_stage_file(self.path, data)
+        self.repo.scm.track_file(relpath(self.path))
+
+
+class Dvcfile:
+    def __new__(cls, repo, path, **kwargs):
+        assert path
+        assert repo
+
+        _, ext = os.path.splitext(path)
+        if ext in [".yaml", ".yml"]:
+            return PipelineFile(repo, path, **kwargs)
+        # fallback to single stage file for better error messages
+        return SingleStageFile(repo, path, **kwargs)
@@ -1,11 +1,20 @@
 """Exceptions raised by the dvc."""
+from funcy import first

-from dvc.utils import relpath, format_link
+from dvc.utils import format_link, relpath


 class DvcException(Exception):
    """Base class for all dvc exceptions."""

+    def __init__(self, msg, *args):
+        assert msg
+        super().__init__(msg, *args)
+
+
+class InvalidArgumentError(ValueError, DvcException):
+    """Thrown if arguments are invalid."""
+

 class OutputDuplicationError(DvcException):
    """Thrown if a file/directory is specified as an output in more than one
@@ -19,12 +28,17 @@ class OutputDuplicationError(DvcException):
    def __init__(self, output, stages):
        assert isinstance(output, str)
        assert all(hasattr(stage, "relpath") for stage in stages)
-        msg = (
-            "file/directory '{}' is specified as an output in more than one "
-            "stage: {}\n"
-            "This is not allowed. Consider using a different output name."
-        ).format(output, "\n    ".join(s.relpath for s in stages))
+        if len(stages) == 1:
+            msg = "output '{}' is already specified in {}.".format(
+                output, first(stages)
+            )
+        else:
+            msg = "output '{}' is already specified in stages:\n{}".format(
+                output, "\n".join(f"\t- {s.addressing}" for s in stages),
+            )
        super().__init__(msg)
+        self.stages = stages
+        self.output = output


 class OutputNotFoundError(DvcException):
@@ -56,8 +70,8 @@ class StagePathAsOutputError(DvcException):
    def __init__(self, stage, output):
        assert isinstance(output, str)
        super().__init__(
-            "'{stage}' is within an output '{output}' of another stage".format(
-                stage=stage.relpath, output=output
+            "{stage} is within an output '{output}' of another stage".format(
+                stage=stage, output=output
            )
        )

@@ -73,10 +87,7 @@ class CircularDependencyError(DvcException):
    def __init__(self, dependency):
        assert isinstance(dependency, str)

-        msg = (
-            "file/directory '{}' is specified as an output and as a "
-            "dependency."
-        )
+        msg = "'{}' is specified as an output and as a dependency."
        super().__init__(msg.format(dependency))


@@ -90,7 +101,7 @@ class ArgumentDuplicationError(DvcException):

    def __init__(self, path):
        assert isinstance(path, str)
-        super().__init__("file '{}' is specified more than once.".format(path))
+        super().__init__(f"file '{path}' is specified more than once.")


 class MoveNotDataSourceError(DvcException):
@@ -125,12 +136,8 @@ class DvcParserError(DvcException):
 class CyclicGraphError(DvcException):
    def __init__(self, stages):
        assert isinstance(stages, list)
-        stages = "\n".join("\t- {}".format(stage.relpath) for stage in stages)
-        msg = (
-            "you've introduced a cycle in your pipeline that involves "
-            "the following stages:"
-            "\n"
-            "{stages}".format(stages=stages)
+        msg = "Pipeline has a cycle involving: {}.".format(
+            ", ".join(s.addressing for s in stages)
        )
        super().__init__(msg)

@@ -138,7 +145,7 @@ class CyclicGraphError(DvcException):
 class ConfirmRemoveError(DvcException):
    def __init__(self, path):
        super().__init__(
-            "unable to remove '{}' without a confirmation from the user. Use "
+            "unable to remove '{}' without a confirmation. Use "
            "`-f` to force.".format(path)
        )

@@ -150,7 +157,7 @@ class InitError(DvcException):
 class ReproductionError(DvcException):
    def __init__(self, dvc_file_name):
        self.path = dvc_file_name
-        super().__init__("failed to reproduce '{}'".format(dvc_file_name))
+        super().__init__(f"failed to reproduce '{dvc_file_name}'")


 class BadMetricError(DvcException):
@@ -158,7 +165,7 @@ class BadMetricError(DvcException):
        super().__init__(
            "the following metrics do not exist, "
            "are not metric files or are malformed: {paths}".format(
-                paths=", ".join("'{}'".format(path) for path in paths)
+                paths=", ".join(f"'{path}'" for path in paths)
            )
        )

@@ -188,22 +195,15 @@ class RecursiveAddingWhileUsingFilename(DvcException):


 class OverlappingOutputPathsError(DvcException):
-    def __init__(self, out_1, out_2):
-        super().__init__(
-            "Paths for outs:\n'{}'('{}')\n'{}'('{}')\noverlap. To avoid "
-            "unpredictable behaviour, rerun command with non overlapping outs "
-            "paths.".format(
-                str(out_1),
-                out_1.stage.relpath,
-                str(out_2),
-                out_2.stage.relpath,
-            )
-        )
+    def __init__(self, parent, overlapping_out, message):
+        self.parent = parent
+        self.overlapping_out = overlapping_out
+        super().__init__(message)


 class CheckoutErrorSuggestGit(DvcException):
    def __init__(self, target):
-        super().__init__("Did you mean `git checkout {}`?".format(target))
+        super().__init__(f"Did you mean `git checkout {target}`?")


 class ETagMismatchError(DvcException):
@@ -217,9 +217,7 @@ class ETagMismatchError(DvcException):
 class FileMissingError(DvcException):
    def __init__(self, path):
        self.path = path
-        super().__init__(
-            "Can't find '{}' neither locally nor on remote".format(path)
-        )
+        super().__init__(f"Can't find '{path}' neither locally nor on remote")


 class DvcIgnoreInCollectedDirError(DvcException):
@@ -244,25 +242,23 @@ class DownloadError(DvcException):
    def __init__(self, amount):
        self.amount = amount

-        super().__init__(
-            "{amount} files failed to download".format(amount=amount)
-        )
+        super().__init__(f"{amount} files failed to download")


 class UploadError(DvcException):
    def __init__(self, amount):
        self.amount = amount

-        super().__init__(
-            "{amount} files failed to upload".format(amount=amount)
-        )
+        super().__init__(f"{amount} files failed to upload")


 class CheckoutError(DvcException):
-    def __init__(self, target_infos):
+    def __init__(self, target_infos, stats=None):
+        self.target_infos = target_infos
+        self.stats = stats
        targets = [str(t) for t in target_infos]
        m = (
-            "Checkout failed for following targets:\n {}\nDid you "
+            "Checkout failed for following targets:\n{}\nDid you "
            "forget to fetch?".format("\n".join(targets))
        )
        super().__init__(m)
@@ -275,7 +271,7 @@ class CollectCacheError(DvcException):
 class NoRemoteInExternalRepoError(DvcException):
    def __init__(self, url):
        super().__init__(
-            "No DVC remote is specified in target repository '{}'.".format(url)
+            f"No DVC remote is specified in target repository '{url}'."
        )


@@ -290,16 +286,27 @@ class NoOutputInExternalRepoError(DvcException):

 class HTTPError(DvcException):
    def __init__(self, code, reason):
-        super().__init__("'{} {}'".format(code, reason))
+        super().__init__(f"'{code} {reason}'")


 class PathMissingError(DvcException):
-    def __init__(self, path, repo):
+    default_msg = (
+        "The path '{}' does not exist in the target repository '{}'"
+        " neither as a DVC output nor as a Git-tracked file."
+    )
+    default_msg_output_only = (
+        "The path '{}' does not exist in the target repository '{}'"
+        " as an DVC output."
+    )
+
+    def __init__(self, path, repo, output_only=False):
        msg = (
-            "The path '{}' does not exist in the target repository '{}'"
-            " neither as an output nor a git-handled file."
+            self.default_msg
+            if not output_only
+            else self.default_msg_output_only
        )
        super().__init__(msg.format(path, repo))
+        self.output_only = output_only


 class RemoteCacheRequiredError(DvcException):
@@ -315,3 +322,7 @@ class RemoteCacheRequiredError(DvcException):
                format_link("https://man.dvc.org/config#cache"),
            )
        )
+
+
+class IsADirectoryError(DvcException):
+    """Raised when a file operation is requested on a directory."""
@@ -1,29 +1,33 @@
 import logging
 import os
 import tempfile
+import threading
 from contextlib import contextmanager
 from distutils.dir_util import copy_tree
-import threading

-from funcy import retry, suppress, wrap_with, cached_property
+from funcy import cached_property, retry, suppress, wrap_with

-from dvc.compat import fspath
+from dvc.config import NoRemoteError, NotDvcRepoError
+from dvc.exceptions import (
+    CheckoutError,
+    FileMissingError,
+    NoOutputInExternalRepoError,
+    NoRemoteInExternalRepoError,
+    OutputNotFoundError,
+    PathMissingError,
+)
+from dvc.path_info import PathInfo
 from dvc.repo import Repo
-from dvc.config import Config, NoRemoteError, NotDvcRepoError
-from dvc.exceptions import NoRemoteInExternalRepoError
-from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
-from dvc.exceptions import FileMissingError, PathMissingError
-from dvc.remote import RemoteConfig
-from dvc.utils.fs import remove, fs_copy
 from dvc.scm.git import Git
-
+from dvc.utils import tmp_fname
+from dvc.utils.fs import fs_copy, move, remove

 logger = logging.getLogger(__name__)


@contextmanager
 def external_repo(url, rev=None, for_write=False):
-    logger.debug("Creating external repo {}@{}", url, rev)
+    logger.debug("Creating external repo %s@%s", url, rev)
    path = _cached_clone(url, rev, for_write=for_write)
    try:
        repo = ExternalRepo(path, url)
@@ -65,38 +69,54 @@ class ExternalRepo(Repo):
        super().__init__(root_dir)
        self.url = url
        self._set_cache_dir()
-        self._set_upstream()
+        self._fix_upstream()

    def pull_to(self, path, to_info):
-        try:
-            out = None
-            with suppress(OutputNotFoundError):
-                out = self.find_out_by_relpath(path)
+        """
+        Pull the corresponding file or directory specified by `path` and
+        checkout it into `to_info`.

+        It works with files tracked by Git and DVC, and also local files
+        outside the repository.
+        """
+        out = None
+        path_info = PathInfo(self.root_dir) / path
+
+        with suppress(OutputNotFoundError):
+            (out,) = self.find_outs_by_path(path_info, strict=False)
+
+        try:
            if out and out.use_cache:
-                self._pull_cached(out, to_info)
+                self._pull_cached(out, path_info, to_info)
                return

-            # Git handled files can't have absolute path
+            # Check if it is handled by Git (it can't have an absolute path)
            if os.path.isabs(path):
                raise FileNotFoundError

-            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
+            fs_copy(path_info, to_info)
        except FileNotFoundError:
            raise PathMissingError(path, self.url)

-    def _pull_cached(self, out, to_info):
+    def _pull_cached(self, out, path_info, dest):
        with self.state:
-            # Only pull unless all needed cache is present
-            if out.changed_cache():
-                self.cloud.pull(out.get_used_cache())
+            tmp = PathInfo(tmp_fname(dest))
+            src = tmp / path_info.relative_to(out.path_info)

-            out.path_info = to_info
-            failed = out.checkout()
-            # This might happen when pull haven't really pulled all the files
-            if failed:
+            out.path_info = tmp
+
+            # Only pull unless all needed cache is present
+            if out.changed_cache(filter_info=src):
+                self.cloud.pull(out.get_used_cache(filter_info=src))
+
+            try:
+                out.checkout(filter_info=src)
+            except CheckoutError:
                raise FileNotFoundError

+            move(src, dest)
+            remove(tmp)
+
    @wrap_with(threading.Lock())
    def _set_cache_dir(self):
        try:
@@ -106,22 +126,43 @@ class ExternalRepo(Repo):

        self.cache.local.cache_dir = cache_dir

-    def _set_upstream(self):
-        # check if the URL is local and no default remote is present
-        # add default remote pointing to the original repo's cache location
-        if os.path.isdir(self.url):
-            rconfig = RemoteConfig(self.config)
-            if not rconfig.has_default():
-                src_repo = Repo(self.url)
-                try:
-                    rconfig.add(
-                        "auto-generated-upstream",
-                        src_repo.cache.local.cache_dir,
-                        default=True,
-                        level=Config.LEVEL_LOCAL,
-                    )
-                finally:
-                    src_repo.close()
+    def _fix_upstream(self):
+        if not os.path.isdir(self.url):
+            return
+
+        try:
+            src_repo = Repo(self.url)
+        except NotDvcRepoError:
+            # If ExternalRepo does not throw NotDvcRepoError and Repo does,
+            # the self.url might be a bare git repo.
+            # NOTE: This will fail to resolve remote with relative path,
+            # same as if it was a remote DVC repo.
+            return
+
+        try:
+            remote_name = self.config["core"].get("remote")
+            if remote_name:
+                self._fix_local_remote(src_repo, remote_name)
+            else:
+                self._add_upstream(src_repo)
+        finally:
+            src_repo.close()
+
+    def _fix_local_remote(self, src_repo, remote_name):
+        # If a remote URL is relative to the source repo,
+        # it will have changed upon config load and made
+        # relative to this new repo. Restore the old one here.
+        new_remote = self.config["remote"][remote_name]
+        old_remote = src_repo.config["remote"][remote_name]
+        if new_remote["url"] != old_remote["url"]:
+            new_remote["url"] = old_remote["url"]
+
+    def _add_upstream(self, src_repo):
+        # Fill the empty upstream entry with a new remote pointing to the
+        # original repo's cache location.
+        cache_dir = src_repo.cache.local.cache_dir
+        self.config["remote"]["auto-generated-upstream"] = {"url": cache_dir}
+        self.config["core"]["remote"] = "auto-generated-upstream"


 class ExternalGitRepo:
@@ -146,12 +187,13 @@ class ExternalGitRepo:
            if os.path.isabs(path):
                raise FileNotFoundError

-            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
+            fs_copy(os.path.join(self.root_dir, path), to_info)
        except FileNotFoundError:
            raise PathMissingError(path, self.url)

    @contextmanager
    def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
+        """Opens a specified resource as a file object."""
        try:
            abs_path = os.path.join(self.root_dir, path)
            with open(abs_path, mode, encoding=encoding) as fd:
@@ -178,6 +220,7 @@ def _cached_clone(url, rev, for_write=False):

    # Copy to a new dir to keep the clone clean
    repo_path = tempfile.mkdtemp("dvc-erepo")
+    logger.debug("erepo: making a copy of %s clone", url)
    copy_tree(clone_path, repo_path)

    # Check out the specified revision
@@ -202,9 +245,11 @@ def _clone_default_branch(url, rev):
        if clone_path:
            git = Git(clone_path)
            # Do not pull for known shas, branches and tags might move
-            if not Git.is_sha(rev) or not git.is_known(rev):
+            if not Git.is_sha(rev) or not git.has_rev(rev):
+                logger.debug("erepo: git pull %s", url)
                git.pull()
        else:
+            logger.debug("erepo: git clone %s to a temporary dir", url)
            clone_path = tempfile.mkdtemp("dvc-clone")
            git = Git.clone(url, clone_path)
            CLONES[url] = clone_path
@@ -216,6 +261,7 @@ def _clone_default_branch(url, rev):


 def _git_checkout(repo_path, rev):
+    logger.debug("erepo: git checkout %s@%s", repo_path, rev)
    git = Git(repo_path)
    try:
        git.checkout(rev)
@@ -11,7 +11,7 @@ from dvc.utils import relpath
 logger = logging.getLogger(__name__)


-class DvcIgnore(object):
+class DvcIgnore:
    DVCIGNORE_FILE = ".dvcignore"

    def __call__(self, root, dirs, files):
@@ -71,10 +71,25 @@ class DvcIgnoreDirs(DvcIgnore):
        return self.basenames == other.basenames


-class DvcIgnoreFilter(object):
+class DvcIgnoreRepo(DvcIgnore):
+    def __call__(self, root, dirs, files):
+        def is_dvc_repo(directory):
+            from dvc.repo import Repo
+
+            return os.path.isdir(os.path.join(directory, Repo.DVC_DIR))
+
+        dirs = [d for d in dirs if not is_dvc_repo(d)]
+
+        return dirs, files
+
+
+class DvcIgnoreFilter:
    def __init__(self, tree):
        self.tree = tree
-        self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
+        self.ignores = {
+            DvcIgnoreDirs([".git", ".hg", ".dvc"]),
+            DvcIgnoreRepo(),
+        }
        for root, dirs, files in self.tree.walk(self.tree.tree_root):
            self._update(root)
            dirs[:], files[:] = self(root, dirs, files)
@@ -115,13 +130,14 @@ class CleanTree(BaseTree):
    def isfile(self, path):
        return self.tree.isfile(path)

+    def isexec(self, path):
+        return self.tree.isexec(path)
+
    def walk(self, top, topdown=True):
        for root, dirs, files in self.tree.walk(top, topdown):
            dirs[:], files[:] = self.dvcignore(root, dirs, files)

            yield root, dirs, files

-    def walk_files(self, top):
-        for root, _, files in self.walk(top):
-            for file in files:
-                yield os.path.join(root, file)
+    def stat(self, path):
+        return self.tree.stat(path)
@@ -5,19 +5,19 @@ import os
 import time
 from datetime import timedelta

-import zc.lockfile
 import flufl.lock
+import zc.lockfile

 from dvc.exceptions import DvcException
-from dvc.utils import format_link
 from dvc.progress import Tqdm
+from dvc.utils import format_link

 DEFAULT_TIMEOUT = 5

 FAILED_TO_LOCK_MESSAGE = (
    "cannot perform the command because another DVC process seems to be "
    "running on this project. If that is not the case, manually remove "
-    "`.dvc/lock` and try again."
+    "`.dvc/tmp/lock` and try again."
 )


@@ -25,7 +25,7 @@ class LockError(DvcException):
    """Thrown when unable to acquire the lock for DVC repo."""


-class Lock(object):
+class Lock:
    """Class for DVC repo lock.

    Uses zc.lockfile as backend.
@@ -47,7 +47,7 @@ class Lock(object):
                disable=not self._friendly,
                desc=(
                    "If DVC froze, see `hardlink_lock` in {}".format(
-                        format_link("man.dvc.org/config#core")
+                        format_link("https://man.dvc.org/config#core")
                    )
                ),
            ):
@@ -1,6 +1,5 @@
 """Manages logging configuration for DVC repo."""

-import traceback
 import logging.config
 import logging.handlers

@@ -8,7 +7,6 @@ import colorama

 from dvc.progress import Tqdm

-
 FOOTER = (
    "\n{yellow}Having any troubles?{nc}"
    " Hit us up at {blue}https://dvc.org/support{nc},"
@@ -37,16 +35,15 @@ class ExcludeInfoFilter(logging.Filter):


 class ColorFormatter(logging.Formatter):
-    """Enable color support when logging to a terminal that supports it.
+    """Spit out colored text in supported terminals.

-    Color support on Windows versions that do not support ANSI color codes is
-    enabled by use of the colorama__ library.
+    colorama__ makes ANSI escape character sequences work under Windows.
    See the colorama documentation for details.

    __ https://pypi.python.org/pypi/colorama

-    For records containing `exc_info`, it will use a custom `_walk_exc` to
-    retrieve the whole traceback.
+    If record has an extra `tb_only` attribute, it will not show the
+    exception cause, just the message and the traceback.
    """

    color_code = {
@@ -57,78 +54,47 @@ class ColorFormatter(logging.Formatter):
    }

    def format(self, record):
-        msg = record.msg.format(*record.args) if record.args else record.msg
-        exception, stack_trace = self._parse_exc(record)
-        return ("{prefix}{description}{stack_trace}").format(
-            prefix=self._prefix(record),
-            description=self._description(msg, exception),
-            stack_trace=stack_trace,
+        record.message = record.getMessage()
+        msg = self.formatMessage(record)
+
+        if record.levelname == "INFO":
+            return msg
+
+        if record.exc_info:
+            if getattr(record, "tb_only", False):
+                cause = ""
+            else:
+                cause = ": ".join(_iter_causes(record.exc_info[1]))
+
+            msg = "{message}{separator}{cause}".format(
+                message=msg or "",
+                separator=" - " if msg and cause else "",
+                cause=cause,
+            )
+
+            if _is_verbose():
+                msg += _stack_trace(record.exc_info)
+
+        return "{asctime}{color}{levelname}{nc}: {msg}".format(
+            asctime=self.formatTime(record, self.datefmt),
+            color=self.color_code[record.levelname],
+            nc=colorama.Fore.RESET,
+            levelname=record.levelname,
+            msg=msg,
        )

-    def _prefix(self, record):
-        if record.levelname == "INFO":
+    def formatTime(self, record, datefmt=None):
+        # only show if current level is set to DEBUG
+        # also, skip INFO as it is used for UI
+        if not _is_verbose() or record.levelno == logging.INFO:
            return ""

-        return "{color}{levelname}{nc}: ".format(
-            color=self.color_code.get(record.levelname, ""),
-            levelname=record.levelname,
+        return "{green}{date}{nc} ".format(
+            green=colorama.Fore.GREEN,
+            date=super().formatTime(record, datefmt),
            nc=colorama.Fore.RESET,
        )

-    def _current_level(self):
-        return logging.getLogger("dvc").getEffectiveLevel()
-
-    def _is_visible(self, record):
-        return record.levelno >= self._current_level()
-
-    def _description(self, message, exception):
-        description = ""
-
-        if exception and message:
-            description = "{message} - {exception}"
-        elif exception:
-            description = "{exception}"
-        elif message:
-            description = "{message}"
-
-        return description.format(message=message, exception=exception)
-
-    def _walk_exc(self, exc_info):
-        exc = exc_info[1]
-
-        exc_list = [str(exc)]
-
-        while hasattr(exc, "__cause__") and exc.__cause__:
-            exc_list.append(str(exc.__cause__))
-            exc = exc.__cause__
-
-        return exc_list
-
-    def _parse_exc(self, record):
-        tb_only = getattr(record, "tb_only", False)
-
-        if not record.exc_info:
-            return (None, "")
-
-        exc_list = self._walk_exc(record.exc_info)
-        tb = traceback.format_exception(*record.exc_info)
-
-        exception = None if tb_only else ": ".join(exc_list)
-
-        if self._current_level() == logging.DEBUG:
-            stack_trace = (
-                "\n" "{red}{line}{nc}\n" "{stack_trace}" "{red}{line}{nc}"
-            ).format(
-                red=colorama.Fore.RED,
-                nc=colorama.Fore.RESET,
-                line="-" * 60,
-                stack_trace="".join(tb),
-            )
-        else:
-            stack_trace = ""
-
-        return (exception, stack_trace)
-

 class LoggerHandler(logging.StreamHandler):
    def handleError(self, record):
@@ -149,6 +115,40 @@ class LoggerHandler(logging.StreamHandler):
            self.handleError(record)


+def _is_verbose():
+    return logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG
+
+
+def _iter_causes(exc):
+    while exc:
+        yield str(exc)
+        exc = exc.__cause__
+
+
+def _stack_trace(exc_info):
+    import traceback
+
+    return (
+        "\n"
+        "{red}{line}{nc}\n"
+        "{trace}"
+        "{red}{line}{nc}".format(
+            red=colorama.Fore.RED,
+            line="-" * 60,
+            trace="".join(traceback.format_exception(*exc_info)),
+            nc=colorama.Fore.RESET,
+        )
+    )
+
+
+def disable_other_loggers():
+    logging.captureWarnings(True)
+    root = logging.root
+    for (logger_name, logger) in root.manager.loggerDict.items():
+        if logger_name != "dvc" and not logger_name.startswith("dvc."):
+            logger.disabled = True
+
+
 def setup(level=logging.INFO):
    colorama.init()

@@ -191,22 +191,7 @@ def setup(level=logging.INFO):
                        "console_errors",
                    ],
                },
-                "paramiko": {
-                    "level": logging.CRITICAL,
-                    "handlers": [
-                        "console_info",
-                        "console_debug",
-                        "console_errors",
-                    ],
-                },
-                "flufl.lock": {
-                    "level": logging.CRITICAL,
-                    "handlers": [
-                        "console_info",
-                        "console_debug",
-                        "console_errors",
-                    ],
-                },
            },
+            "disable_existing_loggers": False,
        }
    )
@@ -6,12 +6,11 @@ import logging
 from dvc import analytics
 from dvc.cli import parse_args
 from dvc.config import ConfigError
-from dvc.exceptions import DvcParserError
-from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
 from dvc.external_repo import clean_repos
-from dvc.logger import FOOTER
-from dvc.utils import format_link
+from dvc.logger import FOOTER, disable_other_loggers
 from dvc.remote.pool import close_pools
+from dvc.utils import format_link

 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
@@ -33,7 +32,7 @@ def main(argv=None):
        int: command's return code.
    """
    args = None
-    cmd = None
+    disable_other_loggers()

    outerLogLevel = logger.level
    try:
@@ -58,6 +57,9 @@ def main(argv=None):
        ret = 253
    except DvcParserError:
        ret = 254
+    except DvcException:
+        ret = 255
+        logger.exception("")
    except Exception as exc:  # pylint: disable=broad-except
        if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
            logger.exception(
@@ -1,34 +1,41 @@
 from urllib.parse import urlparse
-from voluptuous import Any, Required

-from dvc.output.base import OutputBase
-from dvc.output.gs import OutputGS
-from dvc.output.hdfs import OutputHDFS
-from dvc.output.local import OutputLOCAL
-from dvc.output.s3 import OutputS3
-from dvc.output.ssh import OutputSSH
+from voluptuous import And, Any, Coerce, Length, Lower, Required, SetTo
+
+from dvc.output.base import BaseOutput
+from dvc.output.gs import GSOutput
+from dvc.output.hdfs import HDFSOutput
+from dvc.output.local import LocalOutput
+from dvc.output.s3 import S3Output
+from dvc.output.ssh import SSHOutput
 from dvc.remote import Remote
-from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
+from dvc.remote.hdfs import HDFSRemote
+from dvc.remote.local import LocalRemote
+from dvc.remote.s3 import S3Remote
 from dvc.scheme import Schemes

 OUTS = [
-    OutputHDFS,
-    OutputS3,
-    OutputGS,
-    OutputSSH,
-    # NOTE: OutputLOCAL is the default choice
+    HDFSOutput,
+    S3Output,
+    GSOutput,
+    SSHOutput,
+    # NOTE: LocalOutput is the default choice
 ]

 OUTS_MAP = {
-    Schemes.HDFS: OutputHDFS,
-    Schemes.S3: OutputS3,
-    Schemes.GS: OutputGS,
-    Schemes.SSH: OutputSSH,
-    Schemes.LOCAL: OutputLOCAL,
+    Schemes.HDFS: HDFSOutput,
+    Schemes.S3: S3Output,
+    Schemes.GS: GSOutput,
+    Schemes.SSH: SSHOutput,
+    Schemes.LOCAL: LocalOutput,
 }

+CHECKSUM_SCHEMA = Any(
+    None,
+    And(str, Length(max=0), SetTo(None)),
+    And(Any(str, And(int, Coerce(str))), Length(min=3), Lower),
+)
+
 # NOTE: currently there are only 3 possible checksum names:
 #
 #    1) md5 (LOCAL, SSH, GS);
@@ -37,23 +44,21 @@ OUTS_MAP = {
 #
 # so when a few types of outputs share the same name, we only need
 # specify it once.
-CHECKSUM_SCHEMA = {
-    RemoteLOCAL.PARAM_CHECKSUM: Any(str, None),
-    RemoteS3.PARAM_CHECKSUM: Any(str, None),
-    RemoteHDFS.PARAM_CHECKSUM: Any(str, None),
+CHECKSUMS_SCHEMA = {
+    LocalRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    S3Remote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    HDFSRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
 }

-TAGS_SCHEMA = {str: CHECKSUM_SCHEMA}
-
-SCHEMA = CHECKSUM_SCHEMA.copy()
-SCHEMA[Required(OutputBase.PARAM_PATH)] = str
-SCHEMA[OutputBase.PARAM_CACHE] = bool
-SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA
-SCHEMA[OutputBase.PARAM_TAGS] = TAGS_SCHEMA
-SCHEMA[OutputBase.PARAM_PERSIST] = bool
+SCHEMA = CHECKSUMS_SCHEMA.copy()
+SCHEMA[Required(BaseOutput.PARAM_PATH)] = str
+SCHEMA[BaseOutput.PARAM_CACHE] = bool
+SCHEMA[BaseOutput.PARAM_METRIC] = BaseOutput.METRIC_SCHEMA
+SCHEMA[BaseOutput.PARAM_PLOT] = bool
+SCHEMA[BaseOutput.PARAM_PERSIST] = bool


-def _get(stage, p, info, cache, metric, persist=False, tags=None):
+def _get(stage, p, info, cache, metric, plot=False, persist=False):
    parsed = urlparse(p)

    if parsed.scheme == "remote":
@@ -65,8 +70,8 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
            cache=cache,
            remote=remote,
            metric=metric,
+            plot=plot,
            persist=persist,
-            tags=tags,
        )

    for o in OUTS:
@@ -78,29 +83,29 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
                cache=cache,
                remote=None,
                metric=metric,
+                plot=plot,
                persist=persist,
-                tags=tags,
            )
-    return OutputLOCAL(
+    return LocalOutput(
        stage,
        p,
        info,
        cache=cache,
        remote=None,
        metric=metric,
+        plot=plot,
        persist=persist,
-        tags=tags,
    )


 def loadd_from(stage, d_list):
    ret = []
    for d in d_list:
-        p = d.pop(OutputBase.PARAM_PATH)
-        cache = d.pop(OutputBase.PARAM_CACHE, True)
-        metric = d.pop(OutputBase.PARAM_METRIC, False)
-        persist = d.pop(OutputBase.PARAM_PERSIST, False)
-        tags = d.pop(OutputBase.PARAM_TAGS, None)
+        p = d.pop(BaseOutput.PARAM_PATH)
+        cache = d.pop(BaseOutput.PARAM_CACHE, True)
+        metric = d.pop(BaseOutput.PARAM_METRIC, False)
+        plot = d.pop(BaseOutput.PARAM_PLOT, False)
+        persist = d.pop(BaseOutput.PARAM_PERSIST, False)
        ret.append(
            _get(
                stage,
@@ -108,24 +113,25 @@ def loadd_from(stage, d_list):
                info=d,
                cache=cache,
                metric=metric,
+                plot=plot,
                persist=persist,
-                tags=tags,
            )
        )
    return ret


-def loads_from(stage, s_list, use_cache=True, metric=False, persist=False):
-    ret = []
-    for s in s_list:
-        ret.append(
-            _get(
-                stage,
-                s,
-                info={},
-                cache=use_cache,
-                metric=metric,
-                persist=persist,
-            )
+def loads_from(
+    stage, s_list, use_cache=True, metric=False, plot=False, persist=False
+):
+    return [
+        _get(
+            stage,
+            s,
+            info={},
+            cache=use_cache,
+            metric=metric,
+            plot=plot,
+            persist=persist,
        )
-    return ret
+        for s in s_list
+    ]
@@ -1,52 +1,57 @@
 import logging
-from urllib.parse import urlparse
+import os
 from copy import copy
+from urllib.parse import urlparse

 from voluptuous import Any

 import dvc.prompt as prompt
 from dvc.cache import NamedCache
-from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError
-from dvc.exceptions import DvcException
-from dvc.remote.base import RemoteBASE
-
+from dvc.exceptions import (
+    CollectCacheError,
+    DvcException,
+    RemoteCacheRequiredError,
+)
+from dvc.remote.base import BaseRemote

 logger = logging.getLogger(__name__)


 class OutputDoesNotExistError(DvcException):
    def __init__(self, path):
-        msg = "output '{}' does not exist".format(path)
+        msg = f"output '{path}' does not exist"
        super().__init__(msg)


 class OutputIsNotFileOrDirError(DvcException):
    def __init__(self, path):
-        msg = "output '{}' is not a file or directory".format(path)
+        msg = f"output '{path}' is not a file or directory"
        super().__init__(msg)


 class OutputAlreadyTrackedError(DvcException):
    def __init__(self, path):
-        msg = "output '{}' is already tracked by scm (e.g. git)".format(path)
+        msg = f"output '{path}' is already tracked by SCM (e.g. Git)"
        super().__init__(msg)


 class OutputIsStageFileError(DvcException):
    def __init__(self, path):
-        super().__init__("Stage file '{}' cannot be an output.".format(path))
+        super().__init__(f"Stage file '{path}' cannot be an output.")


-class OutputBase(object):
+class BaseOutput:
    IS_DEPENDENCY = False

-    REMOTE = RemoteBASE
+    REMOTE = BaseRemote

    PARAM_PATH = "path"
    PARAM_CACHE = "cache"
    PARAM_METRIC = "metric"
    PARAM_METRIC_TYPE = "type"
    PARAM_METRIC_XPATH = "xpath"
+    PARAM_PLOT = "plot"
+    PARAM_PLOT_TEMPLATE = "template"
    PARAM_PERSIST = "persist"

    METRIC_SCHEMA = Any(
@@ -58,8 +63,6 @@ class OutputBase(object):
        },
    )

-    PARAM_TAGS = "tags"
-
    DoesNotExistError = OutputDoesNotExistError
    IsNotFileOrDirError = OutputIsNotFileOrDirError
    IsStageFileError = OutputIsStageFileError
@@ -74,8 +77,8 @@ class OutputBase(object):
        remote=None,
        cache=True,
        metric=False,
+        plot=False,
        persist=False,
-        tags=None,
    ):
        self._validate_output_path(path)
        # This output (and dependency) objects have too many paths/urls
@@ -95,8 +98,8 @@ class OutputBase(object):
        self.remote = remote or self.REMOTE(self.repo, {})
        self.use_cache = False if self.IS_DEPENDENCY else cache
        self.metric = False if self.IS_DEPENDENCY else metric
+        self.plot = False if self.IS_DEPENDENCY else plot
        self.persist = persist
-        self.tags = None if self.IS_DEPENDENCY else (tags or {})

        self.path_info = self._parse_path(remote, path)
        if self.use_cache and self.cache is None:
@@ -147,10 +150,21 @@ class OutputBase(object):
    def cache_path(self):
        return self.cache.checksum_to_path_info(self.checksum).url

+    @property
+    def checksum_type(self):
+        return self.remote.PARAM_CHECKSUM
+
    @property
    def checksum(self):
        return self.info.get(self.remote.PARAM_CHECKSUM)

+    @checksum.setter
+    def checksum(self, checksum):
+        self.info[self.remote.PARAM_CHECKSUM] = checksum
+
+    def get_checksum(self):
+        return self.remote.get_checksum(self.path_info)
+
    @property
    def is_dir_checksum(self):
        return self.remote.is_dir_checksum(self.checksum)
@@ -159,14 +173,17 @@ class OutputBase(object):
    def exists(self):
        return self.remote.exists(self.path_info)

-    def changed_checksum(self):
-        return self.checksum != self.remote.get_checksum(self.path_info)
+    def save_info(self):
+        return self.remote.save_info(self.path_info)

-    def changed_cache(self):
+    def changed_checksum(self):
+        return self.checksum != self.get_checksum()
+
+    def changed_cache(self, filter_info=None):
        if not self.use_cache or not self.checksum:
            return True

-        return self.cache.changed_cache(self.checksum)
+        return self.cache.changed_cache(self.checksum, filter_info=filter_info)

    def status(self):
        if self.checksum and self.use_cache and self.changed_cache():
@@ -198,6 +215,15 @@ class OutputBase(object):
    def isfile(self):
        return self.remote.isfile(self.path_info)

+    def ignore(self):
+        if not self.use_scm_ignore:
+            return
+
+        if self.repo.scm.is_tracked(self.fspath):
+            raise OutputAlreadyTrackedError(self)
+
+        self.repo.scm.ignore(self.fspath)
+
    def save(self):
        if not self.exists:
            raise self.DoesNotExistError(self)
@@ -206,17 +232,13 @@ class OutputBase(object):
            raise self.IsNotFileOrDirError(self)

        if self.is_empty:
-            logger.warning("'{}' is empty.".format(self))
+            logger.warning(f"'{self}' is empty.")

-        if self.use_scm_ignore:
-            if self.repo.scm.is_tracked(self.fspath):
-                raise OutputAlreadyTrackedError(self)
-
-            self.repo.scm.ignore(self.fspath)
+        self.ignore()

        if not self.use_cache:
-            self.info = self.remote.save_info(self.path_info)
-            if self.metric:
+            self.info = self.save_info()
+            if self.metric or self.plot:
                self.verify_metric()
            if not self.IS_DEPENDENCY:
                logger.info(
@@ -229,12 +251,10 @@ class OutputBase(object):
        assert not self.IS_DEPENDENCY

        if not self.changed():
-            logger.info(
-                "Output '{}' didn't change. Skipping saving.".format(self)
-            )
+            logger.info(f"Output '{self}' didn't change. Skipping saving.")
            return

-        self.info = self.remote.save_info(self.path_info)
+        self.info = self.save_info()

    def commit(self):
        if self.use_cache:
@@ -257,17 +277,16 @@ class OutputBase(object):
                del self.metric[self.PARAM_METRIC_XPATH]

        ret[self.PARAM_METRIC] = self.metric
-        ret[self.PARAM_PERSIST] = self.persist

-        if self.tags:
-            ret[self.PARAM_TAGS] = self.tags
+        if self.plot:
+            ret[self.PARAM_PLOT] = self.plot
+
+        ret[self.PARAM_PERSIST] = self.persist

        return ret

    def verify_metric(self):
-        raise DvcException(
-            "verify metric is not supported for {}".format(self.scheme)
-        )
+        raise DvcException(f"verify metric is not supported for {self.scheme}")

    def download(self, to):
        self.remote.download(self.path_info, to.path_info)
@@ -276,7 +295,6 @@ class OutputBase(object):
        self,
        force=False,
        progress_callback=None,
-        tag=None,
        relink=False,
        filter_info=None,
    ):
@@ -287,14 +305,9 @@ class OutputBase(object):
                )
            return None

-        if tag:
-            info = self.tags[tag]
-        else:
-            info = self.info
-
        return self.cache.checkout(
            self.path_info,
-            info,
+            self.info,
            force=force,
            progress_callback=progress_callback,
            relink=relink,
@@ -334,7 +347,7 @@ class OutputBase(object):
        if self.exists:
            self.remote.unprotect(self.path_info)

-    def _collect_used_dir_cache(
+    def collect_used_dir_cache(
        self, remote=None, force=False, jobs=None, filter_info=None
    ):
        """Get a list of `info`s related to the given directory.
@@ -367,7 +380,7 @@ class OutputBase(object):
                    show_checksums=False,
                )
            except DvcException:
-                logger.debug("failed to pull cache for '{}'".format(self))
+                logger.debug(f"failed to pull cache for '{self}'")

        if self.cache.changed_cache_file(self.checksum):
            msg = (
@@ -383,11 +396,21 @@ class OutputBase(object):
            else:
                return cache

+        path = str(self.path_info)
+        filter_path = str(filter_info) if filter_info else None
+        is_win = os.name == "nt"
        for entry in self.dir_cache:
            checksum = entry[self.remote.PARAM_CHECKSUM]
-            info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            if not filter_info or info.isin_or_eq(filter_info):
-                cache.add(self.scheme, checksum, str(info))
+            entry_relpath = entry[self.remote.PARAM_RELPATH]
+            if is_win:
+                entry_relpath = entry_relpath.replace("/", os.sep)
+            entry_path = os.path.join(path, entry_relpath)
+            if (
+                not filter_path
+                or entry_path == filter_path
+                or entry_path.startswith(filter_path + os.sep)
+            ):
+                cache.add(self.scheme, checksum, entry_path)

        return cache

@@ -409,12 +432,23 @@ class OutputBase(object):
            cache.external[dep.repo_pair].add(dep.def_path)
            return cache

-        if not self.info:
-            logger.warning(
-                "Output '{}'({}) is missing version info. Cache for it will "
-                "not be collected. Use `dvc repro` to get your pipeline up to "
-                "date.".format(self, self.stage)
+        if not self.checksum:
+            msg = (
+                "Output '{}'({}) is missing version info. "
+                "Cache for it will not be collected. "
+                "Use `dvc repro` to get your pipeline up to date.".format(
+                    self, self.stage
+                )
            )
+            if self.exists:
+                msg += (
+                    "\n"
+                    "You can also use `dvc commit {stage.addressing}` "
+                    "to associate existing '{out}' with {stage}.".format(
+                        out=self, stage=self.stage
+                    )
+                )
+            logger.warning(msg)
            return NamedCache()

        ret = NamedCache.make(self.scheme, self.checksum, str(self))
@@ -422,13 +456,15 @@ class OutputBase(object):
        if not self.is_dir_checksum:
            return ret

-        ret.update(self._collect_used_dir_cache(**kwargs))
+        ret.add_child_cache(
+            self.checksum, self.collect_used_dir_cache(**kwargs),
+        )

        return ret

    @classmethod
    def _validate_output_path(cls, path):
-        from dvc.stage import Stage
+        from dvc.dvcfile import is_valid_filename

-        if Stage.is_valid_filename(path):
+        if is_valid_filename(path):
            raise cls.IsStageFileError(path)
@@ -1,6 +1,6 @@
-from dvc.output.s3 import OutputS3
-from dvc.remote.gs import RemoteGS
+from dvc.output.s3 import S3Output
+from dvc.remote.gs import GSRemote


-class OutputGS(OutputS3):
-    REMOTE = RemoteGS
+class GSOutput(S3Output):
+    REMOTE = GSRemote
@@ -1,6 +1,6 @@
-from dvc.output.base import OutputBase
-from dvc.remote.hdfs import RemoteHDFS
+from dvc.output.base import BaseOutput
+from dvc.remote.hdfs import HDFSRemote


-class OutputHDFS(OutputBase):
-    REMOTE = RemoteHDFS
+class HDFSOutput(BaseOutput):
+    REMOTE = HDFSRemote
@@ -4,18 +4,16 @@ from urllib.parse import urlparse

 from dvc.exceptions import DvcException
 from dvc.istextfile import istextfile
-from dvc.output.base import OutputBase
-from dvc.remote.local import RemoteLOCAL
+from dvc.output.base import BaseOutput
+from dvc.remote.local import LocalRemote
 from dvc.utils import relpath
-from dvc.compat import fspath_py35
 from dvc.utils.fs import path_isin

-
 logger = logging.getLogger(__name__)


-class OutputLOCAL(OutputBase):
-    REMOTE = RemoteLOCAL
+class LocalOutput(BaseOutput):
+    REMOTE = LocalRemote
    sep = os.sep

    def __init__(self, stage, path, *args, **kwargs):
@@ -39,7 +37,7 @@ class OutputLOCAL(OutputBase):
            if not p.is_absolute():
                p = self.stage.wdir / p

-        abs_p = os.path.abspath(os.path.normpath(fspath_py35(p)))
+        abs_p = os.path.abspath(os.path.normpath(p))
        return self.REMOTE.path_cls(abs_p)

    def __str__(self):
@@ -73,17 +71,18 @@ class OutputLOCAL(OutputBase):
        return ret

    def verify_metric(self):
-        if not self.metric:
+        if not self.metric or self.plot:
            return

-        path = fspath_py35(self.path_info)
+        path = os.fspath(self.path_info)
        if not os.path.exists(path):
            return

+        name = "metrics" if self.metric else "plot"
        if os.path.isdir(path):
-            msg = "directory '{}' cannot be used as metrics."
-            raise DvcException(msg.format(self.path_info))
+            msg = "directory '{}' cannot be used as {}."
+            raise DvcException(msg.format(self.path_info, name))

        if not istextfile(path):
-            msg = "binary file '{}' cannot be used as metrics."
-            raise DvcException(msg.format(self.path_info))
+            msg = "binary file '{}' cannot be used as {}."
+            raise DvcException(msg.format(self.path_info, name))
@@ -1,6 +1,6 @@
-from dvc.output.base import OutputBase
-from dvc.remote.s3 import RemoteS3
+from dvc.output.base import BaseOutput
+from dvc.remote.s3 import S3Remote


-class OutputS3(OutputBase):
-    REMOTE = RemoteS3
+class S3Output(BaseOutput):
+    REMOTE = S3Remote
@@ -1,6 +1,6 @@
-from dvc.output.base import OutputBase
-from dvc.remote.ssh import RemoteSSH
+from dvc.output.base import BaseOutput
+from dvc.remote.ssh import SSHRemote


-class OutputSSH(OutputBase):
-    REMOTE = RemoteSSH
+class SSHOutput(BaseOutput):
+    REMOTE = SSHRemote
@@ -1,6 +1,6 @@
 import os
-import posixpath
 import pathlib
+import posixpath
 from urllib.parse import urlparse

 from funcy import cached_property
@@ -8,7 +8,7 @@ from funcy import cached_property
 from dvc.utils import relpath


-class _BasePath(object):
+class _BasePath:
    def overlaps(self, other):
        if isinstance(other, (str, bytes)):
            other = self.__class__(other)
@@ -33,10 +33,6 @@ class PathInfo(pathlib.PurePath, _BasePath):
            cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo
        return cls._from_parts(args)

-    @classmethod
-    def from_posix(cls, s):
-        return cls(PosixPathInfo(s))
-
    def as_posix(self):
        f = self._flavour
        # Unlike original implementation [1] that uses `str()` we actually need
@@ -54,7 +50,6 @@ class PathInfo(pathlib.PurePath, _BasePath):
        return "{}: '{}'".format(type(self).__name__, self)

    # This permits passing it to file utils directly in Python 3.6+
-    # With Python 2.7, Python 3.5+ we are stuck with path_info.fspath for now
    def __fspath__(self):
        return pathlib.PurePath.__str__(self)

@@ -92,7 +87,7 @@ class _URLPathInfo(PosixPathInfo):
    __unicode__ = __str__


-class _URLPathParents(object):
+class _URLPathParents:
    def __init__(self, src):
        self.src = src
        self._parents = self.src._path.parents
@@ -104,7 +99,7 @@ class _URLPathParents(object):
        return self.src.replace(path=self._parents[idx])

    def __repr__(self):
-        return "<{}.parents>".format(self.src)
+        return f"<{self.src}.parents>"


 class URLInfo(_BasePath):
@@ -115,7 +110,7 @@ class URLInfo(_BasePath):
        assert not p.query and not p.params and not p.fragment
        assert p.password is None

-        self.fill_parts(p.scheme, p.hostname, p.username, p.port, p.path)
+        self._fill_parts(p.scheme, p.hostname, p.username, p.port, p.path)

    @classmethod
    def from_parts(
@@ -124,13 +119,13 @@ class URLInfo(_BasePath):
        assert bool(host) ^ bool(netloc)

        if netloc is not None:
-            return cls("{}://{}{}".format(scheme, netloc, path))
+            return cls(f"{scheme}://{netloc}{path}")

        obj = cls.__new__(cls)
-        obj.fill_parts(scheme, host, user, port, path)
+        obj._fill_parts(scheme, host, user, port, path)
        return obj

-    def fill_parts(self, scheme, host, user, port, path):
+    def _fill_parts(self, scheme, host, user, port, path):
        assert scheme != "remote"
        assert isinstance(path, (str, bytes, _URLPathInfo))

@@ -158,7 +153,7 @@ class URLInfo(_BasePath):

    @cached_property
    def url(self):
-        return "{}://{}{}".format(self.scheme, self.netloc, self._spath)
+        return f"{self.scheme}://{self.netloc}{self._spath}"

    def __str__(self):
        return self.url
@@ -220,10 +215,10 @@ class URLInfo(_BasePath):
        if isinstance(other, (str, bytes)):
            other = self.__class__(other)
        if self.__class__ != other.__class__:
-            msg = "'{}' has incompatible class with '{}'".format(self, other)
+            msg = f"'{self}' has incompatible class with '{other}'"
            raise ValueError(msg)
        if self._base_parts != other._base_parts:
-            msg = "'{}' does not start with '{}'".format(self, other)
+            msg = f"'{self}' does not start with '{other}'"
            raise ValueError(msg)
        return self._path.relative_to(other._path)

@@ -241,3 +236,78 @@ class CloudURLInfo(URLInfo):
    @property
    def path(self):
        return self._spath.lstrip("/")
+
+
+class HTTPURLInfo(URLInfo):
+    __hash__ = URLInfo.__hash__
+
+    def __init__(self, url):
+        p = urlparse(url)
+        stripped = p._replace(params=None, query=None, fragment=None)
+        super().__init__(stripped.geturl())
+        self.params = p.params
+        self.query = p.query
+        self.fragment = p.fragment
+
+    @classmethod
+    def from_parts(
+        cls,
+        scheme=None,
+        host=None,
+        user=None,
+        port=None,
+        path="",
+        netloc=None,
+        params=None,
+        query=None,
+        fragment=None,
+    ):
+        assert bool(host) ^ bool(netloc)
+
+        if netloc is not None:
+            return cls(
+                "{}://{}{}{}{}{}".format(
+                    scheme,
+                    netloc,
+                    path,
+                    (";" + params) if params else "",
+                    ("?" + query) if query else "",
+                    ("#" + fragment) if fragment else "",
+                )
+            )
+
+        obj = cls.__new__(cls)
+        obj._fill_parts(scheme, host, user, port, path)
+        obj.params = params
+        obj.query = query
+        obj.fragment = fragment
+        return obj
+
+    @property
+    def _extra_parts(self):
+        return (self.params, self.query, self.fragment)
+
+    @property
+    def parts(self):
+        return self._base_parts + self._path.parts + self._extra_parts
+
+    @cached_property
+    def url(self):
+        return "{}://{}{}{}{}{}".format(
+            self.scheme,
+            self.netloc,
+            self._spath,
+            (";" + self.params) if self.params else "",
+            ("?" + self.query) if self.query else "",
+            ("#" + self.fragment) if self.fragment else "",
+        )
+
+    def __eq__(self, other):
+        if isinstance(other, (str, bytes)):
+            other = self.__class__(other)
+        return (
+            self.__class__ == other.__class__
+            and self._base_parts == other._base_parts
+            and self._path == other._path
+            and self._extra_parts == other._extra_parts
+        )
@@ -19,18 +19,17 @@ class Tqdm(tqdm):

    BAR_FMT_DEFAULT = (
        "{percentage:3.0f}% {desc}|{bar}|"
-        "{n_fmt}/{total_fmt}"
-        " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
+        "{postfix[info]}{n_fmt}/{total_fmt}"
+        " [{elapsed}<{remaining}, {rate_fmt:>11}]"
    )
    # nested bars should have fixed bar widths to align nicely
    BAR_FMT_DEFAULT_NESTED = (
        "{percentage:3.0f}%|{bar:10}|{desc:{ncols_desc}.{ncols_desc}}"
-        "{n_fmt}/{total_fmt}"
-        " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
+        "{postfix[info]}{n_fmt}/{total_fmt}"
+        " [{elapsed}<{remaining}, {rate_fmt:>11}]"
    )
    BAR_FMT_NOTOTAL = (
-        "{desc:{ncols_desc}.{ncols_desc}}{n_fmt}"
-        " [{elapsed}, {rate_fmt:>11}{postfix}]"
+        "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
    )
    BYTES_DEFAULTS = dict(
        unit="B", unit_scale=True, unit_divisor=1024, miniters=1
@@ -47,6 +46,7 @@ class Tqdm(tqdm):
        bytes=False,  # pylint: disable=W0622
        file=None,
        total=None,
+        postfix=None,
        **kwargs
    ):
        """
@@ -55,7 +55,8 @@ class Tqdm(tqdm):
        desc  : persists after `close()`
        level  : effective logging level for determining `disable`;
            used only if `disable` is unspecified
-        disable  : If (default: None), will be determined by logging level.
+        disable  : If (default: None) or False,
+            will be determined by logging level.
            May be overridden to `True` due to non-TTY status.
            Skip override by specifying env var `DVC_IGNORE_ISATTY`.
        kwargs  : anything accepted by `tqdm.tqdm()`
@@ -67,9 +68,8 @@ class Tqdm(tqdm):
            kwargs.setdefault("unit_scale", total > 999 if total else True)
        if file is None:
            file = sys.stderr
-        self.desc_persist = desc
        # auto-disable based on `logger.level`
-        if disable is None:
+        if not disable:
            disable = logger.getEffectiveLevel() > level
        # auto-disable based on TTY
        if (
@@ -88,6 +88,7 @@ class Tqdm(tqdm):
            total=total,
            **kwargs
        )
+        self.postfix = postfix or {"info": ""}
        if bar_format is None:
            if self.__len__():
                self.bar_format = (
@@ -101,11 +102,11 @@ class Tqdm(tqdm):
            self.bar_format = bar_format
        self.refresh()

-    def update_desc(self, desc, n=1):
+    def update_msg(self, msg, n=1):
        """
-        Calls `set_description_str(desc)` and `update(n)`
+        Sets `msg` as a postfix and calls `update(n)`.
        """
-        self.set_description_str(desc, refresh=False)
+        self.postfix["info"] = " %s |" % msg
        self.update(n)

    def update_to(self, current, total=None):
@@ -113,13 +114,27 @@ class Tqdm(tqdm):
            self.total = total  # pylint: disable=W0613,W0201
        self.update(current - self.n)

+    def wrap_fn(self, fn, callback=None):
+        """
+        Returns a wrapped `fn` which calls `callback()` on each call.
+        `callback` is `self.update` by default.
+        """
+        if callback is None:
+            callback = self.update
+
+        def wrapped(*args, **kwargs):
+            res = fn(*args, **kwargs)
+            callback()
+            return res
+
+        return wrapped
+
    def close(self):
-        if self.desc_persist is not None:
-            self.set_description_str(self.desc_persist, refresh=False)
-        # unknown/zero ETA
-        self.bar_format = self.bar_format.replace("<{remaining}", "")
-        # remove completed bar
-        self.bar_format = self.bar_format.replace("|{bar:10}|", " ")
+        self.postfix["info"] = ""
+        # remove ETA (either unknown or zero); remove completed bar
+        self.bar_format = self.bar_format.replace("<{remaining}", "").replace(
+            "|{bar:10}|", " "
+        )
        super().close()

    @property
@@ -127,12 +142,15 @@ class Tqdm(tqdm):
        """inject `ncols_desc` to fill the display width (`ncols`)"""
        d = super().format_dict
        ncols = d["ncols"] or 80
-        ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1
-        ncols_desc = max(ncols_desc, 0)
-        if ncols_desc:
-            d["ncols_desc"] = ncols_desc
+        # assumes `bar_format` has max one of ("ncols_desc" & "ncols_info")
+        ncols_left = (
+            ncols - len(self.format_meter(ncols_desc=1, ncols_info=1, **d)) + 1
+        )
+        ncols_left = max(ncols_left, 0)
+        if ncols_left:
+            d["ncols_desc"] = d["ncols_info"] = ncols_left
        else:
            # work-around for zero-width description
-            d["ncols_desc"] = 1
+            d["ncols_desc"] = d["ncols_info"] = 1
            d["prefix"] = ""
        return d
@@ -38,7 +38,7 @@ def confirm(statement):
    Returns:
        bool: whether or not specified statement was confirmed.
    """
-    prompt = "{statement} [y/n]".format(statement=statement)
+    prompt = f"{statement} [y/n]"
    answer = _ask(prompt, limited_to=["yes", "no", "y", "n"])
    return answer and answer.startswith("y")

@@ -52,5 +52,5 @@ def password(statement):
    Returns:
        str: password entered by the user.
    """
-    logger.info("{statement}: ".format(statement=statement))
+    logger.info(f"{statement}: ")
    return getpass("")
@@ -1,42 +1,74 @@
-from .config import RemoteConfig
-from dvc.remote.azure import RemoteAZURE
-from dvc.remote.gdrive import RemoteGDrive
-from dvc.remote.gs import RemoteGS
-from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.http import RemoteHTTP
-from dvc.remote.https import RemoteHTTPS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.oss import RemoteOSS
-from dvc.remote.s3 import RemoteS3
-from dvc.remote.ssh import RemoteSSH
+import posixpath
+from urllib.parse import urlparse

+from dvc.remote.azure import AzureRemote
+from dvc.remote.gdrive import GDriveRemote
+from dvc.remote.gs import GSRemote
+from dvc.remote.hdfs import HDFSRemote
+from dvc.remote.http import HTTPRemote
+from dvc.remote.https import HTTPSRemote
+from dvc.remote.local import LocalRemote
+from dvc.remote.oss import OSSRemote
+from dvc.remote.s3 import S3Remote
+from dvc.remote.ssh import SSHRemote

 REMOTES = [
-    RemoteAZURE,
-    RemoteGDrive,
-    RemoteGS,
-    RemoteHDFS,
-    RemoteHTTP,
-    RemoteHTTPS,
-    RemoteS3,
-    RemoteSSH,
-    RemoteOSS,
-    # NOTE: RemoteLOCAL is the default
+    AzureRemote,
+    GDriveRemote,
+    GSRemote,
+    HDFSRemote,
+    HTTPRemote,
+    HTTPSRemote,
+    S3Remote,
+    SSHRemote,
+    OSSRemote,
+    # NOTE: LocalRemote is the default
 ]


-def _get(config):
+def _get(remote_conf):
    for remote in REMOTES:
-        if remote.supported(config):
+        if remote.supported(remote_conf):
            return remote
-    return RemoteLOCAL
+    return LocalRemote


 def Remote(repo, **kwargs):
    name = kwargs.get("name")
    if name:
-        remote_config = RemoteConfig(repo.config)
-        settings = remote_config.get_settings(name)
+        remote_conf = repo.config["remote"][name.lower()]
    else:
-        settings = kwargs
-    return _get(settings)(repo, settings)
+        remote_conf = kwargs
+    remote_conf = _resolve_remote_refs(repo.config, remote_conf)
+    return _get(remote_conf)(repo, remote_conf)
+
+
+def _resolve_remote_refs(config, remote_conf):
+    # Support for cross referenced remotes.
+    # This will merge the settings, shadowing base ref with remote_conf.
+    # For example, having:
+    #
+    #       dvc remote add server ssh://localhost
+    #       dvc remote modify server user root
+    #       dvc remote modify server ask_password true
+    #
+    #       dvc remote add images remote://server/tmp/pictures
+    #       dvc remote modify images user alice
+    #       dvc remote modify images ask_password false
+    #       dvc remote modify images password asdf1234
+    #
+    # Results on a config dictionary like:
+    #
+    #       {
+    #           "url": "ssh://localhost/tmp/pictures",
+    #           "user": "alice",
+    #           "password": "asdf1234",
+    #           "ask_password": False,
+    #       }
+    parsed = urlparse(remote_conf["url"])
+    if parsed.scheme != "remote":
+        return remote_conf
+
+    base = config["remote"][parsed.netloc]
+    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
+    return {**base, **remote_conf, "url": url}
@@ -1,23 +1,22 @@
 import logging
 import os
+import posixpath
 import re
+import threading
 from datetime import datetime, timedelta
 from urllib.parse import urlparse
-import threading

 from funcy import cached_property, wrap_prop

-from dvc.config import Config
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
+from dvc.remote.base import BaseRemote
 from dvc.scheme import Schemes

-
 logger = logging.getLogger(__name__)


-class RemoteAZURE(RemoteBASE):
+class AzureRemote(BaseRemote):
    scheme = Schemes.AZURE
    path_cls = CloudURLInfo
    REGEX = (
@@ -29,11 +28,12 @@ class RemoteAZURE(RemoteBASE):
    REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
    PARAM_CHECKSUM = "etag"
    COPY_POLL_SECONDS = 5
+    LIST_OBJECT_PAGE_SIZE = 5000

    def __init__(self, repo, config):
        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL, "azure://")
+        url = config.get("url", "azure://")

        match = re.match(self.REGEX, url)  # backward compatibility
        path = match.group("path")
@@ -44,7 +44,7 @@ class RemoteAZURE(RemoteBASE):
        )

        self.connection_string = (
-            config.get(Config.SECTION_AZURE_CONNECTION_STRING)
+            config.get("connection_string")
            or match.group("connection_string")  # backward compatibility
            or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
        )
@@ -67,12 +67,12 @@ class RemoteAZURE(RemoteBASE):
        from azure.storage.blob import BlockBlobService
        from azure.common import AzureMissingResourceHttpError

-        logger.debug("URL {}".format(self.path_info))
-        logger.debug("Connection string {}".format(self.connection_string))
+        logger.debug(f"URL {self.path_info}")
+        logger.debug(f"Connection string {self.connection_string}")
        blob_service = BlockBlobService(
            connection_string=self.connection_string
        )
-        logger.debug("Container name {}".format(self.path_info.bucket))
+        logger.debug(f"Container name {self.path_info.bucket}")
        try:  # verify that container exists
            blob_service.list_blobs(
                self.path_info.bucket, delimiter="/", num_results=1
@@ -85,10 +85,10 @@ class RemoteAZURE(RemoteBASE):
        if path_info.scheme != self.scheme:
            raise NotImplementedError

-        logger.debug("Removing {}".format(path_info))
+        logger.debug(f"Removing {path_info}")
        self.blob_service.delete_blob(path_info.bucket, path_info.path)

-    def _list_paths(self, bucket, prefix):
+    def _list_paths(self, bucket, prefix, progress_callback=None):
        blob_service = self.blob_service
        next_marker = None
        while True:
@@ -97,6 +97,8 @@ class RemoteAZURE(RemoteBASE):
            )

            for blob in blobs:
+                if progress_callback:
+                    progress_callback()
                yield blob.name

            if not blobs.next_marker:
@@ -104,8 +106,16 @@ class RemoteAZURE(RemoteBASE):

            next_marker = blobs.next_marker

-    def list_cache_paths(self):
-        return self._list_paths(self.path_info.bucket, self.path_info.path)
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            prefix = posixpath.join(
+                self.path_info.path, prefix[:2], prefix[2:]
+            )
+        else:
+            prefix = self.path_info.path
+        return self._list_paths(
+            self.path_info.bucket, prefix, progress_callback
+        )

    def _upload(
        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
@@ -1,32 +1,34 @@
 import errno
+import hashlib
 import itertools
 import json
 import logging
 import tempfile
-from urllib.parse import urlparse
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
-from functools import partial
+from functools import partial, wraps
 from multiprocessing import cpu_count
 from operator import itemgetter
+from urllib.parse import urlparse

 from shortuuid import uuid

 import dvc.prompt as prompt
-from dvc.config import Config
 from dvc.exceptions import (
-    DvcException,
+    CheckoutError,
    ConfirmRemoveError,
+    DvcException,
    DvcIgnoreInCollectedDirError,
    RemoteCacheRequiredError,
 )
 from dvc.ignore import DvcIgnore
-from dvc.path_info import PathInfo, URLInfo
+from dvc.path_info import PathInfo, URLInfo, WindowsPathInfo
 from dvc.progress import Tqdm
+from dvc.remote.index import RemoteIndex, RemoteIndexNoop
 from dvc.remote.slow_link_detection import slow_link_guard
 from dvc.state import StateNoop
 from dvc.utils import tmp_fname
-from dvc.utils.fs import move, makedirs
+from dvc.utils.fs import makedirs, move
 from dvc.utils.http import open_url

 logger = logging.getLogger(__name__)
@@ -55,7 +57,7 @@ class RemoteCmdError(DvcException):

 class RemoteActionNotImplemented(DvcException):
    def __init__(self, action, scheme):
-        m = "{} is not supported for {} remotes".format(action, scheme)
+        m = f"{action} is not supported for {scheme} remotes"
        super().__init__(m)


@@ -66,22 +68,40 @@ class RemoteMissingDepsError(DvcException):
 class DirCacheError(DvcException):
    def __init__(self, checksum):
        super().__init__(
-            "Failed to load dir cache for hash value: '{}'.".format(checksum)
+            f"Failed to load dir cache for hash value: '{checksum}'."
        )


-class RemoteBASE(object):
+def index_locked(f):
+    @wraps(f)
+    def wrapper(remote_obj, *args, **kwargs):
+        remote = kwargs.get("remote", remote_obj)
+        with remote.index:
+            return f(remote_obj, *args, **kwargs)
+
+    return wrapper
+
+
+class BaseRemote:
    scheme = "base"
    path_cls = URLInfo
    REQUIRES = {}
    JOBS = 4 * cpu_count()
+    INDEX_CLS = RemoteIndex

    PARAM_RELPATH = "relpath"
    CHECKSUM_DIR_SUFFIX = ".dir"
    CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
    DEFAULT_CACHE_TYPES = ["copy"]
-    DEFAULT_NO_TRAVERSE = True
    DEFAULT_VERIFY = False
+    LIST_OBJECT_PAGE_SIZE = 1000
+    TRAVERSE_WEIGHT_MULTIPLIER = 5
+    TRAVERSE_PREFIX_LEN = 3
+    TRAVERSE_THRESHOLD_SIZE = 500000
+    CAN_TRAVERSE = True
+
+    CACHE_MODE = None
+    SHARED_MODE_MAP = {None: (None, None), "group": (None, None)}

    state = StateNoop()

@@ -89,43 +109,49 @@ class RemoteBASE(object):
        self.repo = repo

        self._check_requires(config)
-        self.checksum_jobs = self._get_checksum_jobs(config)

-        self.protected = False
-        self.no_traverse = config.get(
-            Config.SECTION_REMOTE_NO_TRAVERSE, self.DEFAULT_NO_TRAVERSE
-        )
-        self.verify = config.get(
-            Config.SECTION_REMOTE_VERIFY, self.DEFAULT_VERIFY
+        shared = config.get("shared")
+        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
+
+        self.checksum_jobs = (
+            config.get("checksum_jobs")
+            or (self.repo and self.repo.config["core"].get("checksum_jobs"))
+            or self.CHECKSUM_JOBS
        )
+        self.verify = config.get("verify", self.DEFAULT_VERIFY)
        self._dir_info = {}

-        types = config.get(Config.SECTION_CACHE_TYPE, None)
-        if types:
-            if isinstance(types, str):
-                types = [t.strip() for t in types.split(",")]
-            self.cache_types = types
-        else:
-            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
+        self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES)
        self.cache_type_confirmed = False

-    def _check_requires(self, config):
+        url = config.get("url")
+        if url:
+            index_name = hashlib.sha256(url.encode("utf-8")).hexdigest()
+            self.index = self.INDEX_CLS(
+                self.repo, index_name, dir_suffix=self.CHECKSUM_DIR_SUFFIX
+            )
+        else:
+            self.index = RemoteIndexNoop()
+
+    @classmethod
+    def get_missing_deps(cls):
        import importlib

        missing = []
-
-        for package, module in self.REQUIRES.items():
+        for package, module in cls.REQUIRES.items():
            try:
                importlib.import_module(module)
            except ImportError:
                missing.append(package)

+        return missing
+
+    def _check_requires(self, config):
+        missing = self.get_missing_deps()
        if not missing:
            return

-        url = config.get(
-            Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
-        )
+        url = config.get("url", f"{self.scheme}://")
        msg = (
            "URL '{}' is supported but requires these missing "
            "dependencies: {}. If you have installed dvc using pip, "
@@ -146,19 +172,6 @@ class RemoteBASE(object):
        ).format(url, missing, " ".join(missing), self.scheme)
        raise RemoteMissingDepsError(msg)

-    def _get_checksum_jobs(self, config):
-        checksum_jobs = config.get(Config.SECTION_REMOTE_CHECKSUM_JOBS)
-        if checksum_jobs:
-            return checksum_jobs
-
-        if self.repo:
-            core = self.repo.config.config.get(Config.SECTION_CORE, {})
-            return core.get(
-                Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
-            )
-
-        return self.CHECKSUM_JOBS
-
    def __repr__(self):
        return "{class_name}: '{path_info}'".format(
            class_name=type(self).__name__,
@@ -170,7 +183,7 @@ class RemoteBASE(object):
        if isinstance(config, (str, bytes)):
            url = config
        else:
-            url = config[Config.SECTION_REMOTE_URL]
+            url = config["url"]

        # NOTE: silently skipping remote, calling code should handle that
        parsed = urlparse(url)
@@ -185,14 +198,16 @@ class RemoteBASE(object):

    def _calculate_checksums(self, file_infos):
        file_infos = list(file_infos)
-        with ThreadPoolExecutor(max_workers=self.checksum_jobs) as executor:
-            tasks = executor.map(self.get_file_checksum, file_infos)
-            with Tqdm(
-                tasks,
-                total=len(file_infos),
-                unit="md5",
-                desc="Computing file/dir hashes (only done once)",
-            ) as tasks:
+        with Tqdm(
+            total=len(file_infos),
+            unit="md5",
+            desc="Computing file/dir hashes (only done once)",
+        ) as pbar:
+            worker = pbar.wrap_fn(self.get_file_checksum)
+            with ThreadPoolExecutor(
+                max_workers=self.checksum_jobs
+            ) as executor:
+                tasks = executor.map(worker, file_infos)
                checksums = dict(zip(file_infos, tasks))
        return checksums

@@ -242,7 +257,7 @@ class RemoteBASE(object):
        new_info = self.cache.checksum_to_path_info(checksum)
        if self.cache.changed_cache_file(checksum):
            self.cache.makedirs(new_info.parent)
-            self.cache.move(tmp_info, new_info)
+            self.cache.move(tmp_info, new_info, mode=self.CACHE_MODE)

        self.state.save(path_info, checksum)
        self.state.save(new_info, checksum)
@@ -287,24 +302,29 @@ class RemoteBASE(object):

        if not isinstance(d, list):
            logger.error(
-                "dir cache file format error '{}' [skipping the file]",
+                "dir cache file format error '%s' [skipping the file]",
                path_info,
            )
            return []

-        for info in d:
-            # NOTE: here is a BUG, see comment to .as_posix() below
-            relative_path = PathInfo.from_posix(info[self.PARAM_RELPATH])
-            info[self.PARAM_RELPATH] = relative_path.fspath
+        if self.path_cls == WindowsPathInfo:
+            # only need to convert it for Windows
+            for info in d:
+                # NOTE: here is a BUG, see comment to .as_posix() below
+                info[self.PARAM_RELPATH] = info[self.PARAM_RELPATH].replace(
+                    "/", self.path_cls.sep
+                )

        return d

    @classmethod
    def is_dir_checksum(cls, checksum):
+        if not checksum:
+            return False
        return checksum.endswith(cls.CHECKSUM_DIR_SUFFIX)

    def get_checksum(self, path_info):
-        assert path_info.scheme == self.scheme
+        assert isinstance(path_info, str) or path_info.scheme == self.scheme

        if not self.exists(path_info):
            return None
@@ -355,35 +375,35 @@ class RemoteBASE(object):
        """

        logger.debug(
-            "checking if '{}'('{}') has changed.", path_info, checksum_info
+            "checking if '%s'('%s') has changed.", path_info, checksum_info
        )

        if not self.exists(path_info):
-            logger.debug("'{}' doesn't exist.", path_info)
+            logger.debug("'%s' doesn't exist.", path_info)
            return True

        checksum = checksum_info.get(self.PARAM_CHECKSUM)
        if checksum is None:
-            logger.debug("hash value for '{}' is missing.", path_info)
+            logger.debug("hash value for '%s' is missing.", path_info)
            return True

        if self.changed_cache(checksum):
            logger.debug(
-                "cache for '{}'('{}') has changed.", path_info, checksum
+                "cache for '%s'('%s') has changed.", path_info, checksum
            )
            return True

        actual = self.get_checksum(path_info)
        if checksum != actual:
            logger.debug(
-                "hash value '{}' for '{}' has changed (actual '{}').",
+                "hash value '%s' for '%s' has changed (actual '%s').",
                checksum,
                actual,
                path_info,
            )
            return True

-        logger.debug("'{}' hasn't changed.", path_info)
+        logger.debug("'%s' hasn't changed.", path_info)
        return False

    def link(self, from_info, to_info):
@@ -400,10 +420,10 @@ class RemoteBASE(object):
        if self.cache_type_confirmed:
            return

-        is_link = getattr(self, "is_{}".format(link_type), None)
+        is_link = getattr(self, f"is_{link_type}", None)
        if is_link and not is_link(path_info):
            self.remove(path_info)
-            raise DvcException("failed to verify {}".format(link_type))
+            raise DvcException(f"failed to verify {link_type}")

        self.cache_type_confirmed = True

@@ -418,7 +438,7 @@ class RemoteBASE(object):

            except DvcException as exc:
                logger.debug(
-                    "Cache type '{}' is not supported: {}", link_types[0], exc
+                    "Cache type '%s' is not supported: %s", link_types[0], exc
                )
                del link_types[0]

@@ -426,19 +446,12 @@ class RemoteBASE(object):

    def _do_link(self, from_info, to_info, link_method):
        if self.exists(to_info):
-            raise DvcException("Link '{}' already exists!".format(to_info))
+            raise DvcException(f"Link '{to_info}' already exists!")

        link_method(from_info, to_info)

-        if self.protected:
-            self.protect(to_info)
-
        logger.debug(
-            "Created {}'{}': {} -> {}",
-            "protected " if self.protected else "",
-            self.cache_types[0],
-            from_info,
-            to_info,
+            "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
        )

    def _save_file(self, path_info, checksum, save_link=True):
@@ -446,14 +459,11 @@ class RemoteBASE(object):

        cache_info = self.checksum_to_path_info(checksum)
        if self.changed_cache(checksum):
-            self.move(path_info, cache_info)
+            self.move(path_info, cache_info, mode=self.CACHE_MODE)
            self.link(cache_info, path_info)
        elif self.iscopy(path_info) and self._cache_is_copy(path_info):
            # Default relink procedure involves unneeded copy
-            if self.protected:
-                self.protect(path_info)
-            else:
-                self.unprotect(path_info)
+            self.unprotect(path_info)
        else:
            self.remove(path_info)
            self.link(cache_info, path_info)
@@ -489,7 +499,7 @@ class RemoteBASE(object):
        self.cache_type_confirmed = True
        return self.cache_types[0] == "copy"

-    def _save_dir(self, path_info, checksum):
+    def _save_dir(self, path_info, checksum, save_link=True):
        cache_info = self.checksum_to_path_info(checksum)
        dir_info = self.get_dir_cache(checksum)

@@ -500,7 +510,9 @@ class RemoteBASE(object):
            entry_checksum = entry[self.PARAM_CHECKSUM]
            self._save_file(entry_info, entry_checksum, save_link=False)

-        self.state.save_link(path_info)
+        if save_link:
+            self.state.save_link(path_info)
+
        self.state.save(cache_info, checksum)
        self.state.save(path_info, checksum)

@@ -531,23 +543,22 @@ class RemoteBASE(object):
    def protect(path_info):
        pass

-    def save(self, path_info, checksum_info):
+    def save(self, path_info, checksum_info, save_link=True):
        if path_info.scheme != self.scheme:
            raise RemoteActionNotImplemented(
-                "save {} -> {}".format(path_info.scheme, self.scheme),
-                self.scheme,
+                f"save {path_info.scheme} -> {self.scheme}", self.scheme,
            )

        checksum = checksum_info[self.PARAM_CHECKSUM]
-        self._save(path_info, checksum)
+        self._save(path_info, checksum, save_link)

-    def _save(self, path_info, checksum):
+    def _save(self, path_info, checksum, save_link=True):
        to_info = self.checksum_to_path_info(checksum)
-        logger.debug("Saving '{}' to '{}'.", path_info, to_info)
+        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
        if self.isdir(path_info):
-            self._save_dir(path_info, checksum)
+            self._save_dir(path_info, checksum, save_link)
            return
-        self._save_file(path_info, checksum)
+        self._save_file(path_info, checksum, save_link)

    def _handle_transfer_exception(
        self, from_info, to_info, exception, operation
@@ -556,7 +567,7 @@ class RemoteBASE(object):
            raise exception

        logger.exception(
-            "failed to {} '{}' to '{}'", operation, from_info, to_info
+            "failed to %s '%s' to '%s'", operation, from_info, to_info
        )
        return 1

@@ -570,7 +581,7 @@ class RemoteBASE(object):
        if from_info.scheme != "local":
            raise NotImplementedError

-        logger.debug("Uploading '{}' to '{}'", from_info, to_info)
+        logger.debug("Uploading '%s' to '%s'", from_info, to_info)

        name = name or from_info.name

@@ -626,22 +637,23 @@ class RemoteBASE(object):
            to_info / info.relative_to(from_info) for info in from_infos
        )

-        with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
-            download_files = partial(
-                self._download_file,
-                name=name,
-                no_progress_bar=True,
-                file_mode=file_mode,
-                dir_mode=dir_mode,
+        with Tqdm(
+            total=len(from_infos),
+            desc="Downloading directory",
+            unit="Files",
+            disable=no_progress_bar,
+        ) as pbar:
+            download_files = pbar.wrap_fn(
+                partial(
+                    self._download_file,
+                    name=name,
+                    no_progress_bar=True,
+                    file_mode=file_mode,
+                    dir_mode=dir_mode,
+                )
            )
-            futures = executor.map(download_files, from_infos, to_infos)
-            with Tqdm(
-                futures,
-                total=len(from_infos),
-                desc="Downloading directory",
-                unit="Files",
-                disable=no_progress_bar,
-            ) as futures:
+            with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
+                futures = executor.map(download_files, from_infos, to_infos)
                return sum(futures)

    def _download_file(
@@ -649,7 +661,7 @@ class RemoteBASE(object):
    ):
        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)

-        logger.debug("Downloading '{}' to '{}'", from_info, to_info)
+        logger.debug("Downloading '%s' to '%s'", from_info, to_info)
        name = name or to_info.name

        tmp_file = tmp_fname(to_info)
@@ -677,7 +689,8 @@ class RemoteBASE(object):
    def remove(self, path_info):
        raise RemoteActionNotImplemented("remove", self.scheme)

-    def move(self, from_info, to_info):
+    def move(self, from_info, to_info, mode=None):
+        assert mode is None
        self.copy(from_info, to_info)
        self.remove(from_info)

@@ -700,36 +713,68 @@ class RemoteBASE(object):
        parts = self.path_cls(path).parts[-2:]

        if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
-            raise ValueError("Bad cache file path")
+            raise ValueError(f"Bad cache file path '{path}'")

        return "".join(parts)

    def checksum_to_path_info(self, checksum):
        return self.path_info / checksum[0:2] / checksum[2:]

-    def list_cache_paths(self):
+    # Return path as a string instead of PathInfo for remotes which support
+    # string paths (see local)
+    checksum_to_path = checksum_to_path_info
+
+    def list_cache_paths(self, prefix=None, progress_callback=None):
        raise NotImplementedError

-    def all(self):
-        # NOTE: The list might be way too big(e.g. 100M entries, md5 for each
-        # is 32 bytes, so ~3200Mb list) and we don't really need all of it at
-        # the same time, so it makes sense to use a generator to gradually
-        # iterate over it, without keeping all of it in memory.
-        for path in self.list_cache_paths():
+    def cache_checksums(self, prefix=None, progress_callback=None):
+        """Iterate over remote cache checksums.
+
+        If `prefix` is specified, only checksums which begin with `prefix`
+        will be returned.
+        """
+        for path in self.list_cache_paths(prefix, progress_callback):
            try:
                yield self.path_to_checksum(path)
            except ValueError:
-                # We ignore all the non-cache looking files
-                pass
+                logger.debug(
+                    "'%s' doesn't look like a cache file, skipping", path
+                )

-    def gc(self, named_cache):
+    def all(self, jobs=None, name=None):
+        """Iterate over all checksums in the remote cache.
+
+        Checksums will be fetched in parallel threads according to prefix
+        (except for small remotes) and a progress bar will be displayed.
+        """
+        logger.debug(
+            "Fetching all checksums from '{}'".format(
+                name if name else "remote cache"
+            )
+        )
+
+        if not self.CAN_TRAVERSE:
+            return self.cache_checksums()
+
+        remote_size, remote_checksums = self._estimate_cache_size(name=name)
+        return self._cache_checksums_traverse(
+            remote_size, remote_checksums, jobs, name
+        )
+
+    @index_locked
+    def gc(self, named_cache, jobs=None):
        used = self.extract_used_local_checksums(named_cache)

        if self.scheme != "":
-            used.update(named_cache[self.scheme])
+            used.update(named_cache.scheme_keys(self.scheme))

        removed = False
-        for checksum in self.all():
+        # checksums must be sorted to ensure we always remove .dir files first
+        for checksum in sorted(
+            self.all(jobs, str(self.path_info)),
+            key=self.is_dir_checksum,
+            reverse=True,
+        ):
            if checksum in used:
                continue
            path_info = self.checksum_to_path_info(checksum)
@@ -737,8 +782,13 @@ class RemoteBASE(object):
                self._remove_unpacked_dir(checksum)
            self.remove(path_info)
            removed = True
+        if removed:
+            self.index.clear()
        return removed

+    def is_protected(self, path_info):
+        return False
+
    def changed_cache_file(self, checksum):
        """Compare the given checksum with the (corresponding) actual one.

@@ -751,11 +801,18 @@ class RemoteBASE(object):

        - Remove the file from cache if it doesn't match the actual checksum
        """
-        cache_info = self.checksum_to_path_info(checksum)
+        # Prefer string path over PathInfo when possible due to performance
+        cache_info = self.checksum_to_path(checksum)
+        if self.is_protected(cache_info):
+            logger.debug(
+                "Assuming '%s' is unchanged since it is read-only", cache_info
+            )
+            return False
+
        actual = self.get_checksum(cache_info)

        logger.debug(
-            "cache '{}' expected '{}' actual '{}'",
+            "cache '%s' expected '%s' actual '%s'",
            cache_info,
            checksum,
            actual,
@@ -765,10 +822,13 @@ class RemoteBASE(object):
            return True

        if actual.split(".")[0] == checksum.split(".")[0]:
+            # making cache file read-only so we don't need to check it
+            # next time
+            self.protect(cache_info)
            return False

        if self.exists(cache_info):
-            logger.warning("corrupted cache file '{}'.", cache_info)
+            logger.warning("corrupted cache file '%s'.", cache_info)
            self.remove(cache_info)

        return True
@@ -810,11 +870,13 @@ class RemoteBASE(object):

        There are two ways of performing this check:

-        - Traverse: Get a list of all the files in the remote
+        - Traverse method: Get a list of all the files in the remote
            (traversing the cache directory) and compare it with
-            the given checksums.
+            the given checksums. Cache entries will be retrieved in parallel
+            threads according to prefix (i.e. entries starting with, "00...",
+            "01...", and so on) and a progress bar will be displayed.

-        - No traverse: For each given checksum, run the `exists`
+        - Exists method: For each given checksum, run the `exists`
            method and filter the checksums that aren't on the remote.
            This is done in parallel threads.
            It also shows a progress bar when performing the check.
@@ -825,12 +887,192 @@ class RemoteBASE(object):
        check if particular file exists much quicker, use their own
        implementation of cache_exists (see ssh, local).

+        Which method to use will be automatically determined after estimating
+        the size of the remote cache, and comparing the estimated size with
+        len(checksums). To estimate the size of the remote cache, we fetch
+        a small subset of cache entries (i.e. entries starting with "00...").
+        Based on the number of entries in that subset, the size of the full
+        cache can be estimated, since the cache is evenly distributed according
+        to checksum.
+
        Returns:
            A list with checksums that were found in the remote
        """
-        if not self.no_traverse:
-            return list(set(checksums) & set(self.all()))
+        # Remotes which do not use traverse prefix should override
+        # cache_exists() (see ssh, local)
+        assert self.TRAVERSE_PREFIX_LEN >= 2

+        checksums = set(checksums)
+        indexed_checksums = set(self.index.intersection(checksums))
+        checksums -= indexed_checksums
+        logger.debug(
+            "Matched '{}' indexed checksums".format(len(indexed_checksums))
+        )
+        if not checksums:
+            return indexed_checksums
+
+        if len(checksums) == 1 or not self.CAN_TRAVERSE:
+            remote_checksums = self._cache_object_exists(checksums, jobs, name)
+            return list(indexed_checksums) + remote_checksums
+
+        # Max remote size allowed for us to use traverse method
+        remote_size, remote_checksums = self._estimate_cache_size(
+            checksums, name
+        )
+
+        traverse_pages = remote_size / self.LIST_OBJECT_PAGE_SIZE
+        # For sufficiently large remotes, traverse must be weighted to account
+        # for performance overhead from large lists/sets.
+        # From testing with S3, for remotes with 1M+ files, object_exists is
+        # faster until len(checksums) is at least 10k~100k
+        if remote_size > self.TRAVERSE_THRESHOLD_SIZE:
+            traverse_weight = traverse_pages * self.TRAVERSE_WEIGHT_MULTIPLIER
+        else:
+            traverse_weight = traverse_pages
+        if len(checksums) < traverse_weight:
+            logger.debug(
+                "Large remote ('{}' checksums < '{}' traverse weight), "
+                "using object_exists for remaining checksums".format(
+                    len(checksums), traverse_weight
+                )
+            )
+            return (
+                list(indexed_checksums)
+                + list(checksums & remote_checksums)
+                + self._cache_object_exists(
+                    checksums - remote_checksums, jobs, name
+                )
+            )
+
+        logger.debug(
+            "Querying '{}' checksums via traverse".format(len(checksums))
+        )
+        remote_checksums = set(
+            self._cache_checksums_traverse(
+                remote_size, remote_checksums, jobs, name
+            )
+        )
+        return list(indexed_checksums) + list(
+            checksums & set(remote_checksums)
+        )
+
+    def _checksums_with_limit(
+        self, limit, prefix=None, progress_callback=None
+    ):
+        count = 0
+        for checksum in self.cache_checksums(prefix, progress_callback):
+            yield checksum
+            count += 1
+            if count > limit:
+                logger.debug(
+                    "`cache_checksums()` returned max '{}' checksums, "
+                    "skipping remaining results".format(limit)
+                )
+                return
+
+    def _max_estimation_size(self, checksums):
+        # Max remote size allowed for us to use traverse method
+        return max(
+            self.TRAVERSE_THRESHOLD_SIZE,
+            len(checksums)
+            / self.TRAVERSE_WEIGHT_MULTIPLIER
+            * self.LIST_OBJECT_PAGE_SIZE,
+        )
+
+    def _estimate_cache_size(self, checksums=None, name=None):
+        """Estimate remote cache size based on number of entries beginning with
+        "00..." prefix.
+        """
+        prefix = "0" * self.TRAVERSE_PREFIX_LEN
+        total_prefixes = pow(16, self.TRAVERSE_PREFIX_LEN)
+        if checksums:
+            max_checksums = self._max_estimation_size(checksums)
+        else:
+            max_checksums = None
+
+        with Tqdm(
+            desc="Estimating size of "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            unit="file",
+        ) as pbar:
+
+            def update(n=1):
+                pbar.update(n * total_prefixes)
+
+            if max_checksums:
+                checksums = self._checksums_with_limit(
+                    max_checksums / total_prefixes, prefix, update
+                )
+            else:
+                checksums = self.cache_checksums(prefix, update)
+
+            remote_checksums = set(checksums)
+            if remote_checksums:
+                remote_size = total_prefixes * len(remote_checksums)
+            else:
+                remote_size = total_prefixes
+            logger.debug(f"Estimated remote size: {remote_size} files")
+        return remote_size, remote_checksums
+
+    def _cache_checksums_traverse(
+        self, remote_size, remote_checksums, jobs=None, name=None
+    ):
+        """Iterate over all checksums in the remote cache.
+        Checksums are fetched in parallel according to prefix, except in
+        cases where the remote size is very small.
+
+        All checksums from the remote (including any from the size
+        estimation step passed via the `remote_checksums` argument) will be
+        returned.
+
+        NOTE: For large remotes the list of checksums will be very
+        big(e.g. 100M entries, md5 for each is 32 bytes, so ~3200Mb list)
+        and we don't really need all of it at the same time, so it makes
+        sense to use a generator to gradually iterate over it, without
+        keeping all of it in memory.
+        """
+        num_pages = remote_size / self.LIST_OBJECT_PAGE_SIZE
+        if num_pages < 256 / self.JOBS:
+            # Fetching prefixes in parallel requires at least 255 more
+            # requests, for small enough remotes it will be faster to fetch
+            # entire cache without splitting it into prefixes.
+            #
+            # NOTE: this ends up re-fetching checksums that were already
+            # fetched during remote size estimation
+            traverse_prefixes = [None]
+            initial = 0
+        else:
+            yield from remote_checksums
+            initial = len(remote_checksums)
+            traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
+            if self.TRAVERSE_PREFIX_LEN > 2:
+                traverse_prefixes += [
+                    "{0:0{1}x}".format(i, self.TRAVERSE_PREFIX_LEN)
+                    for i in range(1, pow(16, self.TRAVERSE_PREFIX_LEN - 2))
+                ]
+        with Tqdm(
+            desc="Querying "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            total=remote_size,
+            initial=initial,
+            unit="file",
+        ) as pbar:
+
+            def list_with_update(prefix):
+                return list(
+                    self.cache_checksums(
+                        prefix=prefix, progress_callback=pbar.update
+                    )
+                )
+
+            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
+                in_remote = executor.map(list_with_update, traverse_prefixes,)
+                yield from itertools.chain.from_iterable(in_remote)
+
+    def _cache_object_exists(self, checksums, jobs=None, name=None):
+        logger.debug(
+            "Querying {} checksums via object_exists".format(len(checksums))
+        )
        with Tqdm(
            desc="Querying "
            + ("cache in " + name if name else "remote cache"),
@@ -840,7 +1082,7 @@ class RemoteBASE(object):

            def exists_with_progress(path_info):
                ret = self.exists(path_info)
-                pbar.update_desc(str(path_info))
+                pbar.update_msg(str(path_info))
                return ret

            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
@@ -873,15 +1115,15 @@ class RemoteBASE(object):
        self.remove(path_info)

    def _checkout_file(
-        self, path_info, checksum, force, progress_callback=None
+        self, path_info, checksum, force, progress_callback=None, relink=False
    ):
        """The file is changed we need to checkout a new copy"""
+        added, modified = True, False
        cache_info = self.checksum_to_path_info(checksum)
        if self.exists(path_info):
-            logger.warning(
-                "data '{}' exists. Removing before checkout.", path_info
-            )
+            logger.debug("data '%s' will be replaced.", path_info)
            self.safe_remove(path_info, force=force)
+            added, modified = False, True

        self.link(cache_info, path_info)
        self.state.save_link(path_info)
@@ -889,11 +1131,12 @@ class RemoteBASE(object):
        if progress_callback:
            progress_callback(str(path_info))

+        return added, modified and not relink
+
    def makedirs(self, path_info):
        """Optional: Implement only if the remote needs to create
        directories before copying/linking/moving data
        """
-        pass

    def _checkout_dir(
        self,
@@ -904,14 +1147,16 @@ class RemoteBASE(object):
        relink=False,
        filter_info=None,
    ):
+        added, modified = False, False
        # Create dir separately so that dir is created
        # even if there are no files in it
        if not self.exists(path_info):
+            added = True
            self.makedirs(path_info)

        dir_info = self.get_dir_cache(checksum)

-        logger.debug("Linking directory '{}'.", path_info)
+        logger.debug("Linking directory '%s'.", path_info)

        for entry in dir_info:
            relative_path = entry[self.PARAM_RELPATH]
@@ -924,27 +1169,36 @@ class RemoteBASE(object):

            entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
            if relink or self.changed(entry_info, entry_checksum_info):
+                modified = True
                self.safe_remove(entry_info, force=force)
                self.link(entry_cache_info, entry_info)
                self.state.save(entry_info, entry_checksum)
            if progress_callback:
                progress_callback(str(entry_info))

-        self._remove_redundant_files(path_info, dir_info, force)
+        modified = (
+            self._remove_redundant_files(path_info, dir_info, force)
+            or modified
+        )

        self.state.save_link(path_info)
        self.state.save(path_info, checksum)

+        # relink is not modified, assume it as nochange
+        return added, not added and modified and not relink
+
    def _remove_redundant_files(self, path_info, dir_info, force):
        existing_files = set(self.walk_files(path_info))

        needed_files = {
            path_info / entry[self.PARAM_RELPATH] for entry in dir_info
        }
-
-        for path in existing_files - needed_files:
+        redundant_files = existing_files - needed_files
+        for path in redundant_files:
            self.safe_remove(path, force)

+        return bool(redundant_files)
+
    def checkout(
        self,
        path_info,
@@ -962,21 +1216,21 @@ class RemoteBASE(object):
        skip = False
        if not checksum:
            logger.warning(
-                "No file hash info found for '{}'. " "It won't be created.",
+                "No file hash info found for '%s'. " "It won't be created.",
                path_info,
            )
            self.safe_remove(path_info, force=force)
            failed = path_info

        elif not relink and not self.changed(path_info, checksum_info):
-            logger.debug("Data '{}' didn't change.", path_info)
+            logger.debug("Data '%s' didn't change.", path_info)
            skip = True

        elif self.changed_cache(
            checksum, path_info=path_info, filter_info=filter_info
        ):
            logger.warning(
-                "Cache '{}' not found. File '{}' won't be created.",
+                "Cache '%s' not found. File '%s' won't be created.",
                checksum,
                path_info,
            )
@@ -991,12 +1245,14 @@ class RemoteBASE(object):
                        self.path_info, checksum, filter_info
                    ),
                )
-            return failed
+            if failed:
+                raise CheckoutError([failed])
+            return

-        logger.debug("Checking out '{}' with cache '{}'.", path_info, checksum)
+        logger.debug("Checking out '%s' with cache '%s'.", path_info, checksum)

-        self._checkout(
-            path_info, checksum, force, progress_callback, relink, filter_info
+        return self._checkout(
+            path_info, checksum, force, progress_callback, relink, filter_info,
        )

    def _checkout(
@@ -1010,8 +1266,9 @@ class RemoteBASE(object):
    ):
        if not self.is_dir_checksum(checksum):
            return self._checkout_file(
-                path_info, checksum, force, progress_callback=progress_callback
+                path_info, checksum, force, progress_callback, relink
            )
+
        return self._checkout_dir(
            path_info, checksum, force, progress_callback, relink, filter_info
        )
@@ -1041,7 +1298,7 @@ class RemoteBASE(object):
        return set()

    def extract_used_local_checksums(self, named_cache):
-        used = set(named_cache["local"])
+        used = set(named_cache.scheme_keys("local"))
        unpacked = self._get_unpacked_dir_names(used)
        return used | unpacked

@@ -1,163 +0,0 @@
-import logging
-import os
-import posixpath
-from urllib.parse import urlparse
-
-from dvc.config import Config, ConfigError
-from dvc.utils import relpath
-
-
-logger = logging.getLogger(__name__)
-
-
-class RemoteConfig(object):
-    def __init__(self, config):
-        self.config = config
-
-    def get_settings(self, name):
-        """
-        Args:
-            name (str): The name of the remote that we want to retrieve
-
-        Returns:
-            dict: The content beneath the given remote name.
-
-        Example:
-            >>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
-            >>> get_settings("server")
-            {'url': 'ssh://localhost/'}
-        """
-        settings = self.config.config.get(
-            Config.SECTION_REMOTE_FMT.format(name.lower())
-        )
-
-        if settings is None:
-            raise ConfigError(
-                "Unable to find remote section '{}'".format(name)
-            )
-
-        parsed = urlparse(settings["url"])
-
-        # Support for cross referenced remotes.
-        # This will merge the settings, giving priority to the outer reference.
-        # For example, having:
-        #
-        #       dvc remote add server ssh://localhost
-        #       dvc remote modify server user root
-        #       dvc remote modify server ask_password true
-        #
-        #       dvc remote add images remote://server/tmp/pictures
-        #       dvc remote modify images user alice
-        #       dvc remote modify images ask_password false
-        #       dvc remote modify images password asdf1234
-        #
-        # Results on a config dictionary like:
-        #
-        #       {
-        #           "url": "ssh://localhost/tmp/pictures",
-        #           "user": "alice",
-        #           "password": "asdf1234",
-        #           "ask_password": False,
-        #       }
-        #
-        if parsed.scheme == "remote":
-            reference = self.get_settings(parsed.netloc)
-            url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
-            merged = reference.copy()
-            merged.update(settings)
-            merged["url"] = url
-            return merged
-
-        return settings
-
-    @staticmethod
-    def resolve_path(path, config_file):
-        """Resolve path relative to config file location.
-
-        Args:
-            path: Path to be resolved.
-            config_file: Path to config file, which `path` is specified
-                relative to.
-
-        Returns:
-            Path relative to the `config_file` location. If `path` is an
-            absolute path then it will be returned without change.
-
-        """
-        if os.path.isabs(path):
-            return path
-        return relpath(path, os.path.dirname(config_file))
-
-    def add(self, name, url, default=False, force=False, level=None):
-        from dvc.remote import _get, RemoteLOCAL
-
-        configobj = self.config.get_configobj(level)
-        remote = _get({Config.SECTION_REMOTE_URL: url})
-        if remote == RemoteLOCAL and not url.startswith("remote://"):
-            url = self.resolve_path(url, configobj.filename)
-
-        self.config.set(
-            Config.SECTION_REMOTE_FMT.format(name),
-            Config.SECTION_REMOTE_URL,
-            url,
-            force=force,
-            level=level,
-        )
-        if default:
-            self.config.set(
-                Config.SECTION_CORE,
-                Config.SECTION_CORE_REMOTE,
-                name,
-                level=level,
-            )
-
-    def remove(self, name, level=None):
-        self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level)
-
-        if level is None:
-            level = Config.LEVEL_REPO
-
-        for lev in [
-            Config.LEVEL_LOCAL,
-            Config.LEVEL_REPO,
-            Config.LEVEL_GLOBAL,
-            Config.LEVEL_SYSTEM,
-        ]:
-            self.config.unset(
-                Config.SECTION_CORE,
-                Config.SECTION_CORE_REMOTE,
-                level=lev,
-                force=True,
-            )
-            if lev == level:
-                break
-
-    def modify(self, name, option, value, level=None):
-        # try to get remote settings to verify that section exists
-        self.get_settings(name)
-
-        self.config.set(
-            Config.SECTION_REMOTE_FMT.format(name), option, value, level=level
-        )
-
-    def list(self, level=None):
-        return self.config.list_options(
-            Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level
-        )
-
-    def set_default(self, name, unset=False, level=None):
-        if unset:
-            self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE)
-            return
-        self.config.set(
-            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level
-        )
-
-    def get_default(self, level=None):
-        return self.config.get(
-            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level
-        )
-
-    def has_default(self):
-        core = self.config.config[Config.SECTION_CORE]
-        return bool(core.get(Config.SECTION_CORE_REMOTE))
@@ -1,71 +1,75 @@
+import logging
 import os
 import posixpath
-import logging
 import re
 import threading
+from collections import defaultdict
 from urllib.parse import urlparse

-from funcy import retry, compose, decorator, wrap_with
+from funcy import cached_property, retry, wrap_prop, wrap_with
 from funcy.py3 import cat

-from dvc.progress import Tqdm
-from dvc.scheme import Schemes
-from dvc.path_info import CloudURLInfo
-from dvc.remote.base import RemoteBASE
-from dvc.config import Config
 from dvc.exceptions import DvcException
-from dvc.utils import tmp_fname, format_link
+from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import BaseRemote
+from dvc.scheme import Schemes
+from dvc.utils import format_link, tmp_fname

 logger = logging.getLogger(__name__)
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"


-class GDriveRetriableError(DvcException):
-    pass
-
-
 class GDrivePathNotFound(DvcException):
-    def __init__(self, path_info):
-        super().__init__("Google Drive path '{}' not found.".format(path_info))
+    def __init__(self, path_info, hint):
+        hint = "" if hint is None else f" {hint}"
+        super().__init__(f"GDrive path '{path_info}' not found.{hint}")


-class GDriveAccessTokenRefreshError(DvcException):
-    def __init__(self):
-        super().__init__("Google Drive access token refreshment is failed.")
+class GDriveAuthError(DvcException):
+    def __init__(self, cred_location):
+
+        if cred_location:
+            message = (
+                "GDrive remote auth failed with credentials in '{}'.\n"
+                "Backup first, remove of fix them, and run DVC again.\n"
+                "It should do auth again and refresh the credentials.\n\n"
+                "Details:".format(cred_location)
+            )
+        else:
+            message = "Failed to authenticate GDrive remote"
+
+        super().__init__(message)


-class GDriveMissedCredentialKeyError(DvcException):
-    def __init__(self, path):
-        super().__init__(
-            "Google Drive user credentials file '{}' "
-            "misses value for key.".format(path)
-        )
+def _gdrive_retry(func):
+    def should_retry(exc):
+        from pydrive2.files import ApiRequestError

+        if not isinstance(exc, ApiRequestError):
+            return False

-@decorator
-def _wrap_pydrive_retriable(call):
-    from pydrive2.files import ApiRequestError
+        error_code = exc.error.get("code", 0)
+        result = False
+        if 500 <= error_code < 600:
+            result = True

-    try:
-        result = call()
-    except ApiRequestError as exception:
-        retry_codes = ["403", "500", "502", "503", "504"]
-        if any(
-            "HttpError {}".format(code) in str(exception)
-            for code in retry_codes
-        ):
-            raise GDriveRetriableError("Google API request failed")
-        raise
-    return result
+        if error_code == 403:
+            result = exc.GetField("reason") in [
+                "userRateLimitExceeded",
+                "rateLimitExceeded",
+            ]
+        if result:
+            logger.debug(f"Retrying GDrive API call, error: {exc}.")

+        return result

-gdrive_retry = compose(
-    # 15 tries, start at 0.5s, multiply by golden ratio, cap at 20s
-    retry(
-        15, GDriveRetriableError, timeout=lambda a: min(0.5 * 1.618 ** a, 20)
-    ),
-    _wrap_pydrive_retriable,
-)
+    # 16 tries, start at 0.5s, multiply by golden ratio, cap at 20s
+    return retry(
+        16,
+        timeout=lambda a: min(0.5 * 1.618 ** a, 20),
+        filter_errors=should_retry,
+    )(func)


 class GDriveURLInfo(CloudURLInfo):
@@ -83,180 +87,276 @@ class GDriveURLInfo(CloudURLInfo):
        self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))


-class RemoteGDrive(RemoteBASE):
+class GDriveRemote(BaseRemote):
    scheme = Schemes.GDRIVE
    path_cls = GDriveURLInfo
    REQUIRES = {"pydrive2": "pydrive2"}
-    DEFAULT_NO_TRAVERSE = False
    DEFAULT_VERIFY = True
+    # Always prefer traverse for GDrive since API usage quotas are a concern.
+    TRAVERSE_WEIGHT_MULTIPLIER = 1
+    TRAVERSE_PREFIX_LEN = 2

-    GDRIVE_USER_CREDENTIALS_DATA = "GDRIVE_USER_CREDENTIALS_DATA"
+    GDRIVE_CREDENTIALS_DATA = "GDRIVE_CREDENTIALS_DATA"
    DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"

+    DEFAULT_GDRIVE_CLIENT_ID = "710796635688-iivsgbgsb6uv1fap6635dhvuei09o66c.apps.googleusercontent.com"  # noqa: E501
+    DEFAULT_GDRIVE_CLIENT_SECRET = "a1Fz59uTpVNeG_VGuSKDLJXv"
+
    def __init__(self, repo, config):
        super().__init__(repo, config)
-        url = config[Config.SECTION_REMOTE_URL]
-        self.path_info = self.path_cls(url)
-        self.config = config
+        self.path_info = self.path_cls(config["url"])

        if not self.path_info.bucket:
            raise DvcException(
-                "Empty Google Drive URL '{}'. Learn more at "
-                "{}.".format(
-                    url, format_link("https://man.dvc.org/remote/add")
+                "Empty GDrive URL '{}'. Learn more at {}".format(
+                    config["url"],
+                    format_link("https://man.dvc.org/remote/add"),
                )
            )

        self._bucket = self.path_info.bucket
-        self._client_id = self.config.get(
-            Config.SECTION_GDRIVE_CLIENT_ID, None
+        self._trash_only = config.get("gdrive_trash_only")
+        self._use_service_account = config.get("gdrive_use_service_account")
+        self._service_account_email = config.get(
+            "gdrive_service_account_email"
        )
-        self._client_secret = self.config.get(
-            Config.SECTION_GDRIVE_CLIENT_SECRET, None
+        self._service_account_user_email = config.get(
+            "gdrive_service_account_user_email"
        )
-        if not self._client_id or not self._client_secret:
-            raise DvcException(
-                "Please specify Google Drive's client id and "
-                "secret in DVC config. Learn more at "
-                "{}.".format(format_link("https://man.dvc.org/remote/add"))
-            )
+        self._service_account_p12_file_path = config.get(
+            "gdrive_service_account_p12_file_path"
+        )
+        self._client_id = config.get("gdrive_client_id")
+        self._client_secret = config.get("gdrive_client_secret")
+        self._validate_config()
        self._gdrive_user_credentials_path = (
            tmp_fname(os.path.join(self.repo.tmp_dir, ""))
-            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
-            else self.config.get(
-                Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE,
+            if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA)
+            else config.get(
+                "gdrive_user_credentials_file",
                os.path.join(
                    self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE
                ),
            )
        )

-        self._list_params = None
-        self._gdrive = None
+    def _validate_config(self):
+        # Validate Service Account configuration
+        if self._use_service_account and (
+            not self._service_account_email
+            or not self._service_account_p12_file_path
+        ):
+            raise DvcException(
+                "To use service account, set `gdrive_service_account_email`,\n"
+                "`gdrive_service_account_p12_file_path`, and optionally "
+                "`gdrive_service_account_user_email`\nin DVC config. "
+                "Learn more at {}".format(
+                    format_link("https://man.dvc.org/remote/modify")
+                )
+            )

-        self._cache_initialized = False
-        self._remote_root_id = None
-        self._cached_dirs = None
-        self._cached_ids = None
-
-    @property
-    @wrap_with(threading.RLock())
-    def drive(self):
-        from pydrive2.auth import RefreshError
-
-        if not self._gdrive:
-            from pydrive2.auth import GoogleAuth
-            from pydrive2.drive import GoogleDrive
-
-            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
-                with open(
-                    self._gdrive_user_credentials_path, "w"
-                ) as credentials_file:
-                    credentials_file.write(
-                        os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
+        # Validate OAuth 2.0 Client ID configuration
+        if not self._use_service_account:
+            if bool(self._client_id) != bool(self._client_secret):
+                raise DvcException(
+                    "Please specify GDrive's client ID and secret in "
+                    "DVC config or omit both to use the defaults.\n"
+                    "Learn more at {}".format(
+                        format_link("https://man.dvc.org/remote/modify")
                    )
+                )

-            GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
-            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
-                "client_id": self._client_id,
-                "client_secret": self._client_secret,
+    @cached_property
+    def credentials_location(self):
+        """
+        Helper to determine where will GDrive remote read credentials from.
+        Useful for tests, exception messages, etc. Returns either env variable
+        name if it's set or actual path to the credentials file.
+        """
+        if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
+            return GDriveRemote.GDRIVE_CREDENTIALS_DATA
+        if os.path.exists(self._gdrive_user_credentials_path):
+            return self._gdrive_user_credentials_path
+        return None
+
+    @staticmethod
+    def _validate_credentials(auth, settings):
+        """
+        Detects discrepancy in DVC config and cached credentials file.
+        Usually happens when a second remote is added and it is using
+        the same credentials default file. Or when someones decides to change
+        DVC config client id or secret but forgets to remove the cached
+        credentials file.
+        """
+        if not os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
+            if (
+                settings["client_config"]["client_id"]
+                != auth.credentials.client_id
+                or settings["client_config"]["client_secret"]
+                != auth.credentials.client_secret
+            ):
+                logger.warning(
+                    "Client ID and secret configured do not match the "
+                    "actual ones used\nto access the remote. Do you "
+                    "use multiple GDrive remotes and forgot to\nset "
+                    "`gdrive_user_credentials_file` for one or more of them? "
+                    "Learn more at\n{}.\n".format(
+                        format_link("https://man.dvc.org/remote/modify")
+                    )
+                )
+
+    @wrap_prop(threading.RLock())
+    @cached_property
+    def _drive(self):
+        from pydrive2.auth import GoogleAuth
+        from pydrive2.drive import GoogleDrive
+
+        if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
+            with open(self._gdrive_user_credentials_path, "w") as cred_file:
+                cred_file.write(
+                    os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA)
+                )
+
+        auth_settings = {
+            "client_config_backend": "settings",
+            "save_credentials": True,
+            "save_credentials_backend": "file",
+            "save_credentials_file": self._gdrive_user_credentials_path,
+            "get_refresh_token": True,
+            "oauth_scope": [
+                "https://www.googleapis.com/auth/drive",
+                "https://www.googleapis.com/auth/drive.appdata",
+            ],
+        }
+
+        if self._use_service_account:
+            auth_settings["service_config"] = {
+                "client_service_email": self._service_account_email,
+                "client_user_email": self._service_account_user_email,
+                "client_pkcs12_file_path": self._service_account_p12_file_path,
+            }
+        else:
+            auth_settings["client_config"] = {
+                "client_id": self._client_id or self.DEFAULT_GDRIVE_CLIENT_ID,
+                "client_secret": self._client_secret
+                or self.DEFAULT_GDRIVE_CLIENT_SECRET,
                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
                "token_uri": "https://oauth2.googleapis.com/token",
                "revoke_uri": "https://oauth2.googleapis.com/revoke",
                "redirect_uri": "",
            }
-            GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
-            GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
-            GoogleAuth.DEFAULT_SETTINGS[
-                "save_credentials_file"
-            ] = self._gdrive_user_credentials_path
-            GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
-            GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
-                "https://www.googleapis.com/auth/drive",
-                "https://www.googleapis.com/auth/drive.appdata",
-            ]

-            # Pass non existent settings path to force DEFAULT_SETTINGS loading
-            gauth = GoogleAuth(settings_file="")
+        GoogleAuth.DEFAULT_SETTINGS.update(auth_settings)

-            try:
+        # Pass non existent settings path to force DEFAULT_SETTINGS loadings
+        gauth = GoogleAuth(settings_file="")
+
+        try:
+            logger.debug(
+                "GDrive remote auth with config '{}'.".format(
+                    GoogleAuth.DEFAULT_SETTINGS
+                )
+            )
+            if self._use_service_account:
+                gauth.ServiceAuth()
+            else:
                gauth.CommandLineAuth()
-            except RefreshError as exc:
-                raise GDriveAccessTokenRefreshError from exc
-            except KeyError as exc:
-                raise GDriveMissedCredentialKeyError(
-                    self._gdrive_user_credentials_path
-                ) from exc
-            # Handle pydrive2.auth.AuthenticationError and other auth failures
-            except Exception as exc:
-                raise DvcException(
-                    "Google Drive authentication failed"
-                ) from exc
-            finally:
-                if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
-                    os.remove(self._gdrive_user_credentials_path)
+                GDriveRemote._validate_credentials(gauth, auth_settings)

-            self._gdrive = GoogleDrive(gauth)
+        # Handle AuthenticationError, RefreshError and other auth failures
+        # It's hard to come up with a narrow exception, since PyDrive throws
+        # a lot of different errors - broken credentials file, refresh token
+        # expired, flow failed, etc.
+        except Exception as exc:
+            raise GDriveAuthError(self.credentials_location) from exc
+        finally:
+            if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
+                os.remove(self._gdrive_user_credentials_path)

-        return self._gdrive
+        return GoogleDrive(gauth)

-    @wrap_with(threading.RLock())
-    def _initialize_cache(self):
-        if self._cache_initialized:
-            return
+    @wrap_prop(threading.RLock())
+    @cached_property
+    def _ids_cache(self):
+        cache = {
+            "dirs": defaultdict(list),
+            "ids": {},
+            "root_id": self._get_item_id(
+                self.path_info,
+                use_cache=False,
+                hint="Confirm the directory exists and you can access it.",
+            ),
+        }

-        cached_dirs = {}
-        cached_ids = {}
-        self._remote_root_id = self._get_remote_id(self.path_info)
-        for dir1 in self.gdrive_list_item(
-            "'{}' in parents and trashed=false".format(self._remote_root_id)
+        self._cache_path_id(self.path_info.path, cache["root_id"], cache)
+
+        for item in self._gdrive_list(
+            "'{}' in parents and trashed=false".format(cache["root_id"])
        ):
-            remote_path = posixpath.join(self.path_info.path, dir1["title"])
-            cached_dirs.setdefault(remote_path, []).append(dir1["id"])
-            cached_ids[dir1["id"]] = dir1["title"]
+            item_path = (self.path_info / item["title"]).path
+            self._cache_path_id(item_path, item["id"], cache)

-        self._cached_dirs = cached_dirs
-        self._cached_ids = cached_ids
-        self._cache_initialized = True
+        return cache

-    @property
-    def cached_dirs(self):
-        if not self._cache_initialized:
-            self._initialize_cache()
-        return self._cached_dirs
+    def _cache_path_id(self, path, item_id, cache=None):
+        cache = cache or self._ids_cache
+        cache["dirs"][path].append(item_id)
+        cache["ids"][item_id] = path

-    @property
-    def cached_ids(self):
-        if not self._cache_initialized:
-            self._initialize_cache()
-        return self._cached_ids
-
-    @property
-    def remote_root_id(self):
-        if not self._cache_initialized:
-            self._initialize_cache()
-        return self._remote_root_id
-
-    @property
-    def list_params(self):
-        if not self._list_params:
-            params = {"corpora": "default"}
-            if self._bucket != "root" and self._bucket != "appDataFolder":
-                params["driveId"] = self._get_remote_drive_id(self._bucket)
+    @cached_property
+    def _list_params(self):
+        params = {"corpora": "default"}
+        if self._bucket != "root" and self._bucket != "appDataFolder":
+            drive_id = self._gdrive_shared_drive_id(self._bucket)
+            if drive_id:
+                logger.debug(
+                    "GDrive remote '{}' is using shared drive id '{}'.".format(
+                        self.path_info, drive_id
+                    )
+                )
+                params["driveId"] = drive_id
                params["corpora"] = "drive"
-            self._list_params = params
-        return self._list_params
+        return params

-    @gdrive_retry
-    def gdrive_upload_file(
+    @_gdrive_retry
+    def _gdrive_shared_drive_id(self, item_id):
+        from pydrive2.files import ApiRequestError
+
+        param = {"id": item_id}
+        # it does not create a file on the remote
+        item = self._drive.CreateFile(param)
+        # ID of the shared drive the item resides in.
+        # Only populated for items in shared drives.
+        try:
+            item.FetchMetadata("driveId")
+        except ApiRequestError as exc:
+            error_code = exc.error.get("code", 0)
+            if error_code == 404:
+                raise DvcException(
+                    "'{}' for '{}':\n\n"
+                    "1. Confirm the directory exists and you can access it.\n"
+                    "2. Make sure that credentials in '{}'\n"
+                    "   are correct for this remote e.g. "
+                    "use the `gdrive_user_credentials_file` config\n"
+                    "   option if you use multiple GDrive remotes with "
+                    "different email accounts.\n\nDetails".format(
+                        item_id, self.path_info, self._credentials_location
+                    )
+                ) from exc
+            raise
+
+        return item.get("driveId", None)
+
+    @_gdrive_retry
+    def _gdrive_upload_file(
        self,
        parent_id,
        title,
-        no_progress_bar=True,
+        no_progress_bar=False,
        from_file="",
        progress_name="",
    ):
-        item = self.drive.CreateFile(
+        item = self._drive.CreateFile(
            {"title": title, "parents": [{"id": parent_id}]}
        )

@@ -276,172 +376,188 @@ class RemoteGDrive(RemoteBASE):
                item.Upload()
        return item

-    @gdrive_retry
-    def gdrive_download_file(
-        self, file_id, to_file, progress_name, no_progress_bar
+    @_gdrive_retry
+    def _gdrive_download_file(
+        self, item_id, to_file, progress_desc, no_progress_bar
    ):
-        param = {"id": file_id}
+        param = {"id": item_id}
        # it does not create a file on the remote
-        gdrive_file = self.drive.CreateFile(param)
-        bar_format = (
-            "Downloading {desc:{ncols_desc}.{ncols_desc}}... "
-            + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
-        )
+        gdrive_file = self._drive.CreateFile(param)
+
        with Tqdm(
-            bar_format=bar_format, desc=progress_name, disable=no_progress_bar
-        ):
-            gdrive_file.GetContentFile(to_file)
+            desc=progress_desc,
+            disable=no_progress_bar,
+            bytes=True,
+            # explicit `bar_format` as `total` will be set by `update_to`
+            bar_format=Tqdm.BAR_FMT_DEFAULT,
+        ) as pbar:
+            gdrive_file.GetContentFile(to_file, callback=pbar.update_to)

-    def gdrive_list_item(self, query):
+    @_gdrive_retry
+    def _gdrive_delete_file(self, item_id):
+        from pydrive2.files import ApiRequestError
+
+        param = {"id": item_id}
+        # it does not create a file on the remote
+        item = self._drive.CreateFile(param)
+
+        try:
+            item.Trash() if self._trash_only else item.Delete()
+        except ApiRequestError as exc:
+            http_error_code = exc.error.get("code", 0)
+            if (
+                http_error_code == 403
+                and self._list_params["corpora"] == "drive"
+                and exc.GetField("location") == "file.permissions"
+            ):
+                raise DvcException(
+                    "Insufficient permissions to {}. You should have {} "
+                    "access level for the used shared drive. More details "
+                    "at {}.".format(
+                        "move the file into Trash"
+                        if self._trash_only
+                        else "permanently delete the file",
+                        "Manager or Content Manager"
+                        if self._trash_only
+                        else "Manager",
+                        "https://support.google.com/a/answer/7337554",
+                    )
+                ) from exc
+            raise
+
+    def _gdrive_list(self, query):
        param = {"q": query, "maxResults": 1000}
-        param.update(self.list_params)
+        param.update(self._list_params)
+        file_list = self._drive.ListFile(param)

-        file_list = self.drive.ListFile(param)
+        # Isolate and decorate fetching of remote drive items in pages.
+        get_list = _gdrive_retry(lambda: next(file_list, None))

-        # Isolate and decorate fetching of remote drive items in pages
-        get_list = gdrive_retry(lambda: next(file_list, None))
-
-        # Fetch pages until None is received, lazily flatten the thing
+        # Fetch pages until None is received, lazily flatten the thing.
        return cat(iter(get_list, None))

-    @wrap_with(threading.RLock())
-    def gdrive_create_dir(self, parent_id, title, remote_path):
-        if parent_id == self.remote_root_id:
-            cached = self.cached_dirs.get(remote_path, [])
-            if cached:
-                return cached[0]
-
-        item = self._create_remote_dir(parent_id, title)
-
-        if parent_id == self.remote_root_id:
-            self.cached_dirs.setdefault(remote_path, []).append(item["id"])
-            self.cached_ids[item["id"]] = item["title"]
-
-        return item["id"]
-
-    @gdrive_retry
-    def _create_remote_dir(self, parent_id, title):
+    @_gdrive_retry
+    def _gdrive_create_dir(self, parent_id, title):
        parent = {"id": parent_id}
-        item = self.drive.CreateFile(
+        item = self._drive.CreateFile(
            {"title": title, "parents": [parent], "mimeType": FOLDER_MIME_TYPE}
        )
        item.Upload()
        return item

-    @gdrive_retry
-    def _delete_remote_file(self, remote_id):
-        param = {"id": remote_id}
-        # it does not create a file on the remote
-        item = self.drive.CreateFile(param)
-        item.Delete()
+    @wrap_with(threading.RLock())
+    def _create_dir(self, parent_id, title, remote_path):
+        cached = self._ids_cache["dirs"].get(remote_path)
+        if cached:
+            return cached[0]

-    @gdrive_retry
-    def _get_remote_item(self, name, parents_ids):
-        if not parents_ids:
+        item = self._gdrive_create_dir(parent_id, title)
+
+        if parent_id == self._ids_cache["root_id"]:
+            self._cache_path_id(remote_path, item["id"])
+
+        return item["id"]
+
+    def _get_remote_item_ids(self, parent_ids, title):
+        if not parent_ids:
            return None
-        query = "({})".format(
+        query = "trashed=false and ({})".format(
            " or ".join(
-                "'{}' in parents".format(parent_id)
-                for parent_id in parents_ids
+                f"'{parent_id}' in parents" for parent_id in parent_ids
            )
        )
+        query += " and title='{}'".format(title.replace("'", "\\'"))

-        query += " and trashed=false and title='{}'".format(name)
+        # GDrive list API is case insensitive, we need to compare
+        # all results and pick the ones with the right title
+        return [
+            item["id"]
+            for item in self._gdrive_list(query)
+            if item["title"] == title
+        ]

-        # Remote might contain items with duplicated path (titles).
-        # We thus limit number of items.
-        param = {"q": query, "maxResults": 1}
-        param.update(self.list_params)
-
-        # Limit found remote items count to 1 in response
-        item_list = self.drive.ListFile(param).GetList()
-        return next(iter(item_list), None)
-
-    @gdrive_retry
-    def _get_remote_drive_id(self, remote_id):
-        param = {"id": remote_id}
-        # it does not create a file on the remote
-        item = self.drive.CreateFile(param)
-        item.FetchMetadata("driveId")
-        return item.get("driveId", None)
-
-    def _get_cached_remote_ids(self, path):
+    def _get_cached_item_ids(self, path, use_cache):
        if not path:
            return [self._bucket]
-        if self._cache_initialized:
-            if path == self.path_info.path:
-                return [self.remote_root_id]
-            return self.cached_dirs.get(path, [])
+        if use_cache:
+            return self._ids_cache["dirs"].get(path, [])
        return []

-    def _path_to_remote_ids(self, path, create):
-        remote_ids = self._get_cached_remote_ids(path)
-        if remote_ids:
-            return remote_ids
+    def _path_to_item_ids(self, path, create, use_cache):
+        item_ids = self._get_cached_item_ids(path, use_cache)
+        if item_ids:
+            return item_ids

-        parent_path, part = posixpath.split(path)
-        parent_ids = self._path_to_remote_ids(parent_path, create)
-        item = self._get_remote_item(part, parent_ids)
+        parent_path, title = posixpath.split(path)
+        parent_ids = self._path_to_item_ids(parent_path, create, use_cache)
+        item_ids = self._get_remote_item_ids(parent_ids, title)
+        if item_ids:
+            return item_ids

-        if not item:
-            return (
-                [self.gdrive_create_dir(parent_ids[0], part, path)]
-                if create
-                else []
-            )
+        return (
+            [self._create_dir(min(parent_ids), title, path)] if create else []
+        )

-        return [item["id"]]
-
-    def _get_remote_id(self, path_info, create=False):
+    def _get_item_id(self, path_info, create=False, use_cache=True, hint=None):
        assert path_info.bucket == self._bucket

-        remote_ids = self._path_to_remote_ids(path_info.path, create)
-        if not remote_ids:
-            raise GDrivePathNotFound(path_info)
+        item_ids = self._path_to_item_ids(path_info.path, create, use_cache)
+        if item_ids:
+            return min(item_ids)

-        return remote_ids[0]
+        assert not create
+        raise GDrivePathNotFound(path_info, hint)

    def exists(self, path_info):
        try:
-            self._get_remote_id(path_info)
+            self._get_item_id(path_info)
        except GDrivePathNotFound:
            return False
        else:
            return True

-    def _upload(self, from_file, to_info, name, no_progress_bar):
+    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
        dirname = to_info.parent
        assert dirname
-        parent_id = self._get_remote_id(dirname, True)
+        parent_id = self._get_item_id(dirname, True)

-        self.gdrive_upload_file(
+        self._gdrive_upload_file(
            parent_id, to_info.name, no_progress_bar, from_file, name
        )

-    def _download(self, from_info, to_file, name, no_progress_bar):
-        file_id = self._get_remote_id(from_info)
-        self.gdrive_download_file(file_id, to_file, name, no_progress_bar)
+    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
+        item_id = self._get_item_id(from_info)
+        self._gdrive_download_file(item_id, to_file, name, no_progress_bar)

-    def all(self):
-        if not self.cached_ids:
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        if not self._ids_cache["ids"]:
            return

-        query = "({})".format(
-            " or ".join(
-                "'{}' in parents".format(dir_id) for dir_id in self.cached_ids
-            )
+        if prefix:
+            dir_ids = self._ids_cache["dirs"].get(prefix[:2])
+            if not dir_ids:
+                return
+        else:
+            dir_ids = self._ids_cache["ids"]
+        parents_query = " or ".join(
+            f"'{dir_id}' in parents" for dir_id in dir_ids
        )
+        query = f"({parents_query}) and trashed=false"

-        query += " and trashed=false"
-        for file1 in self.gdrive_list_item(query):
-            parent_id = file1["parents"][0]["id"]
-            path = posixpath.join(self.cached_ids[parent_id], file1["title"])
-            try:
-                yield self.path_to_checksum(path)
-            except ValueError:
-                # We ignore all the non-cache looking files
-                logger.debug('Ignoring path as "non-cache looking"')
+        for item in self._gdrive_list(query):
+            if progress_callback:
+                progress_callback()
+            parent_id = item["parents"][0]["id"]
+            yield posixpath.join(
+                self._ids_cache["ids"][parent_id], item["title"]
+            )

    def remove(self, path_info):
-        remote_id = self._get_remote_id(path_info)
-        self._delete_remote_file(remote_id)
+        item_id = self._get_item_id(path_info)
+        self._gdrive_delete_file(item_id)
+
+    def get_file_checksum(self, path_info):
+        raise NotImplementedError
+
+    def walk_files(self, path_info):
+        raise NotImplementedError
@@ -1,17 +1,16 @@
 import logging
+import os.path
+import posixpath
+import threading
 from datetime import timedelta
 from functools import wraps
-import io
-import os.path
-import threading

 from funcy import cached_property, wrap_prop

-from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
+from dvc.remote.base import BaseRemote
 from dvc.scheme import Schemes

 logger = logging.getLogger(__name__)
@@ -52,10 +51,10 @@ def _upload_to_bucket(
    to_info,
    chunk_size=None,
    name=None,
-    no_progress_bar=True,
+    no_progress_bar=False,
 ):
    blob = bucket.blob(to_info.path, chunk_size=chunk_size)
-    with io.open(from_file, mode="rb") as fobj:
+    with open(from_file, mode="rb") as fobj:
        with Tqdm.wrapattr(
            fobj,
            "read",
@@ -66,7 +65,7 @@ def _upload_to_bucket(
            blob.upload_from_file(wrapped)


-class RemoteGS(RemoteBASE):
+class GSRemote(BaseRemote):
    scheme = Schemes.GS
    path_cls = CloudURLInfo
    REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
@@ -75,11 +74,11 @@ class RemoteGS(RemoteBASE):
    def __init__(self, repo, config):
        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL, "gs:///")
+        url = config.get("url", "gs:///")
        self.path_info = self.path_cls(url)

-        self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
-        self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
+        self.projectname = config.get("projectname", None)
+        self.credentialpath = config.get("credentialpath")

    @wrap_prop(threading.Lock())
    @cached_property
@@ -110,7 +109,7 @@ class RemoteGS(RemoteBASE):
        from_bucket = self.gs.bucket(from_info.bucket)
        blob = from_bucket.get_blob(from_info.path)
        if not blob:
-            msg = "'{}' doesn't exist in the cloud".format(from_info.path)
+            msg = f"'{from_info.path}' doesn't exist in the cloud"
            raise DvcException(msg)

        to_bucket = self.gs.bucket(to_info.bucket)
@@ -120,21 +119,31 @@ class RemoteGS(RemoteBASE):
        if path_info.scheme != "gs":
            raise NotImplementedError

-        logger.debug("Removing gs://{}".format(path_info))
+        logger.debug(f"Removing gs://{path_info}")
        blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
        if not blob:
            return

        blob.delete()

-    def _list_paths(self, path_info, max_items=None):
+    def _list_paths(
+        self, path_info, max_items=None, prefix=None, progress_callback=None
+    ):
+        if prefix:
+            prefix = posixpath.join(path_info.path, prefix[:2], prefix[2:])
+        else:
+            prefix = path_info.path
        for blob in self.gs.bucket(path_info.bucket).list_blobs(
            prefix=path_info.path, max_results=max_items
        ):
+            if progress_callback:
+                progress_callback()
            yield blob.name

-    def list_cache_paths(self):
-        return self._list_paths(self.path_info)
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        return self._list_paths(
+            self.path_info, prefix=prefix, progress_callback=progress_callback
+        )

    def walk_files(self, path_info):
        for fname in self._list_paths(path_info / ""):
@@ -144,6 +153,9 @@ class RemoteGS(RemoteBASE):
            yield path_info.replace(fname)

    def makedirs(self, path_info):
+        if not path_info.path:
+            return
+
        self.gs.bucket(path_info.bucket).blob(
            (path_info / "").path
        ).upload_from_string("")
@@ -167,7 +179,7 @@ class RemoteGS(RemoteBASE):
        """
        return self.isfile(path_info) or self.isdir(path_info)

-    def _upload(self, from_file, to_info, name=None, no_progress_bar=True):
+    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
        bucket = self.gs.bucket(to_info.bucket)
        _upload_to_bucket(
            bucket,
@@ -177,10 +189,10 @@ class RemoteGS(RemoteBASE):
            no_progress_bar=no_progress_bar,
        )

-    def _download(self, from_info, to_file, name=None, no_progress_bar=True):
+    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
        bucket = self.gs.bucket(from_info.bucket)
        blob = bucket.get_blob(from_info.path)
-        with io.open(to_file, mode="wb") as fobj:
+        with open(to_file, mode="wb") as fobj:
            with Tqdm.wrapattr(
                fobj,
                "write",
@@ -8,31 +8,31 @@ from collections import deque
 from contextlib import closing, contextmanager
 from urllib.parse import urlparse

-from .base import RemoteBASE, RemoteCmdError
-from .pool import get_connection
-from dvc.config import Config
 from dvc.scheme import Schemes
 from dvc.utils import fix_env, tmp_fname

+from .base import BaseRemote, RemoteCmdError
+from .pool import get_connection
+
 logger = logging.getLogger(__name__)


-class RemoteHDFS(RemoteBASE):
+class HDFSRemote(BaseRemote):
    scheme = Schemes.HDFS
    REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
    PARAM_CHECKSUM = "checksum"
    REQUIRES = {"pyarrow": "pyarrow"}
+    TRAVERSE_PREFIX_LEN = 2

    def __init__(self, repo, config):
        super().__init__(repo, config)
        self.path_info = None
-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
        if not url:
            return

        parsed = urlparse(url)
-
-        user = parsed.username or config.get(Config.SECTION_REMOTE_USER)
+        user = parsed.username or config.get("user")

        self.path_info = self.path_cls.from_parts(
            scheme=self.scheme,
@@ -55,7 +55,7 @@ class RemoteHDFS(RemoteBASE):
    def hadoop_fs(self, cmd, user=None):
        cmd = "hadoop fs -" + cmd
        if user:
-            cmd = "HADOOP_USER_NAME={} ".format(user) + cmd
+            cmd = f"HADOOP_USER_NAME={user} " + cmd

        # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
        # See https://github.com/iterative/dvc/issues/1197.
@@ -87,7 +87,7 @@ class RemoteHDFS(RemoteBASE):
        # NOTE: pyarrow doesn't support checksum, so we need to use hadoop
        regex = r".*\t.*\t(?P<checksum>.*)"
        stdout = self.hadoop_fs(
-            "checksum {}".format(path_info.path), user=path_info.user
+            f"checksum {path_info.path}", user=path_info.user
        )
        return self._group(regex, stdout, "checksum")

@@ -112,7 +112,7 @@ class RemoteHDFS(RemoteBASE):
            raise NotImplementedError

        if self.exists(path_info):
-            logger.debug("Removing {}".format(path_info.path))
+            logger.debug(f"Removing {path_info.path}")
            with self.hdfs(path_info) as hdfs:
                hdfs.rm(path_info.path)

@@ -147,23 +147,36 @@ class RemoteHDFS(RemoteBASE):
                    yield fd
                else:
                    yield io.TextIOWrapper(fd, encoding=encoding)
-        except IOError as e:
+        except OSError as e:
            # Empty .errno and not specific enough error class in pyarrow,
            # see https://issues.apache.org/jira/browse/ARROW-6248
            if "file does not exist" in str(e):
                raise FileNotFoundError(*e.args)
            raise

-    def list_cache_paths(self):
+    def list_cache_paths(self, prefix=None, progress_callback=None):
        if not self.exists(self.path_info):
            return

-        dirs = deque([self.path_info.path])
+        if prefix:
+            root = posixpath.join(self.path_info.path, prefix[:2])
+        else:
+            root = self.path_info.path
+        dirs = deque([root])

        with self.hdfs(self.path_info) as hdfs:
            while dirs:
-                for entry in hdfs.ls(dirs.pop(), detail=True):
-                    if entry["kind"] == "directory":
-                        dirs.append(urlparse(entry["name"]).path)
-                    elif entry["kind"] == "file":
-                        yield urlparse(entry["name"]).path
+                try:
+                    entries = hdfs.ls(dirs.pop(), detail=True)
+                    for entry in entries:
+                        if entry["kind"] == "directory":
+                            dirs.append(urlparse(entry["name"]).path)
+                        elif entry["kind"] == "file":
+                            if progress_callback:
+                                progress_callback()
+                            yield urlparse(entry["name"]).path
+                except OSError as e:
+                    # When searching for a specific prefix pyarrow raises an
+                    # exception if the specified cache dir does not exist
+                    if not prefix:
+                        raise e
@@ -1,53 +1,96 @@
 import logging
+import os.path
 import threading

-from funcy import cached_property, wrap_prop
+from funcy import cached_property, memoize, wrap_prop, wrap_with

-from dvc.config import Config
-from dvc.config import ConfigError
+import dvc.prompt as prompt
 from dvc.exceptions import DvcException, HTTPError
+from dvc.path_info import HTTPURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
+from dvc.remote.base import BaseRemote
 from dvc.scheme import Schemes

 logger = logging.getLogger(__name__)


-class RemoteHTTP(RemoteBASE):
+@wrap_with(threading.Lock())
+@memoize
+def ask_password(host, user):
+    return prompt.password(
+        "Enter a password for "
+        "host '{host}' user '{user}'".format(host=host, user=user)
+    )
+
+
+class HTTPRemote(BaseRemote):
    scheme = Schemes.HTTP
+    path_cls = HTTPURLInfo
    SESSION_RETRIES = 5
    SESSION_BACKOFF_FACTOR = 0.1
    REQUEST_TIMEOUT = 10
    CHUNK_SIZE = 2 ** 16
    PARAM_CHECKSUM = "etag"
+    CAN_TRAVERSE = False

    def __init__(self, repo, config):
        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL)
-        self.path_info = self.path_cls(url) if url else None
+        url = config.get("url")
+        if url:
+            self.path_info = self.path_cls(url)
+            user = config.get("user", None)
+            if user:
+                self.path_info.user = user
+        else:
+            self.path_info = None

-        if not self.no_traverse:
-            raise ConfigError(
-                "HTTP doesn't support traversing the remote to list existing "
-                "files. Use: `dvc remote modify <name> no_traverse true`"
-            )
+        self.auth = config.get("auth", None)
+        self.custom_auth_header = config.get("custom_auth_header", None)
+        self.password = config.get("password", None)
+        self.ask_password = config.get("ask_password", False)
+        self.headers = {}

    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
        response = self._request("GET", from_info.url, stream=True)
        if response.status_code != 200:
            raise HTTPError(response.status_code, response.reason)
-        with Tqdm(
-            total=None if no_progress_bar else self._content_length(response),
-            leave=False,
-            bytes=True,
-            desc=from_info.url if name is None else name,
-            disable=no_progress_bar,
-        ) as pbar:
-            with open(to_file, "wb") as fd:
+        with open(to_file, "wb") as fd:
+            with Tqdm.wrapattr(
+                fd,
+                "write",
+                total=None
+                if no_progress_bar
+                else self._content_length(response),
+                leave=False,
+                desc=from_info.url if name is None else name,
+                disable=no_progress_bar,
+            ) as fd_wrapped:
                for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE):
-                    fd.write(chunk)
-                    pbar.update(len(chunk))
+                    fd_wrapped.write(chunk)
+
+    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
+        def chunks():
+            with open(from_file, "rb") as fd:
+                with Tqdm.wrapattr(
+                    fd,
+                    "read",
+                    total=None
+                    if no_progress_bar
+                    else os.path.getsize(from_file),
+                    leave=False,
+                    desc=to_info.url if name is None else name,
+                    disable=no_progress_bar,
+                ) as fd_wrapped:
+                    while True:
+                        chunk = fd_wrapped.read(self.CHUNK_SIZE)
+                        if not chunk:
+                            break
+                        yield chunk
+
+        response = self._request("POST", to_info.url, data=chunks())
+        if response.status_code not in (200, 201):
+            raise HTTPError(response.status_code, response.reason)

    def exists(self, path_info):
        return bool(self._request("HEAD", path_info.url))
@@ -75,6 +118,24 @@ class RemoteHTTP(RemoteBASE):

        return etag

+    def auth_method(self, path_info=None):
+        from requests.auth import HTTPBasicAuth, HTTPDigestAuth
+
+        if path_info is None:
+            path_info = self.path_info
+
+        if self.auth:
+            if self.ask_password and self.password is None:
+                host, user = path_info.host, path_info.user
+                self.password = ask_password(host, user)
+            if self.auth == "basic":
+                return HTTPBasicAuth(path_info.user, self.password)
+            if self.auth == "digest":
+                return HTTPDigestAuth(path_info.user, self.password)
+            if self.auth == "custom" and self.custom_auth_header:
+                self.headers.update({self.custom_auth_header: self.password})
+        return None
+
    @wrap_prop(threading.Lock())
    @cached_property
    def _session(self):
@@ -101,7 +162,13 @@ class RemoteHTTP(RemoteBASE):
        kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)

        try:
-            res = self._session.request(method, url, **kwargs)
+            res = self._session.request(
+                method,
+                url,
+                auth=self.auth_method(),
+                headers=self.headers,
+                **kwargs,
+            )

            redirect_no_location = (
                kwargs["allow_redirects"]
@@ -118,7 +185,7 @@ class RemoteHTTP(RemoteBASE):
            return res

        except requests.exceptions.RequestException:
-            raise DvcException("could not perform a {} request".format(method))
+            raise DvcException(f"could not perform a {method} request")

    def gc(self):
        raise NotImplementedError
@@ -1,6 +1,7 @@
-from .http import RemoteHTTP
 from dvc.scheme import Schemes

+from .http import HTTPRemote

-class RemoteHTTPS(RemoteHTTP):
+
+class HTTPSRemote(HTTPRemote):
    scheme = Schemes.HTTPS
@@ -0,0 +1,212 @@
+import logging
+import os
+import sqlite3
+import threading
+
+from funcy import lchunks
+
+from dvc.state import _connect_sqlite
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteIndexNoop:
+    """No-op class for remotes which are not indexed (i.e. local)."""
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, typ, value, tbck):
+        pass
+
+    def __iter__(self):
+        return iter([])
+
+    def __contains__(self, checksum):
+        return False
+
+    @staticmethod
+    def checksums():
+        return []
+
+    @staticmethod
+    def dir_checksums():
+        return []
+
+    def load(self):
+        pass
+
+    def dump(self):
+        pass
+
+    def clear(self):
+        pass
+
+    def update(self, *args):
+        pass
+
+    @staticmethod
+    def intersection(*args):
+        return []
+
+
+class RemoteIndex:
+    """Class for indexing remote checksums in a sqlite3 database.
+
+    Args:
+        repo: repo for this remote index.
+        name: name for this index. Index db will be loaded from and saved to
+            ``.dvc/tmp/index/{name}.idx``.
+        dir_suffix: suffix used for naming directory checksums
+    """
+
+    INDEX_SUFFIX = ".idx"
+    VERSION = 1
+    INDEX_TABLE = "remote_index"
+    INDEX_TABLE_LAYOUT = "checksum TEXT PRIMARY KEY, " "dir INTEGER NOT NULL"
+
+    def __init__(self, repo, name, dir_suffix=".dir"):
+        self.path = os.path.join(repo.index_dir, f"{name}{self.INDEX_SUFFIX}")
+
+        self.dir_suffix = dir_suffix
+        self.database = None
+        self.cursor = None
+        self.modified = False
+        self.lock = threading.Lock()
+
+    def __iter__(self):
+        cmd = f"SELECT checksum FROM {self.INDEX_TABLE}"
+        for (checksum,) in self._execute(cmd):
+            yield checksum
+
+    def __enter__(self):
+        self.lock.acquire()
+        self.load()
+
+    def __exit__(self, typ, value, tbck):
+        self.dump()
+        self.lock.release()
+
+    def __contains__(self, checksum):
+        cmd = "SELECT checksum FROM {} WHERE checksum = (?)".format(
+            self.INDEX_TABLE
+        )
+        self._execute(cmd, (checksum,))
+        return self.cursor.fetchone() is not None
+
+    def checksums(self):
+        """Iterate over checksums stored in the index."""
+        return iter(self)
+
+    def dir_checksums(self):
+        """Iterate over .dir checksums stored in the index."""
+        cmd = f"SELECT checksum FROM {self.INDEX_TABLE} WHERE dir = 1"
+        for (checksum,) in self._execute(cmd):
+            yield checksum
+
+    def is_dir_checksum(self, checksum):
+        return checksum.endswith(self.dir_suffix)
+
+    def _execute(self, cmd, parameters=()):
+        return self.cursor.execute(cmd, parameters)
+
+    def _executemany(self, cmd, seq_of_parameters):
+        return self.cursor.executemany(cmd, seq_of_parameters)
+
+    def _prepare_db(self, empty=False):
+        if not empty:
+            cmd = "PRAGMA user_version;"
+            self._execute(cmd)
+            ret = self.cursor.fetchall()
+            assert len(ret) == 1
+            assert len(ret[0]) == 1
+            assert isinstance(ret[0][0], int)
+            version = ret[0][0]
+
+            if version != self.VERSION:
+                logger.error(
+                    "Index file version '{}' will be reformatted "
+                    "to the current version '{}'.".format(
+                        version, self.VERSION,
+                    )
+                )
+                cmd = "DROP TABLE IF EXISTS {};"
+                self._execute(cmd.format(self.INDEX_TABLE))
+
+        cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
+        self._execute(cmd.format(self.INDEX_TABLE, self.INDEX_TABLE_LAYOUT))
+
+        cmd = "PRAGMA user_version = {};"
+        self._execute(cmd.format(self.VERSION))
+
+    def load(self):
+        """(Re)load this index database."""
+        retries = 1
+        while True:
+            assert self.database is None
+            assert self.cursor is None
+
+            empty = not os.path.isfile(self.path)
+            self.database = _connect_sqlite(self.path, {"nolock": 1})
+            self.cursor = self.database.cursor()
+
+            try:
+                self._prepare_db(empty=empty)
+                return
+            except sqlite3.DatabaseError:
+                self.cursor.close()
+                self.database.close()
+                self.database = None
+                self.cursor = None
+                if retries > 0:
+                    os.unlink(self.path)
+                    retries -= 1
+                else:
+                    raise
+
+    def dump(self):
+        """Save this index database."""
+        assert self.database is not None
+
+        self.database.commit()
+        self.cursor.close()
+        self.database.close()
+        self.database = None
+        self.cursor = None
+
+    def clear(self):
+        """Clear this index (to force re-indexing later).
+
+        Changes to the index will not committed until dump() is called.
+        """
+        cmd = f"DELETE FROM {self.INDEX_TABLE}"
+        self._execute(cmd)
+
+    def update(self, dir_checksums, file_checksums):
+        """Update this index, adding the specified checksums.
+
+        Changes to the index will not committed until dump() is called.
+        """
+        cmd = "INSERT OR IGNORE INTO {} (checksum, dir) VALUES (?, ?)".format(
+            self.INDEX_TABLE
+        )
+        self._executemany(
+            cmd, ((checksum, True) for checksum in dir_checksums)
+        )
+        self._executemany(
+            cmd, ((checksum, False) for checksum in file_checksums)
+        )
+
+    def intersection(self, checksums):
+        """Iterate over values from `checksums` which exist in the index."""
+        # sqlite has a compile time limit of 999, see:
+        # https://www.sqlite.org/c3ref/c_limit_attached.html#sqlitelimitvariablenumber
+        for chunk in lchunks(999, checksums):
+            cmd = "SELECT checksum FROM {} WHERE checksum IN ({})".format(
+                self.INDEX_TABLE, ",".join("?" for checksum in chunk)
+            )
+            for (checksum,) in self._execute(cmd, chunk):
+                yield checksum
@@ -2,66 +2,51 @@ import errno
 import logging
 import os
 import stat
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from functools import partial

+from funcy import cached_property, concat
 from shortuuid import uuid

-from dvc.config import Config
-from dvc.exceptions import DownloadError
-from dvc.exceptions import DvcException
-from dvc.exceptions import UploadError
+from dvc.exceptions import DownloadError, DvcException, UploadError
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
-from dvc.remote.base import STATUS_DELETED
-from dvc.remote.base import STATUS_MAP
-from dvc.remote.base import STATUS_MISSING
-from dvc.remote.base import STATUS_NEW
+from dvc.remote.base import (
+    STATUS_DELETED,
+    STATUS_MAP,
+    STATUS_MISSING,
+    STATUS_NEW,
+    BaseRemote,
+    index_locked,
+)
+from dvc.remote.index import RemoteIndexNoop
 from dvc.scheme import Schemes
 from dvc.scm.tree import is_working_tree
 from dvc.system import System
-from dvc.utils.fs import copyfile
-from dvc.utils import file_md5
-from dvc.utils import relpath
-from dvc.utils import tmp_fname
-from dvc.compat import fspath_py35
-from dvc.utils.fs import move, makedirs, remove
-from dvc.utils.fs import walk_files
+from dvc.utils import file_md5, relpath, tmp_fname
+from dvc.utils.fs import copyfile, makedirs, move, remove, walk_files

 logger = logging.getLogger(__name__)


-class RemoteLOCAL(RemoteBASE):
+class LocalRemote(BaseRemote):
    scheme = Schemes.LOCAL
    path_cls = PathInfo
    PARAM_CHECKSUM = "md5"
    PARAM_PATH = "path"
+    TRAVERSE_PREFIX_LEN = 2
+    INDEX_CLS = RemoteIndexNoop

    UNPACKED_DIR_SUFFIX = ".unpacked"

    DEFAULT_CACHE_TYPES = ["reflink", "copy"]

+    CACHE_MODE = 0o444
    SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}

    def __init__(self, repo, config):
        super().__init__(repo, config)
-        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
-
-        shared = config.get(Config.SECTION_CACHE_SHARED)
-        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
-
-        if self.protected:
-            # cache files are set to be read-only for everyone
-            self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
-
-        cache_dir = config.get(Config.SECTION_REMOTE_URL)
-
-        if cache_dir is not None and not os.path.isabs(cache_dir):
-            cwd = config[Config.PRIVATE_CWD]
-            cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
-
-        self.cache_dir = cache_dir
+        self.cache_dir = config.get("url")
        self._dir_info = {}

    @property
@@ -80,9 +65,25 @@ class RemoteLOCAL(RemoteBASE):
    def supported(cls, config):
        return True

-    def list_cache_paths(self):
+    @cached_property
+    def cache_path(self):
+        return os.path.abspath(self.cache_dir)
+
+    def checksum_to_path(self, checksum):
+        return os.path.join(self.cache_path, checksum[0:2], checksum[2:])
+
+    def list_cache_paths(self, prefix=None, progress_callback=None):
        assert self.path_info is not None
-        return walk_files(self.path_info)
+        if prefix:
+            path_info = self.path_info / prefix[:2]
+        else:
+            path_info = self.path_info
+        if progress_callback:
+            for path in walk_files(path_info):
+                progress_callback()
+                yield path
+        else:
+            yield from walk_files(path_info)

    def get(self, md5):
        if not md5:
@@ -90,10 +91,10 @@ class RemoteLOCAL(RemoteBASE):

        return self.checksum_to_path_info(md5).url

-    @staticmethod
-    def exists(path_info):
-        assert path_info.scheme == "local"
-        return os.path.lexists(fspath_py35(path_info))
+    def exists(self, path_info):
+        assert is_working_tree(self.repo.tree)
+        assert isinstance(path_info, str) or path_info.scheme == "local"
+        return self.repo.tree.exists(path_info)

    def makedirs(self, path_info):
        makedirs(path_info, exist_ok=True, mode=self._dir_mode)
@@ -108,6 +109,12 @@ class RemoteLOCAL(RemoteBASE):

        return not self.changed_cache(current_md5)

+    def _verify_link(self, path_info, link_type):
+        if link_type == "hardlink" and self.getsize(path_info) == 0:
+            return
+
+        super()._verify_link(path_info, link_type)
+
    def is_empty(self, path_info):
        path = path_info.fspath

@@ -121,11 +128,11 @@ class RemoteLOCAL(RemoteBASE):

    @staticmethod
    def isfile(path_info):
-        return os.path.isfile(fspath_py35(path_info))
+        return os.path.isfile(path_info)

    @staticmethod
    def isdir(path_info):
-        return os.path.isdir(fspath_py35(path_info))
+        return os.path.isdir(path_info)

    def iscopy(self, path_info):
        return not (
@@ -134,7 +141,7 @@ class RemoteLOCAL(RemoteBASE):

    @staticmethod
    def getsize(path_info):
-        return os.path.getsize(fspath_py35(path_info))
+        return os.path.getsize(path_info)

    def walk_files(self, path_info):
        assert is_working_tree(self.repo.tree)
@@ -146,22 +153,27 @@ class RemoteLOCAL(RemoteBASE):
        return file_md5(path_info)[0]

    def remove(self, path_info):
-        if path_info.scheme != "local":
-            raise NotImplementedError
+        if isinstance(path_info, PathInfo):
+            if path_info.scheme != "local":
+                raise NotImplementedError
+            path = path_info.fspath
+        else:
+            path = path_info

-        if self.exists(path_info):
-            remove(path_info.fspath)
+        if self.exists(path):
+            remove(path)

-    def move(self, from_info, to_info):
+    def move(self, from_info, to_info, mode=None):
        if from_info.scheme != "local" or to_info.scheme != "local":
            raise NotImplementedError

        self.makedirs(to_info.parent)

-        if self.isfile(from_info):
-            mode = self._file_mode
-        else:
-            mode = self._dir_mode
+        if mode is None:
+            if self.isfile(from_info):
+                mode = self._file_mode
+            else:
+                mode = self._dir_mode

        move(from_info, to_info, mode=mode)

@@ -169,7 +181,8 @@ class RemoteLOCAL(RemoteBASE):
        tmp_info = to_info.parent / tmp_fname(to_info.name)
        try:
            System.copy(from_info, tmp_info)
-            os.rename(fspath_py35(tmp_info), fspath_py35(to_info))
+            os.chmod(tmp_info, self._file_mode)
+            os.rename(tmp_info, to_info)
        except Exception:
            self.remove(tmp_info)
            raise
@@ -212,9 +225,13 @@ class RemoteLOCAL(RemoteBASE):
    def is_hardlink(path_info):
        return System.is_hardlink(path_info)

-    @staticmethod
-    def reflink(from_info, to_info):
-        System.reflink(from_info, to_info)
+    def reflink(self, from_info, to_info):
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        System.reflink(from_info, tmp_info)
+        # NOTE: reflink has its own separate inode, so you can set permissions
+        # that are different from the source.
+        os.chmod(tmp_info, self._file_mode)
+        os.rename(tmp_info, to_info)

    def cache_exists(self, checksums, jobs=None, name=None):
        return [
@@ -237,7 +254,7 @@ class RemoteLOCAL(RemoteBASE):
        copyfile(
            from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
        )
-        os.rename(tmp_file, fspath_py35(to_info))
+        os.rename(tmp_file, to_info)

    def _download(
        self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
@@ -248,8 +265,9 @@ class RemoteLOCAL(RemoteBASE):

    @staticmethod
    def open(path_info, mode="r", encoding=None):
-        return open(fspath_py35(path_info), mode=mode, encoding=encoding)
+        return open(path_info, mode=mode, encoding=encoding)

+    @index_locked
    def status(
        self,
        named_cache,
@@ -258,37 +276,128 @@ class RemoteLOCAL(RemoteBASE):
        show_checksums=False,
        download=False,
    ):
-        logger.debug(
-            "Preparing to collect status from {}".format(remote.path_info)
+        # Return flattened dict containing all status info
+        dir_status, file_status, _ = self._status(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
        )
-        md5s = list(named_cache[self.scheme])
+        return dict(dir_status, **file_status)
+
+    def _status(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        """Return a tuple of (dir_status_info, file_status_info, dir_contents).
+
+        dir_status_info contains status for .dir files, file_status_info
+        contains status for all other files, and dir_contents is a dict of
+        {dir_checksum: set(file_checksum, ...)} which can be used to map
+        a .dir file to its file contents.
+        """
+        logger.debug(f"Preparing to collect status from {remote.path_info}")
+        md5s = set(named_cache.scheme_keys(self.scheme))

        logger.debug("Collecting information from local cache...")
-        local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
+        local_exists = frozenset(
+            self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
+        )

        # This is a performance optimization. We can safely assume that,
        # if the resources that we want to fetch are already cached,
        # there's no need to check the remote storage for the existence of
        # those files.
-        if download and sorted(local_exists) == sorted(md5s):
+        if download and local_exists == md5s:
            remote_exists = local_exists
        else:
            logger.debug("Collecting information from remote cache...")
-            remote_exists = list(
-                remote.cache_exists(
-                    md5s, jobs=jobs, name=str(remote.path_info)
+            remote_exists = set()
+            dir_md5s = set(named_cache.dir_keys(self.scheme))
+            if dir_md5s:
+                remote_exists.update(
+                    self._indexed_dir_checksums(named_cache, remote, dir_md5s)
                )
+                md5s.difference_update(remote_exists)
+            if md5s:
+                remote_exists.update(
+                    remote.cache_exists(
+                        md5s, jobs=jobs, name=str(remote.path_info)
+                    )
+                )
+        return self._make_status(
+            named_cache, show_checksums, local_exists, remote_exists
+        )
+
+    def _make_status(
+        self, named_cache, show_checksums, local_exists, remote_exists
+    ):
+        def make_names(checksum, names):
+            return {"name": checksum if show_checksums else " ".join(names)}
+
+        dir_status = {}
+        file_status = {}
+        dir_contents = {}
+        for checksum, item in named_cache[self.scheme].items():
+            if item.children:
+                dir_status[checksum] = make_names(checksum, item.names)
+                dir_contents[checksum] = set()
+                for child_checksum, child in item.children.items():
+                    file_status[child_checksum] = make_names(
+                        child_checksum, child.names
+                    )
+                    dir_contents[checksum].add(child_checksum)
+            else:
+                file_status[checksum] = make_names(checksum, item.names)
+
+        self._fill_statuses(dir_status, local_exists, remote_exists)
+        self._fill_statuses(file_status, local_exists, remote_exists)
+
+        self._log_missing_caches(dict(dir_status, **file_status))
+
+        return dir_status, file_status, dir_contents
+
+    def _indexed_dir_checksums(self, named_cache, remote, dir_md5s):
+        # Validate our index by verifying all indexed .dir checksums
+        # still exist on the remote
+        indexed_dirs = set(remote.index.dir_checksums())
+        indexed_dir_exists = set()
+        if indexed_dirs:
+            indexed_dir_exists.update(
+                remote._cache_object_exists(indexed_dirs)
            )
+            missing_dirs = indexed_dirs.difference(indexed_dir_exists)
+            if missing_dirs:
+                logger.debug(
+                    "Remote cache missing indexed .dir checksums '{}', "
+                    "clearing remote index".format(", ".join(missing_dirs))
+                )
+                remote.index.clear()

-        ret = {
-            checksum: {"name": checksum if show_checksums else " ".join(names)}
-            for checksum, names in named_cache[self.scheme].items()
-        }
-        self._fill_statuses(ret, local_exists, remote_exists)
+        # Check if non-indexed (new) dir checksums exist on remote
+        dir_exists = dir_md5s.intersection(indexed_dir_exists)
+        dir_exists.update(remote._cache_object_exists(dir_md5s - dir_exists))

-        self._log_missing_caches(ret)
-
-        return ret
+        # If .dir checksum exists on the remote, assume directory contents
+        # still exists on the remote
+        for dir_checksum in dir_exists:
+            file_checksums = list(
+                named_cache.child_keys(self.scheme, dir_checksum)
+            )
+            if dir_checksum not in remote.index:
+                logger.debug(
+                    "Indexing new .dir '{}' with '{}' nested files".format(
+                        dir_checksum, len(file_checksums)
+                    )
+                )
+                remote.index.update([dir_checksum], file_checksums)
+            yield dir_checksum
+            yield from file_checksums

    @staticmethod
    def _fill_statuses(checksum_info_dir, local_exists, remote_exists):
@@ -304,6 +413,7 @@ class RemoteLOCAL(RemoteBASE):
        cache = []
        path_infos = []
        names = []
+        checksums = []
        for md5, info in Tqdm(
            status_info.items(), desc="Analysing status", unit="file"
        ):
@@ -311,6 +421,7 @@ class RemoteLOCAL(RemoteBASE):
                cache.append(self.checksum_to_path_info(md5))
                path_infos.append(remote.checksum_to_path_info(md5))
                names.append(info["name"])
+                checksums.append(md5)

        if download:
            to_infos = cache
@@ -319,7 +430,7 @@ class RemoteLOCAL(RemoteBASE):
            to_infos = path_infos
            from_infos = cache

-        return from_infos, to_infos, names
+        return from_infos, to_infos, names, checksums

    def _process(
        self,
@@ -343,14 +454,16 @@ class RemoteLOCAL(RemoteBASE):
                file_mode=self._file_mode,
            )
            status = STATUS_DELETED
+            desc = "Downloading"
        else:
            func = remote.upload
            status = STATUS_NEW
+            desc = "Uploading"

        if jobs is None:
            jobs = remote.JOBS

-        status_info = self.status(
+        dir_status, file_status, dir_contents = self._status(
            named_cache,
            remote,
            jobs=jobs,
@@ -358,24 +471,87 @@ class RemoteLOCAL(RemoteBASE):
            download=download,
        )

-        plans = self._get_plans(download, remote, status_info, status)
+        dir_plans = self._get_plans(download, remote, dir_status, status)
+        file_plans = self._get_plans(download, remote, file_status, status)

-        if len(plans[0]) == 0:
+        total = len(dir_plans[0]) + len(file_plans[0])
+        if total == 0:
            return 0

-        if jobs > 1:
+        with Tqdm(total=total, unit="file", desc=desc) as pbar:
+            func = pbar.wrap_fn(func)
            with ThreadPoolExecutor(max_workers=jobs) as executor:
-                fails = sum(executor.map(func, *plans))
-        else:
-            fails = sum(map(func, *plans))
+                if download:
+                    fails = sum(executor.map(func, *dir_plans))
+                    fails += sum(executor.map(func, *file_plans))
+                else:
+                    # for uploads, push files first, and any .dir files last
+
+                    file_futures = {}
+                    for from_info, to_info, name, checksum in zip(*file_plans):
+                        file_futures[checksum] = executor.submit(
+                            func, from_info, to_info, name
+                        )
+                    dir_futures = {}
+                    for from_info, to_info, name, dir_checksum in zip(
+                        *dir_plans
+                    ):
+                        wait_futures = {
+                            future
+                            for file_checksum, future in file_futures.items()
+                            if file_checksum in dir_contents[dir_checksum]
+                        }
+                        dir_futures[dir_checksum] = executor.submit(
+                            self._dir_upload,
+                            func,
+                            wait_futures,
+                            from_info,
+                            to_info,
+                            name,
+                        )
+                    fails = sum(
+                        future.result()
+                        for future in concat(
+                            file_futures.values(), dir_futures.values()
+                        )
+                    )

        if fails:
            if download:
+                remote.index.clear()
                raise DownloadError(fails)
            raise UploadError(fails)

-        return len(plans[0])
+        if not download:
+            # index successfully pushed dirs
+            for dir_checksum, future in dir_futures.items():
+                if future.result() == 0:
+                    file_checksums = dir_contents[dir_checksum]
+                    logger.debug(
+                        "Indexing pushed dir '{}' with "
+                        "'{}' nested files".format(
+                            dir_checksum, len(file_checksums)
+                        )
+                    )
+                    remote.index.update([dir_checksum], file_checksums)

+        return len(dir_plans[0]) + len(file_plans[0])
+
+    @staticmethod
+    def _dir_upload(func, futures, from_info, to_info, name):
+        for future in as_completed(futures):
+            if future.result():
+                # do not upload this .dir file if any file in this
+                # directory failed to upload
+                logger.debug(
+                    "failed to upload full contents of '{}', "
+                    "aborting .dir file upload".format(name)
+                )
+                logger.error(f"failed to upload '{from_info}' to '{to_info}'")
+                return 1
+        return func(from_info, to_info, name)
+
+    @index_locked
    def push(self, named_cache, remote, jobs=None, show_checksums=False):
        return self._process(
            named_cache,
@@ -385,6 +561,7 @@ class RemoteLOCAL(RemoteBASE):
            download=False,
        )

+    @index_locked
    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
        return self._process(
            named_cache,
@@ -412,10 +589,9 @@ class RemoteLOCAL(RemoteBASE):
            )
            logger.warning(msg)

-    @staticmethod
-    def _unprotect_file(path):
+    def _unprotect_file(self, path):
        if System.is_symlink(path) or System.is_hardlink(path):
-            logger.debug("Unprotecting '{}'".format(path))
+            logger.debug(f"Unprotecting '{path}'")
            tmp = os.path.join(os.path.dirname(path), "." + uuid())

            # The operations order is important here - if some application
@@ -433,41 +609,42 @@ class RemoteLOCAL(RemoteBASE):
                "a symlink or a hardlink.".format(path)
            )

-        os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
+        os.chmod(path, self._file_mode)

    def _unprotect_dir(self, path):
        assert is_working_tree(self.repo.tree)

        for fname in self.repo.tree.walk_files(path):
-            RemoteLOCAL._unprotect_file(fname)
+            self._unprotect_file(fname)

    def unprotect(self, path_info):
        path = path_info.fspath
        if not os.path.exists(path):
-            raise DvcException(
-                "can't unprotect non-existing data '{}'".format(path)
-            )
+            raise DvcException(f"can't unprotect non-existing data '{path}'")

        if os.path.isdir(path):
            self._unprotect_dir(path)
        else:
-            RemoteLOCAL._unprotect_file(path)
+            self._unprotect_file(path)

-    @staticmethod
-    def protect(path_info):
-        path = fspath_py35(path_info)
-        mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
+    def protect(self, path_info):
+        path = os.fspath(path_info)
+        mode = self.CACHE_MODE

        try:
            os.chmod(path, mode)
        except OSError as exc:
-            # In share cache scenario, we might not own the cache file, so we
+            # There is nothing we need to do in case of a read-only file system
+            if exc.errno == errno.EROFS:
+                return
+
+            # In shared cache scenario, we might not own the cache file, so we
            # need to check if cache file is already protected.
            if exc.errno not in [errno.EPERM, errno.EACCES]:
                raise

-            actual = os.stat(path).st_mode
-            if actual & mode != mode:
+            actual = stat.S_IMODE(os.stat(path).st_mode)
+            if actual != mode:
                raise

    def _get_unpacked_dir_path_info(self, checksum):
@@ -495,7 +672,7 @@ class RemoteLOCAL(RemoteBASE):
            dir_info = self.get_dir_cache(checksum)
            self._create_unpacked_dir(checksum, dir_info, unpacked_dir_info)
        except DvcException:
-            logger.warning("Could not create '{}'".format(unpacked_dir_info))
+            logger.warning(f"Could not create '{unpacked_dir_info}'")

            self.remove(unpacked_dir_info)

@@ -529,3 +706,11 @@ class RemoteLOCAL(RemoteBASE):
            if self.is_dir_checksum(c):
                unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
        return unpacked
+
+    def is_protected(self, path_info):
+        if not self.exists(path_info):
+            return False
+
+        mode = os.stat(path_info).st_mode
+
+        return stat.S_IMODE(mode) == self.CACHE_MODE
@@ -1,20 +1,19 @@
 import logging
 import os
+import posixpath
 import threading

 from funcy import cached_property, wrap_prop

-from dvc.config import Config
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
+from dvc.remote.base import BaseRemote
 from dvc.scheme import Schemes

-
 logger = logging.getLogger(__name__)


-class RemoteOSS(RemoteBASE):
+class OSSRemote(BaseRemote):
    """
    oss2 document:
    https://www.alibabacloud.com/help/doc-detail/32026.htm
@@ -38,25 +37,24 @@ class RemoteOSS(RemoteBASE):
    REQUIRES = {"oss2": "oss2"}
    PARAM_CHECKSUM = "etag"
    COPY_POLL_SECONDS = 5
+    LIST_OBJECT_PAGE_SIZE = 100

    def __init__(self, repo, config):
        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
        self.path_info = self.path_cls(url) if url else None

-        self.endpoint = config.get(Config.SECTION_OSS_ENDPOINT) or os.getenv(
-            "OSS_ENDPOINT"
-        )
+        self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")

        self.key_id = (
-            config.get(Config.SECTION_OSS_ACCESS_KEY_ID)
+            config.get("oss_key_id")
            or os.getenv("OSS_ACCESS_KEY_ID")
            or "defaultId"
        )

        self.key_secret = (
-            config.get(Config.SECTION_OSS_ACCESS_KEY_SECRET)
+            config.get("oss_key_secret")
            or os.getenv("OSS_ACCESS_KEY_SECRET")
            or "defaultSecret"
        )
@@ -66,9 +64,9 @@ class RemoteOSS(RemoteBASE):
    def oss_service(self):
        import oss2

-        logger.debug("URL: {}".format(self.path_info))
-        logger.debug("key id: {}".format(self.key_id))
-        logger.debug("key secret: {}".format(self.key_secret))
+        logger.debug(f"URL: {self.path_info}")
+        logger.debug(f"key id: {self.key_id}")
+        logger.debug(f"key secret: {self.key_secret}")

        auth = oss2.Auth(self.key_id, self.key_secret)
        bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)
@@ -89,17 +87,25 @@ class RemoteOSS(RemoteBASE):
        if path_info.scheme != self.scheme:
            raise NotImplementedError

-        logger.debug("Removing oss://{}".format(path_info))
+        logger.debug(f"Removing oss://{path_info}")
        self.oss_service.delete_object(path_info.path)

-    def _list_paths(self, prefix):
+    def _list_paths(self, prefix, progress_callback=None):
        import oss2

        for blob in oss2.ObjectIterator(self.oss_service, prefix=prefix):
+            if progress_callback:
+                progress_callback()
            yield blob.key

-    def list_cache_paths(self):
-        return self._list_paths(self.path_info.path)
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            prefix = posixpath.join(
+                self.path_info.path, prefix[:2], prefix[2:]
+            )
+        else:
+            prefix = self.path_info.path
+        return self._list_paths(prefix, progress_callback)

    def _upload(
        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
@@ -1,6 +1,6 @@
+import threading
 from collections import deque
 from contextlib import contextmanager
-import threading

 from funcy import memoize, wrap_with

@@ -30,7 +30,7 @@ def close_pools():
    get_pool.memory.clear()


-class Pool(object):
+class Pool:
    def __init__(self, conn_func, *conn_args, **conn_kwargs):
        self._conn_func = conn_func
        self._conn_args = conn_args
@@ -1,23 +1,21 @@
-# -*- coding: utf-8 -*-
-
 import logging
 import os
+import posixpath
 import threading

 from funcy import cached_property, wrap_prop

-from dvc.config import Config, ConfigError
-from dvc.exceptions import DvcException
-from dvc.exceptions import ETagMismatchError
+from dvc.config import ConfigError
+from dvc.exceptions import DvcException, ETagMismatchError
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
+from dvc.remote.base import BaseRemote
 from dvc.scheme import Schemes

 logger = logging.getLogger(__name__)


-class RemoteS3(RemoteBASE):
+class S3Remote(BaseRemote):
    scheme = Schemes.S3
    path_cls = CloudURLInfo
    REQUIRES = {"boto3": "boto3"}
@@ -26,35 +24,37 @@ class RemoteS3(RemoteBASE):
    def __init__(self, repo, config):
        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL, "s3://")
+        url = config.get("url", "s3://")
        self.path_info = self.path_cls(url)

-        self.region = config.get(Config.SECTION_AWS_REGION)
+        self.region = config.get("region")
+        self.profile = config.get("profile")
+        self.endpoint_url = config.get("endpointurl")

-        self.profile = config.get(Config.SECTION_AWS_PROFILE)
-
-        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
-
-        if config.get(Config.SECTION_AWS_LIST_OBJECTS):
+        if config.get("listobjects"):
            self.list_objects_api = "list_objects"
        else:
            self.list_objects_api = "list_objects_v2"

-        self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
+        self.use_ssl = config.get("use_ssl", True)

        self.extra_args = {}

-        self.sse = config.get(Config.SECTION_AWS_SSE, "")
+        self.sse = config.get("sse")
        if self.sse:
            self.extra_args["ServerSideEncryption"] = self.sse

-        self.acl = config.get(Config.SECTION_AWS_ACL, "")
+        self.sse_kms_key_id = config.get("sse_kms_key_id")
+        if self.sse_kms_key_id:
+            self.extra_args["SSEKMSKeyId"] = self.sse_kms_key_id
+
+        self.acl = config.get("acl")
        if self.acl:
            self.extra_args["ACL"] = self.acl

        self._append_aws_grants_to_extra_args(config)

-        shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
+        shared_creds = config.get("credentialpath")
        if shared_creds:
            os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)

@@ -86,9 +86,7 @@ class RemoteS3(RemoteBASE):
        try:
            obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
        except Exception as exc:
-            raise DvcException(
-                "s3://{}/{} does not exist".format(bucket, path)
-            ) from exc
+            raise DvcException(f"s3://{bucket}/{path} does not exist") from exc
        return obj

    @classmethod
@@ -111,7 +109,7 @@ class RemoteS3(RemoteBASE):
            if lastbyte > size:
                lastbyte = size - 1

-            srange = "bytes={}-{}".format(byte_position, lastbyte)
+            srange = f"bytes={byte_position}-{lastbyte}"

            part = s3.upload_part_copy(
                Bucket=to_info.bucket,
@@ -189,27 +187,44 @@ class RemoteS3(RemoteBASE):
        if path_info.scheme != "s3":
            raise NotImplementedError

-        logger.debug("Removing {}".format(path_info))
+        logger.debug(f"Removing {path_info}")
        self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)

-    def _list_objects(self, path_info, max_items=None):
+    def _list_objects(
+        self, path_info, max_items=None, prefix=None, progress_callback=None
+    ):
        """ Read config for list object api, paginate through list objects."""
        kwargs = {
            "Bucket": path_info.bucket,
            "Prefix": path_info.path,
            "PaginationConfig": {"MaxItems": max_items},
        }
+        if prefix:
+            kwargs["Prefix"] = posixpath.join(path_info.path, prefix[:2])
        paginator = self.s3.get_paginator(self.list_objects_api)
        for page in paginator.paginate(**kwargs):
-            yield from page.get("Contents", ())
+            contents = page.get("Contents", ())
+            if progress_callback:
+                for item in contents:
+                    progress_callback()
+                    yield item
+            else:
+                yield from contents

-    def _list_paths(self, path_info, max_items=None):
+    def _list_paths(
+        self, path_info, max_items=None, prefix=None, progress_callback=None
+    ):
        return (
-            item["Key"] for item in self._list_objects(path_info, max_items)
+            item["Key"]
+            for item in self._list_objects(
+                path_info, max_items, prefix, progress_callback
+            )
        )

-    def list_cache_paths(self):
-        return self._list_paths(self.path_info)
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        return self._list_paths(
+            self.path_info, prefix=prefix, progress_callback=progress_callback
+        )

    def isfile(self, path_info):
        from botocore.exceptions import ClientError
@@ -240,6 +255,9 @@ class RemoteS3(RemoteBASE):
        #
        # We are not creating directory objects for every parent prefix,
        # as it is not required.
+        if not path_info.path:
+            return
+
        dir_path = path_info / ""
        self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")

@@ -322,14 +340,14 @@ class RemoteS3(RemoteBASE):
        """

        grants = {
-            Config.SECTION_AWS_GRANT_FULL_CONTROL: "GrantFullControl",
-            Config.SECTION_AWS_GRANT_READ: "GrantRead",
-            Config.SECTION_AWS_GRANT_READ_ACP: "GrantReadACP",
-            Config.SECTION_AWS_GRANT_WRITE_ACP: "GrantWriteACP",
+            "grant_full_control": "GrantFullControl",
+            "grant_read": "GrantRead",
+            "grant_read_acp": "GrantReadACP",
+            "grant_write_acp": "GrantWriteACP",
        }

        for grant_option, extra_args_key in grants.items():
-            if config.get(grant_option, ""):
+            if config.get(grant_option):
                if self.acl:
                    raise ConfigError(
                        "`acl` and `grant_*` AWS S3 config options "
@@ -5,9 +5,6 @@ from functools import wraps

 import colorama

-from dvc.config import Config
-
-
 logger = logging.getLogger(__name__)
 this = sys.modules[__name__]

@@ -31,11 +28,9 @@ def slow_link_guard(f):
        if this.already_displayed:
            return f(remote, *args, **kwargs)

-        config = remote.repo.config.config.get(Config.SECTION_CACHE, {})
-        cache_type = config.get(Config.SECTION_CACHE_TYPE)
-        should_warn = config.get(Config.SECTION_CACHE_SLOW_LINK_WARNING, True)
-
-        if not should_warn or cache_type:
+        cache_conf = remote.repo.config["cache"]
+        slow_link_warning = cache_conf.get("slow_link_warning", True)
+        if not slow_link_warning or cache_conf.get("type"):
            return f(remote, *args, **kwargs)

        start = time.time()
@@ -4,17 +4,17 @@ import io
 import itertools
 import logging
 import os
+import posixpath
 import threading
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import closing, contextmanager
 from urllib.parse import urlparse

-from funcy import memoize, wrap_with
+from funcy import first, memoize, silent, wrap_with

 import dvc.prompt as prompt
-from dvc.config import Config
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
+from dvc.remote.base import BaseRemote
 from dvc.remote.pool import get_connection
 from dvc.scheme import Schemes
 from dvc.utils import to_chunks
@@ -33,7 +33,7 @@ def ask_password(host, user, port):
    )


-class RemoteSSH(RemoteBASE):
+class SSHRemote(BaseRemote):
    scheme = Schemes.SSH
    REQUIRES = {"paramiko": "paramiko"}

@@ -45,25 +45,26 @@ class RemoteSSH(RemoteBASE):
    # paramiko stuff, so we would ideally have it double of server processors.
    # We use conservative setting of 4 instead to not exhaust max sessions.
    CHECKSUM_JOBS = 4
+    TRAVERSE_PREFIX_LEN = 2

    DEFAULT_CACHE_TYPES = ["copy"]

    def __init__(self, repo, config):
        super().__init__(repo, config)
-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
        if url:
            parsed = urlparse(url)
            user_ssh_config = self._load_user_ssh_config(parsed.hostname)

            host = user_ssh_config.get("hostname", parsed.hostname)
            user = (
-                config.get(Config.SECTION_REMOTE_USER)
+                config.get("user")
                or parsed.username
                or user_ssh_config.get("user")
                or getpass.getuser()
            )
            port = (
-                config.get(Config.SECTION_REMOTE_PORT)
+                config.get("port")
                or parsed.port
                or self._try_get_ssh_config_port(user_ssh_config)
                or self.DEFAULT_PORT
@@ -80,14 +81,19 @@ class RemoteSSH(RemoteBASE):
            user_ssh_config = {}

        self.keyfile = config.get(
-            Config.SECTION_REMOTE_KEY_FILE
+            "keyfile"
        ) or self._try_get_ssh_config_keyfile(user_ssh_config)
-        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
-        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
-        self.ask_password = config.get(
-            Config.SECTION_REMOTE_ASK_PASSWORD, False
-        )
-        self.gss_auth = config.get(Config.SECTION_REMOTE_GSS_AUTH, False)
+        self.timeout = config.get("timeout", self.TIMEOUT)
+        self.password = config.get("password", None)
+        self.ask_password = config.get("ask_password", False)
+        self.gss_auth = config.get("gss_auth", False)
+        proxy_command = user_ssh_config.get("proxycommand", False)
+        if proxy_command:
+            import paramiko
+
+            self.sock = paramiko.ProxyCommand(proxy_command)
+        else:
+            self.sock = None

    @staticmethod
    def ssh_config_filename():
@@ -97,7 +103,7 @@ class RemoteSSH(RemoteBASE):
    def _load_user_ssh_config(hostname):
        import paramiko

-        user_config_file = RemoteSSH.ssh_config_filename()
+        user_config_file = SSHRemote.ssh_config_filename()
        user_ssh_config = {}
        if hostname and os.path.exists(user_config_file):
            ssh_config = paramiko.SSHConfig()
@@ -110,17 +116,11 @@ class RemoteSSH(RemoteBASE):

    @staticmethod
    def _try_get_ssh_config_port(user_ssh_config):
-        try:
-            return int(user_ssh_config.get("port"))
-        except (ValueError, TypeError):
-            return None
+        return silent(int)(user_ssh_config.get("port"))

    @staticmethod
    def _try_get_ssh_config_keyfile(user_ssh_config):
-        identity_file = user_ssh_config.get("identityfile")
-        if identity_file and len(identity_file) > 0:
-            return identity_file[0]
-        return None
+        return first(user_ssh_config.get("identityfile") or ())

    def ensure_credentials(self, path_info=None):
        if path_info is None:
@@ -145,6 +145,7 @@ class RemoteSSH(RemoteBASE):
            timeout=self.timeout,
            password=self.password,
            gss_auth=self.gss_auth,
+            sock=self.sock,
        )

    def exists(self, path_info):
@@ -218,7 +219,8 @@ class RemoteSSH(RemoteBASE):
        with self.ssh(path_info) as ssh:
            ssh.remove(path_info.path)

-    def move(self, from_info, to_info):
+    def move(self, from_info, to_info, mode=None):
+        assert mode is None
        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
            raise NotImplementedError

@@ -257,10 +259,19 @@ class RemoteSSH(RemoteBASE):
            else:
                yield io.TextIOWrapper(fd, encoding=encoding)

-    def list_cache_paths(self):
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            root = posixpath.join(self.path_info.path, prefix[:2])
+        else:
+            root = self.path_info.path
        with self.ssh(self.path_info) as ssh:
            # If we simply return an iterator then with above closes instantly
-            yield from ssh.walk_files(self.path_info.path)
+            if progress_callback:
+                for path in ssh.walk_files(root):
+                    progress_callback()
+                    yield path
+            else:
+                yield from ssh.walk_files(root)

    def walk_files(self, path_info):
        with self.ssh(path_info) as ssh:
@@ -279,7 +290,7 @@ class RemoteSSH(RemoteBASE):
                try:
                    channel.stat(path)
                    ret.append(True)
-                except IOError as exc:
+                except OSError as exc:
                    if exc.errno != errno.ENOENT:
                        raise
                    ret.append(False)
@@ -305,7 +316,7 @@ class RemoteSSH(RemoteBASE):
        faster than current approach (relying on exists(path_info)) applied in
        remote/base.
        """
-        if not self.no_traverse:
+        if not self.CAN_TRAVERSE:
            return list(set(checksums) & set(self.all()))

        # possibly prompt for credentials before "Querying" progress output
@@ -319,7 +330,7 @@ class RemoteSSH(RemoteBASE):
        ) as pbar:

            def exists_with_progress(chunks):
-                return self.batch_exists(chunks, callback=pbar.update_desc)
+                return self.batch_exists(chunks, callback=pbar.update_msg)

            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
                path_infos = [self.checksum_to_path_info(x) for x in checksums]
--- a/Show More
+++ b/Show More