dvc: bump to 0.90.2

pipeline: fix --outs output (#3507 )
Fixes #3497
2026-11-02 07:18:25 +01:00 · 2020-03-18 23:11:17 +02:00 · 2020-03-18 23:03:41 +02:00 · 2020-03-18 21:34:46 +02:00 · 2020-03-18 19:20:22 +02:00 · 2020-03-18 19:18:58 +02:00
155 changed files with 4964 additions and 2420 deletions
@@ -1,7 +1,7 @@
-* [ ] ❗ Have you followed the guidelines in the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) list?
+* [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.

-* [ ] 📖 Check this box if this PR **does not** require [documentation](https://dvc.org/doc) updates, or if it does **and** you have created a separate PR in [dvc.org](https://github.com/iterative/dvc.org) with such updates (or at least opened an issue about it in that repo). Please link below to your PR (or issue) in the [dvc.org](https://github.com/iterative/dvc.org) repo.
+* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here. If the CLI API is changed, I have updated [tab completion scripts](https://github.com/iterative/dvc/tree/master/scripts/completion).

-* [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect everything to be addressed. Please review them carefully and fix those that actually improve code or fix bugs.
+* [ ] ❌ I will check DeepSource, CodeClimate, and other sanity checks below. (We consider them recommendatory and don't expect everything to be addressed. Please fix things that actually improve code or fix bugs.)

 Thank you for the contribution - we'll try to review it as soon as possible. 🙏
@@ -26,7 +26,7 @@ innosetup/config.ini
 .coverage
 .coverage.*

-*.swp
+*.sw?

 pip-wheel-metadata/
 .vscode/
@@ -1,17 +1,35 @@
 repos:
-   repo: https://github.com/ambv/black
-    rev: 19.3b0
-    hooks:
-    - id: black
-      language_version: python3
-   repo: https://gitlab.com/pycqa/flake8
-    rev: master
-    hooks:
-    - id: flake8
-      language_version: python3
-   repo: https://github.com/lovesegfault/beautysh
-    rev: master
-    hooks:
-    - id: beautysh
-      language_version: python3
-      args: [-i, '2']  # 2-space indentaion
+- hooks:
+  - id: black
+    language_version: python3
+  repo: https://github.com/ambv/black
+  rev: 19.10b0
+- hooks:
+  - id: flake8
+    language_version: python3
+  repo: https://gitlab.com/pycqa/flake8
+  rev: master
+- hooks:
+  - args:
+    - -i
+    - '2'
+    id: beautysh
+    language_version: python3
+  repo: https://github.com/lovesegfault/beautysh
+  rev: master
+- hooks:
+  - id: dvc-pre-commit
+    language_version: python3
+    stages:
+    - commit
+  - id: dvc-pre-push
+    language_version: python3
+    stages:
+    - push
+  - always_run: true
+    id: dvc-post-checkout
+    language_version: python3
+    stages:
+    - post-checkout
+  repo: https://github.com/iterative/dvc
+  rev: master
@@ -0,0 +1,32 @@
+- args:
+  - git-hook
+  - pre-commit
+  entry: dvc
+  id: dvc-pre-commit
+  language: python
+  language_version: python3
+  name: DVC pre-commit
+  stages:
+  - commit
+- args:
+  - git-hook
+  - pre-push
+  entry: dvc
+  id: dvc-pre-push
+  language: python
+  language_version: python3
+  name: DVC pre-push
+  stages:
+  - push
+- always_run: true
+  args:
+  - git-hook
+  - post-checkout
+  entry: dvc
+  id: dvc-post-checkout
+  language: python
+  language_version: python3
+  minimum_pre_commit_version: 2.2.0
+  name: DVC post-checkout
+  stages:
+  - post-checkout
@@ -1,7 +1,7 @@
 ---
 restylers:
  - name: black
-    image: restyled/restyler-black:v19.3b0
+    image: restyled/restyler-black:v19.10b0
    command:
    - black
    arguments: []
@@ -37,6 +37,15 @@ jobs:
    stage: test
    os: windows           # Windows 10.0.17134 N/A Build 17134
    language: shell       # 'language: python' is an error on Travis CI Windows
+    env:
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
+  - name: "3.8 on Windows"
+    os: windows           # Windows 10.0.17134 N/A Build 17134
+    language: shell       # 'language: python' is an error on Travis CI Windows
+    env:
+      - PYTHON_VERSION=3.8.0
+      - PATH=/c/Python38:/c/Python38/Scripts:$PATH
  - name: "3.5 on Linux"
    language: python
    python: 3.5
@@ -46,6 +55,9 @@ jobs:
  - name: "3.7 on Linux"
    language: python
    python: 3.7
+  - name: "3.8 on Linux"
+    language: python
+    python: 3.8
  - name: "3.7 on Mac"
    os: osx
    osx_image: xcode11.3
@@ -63,6 +75,9 @@ jobs:
    language: shell
    install:
    script: ./scripts/build_windows.cmd
+    env:
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
  - name: "Linux pkgs"
    language: python
    python: 3.7
@@ -71,16 +86,18 @@ jobs:
    script: ./scripts/build_posix.sh
  - name: "PyPI pkgs"
    language: python
-    python: 3.7
+    python: 3.8
    script: ./scripts/build_package.sh
  - name: Snapcraft snap
    language: python
-    python: 3.7
+    python: 3.8
    addons:
     snaps:
     - name: snapcraft
       channel: stable
       confinement: classic
+     - name: review-tools
+       channel: stable
     - name: lxd
       channel: stable
    env:
@@ -0,0 +1,8 @@
+{
+  "title": "DVC: Data Version Control - Git for Data & Models",
+  "keywords": [
+    "data-science", "data-version-control", "machine-learning", "git",
+    "developer-tools", "reproducibility", "collaboration", "ai", "python"],
+  "contributors": [
+    {"name": "DVC team", "type": "Other", "affiliation": "Iterative"}]
+}
@@ -1,6 +1,4 @@
-.. image:: https://dvc.org/static/img/logo-github-readme.png
-  :target: https://dvc.org
-  :alt: DVC logo
+|Banner|

 `Website <https://dvc.org>`_
 • `Docs <https://dvc.org/doc>`_
@@ -10,33 +8,9 @@
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_

-.. image:: https://img.shields.io/badge/release-ok-brightgreen
-  :target: https://travis-ci.com/iterative/dvc
-  :alt: Release
+|Release| |CI| |Maintainability| |Coverage| |Donate| |DOI|

-.. image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev
-  :target: https://travis-ci.com/iterative/dvc
-  :alt: Travis dev branch
-
-.. image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
-  :target: https://codeclimate.com/github/iterative/dvc
-  :alt: Code Climate
-
-.. image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
-  :target: https://codecov.io/gh/iterative/dvc
-  :alt: Codecov
-
-.. image:: https://img.shields.io/badge/patreon-donate-green.svg
-  :target: https://www.patreon.com/DVCorg/overview
-  :alt: Donate
-
-.. image:: https://anaconda.org/conda-forge/dvc/badges/version.svg
-  :target: https://anaconda.org/conda-forge/dvc
-  :alt: Conda-forge
-
-.. image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
-  :target: https://snapcraft.io/dvc
-  :alt: Snapcraft
+|Snap| |Choco| |Brew| |Conda| |PyPI| |Packages|

 |

@@ -79,9 +53,7 @@ to store data and model files seamlessly out of Git, while preserving almost the
 were stored in Git itself. To store and share the data cache, DVC supports multiple remotes - any cloud (S3, Azure,
 Google Cloud, etc) or any on-premise network storage (via SSH, for example).

-.. image:: https://dvc.org/static/img/flow.gif
-   :target: https://dvc.org/static/img/flow.gif
-   :alt: how_dvc_works
+|Flowchart|

 The DVC pipelines (computational graph) feature connects code and data together. It is possible to explicitly
 specify all steps required to produce a model: input dependencies including data, commands to run,
@@ -119,9 +91,52 @@ Installation
 There are four options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package.
 Full instructions are `available here <https://dvc.org/doc/get-started/install>`_.

+Snap (Snapcraft/Linux)
+----------------------
+
+|Snap|
+
+.. code-block:: bash
+
+   snap install dvc --classic
+
+This corresponds to the latest tagged release.
+Add ``--edge`` for the latest ``master`` version.
+
+Choco (Chocolatey/Windows)
+--------------------------
+
+|Choco|
+
+.. code-block:: bash
+
+   choco install dvc
+
+Brew (Homebrew/Mac OS)
+----------------------
+
+|Brew|
+
+.. code-block:: bash
+
+   brew install dvc
+
+Conda (Anaconda)
+----------------
+
+|Conda|
+
+.. code-block:: bash
+
+   conda install -c conda-forge dvc
+
+Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
+
 pip (PyPI)
 ----------

+|PyPI|
+
 .. code-block:: bash

   pip install dvc
@@ -137,36 +152,11 @@ To install the development version, run:

   pip install git+git://github.com/iterative/dvc

-Homebrew
--------
-
-.. code-block:: bash
-
-   brew install dvc
-
-
-Conda (Anaconda)
----------------
-
-.. code-block:: bash
-
-   conda install -c conda-forge dvc
-
-Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
-
-Snap (Snapcraft)
----------------
-
-.. code-block:: bash
-
-   snap install dvc --classic
-
-This corresponds to the latest tagged release.
-Add ``--edge`` for the latest ``master`` version.
-
 Package
 -------

+|Packages|
+
 Self-contained packages for Linux, Windows, and Mac are available. The latest version of the packages
 can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.

@@ -206,40 +196,43 @@ Comparison to related technologies

 Contributing
 ============
+
+|Maintainability| |Donate|
+
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
-  :alt: 0
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+   :alt: 0

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
-  :alt: 1
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+   :alt: 1

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
-  :alt: 2
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+   :alt: 2

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
-  :alt: 3
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
+   :alt: 3

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
-  :alt: 4
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+   :alt: 4

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
-  :alt: 5
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+   :alt: 5

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
-  :alt: 6
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+   :alt: 6

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
-  :alt: 7
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+   :alt: 7

 Mailing List
 ============
@@ -253,3 +246,67 @@ This project is distributed under the Apache license version 2.0 (see the LICENS

 By submitting a pull request to this project, you agree to license your contribution under the Apache license version
 2.0 to this project.
+
+Citation
+========
+
+|DOI|
+
+Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
+`DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
+
+.. |Banner| image:: https://dvc.org/static/img/logo-github-readme.png
+   :target: https://dvc.org
+   :alt: DVC logo
+
+.. |Release| image:: https://img.shields.io/badge/release-ok-brightgreen
+   :target: https://travis-ci.com/iterative/dvc/branches
+   :alt: Release
+
+.. |CI| image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev&logo=travis
+   :target: https://travis-ci.com/iterative/dvc/builds
+   :alt: Travis dev branch
+
+.. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
+   :target: https://codeclimate.com/github/iterative/dvc
+   :alt: Code Climate
+
+.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/iterative/dvc
+   :alt: Codecov
+
+.. |Donate| image:: https://img.shields.io/badge/patreon-donate-green.svg?logo=patreon
+   :target: https://www.patreon.com/DVCorg/overview
+   :alt: Donate
+
+.. |Snap| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
+   :target: https://snapcraft.io/dvc
+   :alt: Snapcraft
+
+.. |Choco| image:: https://img.shields.io/chocolatey/v/dvc?label=choco
+   :target: https://chocolatey.org/packages/dvc
+   :alt: Chocolatey
+
+.. |Brew| image:: https://img.shields.io/homebrew/v/dvc?label=brew
+   :target: https://formulae.brew.sh/formula/dvc
+   :alt: Homebrew
+
+.. |Conda| image:: https://img.shields.io/conda/v/conda-forge/dvc.svg?label=conda&logo=conda-forge
+   :target: https://anaconda.org/conda-forge/dvc
+   :alt: Conda-forge
+
+.. |PyPI| image:: https://img.shields.io/pypi/v/dvc.svg?label=pip&logo=PyPI&logoColor=white
+   :target: https://pypi.org/project/dvc
+   :alt: PyPI
+
+.. |Packages| image:: https://img.shields.io/github/v/release/iterative/dvc?label=deb|pkg|rpm|exe&logo=GitHub
+   :target: https://github.com/iterative/dvc/releases/latest
+   :alt: deb|pkg|rpm|exe
+
+.. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.3677553-blue.svg
+   :target: https://doi.org/10.5281/zenodo.3677553
+   :alt: DOI
+
+.. |Flowchart| image:: https://dvc.org/static/img/flow.gif
+   :target: https://dvc.org/static/img/flow.gif
+   :alt: how_dvc_works
@@ -15,7 +15,8 @@ from dvc.daemon import daemon
 from dvc.exceptions import NotDvcRepoError
 from dvc.lock import Lock, LockError
 from dvc.repo import Repo
-from dvc.scm import SCM
+from dvc.scm import SCM, NoSCM
+from dvc.scm.base import SCMError
 from dvc.utils import env2bool, is_binary
 from dvc.utils.fs import makedirs

@@ -54,11 +55,8 @@ def is_enabled():
        return False

    enabled = to_bool(
-        Config(validate=False)
-        .config.get(Config.SECTION_CORE, {})
-        .get(Config.SECTION_CORE_ANALYTICS, "true")
+        Config(validate=False).get("core", {}).get("analytics", "true")
    )
-
    logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))

    return enabled
@@ -76,7 +74,10 @@ def send(report):
    headers = {"content-type": "application/json"}

    with open(report, "rb") as fobj:
-        requests.post(url, data=fobj, headers=headers, timeout=5)
+        try:
+            requests.post(url, data=fobj, headers=headers, timeout=5)
+        except requests.exceptions.RequestException:
+            logger.debug("failed to send analytics report", exc_info=True)

    os.remove(report)

@@ -85,6 +86,8 @@ def _scm_in_use():
    try:
        scm = SCM(root_dir=Repo.find_root())
        return type(scm).__name__
+    except SCMError:
+        return NoSCM.__name__
    except NotDvcRepoError:
        pass

@@ -141,7 +144,7 @@ def _find_or_create_user_id():

    IDs are generated randomly with UUID.
    """
-    config_dir = Config.get_global_config_dir()
+    config_dir = Config.get_dir("global")
    fname = os.path.join(config_dir, "user_id")
    lockfile = os.path.join(config_dir, "user_id.lock")

@@ -7,11 +7,7 @@ from dvc.external_repo import external_repo


 class UrlNotDvcRepoError(DvcException):
-    """Thrown if given url is not a DVC repository.
-
-    Args:
-        url (str): URL to the repository
-    """
+    """Thrown if the given URL is not a DVC repository."""

    def __init__(self, url):
        super().__init__("'{}' is not a DVC repository.".format(url))
@@ -19,19 +15,36 @@ class UrlNotDvcRepoError(DvcException):

 def get_url(path, repo=None, rev=None, remote=None):
    """
-    Returns the full URL to the data artifact specified by its `path` in a
-    `repo`.
-    NOTE: There is no guarantee that the file actually exists in that location.
+    Returns the URL to the storage location of a data file or directory tracked
+    in a DVC repo. For Git repos, HEAD is used unless a rev argument is
+    supplied. The default remote is tried unless a remote argument is supplied.
+
+    Raises UrlNotDvcRepoError if repo is not a DVC project.
+
+    NOTE: This function does not check for the actual existence of the file or
+    directory in the remote storage.
    """
    with _make_repo(repo, rev=rev) as _repo:
-        _require_dvc(_repo)
+        if not isinstance(_repo, Repo):
+            raise UrlNotDvcRepoError(_repo.url)
        out = _repo.find_out_by_relpath(path)
        remote_obj = _repo.cloud.get_remote(remote)
        return str(remote_obj.checksum_to_path_info(out.checksum))


 def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Context manager to open a file artifact as a file object."""
+    """
+    Open file in the supplied path tracked in a repo (both DVC projects and
+    plain Git repos are supported). For Git repos, HEAD is used unless a rev
+    argument is supplied. The default remote is tried unless a remote argument
+    is supplied. It may only be used as a context manager:
+
+        with dvc.api.open(
+                'path/to/file',
+                repo='https://example.com/url/to/repo'
+                ) as fd:
+            # ... Handle file object fd
+    """
    args = (path,)
    kwargs = {
        "repo": repo,
@@ -63,7 +76,11 @@ def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):


 def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Returns the contents of a file artifact."""
+    """
+    Returns the contents of a tracked file (by DVC or Git). For Git repos, HEAD
+    is used unless a rev argument is supplied. The default remote is tried
+    unless a remote argument is supplied.
+    """
    with open(
        path, repo=repo, rev=rev, remote=remote, mode=mode, encoding=encoding
    ) as fd:
@@ -81,8 +98,3 @@ def _make_repo(repo_url=None, rev=None):
            pass  # fallthrough to external_repo
    with external_repo(url=repo_url, rev=rev) as repo:
        yield repo
-
-
-def _require_dvc(repo):
-    if not isinstance(repo, Repo):
-        raise UrlNotDvcRepoError(repo.url)
@@ -1,25 +1,8 @@
 """Manages cache of a DVC repo."""
-import os
 from collections import defaultdict

 from funcy import cached_property

-from dvc.config import Config
-
-
-class CacheConfig(object):
-    def __init__(self, config):
-        self.config = config
-
-    def set_dir(self, dname, level=None):
-        from dvc.remote.config import RemoteConfig
-
-        configobj = self.config.get_configobj(level)
-        path = RemoteConfig.resolve_path(dname, configobj.filename)
-        self.config.set(
-            Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level
-        )
-

 def _make_remote_property(name):
    """
@@ -70,37 +53,22 @@ class Cache(object):
        from dvc.remote import Remote

        self.repo = repo
+        self.config = config = repo.config["cache"]

-        self.config = config = repo.config.config[Config.SECTION_CACHE]
-        local = config.get(Config.SECTION_CACHE_LOCAL)
+        local = config.get("local")

        if local:
-            name = Config.SECTION_REMOTE_FMT.format(local)
-            settings = repo.config.config[name]
+            settings = {"name": local}
        else:
-            default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
-            cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
-            cache_type = config.get(Config.SECTION_CACHE_TYPE)
-            protected = config.get(Config.SECTION_CACHE_PROTECTED)
-            shared = config.get(Config.SECTION_CACHE_SHARED)
-
-            settings = {
-                Config.PRIVATE_CWD: config.get(
-                    Config.PRIVATE_CWD, repo.dvc_dir
-                ),
-                Config.SECTION_REMOTE_URL: cache_dir,
-                Config.SECTION_CACHE_TYPE: cache_type,
-                Config.SECTION_CACHE_PROTECTED: protected,
-                Config.SECTION_CACHE_SHARED: shared,
-            }
+            settings = {**config, "url": config["dir"]}

        self.local = Remote(repo, **settings)

-    s3 = _make_remote_property(Config.SECTION_CACHE_S3)
-    gs = _make_remote_property(Config.SECTION_CACHE_GS)
-    ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
-    hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
-    azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
+    s3 = _make_remote_property("s3")
+    gs = _make_remote_property("gs")
+    ssh = _make_remote_property("ssh")
+    hdfs = _make_remote_property("hdfs")
+    azure = _make_remote_property("azure")


 class NamedCache(object):
@@ -20,6 +20,7 @@ from .command import (
    imp_url,
    init,
    install,
+    ls,
    lock,
    metrics,
    move,
@@ -33,6 +34,7 @@ from .command import (
    unprotect,
    update,
    version,
+    git_hook,
 )
 from .command.base import fix_subparsers
 from .exceptions import DvcParserError
@@ -62,6 +64,7 @@ COMMANDS = [
    metrics,
    install,
    root,
+    ls,
    lock,
    pipeline,
    daemon,
@@ -70,6 +73,7 @@ COMMANDS = [
    diff,
    version,
    update,
+    git_hook,
 ]


@@ -1,11 +1,8 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.exceptions import DvcException
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
-
+from dvc.command.base import append_doc_link, CmdBase
+from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename

 logger = logging.getLogger(__name__)

@@ -35,7 +35,7 @@ class CmdBase(object):
        self.repo = Repo()
        self.config = self.repo.config
        self.args = args
-        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        hardlink_lock = self.config["core"].get("hardlink_lock", False)
        updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
        updater.check()

@@ -1,18 +1,13 @@
 import argparse

-from dvc.cache import CacheConfig
-from dvc.command.base import append_doc_link
-from dvc.command.base import fix_subparsers
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig


 class CmdCacheDir(CmdConfig):
-    def __init__(self, args):
-        super().__init__(args)
-        self.cache_config = CacheConfig(self.config)
-
    def run(self):
-        self.cache_config.set_dir(self.args.value, level=self.args.level)
+        with self.config.edit(level=self.args.level) as edit:
+            edit["cache"]["dir"] = self.args.value
        return 0


@@ -1,18 +1,86 @@
 import argparse
+import logging
+
+import colorama

 from dvc.command.base import append_doc_link
 from dvc.command.base import CmdBase
+from dvc.exceptions import CheckoutError
+
+logger = logging.getLogger(__name__)
+
+
+def _human_join(words):
+    words = list(words)
+    if not words:
+        return ""
+
+    return (
+        "{before} and {after}".format(
+            before=", ".join(words[:-1]), after=words[-1],
+        )
+        if len(words) > 1
+        else words[0]
+    )
+
+
+def log_summary(stats):
+    states = ("added", "deleted", "modified")
+    summary = (
+        "{} {}".format(len(stats[state]), state)
+        for state in states
+        if stats.get(state)
+    )
+    logger.info(_human_join(summary))
+
+
+def log_changes(stats):
+    colors = [
+        ("modified", colorama.Fore.YELLOW,),
+        ("added", colorama.Fore.GREEN),
+        ("deleted", colorama.Fore.RED,),
+    ]
+
+    for state, color in colors:
+        entries = stats.get(state)
+
+        if not entries:
+            continue
+
+        for entry in entries:
+            logger.info(
+                "{color}{state}{nc}{spacing}{entry}".format(
+                    color=color,
+                    state=state[0].upper(),
+                    nc=colorama.Fore.RESET,
+                    spacing="\t",
+                    entry=entry,
+                )
+            )


 class CmdCheckout(CmdBase):
    def run(self):
-        self.repo.checkout(
-            targets=self.args.targets,
-            with_deps=self.args.with_deps,
-            force=self.args.force,
-            relink=self.args.relink,
-            recursive=self.args.recursive,
-        )
+        stats, exc = None, None
+        try:
+            stats = self.repo.checkout(
+                targets=self.args.targets,
+                with_deps=self.args.with_deps,
+                force=self.args.force,
+                relink=self.args.relink,
+                recursive=self.args.recursive,
+            )
+        except CheckoutError as _exc:
+            exc = _exc
+            stats = exc.stats
+
+        if self.args.summary:
+            log_summary(stats)
+        else:
+            log_changes(stats)
+
+        if exc:
+            raise exc
        return 0


@@ -26,6 +94,12 @@ def add_parser(subparsers, parent_parser):
        help=CHECKOUT_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
+    checkout_parser.add_argument(
+        "--summary",
+        action="store_true",
+        default=False,
+        help="Show summary of the changes.",
+    )
    checkout_parser.add_argument(
        "-d",
        "--with-deps",
@@ -1,10 +1,8 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
-from dvc.config import Config
-
+from dvc.command.base import append_doc_link, CmdBaseNoRepo
+from dvc.config import Config, ConfigError

 logger = logging.getLogger(__name__)

@@ -18,17 +16,21 @@ class CmdConfig(CmdBaseNoRepo):
    def run(self):
        section, opt = self.args.name.lower().strip().split(".", 1)

-        if self.args.unset:
-            self.config.unset(section, opt, level=self.args.level)
-        elif self.args.value is None:
-            logger.info(self.config.get(section, opt, level=self.args.level))
-        else:
-            self.config.set(
-                section, opt, self.args.value, level=self.args.level
-            )
+        if self.args.value is None and not self.args.unset:
+            conf = self.config.load_one(self.args.level)
+            self._check(conf, section, opt)
+            logger.info(conf[section][opt])
+            return 0

-        is_write = self.args.unset or self.args.value is not None
-        if is_write and self.args.name == "cache.type":
+        with self.config.edit(self.args.level) as conf:
+            if self.args.unset:
+                self._check(conf, section, opt)
+                del conf[section][opt]
+            else:
+                self._check(conf, section)
+                conf[section][opt] = self.args.value
+
+        if self.args.name == "cache.type":
            logger.warning(
                "You have changed the 'cache.type' option. This doesn't update"
                " any existing workspace file links, but it can be done with:"
@@ -37,30 +39,39 @@ class CmdConfig(CmdBaseNoRepo):

        return 0

+    def _check(self, conf, section, opt=None):
+        if section not in conf:
+            msg = "section {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+
+        if opt and opt not in conf[section]:
+            msg = "option {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+

 parent_config_parser = argparse.ArgumentParser(add_help=False)
 parent_config_parser.add_argument(
    "--global",
    dest="level",
    action="store_const",
-    const=Config.LEVEL_GLOBAL,
+    const="global",
    help="Use global config.",
 )
 parent_config_parser.add_argument(
    "--system",
    dest="level",
    action="store_const",
-    const=Config.LEVEL_SYSTEM,
+    const="system",
    help="Use system config.",
 )
 parent_config_parser.add_argument(
    "--local",
    dest="level",
    action="store_const",
-    const=Config.LEVEL_LOCAL,
+    const="local",
    help="Use local config.",
 )
-parent_config_parser.set_defaults(level=Config.LEVEL_REPO)
+parent_config_parser.set_defaults(level="repo")


 def add_parser(subparsers, parent_parser):
@@ -16,9 +16,7 @@ class CmdDaemonUpdater(CmdDaemonBase):
        root_dir = Repo.find_root()
        dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
        config = Config(dvc_dir, validate=False)
-        hardlink_lock = config.config.get("core", {}).get(
-            "hardlink_lock", False
-        )
+        hardlink_lock = config.get("core", {}).get("hardlink_lock", False)
        updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
        updater.fetch(detach=False)

@@ -3,7 +3,8 @@ import logging

 from dvc.command.base import append_doc_link
 from dvc.command.base import CmdBase
-from dvc.exceptions import DvcException
+from dvc.command.checkout import log_summary
+from dvc.exceptions import DvcException, CheckoutError


 logger = logging.getLogger(__name__)
@@ -19,7 +20,7 @@ class CmdDataBase(CmdBase):
 class CmdDataPull(CmdDataBase):
    def run(self):
        try:
-            processed_files_count = self.repo.pull(
+            stats = self.repo.pull(
                targets=self.args.targets,
                jobs=self.args.jobs,
                remote=self.args.remote,
@@ -29,10 +30,13 @@ class CmdDataPull(CmdDataBase):
                force=self.args.force,
                recursive=self.args.recursive,
            )
-        except DvcException:
+            log_summary(stats)
+        except (CheckoutError, DvcException) as exc:
+            log_summary(getattr(exc, "stats", {}))
            logger.exception("failed to pull data from the cloud")
            return 1
-        self.check_up_to_date(processed_files_count)
+
+        self.check_up_to_date(stats["downloaded"])
        return 0


@@ -148,7 +152,8 @@ def add_parser(subparsers, _parent_parser):
    pull_parser.set_defaults(func=CmdDataPull)

    # Push
-    PUSH_HELP = "Push data files to a DVC remote storage."
+    PUSH_HELP = "Uploads files or directories tracked by DVC"
+    " to remote storage."

    push_parser = subparsers.add_parser(
        "push",
@@ -191,7 +196,8 @@ def add_parser(subparsers, _parent_parser):
    push_parser.set_defaults(func=CmdDataPush)

    # Fetch
-    FETCH_HELP = "Fetch data files from a DVC remote storage."
+    FETCH_HELP = "Get files or directories tracked by DVC from remote storage"
+    " into the cache."

    fetch_parser = subparsers.add_parser(
        "fetch",
@@ -28,7 +28,7 @@ class CmdDiff(CmdBase):
                dir/
                dir/1

-        An example of a diff formatted when entries contain checksums:
+        An example of a diff formatted when entries contain hash:

            Added:
                d3b07384 foo
@@ -66,7 +66,7 @@ class CmdDiff(CmdBase):

            for entry in entries:
                path = entry["path"]
-                checksum = entry.get("checksum")
+                checksum = entry.get("hash")
                summary[state] += 1 if not path.endswith(os.sep) else 0
                content.append(
                    "{space}{checksum}{separator}{path}".format(
@@ -95,15 +95,15 @@ class CmdDiff(CmdBase):

    def run(self):
        try:
-            diff = self.repo.diff(self.args.a_ref, self.args.b_ref)
+            diff = self.repo.diff(self.args.a_rev, self.args.b_rev)

            if not any(diff.values()):
                return 0

-            if not self.args.checksums:
+            if not self.args.show_hash:
                for _, entries in diff.items():
                    for entry in entries:
-                        del entry["checksum"]
+                        del entry["hash"]

            if self.args.show_json:
                res = json.dumps(diff)
@@ -120,8 +120,9 @@ class CmdDiff(CmdBase):

 def add_parser(subparsers, parent_parser):
    DIFF_DESCRIPTION = (
-        "Compare two versions of the DVC repository.\n"
-        "Shows the list of paths added, modified, or deleted"
+        "Show changes between commits in the DVC repository,"
+        " or between a commit and the workspace."
+        "Prints the list of paths added, modified, or deleted."
    )
    diff_parser = subparsers.add_parser(
        "diff",
@@ -131,20 +132,14 @@ def add_parser(subparsers, parent_parser):
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    diff_parser.add_argument(
-        "a_ref",
-        help=(
-            "Git reference to the older version to compare "
-            "(defaults to HEAD)"
-        ),
+        "a_rev",
+        help="Old Git commit to compare (defaults to HEAD)",
        nargs="?",
        default="HEAD",
    )
    diff_parser.add_argument(
-        "b_ref",
-        help=(
-            "Git reference to the newer version to compare "
-            "(defaults to the current workspace)"
-        ),
+        "b_rev",
+        help=("New Git commit to compare (defaults to the current workspace)"),
        nargs="?",
    )
    diff_parser.add_argument(
@@ -154,7 +149,7 @@ def add_parser(subparsers, parent_parser):
        default=False,
    )
    diff_parser.add_argument(
-        "--checksums",
+        "--show-hash",
        help="Display hash value for each entry",
        action="store_true",
        default=False,
@@ -12,6 +12,15 @@ logger = logging.getLogger(__name__)

 class CmdGC(CmdBase):
    def run(self):
+        from dvc.repo.gc import _raise_error_if_all_disabled
+
+        _raise_error_if_all_disabled(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+        )
+
        msg = "This will remove all cache except items used in "

        msg += "the working tree"
@@ -47,6 +56,7 @@ class CmdGC(CmdBase):
            force=self.args.force,
            jobs=self.args.jobs,
            repos=self.args.repos,
+            workspace=self.args.workspace,
        )
        return 0

@@ -64,25 +74,32 @@ def add_parser(subparsers, parent_parser):
        help=GC_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
+    gc_parser.add_argument(
+        "-w",
+        "--workspace",
+        action="store_true",
+        default=False,
+        help="Keep data files used in the current workspace (working tree).",
+    )
    gc_parser.add_argument(
        "-a",
        "--all-branches",
        action="store_true",
        default=False,
-        help="Keep data files for the tips of all git branches.",
+        help="Keep data files for the tips of all Git branches.",
    )
    gc_parser.add_argument(
        "-T",
        "--all-tags",
        action="store_true",
        default=False,
-        help="Keep data files for all git tags.",
+        help="Keep data files for all Git tags.",
    )
    gc_parser.add_argument(
        "--all-commits",
        action="store_true",
        default=False,
-        help=argparse.SUPPRESS,
+        help="Keep data files for all Git commits.",
    )
    gc_parser.add_argument(
        "-c",
@@ -51,9 +51,7 @@ class CmdGet(CmdBaseNoRepo):


 def add_parser(subparsers, parent_parser):
-    GET_HELP = (
-        "Download file or directory from any DVC project or Git repository."
-    )
+    GET_HELP = "Download file or directory tracked by DVC or by Git."
    get_parser = subparsers.add_parser(
        "get",
        parents=[parent_parser],
@@ -62,12 +60,10 @@ def add_parser(subparsers, parent_parser):
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    get_parser.add_argument(
-        "url",
-        help="Location of DVC project or Git repository to download from",
+        "url", help="Location of DVC or Git repository to download from"
    )
    get_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the project or repository",
+        "path", help="Path to a file or directory within the repository"
    )
    get_parser.add_argument(
        "-o", "--out", nargs="?", help="Destination path to download files to"
@@ -0,0 +1,112 @@
+import logging
+import os
+
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.exceptions import NotDvcRepoError
+
+logger = logging.getLogger(__name__)
+
+
+class CmdHookBase(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            repo = Repo()
+            repo.close()
+        except NotDvcRepoError:
+            return 0
+
+        return self._run()
+
+
+class CmdPreCommit(CmdHookBase):
+    def _run(self):
+        from dvc.main import main
+
+        return main(["status"])
+
+
+class CmdPostCheckout(CmdHookBase):
+    def run(self):
+        # when we are running from pre-commit tool, it doesn't provide CLI
+        # flags, but instead provides respective env vars that we could use.
+        flag = os.environ.get("PRE_COMMIT_CHECKOUT_TYPE")
+        if flag is None and len(self.args.args) >= 3:
+            # see https://git-scm.com/docs/githooks#_post_checkout
+            flag = self.args.args[2]
+
+        # checking out some reference and not specific file.
+        if flag != "1":
+            return 0
+
+        # make sure we are not in the middle of a rebase/merge, so we
+        # don't accidentally break it with an unsuccessful checkout.
+        # Note that git hooks are always running in repo root.
+        if os.path.isdir(os.path.join(".git", "rebase-merge")):
+            return 0
+
+        from dvc.main import main
+
+        return main(["checkout"])
+
+
+class CmdPrePush(CmdHookBase):
+    def run(self):
+        from dvc.main import main
+
+        return main(["push"])
+
+
+def add_parser(subparsers, parent_parser):
+    GIT_HOOK_HELP = "Run GIT hook."
+
+    git_hook_parser = subparsers.add_parser(
+        "git-hook",
+        parents=[parent_parser],
+        description=GIT_HOOK_HELP,
+        add_help=False,
+    )
+
+    git_hook_subparsers = git_hook_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc daemon CMD --help` for command-specific help.",
+    )
+
+    fix_subparsers(git_hook_subparsers)
+
+    PRE_COMMIT_HELP = "Run pre-commit GIT hook."
+    pre_commit_parser = git_hook_subparsers.add_parser(
+        "pre-commit",
+        parents=[parent_parser],
+        description=PRE_COMMIT_HELP,
+        help=PRE_COMMIT_HELP,
+    )
+    pre_commit_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    pre_commit_parser.set_defaults(func=CmdPreCommit)
+
+    POST_CHECKOUT_HELP = "Run post-checkout GIT hook."
+    post_checkout_parser = git_hook_subparsers.add_parser(
+        "post-checkout",
+        parents=[parent_parser],
+        description=POST_CHECKOUT_HELP,
+        help=POST_CHECKOUT_HELP,
+    )
+    post_checkout_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    post_checkout_parser.set_defaults(func=CmdPostCheckout)
+
+    PRE_PUSH_HELP = "Run pre-push GIT hook."
+    pre_push_parser = git_hook_subparsers.add_parser(
+        "pre-push",
+        parents=[parent_parser],
+        description=PRE_PUSH_HELP,
+        help=PRE_PUSH_HELP,
+    )
+    pre_push_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    pre_push_parser.set_defaults(func=CmdPrePush)
@@ -30,7 +30,7 @@ class CmdImport(CmdBase):

 def add_parser(subparsers, parent_parser):
    IMPORT_HELP = (
-        "Download file or directory from any DVC project or Git repository "
+        "Download file or directory tracked by DVC or by Git "
        "into the workspace, and track it."
    )

@@ -42,12 +42,10 @@ def add_parser(subparsers, parent_parser):
        formatter_class=argparse.RawTextHelpFormatter,
    )
    import_parser.add_argument(
-        "url",
-        help="Location of DVC project or Git repository to download from",
+        "url", help="Location of DVC or Git repository to download from"
    )
    import_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the project or repository",
+        "path", help="Path to a file or directory within the repository"
    )
    import_parser.add_argument(
        "-o", "--out", nargs="?", help="Destination path to download files to"
@@ -15,7 +15,10 @@ class CmdInit(CmdBaseNoRepo):

        try:
            self.repo = Repo.init(
-                ".", no_scm=self.args.no_scm, force=self.args.force
+                ".",
+                no_scm=self.args.no_scm,
+                force=self.args.force,
+                subdir=self.args.subdir,
            )
            self.config = self.repo.config
        except InitError:
@@ -56,4 +59,13 @@ def add_parser(subparsers, parent_parser):
            "This operation removes local cache."
        ),
    )
+    init_parser.add_argument(
+        "--subdir",
+        action="store_true",
+        default=False,
+        help=(
+            "Necessary for running this command inside a subdirectory of a "
+            "parent SCM repository."
+        ),
+    )
    init_parser.set_defaults(func=CmdInit)
@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
 class CmdInstall(CmdBase):
    def run(self):
        try:
-            self.repo.install()
+            self.repo.install(self.args.use_pre_commit_tool)
        except Exception:
            logger.exception("failed to install DVC Git hooks")
            return 1
@@ -27,4 +27,11 @@ def add_parser(subparsers, parent_parser):
        help=INSTALL_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
+    install_parser.add_argument(
+        "--use-pre-commit-tool",
+        action="store_true",
+        default=False,
+        help="Install DVC hooks using pre-commit "
+        "(https://pre-commit.com) if it is installed.",
+    )
    install_parser.set_defaults(func=CmdInstall)
@@ -0,0 +1,77 @@
+import argparse
+import logging
+import sys
+
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
+from dvc.command.ls.ls_colors import LsColors
+from dvc.exceptions import DvcException
+
+
+logger = logging.getLogger(__name__)
+
+
+def _prettify(entries, with_color=False):
+    if with_color:
+        ls_colors = LsColors()
+        fmt = ls_colors.format
+    else:
+
+        def fmt(entry):
+            return entry["path"]
+
+    return [fmt(entry) for entry in entries]
+
+
+class CmdList(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            entries = Repo.ls(
+                self.args.url,
+                self.args.path,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+                outs_only=self.args.outs_only,
+            )
+            if entries:
+                entries = _prettify(entries, sys.stdout.isatty())
+                logger.info("\n".join(entries))
+            return 0
+        except DvcException:
+            logger.exception("failed to list '{}'".format(self.args.url))
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    LIST_HELP = (
+        "List repository contents, including files"
+        " and directories tracked by DVC and by Git."
+    )
+    list_parser = subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(LIST_HELP, "list"),
+        help=LIST_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    list_parser.add_argument("url", help="Location of DVC repository to list")
+    list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        help="Recursively list files.",
+    )
+    list_parser.add_argument(
+        "--outs-only", action="store_true", help="Show only DVC outputs."
+    )
+    list_parser.add_argument(
+        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
+    )
+    list_parser.add_argument(
+        "path",
+        nargs="?",
+        help="Path to directory within the repository to list outputs for",
+    )
+    list_parser.set_defaults(func=CmdList)
@@ -0,0 +1,48 @@
+import os
+
+
+class LsColors(object):
+    default = "rs=0:di=01;34:ex=01;32"
+
+    def __init__(self, lscolors=None):
+        self._extensions = {}
+        self._codes = {}
+        self._load(lscolors or os.environ.get("LS_COLORS") or LsColors.default)
+
+    def _load(self, lscolors):
+        for item in lscolors.split(":"):
+            try:
+                code, color = item.split("=", 1)
+            except ValueError:
+                continue
+            if code.startswith("*."):
+                self._extensions[code[1:]] = color
+            else:
+                self._codes[code] = color
+
+    def format(self, entry):
+        text = entry["path"]
+
+        if entry.get("isout", False) and "out" in self._codes:
+            return self._format(text, code="out")
+
+        if entry.get("isdir", False):
+            return self._format(text, code="di")
+
+        if entry.get("isexec", False):
+            return self._format(text, code="ex")
+
+        _, ext = os.path.splitext(text)
+        return self._format(text, ext=ext)
+
+    def _format(self, text, code=None, ext=None):
+        val = None
+        if ext:
+            val = self._extensions.get(ext, None)
+        if code:
+            val = self._codes.get(code, None)
+
+        if not val:
+            return text
+        rs = self._codes.get("rs", 0)
+        return "\033[{}m{}\033[{}m".format(val, text, rs)
@@ -136,8 +136,8 @@ class CmdMetricsDiff(CmdBase):
    def run(self):
        try:
            diff = self.repo.metrics.diff(
-                a_ref=self.args.a_ref,
-                b_ref=self.args.b_ref,
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
                targets=self.args.targets,
                typ=self.args.type,
                xpath=self.args.xpath,
@@ -187,8 +187,7 @@ def add_parser(subparsers, parent_parser):
    metrics_show_parser.add_argument(
        "targets",
        nargs="*",
-        help="Metric files or directories (see -R) to show "
-        "(leave empty to display all)",
+        help="Metric files or directories (see -R) to show",
    )
    metrics_show_parser.add_argument(
        "-t",
@@ -273,8 +272,8 @@ def add_parser(subparsers, parent_parser):
    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)

-    METRICS_DIFF_HELP = "Show a table of changes between metrics among "
-    "versions of the DVC repository."
+    METRICS_DIFF_HELP = "Show changes in metrics between commits"
+    " in the DVC repository, or between a commit and the workspace."
    metrics_diff_parser = metrics_subparsers.add_parser(
        "diff",
        parents=[parent_parser],
@@ -283,20 +282,12 @@ def add_parser(subparsers, parent_parser):
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    metrics_diff_parser.add_argument(
-        "a_ref",
-        nargs="?",
-        help=(
-            "Git reference to the older version to compare "
-            "(defaults to HEAD)"
-        ),
+        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
    )
    metrics_diff_parser.add_argument(
-        "b_ref",
+        "b_rev",
        nargs="?",
-        help=(
-            "Git reference to the newer version to compare "
-            "(defaults to the current workspace)"
-        ),
+        help=("New Git commit to compare (defaults to the current workspace)"),
    )
    metrics_diff_parser.add_argument(
        "--targets",
@@ -48,26 +48,21 @@ class CmdPipelineShow(CmdBase):
            elif outs:
                for out in stage.outs:
                    nodes.add(str(out))
-                for dep in stage.deps:
-                    nodes.add(str(dep))
            else:
                nodes.add(stage.relpath)

        edges = []
-
-        if outs:
-            for stage in networkx.dfs_preorder_nodes(G, target_stage):
-                for dep in stage.deps:
-                    for out in stage.outs:
-                        edges.append((str(out), str(dep)))
-        else:
-            for from_stage, to_stage in networkx.dfs_edges(G, target_stage):
-                if commands:
-                    if to_stage.cmd is None:
-                        continue
-                    edges.append((from_stage.cmd, to_stage.cmd))
-                else:
-                    edges.append((from_stage.relpath, to_stage.relpath))
+        for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
+            if commands:
+                if to_stage.cmd is None:
+                    continue
+                edges.append((from_stage.cmd, to_stage.cmd))
+            elif outs:
+                for from_out in from_stage.outs:
+                    for to_out in to_stage.outs:
+                        edges.append((str(from_out), str(to_out)))
+            else:
+                edges.append((from_stage.relpath, to_stage.relpath))

        return list(nodes), edges, networkx.is_tree(G)

@@ -1,72 +1,99 @@
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import fix_subparsers
+from dvc.config import ConfigError
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.remote.config import RemoteConfig
 from dvc.utils import format_link

 logger = logging.getLogger(__name__)


-class CmdRemoteConfig(CmdConfig):
+class CmdRemote(CmdConfig):
    def __init__(self, args):
        super().__init__(args)
-        self.remote_config = RemoteConfig(self.config)
+
+        if getattr(self.args, "name", None):
+            self.args.name = self.args.name.lower()
+
+    def _check_exists(self, conf):
+        if self.args.name not in conf["remote"]:
+            raise ConfigError(
+                "remote '{}' doesn't exists.".format(self.args.name)
+            )


-class CmdRemoteAdd(CmdRemoteConfig):
+class CmdRemoteAdd(CmdRemote):
    def run(self):
        if self.args.default:
            logger.info(
                "Setting '{}' as a default remote.".format(self.args.name)
            )
-        self.remote_config.add(
-            self.args.name,
-            self.args.url,
-            force=self.args.force,
-            default=self.args.default,
-            level=self.args.level,
-        )
+
+        with self.config.edit(self.args.level) as conf:
+            if self.args.name in conf["remote"] and not self.args.force:
+                raise ConfigError(
+                    "remote '{}' already exists. Use `-f|--force` to "
+                    "overwrite it.".format(self.args.name)
+                )
+
+            conf["remote"][self.args.name] = {"url": self.args.url}
+            if self.args.default:
+                conf["core"]["remote"] = self.args.name
+
        return 0


-class CmdRemoteRemove(CmdRemoteConfig):
+class CmdRemoteRemove(CmdRemote):
    def run(self):
-        self.remote_config.remove(self.args.name, level=self.args.level)
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            del conf["remote"][self.args.name]
+
+        # Remove core.remote refs to this remote in any shadowing configs
+        for level in reversed(self.config.LEVELS):
+            with self.config.edit(level) as conf:
+                if conf["core"].get("remote") == self.args.name:
+                    del conf["core"]["remote"]
+
+            if level == self.args.level:
+                break
+
        return 0


-class CmdRemoteModify(CmdRemoteConfig):
+class CmdRemoteModify(CmdRemote):
    def run(self):
-        self.remote_config.modify(
-            self.args.name,
-            self.args.option,
-            self.args.value,
-            level=self.args.level,
-        )
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            conf["remote"][self.args.name][self.args.option] = self.args.value
        return 0


-class CmdRemoteDefault(CmdRemoteConfig):
+class CmdRemoteDefault(CmdRemote):
    def run(self):
+
        if self.args.name is None and not self.args.unset:
-            name = self.remote_config.get_default(level=self.args.level)
-            print(name)
+            conf = self.config.load_one(self.args.level)
+            try:
+                print(conf["core"]["remote"])
+            except KeyError:
+                logger.info("No default remote set")
+                return 1
        else:
-            self.remote_config.set_default(
-                self.args.name, unset=self.args.unset, level=self.args.level
-            )
+            with self.config.edit(self.args.level) as conf:
+                if self.args.unset:
+                    conf["core"].pop("remote", None)
+                else:
+                    conf["core"]["remote"] = self.args.name
        return 0


-class CmdRemoteList(CmdRemoteConfig):
+class CmdRemoteList(CmdRemote):
    def run(self):
-        for name, url in self.remote_config.list(
-            level=self.args.level
-        ).items():
-            logger.info("{}\t{}".format(name, url))
+        conf = self.config.load_one(self.args.level)
+        for name, conf in conf["remote"].items():
+            logger.info("{}\t{}".format(name, conf["url"]))
        return 0


@@ -22,7 +22,10 @@ class CmdUnprotect(CmdBase):


 def add_parser(subparsers, parent_parser):
-    UNPROTECT_HELP = "Unprotect data files or directories."
+    UNPROTECT_HELP = (
+        "Unprotect tracked files or directories (when hardlinks or symlinks "
+        "have been enabled with `dvc config cache.type`)"
+    )
    unprotect_parser = subparsers.add_parser(
        "unprotect",
        parents=[parent_parser],
@@ -14,7 +14,7 @@ class CmdUpdate(CmdBase):
        ret = 0
        for target in self.args.targets:
            try:
-                self.repo.update(target)
+                self.repo.update(target, self.args.rev)
            except DvcException:
                logger.exception("failed to update '{}'.".format(target))
                ret = 1
@@ -33,4 +33,7 @@ def add_parser(subparsers, parent_parser):
    update_parser.add_argument(
        "targets", nargs="+", help="DVC-files to update."
    )
+    update_parser.add_argument(
+        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+    )
    update_parser.set_defaults(func=CmdUpdate)
@@ -31,6 +31,7 @@ class CmdVersion(CmdBaseNoRepo):
            "Platform: {}".format(platform.platform()),
            "Binary: {}".format(is_binary()),
            "Package: {}".format(PKG),
+            "Supported remotes: {}".format(self.get_supported_remotes()),
        ]

        try:
@@ -112,6 +113,17 @@ class CmdVersion(CmdBaseNoRepo):

        return ", ".join(cache)

+    @staticmethod
+    def get_supported_remotes():
+        from dvc.remote import REMOTES
+
+        supported_remotes = []
+        for remote in REMOTES:
+            if not remote.get_missing_deps():
+                supported_remotes.append(remote.scheme)
+
+        return ", ".join(supported_remotes)
+

 def add_parser(subparsers, parent_parser):
    VERSION_HELP = "Show DVC version and system/environment information."
@@ -1,17 +1,17 @@
 """DVC config objects."""
-
-import copy
-import errno
+from contextlib import contextmanager
 import logging
 import os
 import re
+from urllib.parse import urlparse

+from funcy import cached_property, re_find, walk_values, compact
 import configobj
-from voluptuous import Schema, Required, Optional, Invalid
-from voluptuous import All, Any, Lower, Range, Coerce, Match
+from voluptuous import Schema, Optional, Invalid, ALLOW_EXTRA
+from voluptuous import All, Any, Lower, Range, Coerce

-from dvc.exceptions import DvcException
-from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.utils import relpath

 logger = logging.getLogger(__name__)

@@ -48,13 +48,12 @@ def supported_cache_type(types):


 # Checks that value is either true or false and converts it to bool
-Bool = All(
+to_bool = Bool = All(
    Lower,
    Any("true", "false"),
    lambda v: v == "true",
    msg="expected true or false",
 )
-to_bool = Schema(Bool)


 def Choices(*choices):
@@ -67,7 +66,136 @@ def Choices(*choices):
    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))


-class Config(object):  # pylint: disable=too-many-instance-attributes
+def ByUrl(mapping):
+    schemas = walk_values(Schema, mapping)
+
+    def validate(data):
+        if "url" not in data:
+            raise Invalid("expected 'url'")
+
+        parsed = urlparse(data["url"])
+        # Windows absolute paths should really have scheme == "" (local)
+        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+            return schemas[""](data)
+        if parsed.scheme not in schemas:
+            raise Invalid("Unsupported URL type {}://".format(parsed.scheme))
+
+        return schemas[parsed.scheme](data)
+
+    return validate
+
+
+class RelPath(str):
+    pass
+
+
+REMOTE_COMMON = {
+    "url": str,
+    "checksum_jobs": All(Coerce(int), Range(1)),
+    "no_traverse": Bool,
+    "verify": Bool,
+}
+LOCAL_COMMON = {
+    "type": supported_cache_type,
+    Optional("protected", default=False): Bool,  # obsoleted
+    "shared": All(Lower, Choices("group")),
+    Optional("slow_link_warning", default=True): Bool,
+}
+HTTP_COMMON = {
+    "auth": All(Lower, Choices("basic", "digest", "custom")),
+    "custom_auth_header": str,
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+}
+SCHEMA = {
+    "core": {
+        "remote": Lower,
+        "checksum_jobs": All(Coerce(int), Range(1)),
+        Optional("interactive", default=False): Bool,
+        Optional("analytics", default=True): Bool,
+        Optional("hardlink_lock", default=False): Bool,
+        Optional("no_scm", default=False): Bool,
+    },
+    "cache": {
+        "local": str,
+        "s3": str,
+        "gs": str,
+        "hdfs": str,
+        "ssh": str,
+        "azure": str,
+        # This is for default local cache
+        "dir": str,
+        **LOCAL_COMMON,
+    },
+    "remote": {
+        str: ByUrl(
+            {
+                "": {**LOCAL_COMMON, **REMOTE_COMMON},
+                "s3": {
+                    "region": str,
+                    "profile": str,
+                    "credentialpath": str,
+                    "endpointurl": str,
+                    Optional("listobjects", default=False): Bool,
+                    Optional("use_ssl", default=True): Bool,
+                    "sse": str,
+                    "acl": str,
+                    "grant_read": str,
+                    "grant_read_acp": str,
+                    "grant_write_acp": str,
+                    "grant_full_control": str,
+                    **REMOTE_COMMON,
+                },
+                "gs": {
+                    "projectname": str,
+                    "credentialpath": str,
+                    **REMOTE_COMMON,
+                },
+                "ssh": {
+                    "type": supported_cache_type,
+                    "port": Coerce(int),
+                    "user": str,
+                    "password": str,
+                    "ask_password": Bool,
+                    "keyfile": str,
+                    "timeout": Coerce(int),
+                    "gss_auth": Bool,
+                    **REMOTE_COMMON,
+                },
+                "hdfs": {"user": str, **REMOTE_COMMON},
+                "azure": {"connection_string": str, **REMOTE_COMMON},
+                "oss": {
+                    "oss_key_id": str,
+                    "oss_key_secret": str,
+                    "oss_endpoint": str,
+                    **REMOTE_COMMON,
+                },
+                "gdrive": {
+                    "gdrive_use_service_account": Bool,
+                    "gdrive_client_id": str,
+                    "gdrive_client_secret": str,
+                    "gdrive_user_credentials_file": str,
+                    "gdrive_service_account_email": str,
+                    "gdrive_service_account_user_email": str,
+                    "gdrive_service_account_p12_file_path": str,
+                    **REMOTE_COMMON,
+                },
+                "http": {**HTTP_COMMON, **REMOTE_COMMON},
+                "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "remote": {str: object},  # Any of the above options are valid
+            }
+        )
+    },
+    "state": {
+        "row_limit": All(Coerce(int), Range(1)),
+        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
+    },
+}
+COMPILED_SCHEMA = Schema(SCHEMA)
+
+
+class Config(dict):
    """Class that manages configuration files for a DVC repo.

    Args:
@@ -77,176 +205,21 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
        validate (bool): optional flag to tell dvc if it should validate the
            config or just load it as is. 'True' by default.

-
    Raises:
-        ConfigError: thrown when config has an invalid format.
+        ConfigError: thrown if config has an invalid format.
    """

    APPNAME = "dvc"
    APPAUTHOR = "iterative"

-    # NOTE: used internally in RemoteLOCAL to know config
-    # location, that url should resolved relative to.
-    PRIVATE_CWD = "_cwd"
+    # In the order they shadow each other
+    LEVELS = ("system", "global", "repo", "local")

    CONFIG = "config"
    CONFIG_LOCAL = "config.local"

-    CREDENTIALPATH = "credentialpath"
-
-    LEVEL_LOCAL = 0
-    LEVEL_REPO = 1
-    LEVEL_GLOBAL = 2
-    LEVEL_SYSTEM = 3
-
-    SECTION_CORE = "core"
-    SECTION_CORE_LOGLEVEL = "loglevel"
-    SECTION_CORE_LOGLEVEL_SCHEMA = All(
-        Lower, Choices("info", "debug", "warning", "error")
-    )
-    SECTION_CORE_REMOTE = "remote"
-    SECTION_CORE_INTERACTIVE = "interactive"
-    SECTION_CORE_ANALYTICS = "analytics"
-    SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_CORE_HARDLINK_LOCK = "hardlink_lock"
-
-    SECTION_CACHE = "cache"
-    SECTION_CACHE_DIR = "dir"
-    SECTION_CACHE_TYPE = "type"
-    SECTION_CACHE_PROTECTED = "protected"
-    SECTION_CACHE_SHARED = "shared"
-    SECTION_CACHE_SHARED_SCHEMA = All(Lower, Choices("group"))
-    SECTION_CACHE_LOCAL = "local"
-    SECTION_CACHE_S3 = "s3"
-    SECTION_CACHE_GS = "gs"
-    SECTION_CACHE_SSH = "ssh"
-    SECTION_CACHE_HDFS = "hdfs"
-    SECTION_CACHE_AZURE = "azure"
-    SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
-    SECTION_CACHE_SCHEMA = {
-        SECTION_CACHE_LOCAL: str,
-        SECTION_CACHE_S3: str,
-        SECTION_CACHE_GS: str,
-        SECTION_CACHE_HDFS: str,
-        SECTION_CACHE_SSH: str,
-        SECTION_CACHE_AZURE: str,
-        SECTION_CACHE_DIR: str,
-        SECTION_CACHE_TYPE: supported_cache_type,
-        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
-        SECTION_CACHE_SHARED: SECTION_CACHE_SHARED_SCHEMA,
-        PRIVATE_CWD: str,
-        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): Bool,
-    }
-
-    SECTION_CORE_SCHEMA = {
-        SECTION_CORE_LOGLEVEL: SECTION_CORE_LOGLEVEL_SCHEMA,
-        SECTION_CORE_REMOTE: Lower,
-        Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
-        Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
-        SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
-        Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool,
-    }
-
-    # aws specific options
-    SECTION_AWS_CREDENTIALPATH = CREDENTIALPATH
-    SECTION_AWS_ENDPOINT_URL = "endpointurl"
-    SECTION_AWS_LIST_OBJECTS = "listobjects"
-    SECTION_AWS_REGION = "region"
-    SECTION_AWS_PROFILE = "profile"
-    SECTION_AWS_USE_SSL = "use_ssl"
-    SECTION_AWS_SSE = "sse"
-    SECTION_AWS_ACL = "acl"
-    SECTION_AWS_GRANT_READ = "grant_read"
-    SECTION_AWS_GRANT_READ_ACP = "grant_read_acp"
-    SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp"
-    SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control"
-
-    # gcp specific options
-    SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH
-    SECTION_GCP_PROJECTNAME = "projectname"
-
-    # azure specific option
-    SECTION_AZURE_CONNECTION_STRING = "connection_string"
-
-    # Alibabacloud oss options
-    SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
-    SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
-    SECTION_OSS_ENDPOINT = "oss_endpoint"
-
-    # GDrive options
-    SECTION_GDRIVE_CLIENT_ID = "gdrive_client_id"
-    SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret"
-    SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file"
-
-    SECTION_REMOTE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
-    SECTION_REMOTE_FMT = 'remote "{}"'
-    SECTION_REMOTE_URL = "url"
-    SECTION_REMOTE_USER = "user"
-    SECTION_REMOTE_PORT = "port"
-    SECTION_REMOTE_KEY_FILE = "keyfile"
-    SECTION_REMOTE_TIMEOUT = "timeout"
-    SECTION_REMOTE_PASSWORD = "password"
-    SECTION_REMOTE_ASK_PASSWORD = "ask_password"
-    SECTION_REMOTE_GSS_AUTH = "gss_auth"
-    SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
-    SECTION_REMOTE_VERIFY = "verify"
-    SECTION_REMOTE_SCHEMA = {
-        Required(SECTION_REMOTE_URL): str,
-        SECTION_AWS_REGION: str,
-        SECTION_AWS_PROFILE: str,
-        SECTION_AWS_CREDENTIALPATH: str,
-        SECTION_AWS_ENDPOINT_URL: str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): Bool,
-        Optional(SECTION_AWS_USE_SSL, default=True): Bool,
-        SECTION_AWS_SSE: str,
-        SECTION_AWS_ACL: str,
-        SECTION_AWS_GRANT_READ: str,
-        SECTION_AWS_GRANT_READ_ACP: str,
-        SECTION_AWS_GRANT_WRITE_ACP: str,
-        SECTION_AWS_GRANT_FULL_CONTROL: str,
-        SECTION_GCP_PROJECTNAME: str,
-        SECTION_CACHE_TYPE: supported_cache_type,
-        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
-        SECTION_REMOTE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
-        SECTION_REMOTE_USER: str,
-        SECTION_REMOTE_PORT: Coerce(int),
-        SECTION_REMOTE_KEY_FILE: str,
-        SECTION_REMOTE_TIMEOUT: Coerce(int),
-        SECTION_REMOTE_PASSWORD: str,
-        SECTION_REMOTE_ASK_PASSWORD: Bool,
-        SECTION_REMOTE_GSS_AUTH: Bool,
-        SECTION_AZURE_CONNECTION_STRING: str,
-        SECTION_OSS_ACCESS_KEY_ID: str,
-        SECTION_OSS_ACCESS_KEY_SECRET: str,
-        SECTION_OSS_ENDPOINT: str,
-        SECTION_GDRIVE_CLIENT_ID: str,
-        SECTION_GDRIVE_CLIENT_SECRET: str,
-        SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
-        PRIVATE_CWD: str,
-        SECTION_REMOTE_NO_TRAVERSE: Bool,
-        SECTION_REMOTE_VERIFY: Bool,
-    }
-
-    SECTION_STATE = "state"
-    SECTION_STATE_ROW_LIMIT = "row_limit"
-    SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
-    SECTION_STATE_SCHEMA = {
-        SECTION_STATE_ROW_LIMIT: All(Coerce(int), Range(1)),
-        SECTION_STATE_ROW_CLEANUP_QUOTA: All(Coerce(int), Range(0, 100)),
-    }
-
-    SCHEMA = {
-        Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
-        Match(SECTION_REMOTE_REGEX): SECTION_REMOTE_SCHEMA,
-        Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
-        Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
-    }
-    COMPILED_SCHEMA = Schema(SCHEMA)
-
    def __init__(self, dvc_dir=None, validate=True):
        self.dvc_dir = dvc_dir
-        self.should_validate = validate

        if not dvc_dir:
            try:
@@ -258,33 +231,31 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
        else:
            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))

-        self.load()
+        self.load(validate=validate)

-    @staticmethod
-    def get_global_config_dir():
-        """Returns global config location. E.g. ~/.config/dvc/config.
+    @classmethod
+    def get_dir(cls, level):
+        from appdirs import user_config_dir, site_config_dir

-        Returns:
-            str: path to the global config directory.
-        """
-        from appdirs import user_config_dir
+        assert level in ("global", "system")

-        return user_config_dir(
-            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
-        )
+        if level == "global":
+            return user_config_dir(cls.APPNAME, cls.APPAUTHOR)
+        if level == "system":
+            return site_config_dir(cls.APPNAME, cls.APPAUTHOR)

-    @staticmethod
-    def get_system_config_dir():
-        """Returns system config location. E.g. /etc/dvc.conf.
+    @cached_property
+    def files(self):
+        files = {
+            level: os.path.join(self.get_dir(level), self.CONFIG)
+            for level in ("system", "global")
+        }

-        Returns:
-            str: path to the system config directory.
-        """
-        from appdirs import site_config_dir
+        if self.dvc_dir is not None:
+            files["repo"] = os.path.join(self.dvc_dir, self.CONFIG)
+            files["local"] = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)

-        return site_config_dir(
-            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
-        )
+        return files

    @staticmethod
    def init(dvc_dir):
@@ -300,250 +271,137 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
        open(config_file, "w+").close()
        return Config(dvc_dir)

-    def _resolve_cache_path(self, config):
-        cache = config.get(self.SECTION_CACHE)
-        if cache is None:
-            return
-
-        cache_dir = cache.get(self.SECTION_CACHE_DIR)
-        if cache_dir is None:
-            return
-
-        cache[self.PRIVATE_CWD] = os.path.dirname(config.filename)
-
-    def _resolve_paths(self, config):
-        if config.filename is None:
-            return config
-
-        ret = copy.deepcopy(config)
-        self._resolve_cache_path(ret)
-
-        for section in ret.values():
-            if self.SECTION_REMOTE_URL not in section.keys():
-                continue
-
-            section[self.PRIVATE_CWD] = os.path.dirname(ret.filename)
-
-        return ret
-
-    def _load_configs(self):
-        system_config_file = os.path.join(
-            self.get_system_config_dir(), self.CONFIG
-        )
-
-        global_config_file = os.path.join(
-            self.get_global_config_dir(), self.CONFIG
-        )
-
-        self._system_config = configobj.ConfigObj(system_config_file)
-        self._global_config = configobj.ConfigObj(global_config_file)
-        self._repo_config = configobj.ConfigObj()
-        self._local_config = configobj.ConfigObj()
-
-        if not self.dvc_dir:
-            return
-
-        config_file = os.path.join(self.dvc_dir, self.CONFIG)
-        config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
-
-        self._repo_config = configobj.ConfigObj(config_file)
-        self._local_config = configobj.ConfigObj(config_local_file)
-
-    @property
-    def config_local_file(self):
-        return self._local_config.filename
-
-    @property
-    def config_file(self):
-        return self._repo_config.filename
-
-    def load(self):
+    def load(self, validate=True):
        """Loads config from all the config files.

        Raises:
-            dvc.config.ConfigError: thrown if config has invalid format.
+            ConfigError: thrown if config has an invalid format.
        """
-        self._load_configs()
+        conf = {}
+        for level in self.LEVELS:
+            if level in self.files:
+                _merge(conf, self.load_one(level))

-        self.config = configobj.ConfigObj()
-        for c in [
-            self._system_config,
-            self._global_config,
-            self._repo_config,
-            self._local_config,
-        ]:
-            c = self._resolve_paths(c)
-            c = self._lower(c)
-            self.config.merge(c)
+        if validate:
+            conf = self.validate(conf)

-        if not self.should_validate:
-            return
+        self.clear()
+        self.update(conf)

-        d = self.validate(self.config)
-        self.config = configobj.ConfigObj(d, write_empty_values=True)
+        # Add resolved default cache.dir
+        if not self["cache"].get("dir") and self.dvc_dir:
+            self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")

-    def save(self, config=None):
-        """Saves config to config files.
+    def load_one(self, level):
+        conf = _load_config(self.files[level])
+        conf = self._load_paths(conf, self.files[level])

-        Raises:
-            dvc.config.ConfigError: thrown if failed to write config file.
-        """
-        if config is not None:
-            clist = [config]
-        else:
-            clist = [
-                self._system_config,
-                self._global_config,
-                self._repo_config,
-                self._local_config,
-            ]
+        # Autovivify sections
+        for key in COMPILED_SCHEMA.schema:
+            conf.setdefault(key, {})

-        for conf in clist:
-            self._save(conf)
+        return conf

+    @staticmethod
+    def _load_paths(conf, filename):
+        abs_conf_dir = os.path.abspath(os.path.dirname(filename))
+
+        def resolve(path):
+            if os.path.isabs(path) or re.match(r"\w+://", path):
+                return path
+            return RelPath(os.path.join(abs_conf_dir, path))
+
+        return Config._map_dirs(conf, resolve)
+
+    @staticmethod
+    def _save_paths(conf, filename):
+        conf_dir = os.path.dirname(filename)
+
+        def rel(path):
+            if re.match(r"\w+://", path):
+                return path
+
+            if isinstance(path, RelPath) or not os.path.isabs(path):
+                return relpath(path, conf_dir)
+            return path
+
+        return Config._map_dirs(conf, rel)
+
+    @staticmethod
+    def _map_dirs(conf, func):
+        dirs_schema = {"cache": {"dir": func}, "remote": {str: {"url": func}}}
+        return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
+
+    @contextmanager
+    def edit(self, level="repo"):
+        if level in {"repo", "local"} and self.dvc_dir is None:
+            raise ConfigError("Not inside a DVC repo")
+
+        conf = self.load_one(level)
+        yield conf
+
+        conf = self._save_paths(conf, self.files[level])
+        _save_config(self.files[level], conf)
        self.load()

    @staticmethod
-    def _save(config):
-        if config.filename is None:
-            return
-
-        logger.debug("Writing '{}'.".format(config.filename))
-        dname = os.path.dirname(os.path.abspath(config.filename))
+    def validate(data):
        try:
-            os.makedirs(dname)
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                raise
-        config.write()
-
-    def validate(self, config):
-        try:
-            return self.COMPILED_SCHEMA(config.dict())
+            return COMPILED_SCHEMA(data)
        except Invalid as exc:
-            raise ConfigError(str(exc)) from exc
+            raise ConfigError(str(exc)) from None

-    def unset(self, section, opt=None, level=None, force=False):
-        """Unsets specified option and/or section in the config.

-        Args:
-            section (str): section name.
-            opt (str): optional option name.
-            level (int): config level to use.
-            force (bool): don't error-out even if section doesn't exist. False
-                by default.
+def _load_config(filename):
+    conf_obj = configobj.ConfigObj(filename)
+    return _parse_remotes(_lower_keys(conf_obj.dict()))

-        Raises:
-            dvc.config.ConfigError: thrown if section doesn't exist and
-                `force != True`.
-        """
-        config = self.get_configobj(level)

-        if section not in config.keys():
-            if force:
-                return
-            raise ConfigError("section '{}' doesn't exist".format(section))
+def _save_config(filename, conf_dict):
+    logger.debug("Writing '{}'.".format(filename))
+    os.makedirs(os.path.dirname(filename), exist_ok=True)

-        if opt:
-            if opt not in config[section].keys():
-                if force:
-                    return
-                raise ConfigError(
-                    "option '{}.{}' doesn't exist".format(section, opt)
-                )
-            del config[section][opt]
+    config = configobj.ConfigObj(_pack_remotes(conf_dict))
+    config.filename = filename
+    config.write()

-            if not config[section]:
-                del config[section]
+
+def _parse_remotes(conf):
+    result = {"remote": {}}
+
+    for section, val in conf.items():
+        name = re_find(r'^\s*remote\s*"(.*)"\s*$', section)
+        if name:
+            result["remote"][name] = val
        else:
-            del config[section]
+            result[section] = val

-        self.save(config)
+    return result

-    def set(self, section, opt, value, level=None, force=True):
-        """Sets specified option in the config.

-        Args:
-            section (str): section name.
-            opt (str): option name.
-            value: value to set option to.
-            level (int): config level to use.
-            force (bool): set option even if section already exists. True by
-                default.
+def _pack_remotes(conf):
+    # Drop empty sections
+    result = compact(conf)

-        Raises:
-            dvc.config.ConfigError: thrown if section already exists and
-                `force != True`.
+    # Transform remote.name -> 'remote "name"'
+    for name, val in conf["remote"].items():
+        result['remote "{}"'.format(name)] = val
+    result.pop("remote", None)

-        """
-        config = self.get_configobj(level)
+    return result

-        if section not in config.keys():
-            config[section] = {}
-        elif not force:
-            raise ConfigError(
-                "Section '{}' already exists. Use `-f|--force` to overwrite "
-                "section with new value.".format(section)
-            )

-        config[section][opt] = value
+def _merge(into, update):
+    """Merges second dict into first recursively"""
+    for key, val in update.items():
+        if isinstance(into.get(key), dict) and isinstance(val, dict):
+            _merge(into[key], val)
+        else:
+            into[key] = val

-        result = copy.deepcopy(self.config)
-        result.merge(config)
-        self.validate(result)

-        self.save(config)
-
-    def get(self, section, opt=None, level=None):
-        """Return option value from the config.
-
-        Args:
-            section (str): section name.
-            opt (str): option name.
-            level (int): config level to use.
-
-        Returns:
-            value (str, int): option value.
-        """
-        config = self.get_configobj(level)
-
-        if section not in config.keys():
-            raise ConfigError("section '{}' doesn't exist".format(section))
-
-        if opt not in config[section].keys():
-            raise ConfigError(
-                "option '{}.{}' doesn't exist".format(section, opt)
-            )
-
-        return config[section][opt]
-
-    @staticmethod
-    def _lower(config):
-        new_config = configobj.ConfigObj()
-        for s_key, s_value in config.items():
-            new_s = {}
-            for key, value in s_value.items():
-                new_s[key.lower()] = str(value)
-            new_config[s_key.lower()] = new_s
-        return new_config
-
-    def get_configobj(self, level):
-        configs = {
-            self.LEVEL_LOCAL: self._local_config,
-            self.LEVEL_REPO: self._repo_config,
-            self.LEVEL_GLOBAL: self._global_config,
-            self.LEVEL_SYSTEM: self._system_config,
-        }
-
-        return configs.get(level, self._repo_config)
-
-    def list_options(self, section_regex, option, level=None):
-        ret = {}
-        config = self.get_configobj(level)
-        for section in config.keys():
-            r = re.match(section_regex, section)
-            if r:
-                name = r.group("name")
-                value = config[section].get(option, "")
-                ret[name] = value
-        return ret
+def _lower_keys(data):
+    return {
+        k.lower(): _lower_keys(v) if isinstance(v, dict) else v
+        for k, v in data.items()
+    }
@@ -2,7 +2,6 @@

 import logging

-from dvc.config import Config
 from dvc.config import NoRemoteError
 from dvc.remote import Remote

@@ -24,28 +23,14 @@ class DataCloud(object):
    def __init__(self, repo):
        self.repo = repo

-    @property
-    def _config(self):
-        return self.repo.config.config
-
-    @property
-    def _core(self):
-        return self._config.get(Config.SECTION_CORE, {})
-
    def get_remote(self, remote=None, command="<command>"):
        if not remote:
-            remote = self._core.get(Config.SECTION_CORE_REMOTE)
+            remote = self.repo.config["core"].get("remote")

        if remote:
            return self._init_remote(remote)

-        has_non_default_remote = bool(
-            self.repo.config.list_options(
-                Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL
-            )
-        )
-
-        if has_non_default_remote:
+        if bool(self.repo.config["remote"]):
            error_msg = (
                "no remote specified. Setup default remote with\n"
                "    dvc remote default <remote name>\n"
@@ -27,5 +27,5 @@ class DependencyBase(object):
    IsNotFileOrDirError = DependencyIsNotFileOrDirError
    IsStageFileError = DependencyIsStageFileError

-    def update(self):
+    def update(self, rev=None):
        pass
@@ -80,6 +80,9 @@ class DependencyREPO(DependencyLOCAL):

            repo.pull_to(self.def_path, to.path_info)

-    def update(self):
+    def update(self, rev=None):
+        if rev:
+            self.def_repo[self.PARAM_REV] = rev
+
        with self._make_repo(locked=False) as repo:
            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
@@ -7,6 +7,10 @@ class DvcException(Exception):
    """Base class for all dvc exceptions."""


+class InvalidArgumentError(ValueError, DvcException):
+    """Thrown if arguments are invalid."""
+
+
 class OutputDuplicationError(DvcException):
    """Thrown if a file/directory is specified as an output in more than one
    stage.
@@ -112,18 +116,7 @@ class MoveNotDataSourceError(DvcException):


 class NotDvcRepoError(DvcException):
-    """Thrown if a directory is not a DVC repo.
-
-    Args:
-        root (str): path to the directory.
-    """
-
-    def __init__(self, root):
-        msg = (
-            "you are not inside of a DVC repository "
-            "(checked up to mount point '{}')"
-        )
-        super().__init__(msg.format(root))
+    """Thrown if a directory is not a DVC repo"""


 class DvcParserError(DvcException):
@@ -199,17 +192,10 @@ class RecursiveAddingWhileUsingFilename(DvcException):


 class OverlappingOutputPathsError(DvcException):
-    def __init__(self, out_1, out_2):
-        super().__init__(
-            "Paths for outs:\n'{}'('{}')\n'{}'('{}')\noverlap. To avoid "
-            "unpredictable behaviour, rerun command with non overlapping outs "
-            "paths.".format(
-                str(out_1),
-                out_1.stage.relpath,
-                str(out_2),
-                out_2.stage.relpath,
-            )
-        )
+    def __init__(self, parent, overlapping_out, message):
+        self.parent = parent
+        self.overlapping_out = overlapping_out
+        super().__init__(message)


 class CheckoutErrorSuggestGit(DvcException):
@@ -270,10 +256,12 @@ class UploadError(DvcException):


 class CheckoutError(DvcException):
-    def __init__(self, target_infos):
+    def __init__(self, target_infos, stats=None):
+        self.target_infos = target_infos
+        self.stats = stats
        targets = [str(t) for t in target_infos]
        m = (
-            "Checkout failed for following targets:\n {}\nDid you "
+            "Checkout failed for following targets:\n{}\nDid you "
            "forget to fetch?".format("\n".join(targets))
        )
        super().__init__(m)
@@ -305,12 +293,23 @@ class HTTPError(DvcException):


 class PathMissingError(DvcException):
-    def __init__(self, path, repo):
+    default_msg = (
+        "The path '{}' does not exist in the target repository '{}'"
+        " neither as a DVC output nor as a Git-tracked file."
+    )
+    default_msg_output_only = (
+        "The path '{}' does not exist in the target repository '{}'"
+        " as an DVC output."
+    )
+
+    def __init__(self, path, repo, output_only=False):
        msg = (
-            "The path '{}' does not exist in the target repository '{}'"
-            " neither as an output nor a git-handled file."
+            self.default_msg
+            if not output_only
+            else self.default_msg_output_only
        )
        super().__init__(msg.format(path, repo))
+        self.output_only = output_only


 class RemoteCacheRequiredError(DvcException):
@@ -326,3 +325,7 @@ class RemoteCacheRequiredError(DvcException):
                format_link("https://man.dvc.org/config#cache"),
            )
        )
+
+
+class IsADirectoryError(DvcException):
+    """Raised when a file operation is requested on a directory."""
@@ -7,23 +7,25 @@ import threading

 from funcy import retry, suppress, wrap_with, cached_property

+from dvc.path_info import PathInfo
 from dvc.compat import fspath
 from dvc.repo import Repo
-from dvc.config import Config, NoRemoteError, NotDvcRepoError
-from dvc.exceptions import NoRemoteInExternalRepoError
+from dvc.config import NoRemoteError, NotDvcRepoError
+from dvc.exceptions import NoRemoteInExternalRepoError, CheckoutError
 from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
 from dvc.exceptions import FileMissingError, PathMissingError
-from dvc.remote import RemoteConfig
-from dvc.utils.fs import remove, fs_copy
-from dvc.scm import SCM
+from dvc.utils.fs import remove, fs_copy, move
+from dvc.utils import tmp_fname
+from dvc.scm.git import Git


 logger = logging.getLogger(__name__)


@contextmanager
-def external_repo(url, rev=None):
-    path = _cached_clone(url, rev)
+def external_repo(url, rev=None, for_write=False):
+    logger.debug("Creating external repo %s@%s", url, rev)
+    path = _cached_clone(url, rev, for_write=for_write)
    try:
        repo = ExternalRepo(path, url)
    except NotDvcRepoError:
@@ -41,7 +43,8 @@ def external_repo(url, rev=None):
        raise PathMissingError(exc.path, url)
    finally:
        repo.close()
-        _remove(path)
+        if for_write:
+            _remove(path)


 CLONES = {}
@@ -63,38 +66,54 @@ class ExternalRepo(Repo):
        super().__init__(root_dir)
        self.url = url
        self._set_cache_dir()
-        self._set_upstream()
+        self._fix_upstream()

    def pull_to(self, path, to_info):
-        try:
-            out = None
-            with suppress(OutputNotFoundError):
-                out = self.find_out_by_relpath(path)
+        """
+        Pull the corresponding file or directory specified by `path` and
+        checkout it into `to_info`.

+        It works with files tracked by Git and DVC, and also local files
+        outside the repository.
+        """
+        out = None
+        path_info = PathInfo(self.root_dir) / path
+
+        with suppress(OutputNotFoundError):
+            (out,) = self.find_outs_by_path(fspath(path_info), strict=False)
+
+        try:
            if out and out.use_cache:
-                self._pull_cached(out, to_info)
+                self._pull_cached(out, path_info, to_info)
                return

-            # Git handled files can't have absolute path
+            # Check if it is handled by Git (it can't have an absolute path)
            if os.path.isabs(path):
                raise FileNotFoundError

-            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
+            fs_copy(fspath(path_info), fspath(to_info))
        except FileNotFoundError:
            raise PathMissingError(path, self.url)

-    def _pull_cached(self, out, to_info):
+    def _pull_cached(self, out, path_info, dest):
        with self.state:
-            # Only pull unless all needed cache is present
-            if out.changed_cache():
-                self.cloud.pull(out.get_used_cache())
+            tmp = PathInfo(tmp_fname(dest))
+            src = tmp / path_info.relative_to(out.path_info)

-            out.path_info = to_info
-            failed = out.checkout()
-            # This might happen when pull haven't really pulled all the files
-            if failed:
+            out.path_info = tmp
+
+            # Only pull unless all needed cache is present
+            if out.changed_cache(filter_info=src):
+                self.cloud.pull(out.get_used_cache(filter_info=src))
+
+            try:
+                out.checkout(filter_info=src)
+            except CheckoutError:
                raise FileNotFoundError

+            move(src, dest)
+            remove(tmp)
+
    @wrap_with(threading.Lock())
    def _set_cache_dir(self):
        try:
@@ -104,22 +123,35 @@ class ExternalRepo(Repo):

        self.cache.local.cache_dir = cache_dir

-    def _set_upstream(self):
-        # check if the URL is local and no default remote is present
-        # add default remote pointing to the original repo's cache location
-        if os.path.isdir(self.url):
-            rconfig = RemoteConfig(self.config)
-            if not rconfig.has_default():
-                src_repo = Repo(self.url)
-                try:
-                    rconfig.add(
-                        "auto-generated-upstream",
-                        src_repo.cache.local.cache_dir,
-                        default=True,
-                        level=Config.LEVEL_LOCAL,
-                    )
-                finally:
-                    src_repo.close()
+    def _fix_upstream(self):
+        if not os.path.isdir(self.url):
+            return
+
+        remote_name = self.config["core"].get("remote")
+        src_repo = Repo(self.url)
+        try:
+            if remote_name:
+                self._fix_local_remote(src_repo, remote_name)
+            else:
+                self._add_upstream(src_repo)
+        finally:
+            src_repo.close()
+
+    def _fix_local_remote(self, src_repo, remote_name):
+        # If a remote URL is relative to the source repo,
+        # it will have changed upon config load and made
+        # relative to this new repo. Restore the old one here.
+        new_remote = self.config["remote"][remote_name]
+        old_remote = src_repo.config["remote"][remote_name]
+        if new_remote["url"] != old_remote["url"]:
+            new_remote["url"] = old_remote["url"]
+
+    def _add_upstream(self, src_repo):
+        # Fill the empty upstream entry with a new remote pointing to the
+        # original repo's cache location.
+        cache_dir = src_repo.cache.local.cache_dir
+        self.config["remote"]["auto-generated-upstream"] = {"url": cache_dir}
+        self.config["core"]["remote"] = "auto-generated-upstream"


 class ExternalGitRepo:
@@ -129,7 +161,7 @@ class ExternalGitRepo:

    @cached_property
    def scm(self):
-        return SCM(self.root_dir)
+        return Git(self.root_dir)

    def close(self):
        if "scm" in self.__dict__:
@@ -158,45 +190,66 @@ class ExternalGitRepo:
            raise PathMissingError(path, self.url)


-@wrap_with(threading.Lock())
-def _cached_clone(url, rev):
+def _cached_clone(url, rev, for_write=False):
    """Clone an external git repo to a temporary directory.

    Returns the path to a local temporary directory with the specified
-    revision checked out.
+    revision checked out. If for_write is set prevents reusing this dir via
+    cache.
    """
-    # Get or create a clean clone
-    clone_path = CLONES.get(url)
-    if clone_path:
-        # Do not pull for known shas, branches and tags might move
-        if not _is_known_sha(clone_path, rev):
-            _git_pull(clone_path)
-    else:
-        clone_path = tempfile.mkdtemp("dvc-clone")
-        _git_clone(url, clone_path)
-        CLONES[url] = clone_path
+    if not for_write and Git.is_sha(rev) and (url, rev) in CLONES:
+        return CLONES[url, rev]
+
+    clone_path = _clone_default_branch(url, rev)
+    rev_sha = Git(clone_path).resolve_rev(rev or "HEAD")
+
+    if not for_write and (url, rev_sha) in CLONES:
+        return CLONES[url, rev_sha]

    # Copy to a new dir to keep the clone clean
    repo_path = tempfile.mkdtemp("dvc-erepo")
+    logger.debug("erepo: making a copy of %s clone", url)
    copy_tree(clone_path, repo_path)

    # Check out the specified revision
    if rev is not None:
        _git_checkout(repo_path, rev)

+    if not for_write:
+        CLONES[url, rev_sha] = repo_path
    return repo_path


-def _git_clone(url, path):
-    from dvc.scm.git import Git
+@wrap_with(threading.Lock())
+def _clone_default_branch(url, rev):
+    """Get or create a clean clone of the url.

-    git = Git.clone(url, path)
-    git.close()
+    The cloned is reactualized with git pull unless rev is a known sha.
+    """
+    clone_path = CLONES.get(url)
+
+    git = None
+    try:
+        if clone_path:
+            git = Git(clone_path)
+            # Do not pull for known shas, branches and tags might move
+            if not Git.is_sha(rev) or not git.has_rev(rev):
+                logger.debug("erepo: git pull %s", url)
+                git.pull()
+        else:
+            logger.debug("erepo: git clone %s to a temporary dir", url)
+            clone_path = tempfile.mkdtemp("dvc-clone")
+            git = Git.clone(url, clone_path)
+            CLONES[url] = clone_path
+    finally:
+        if git:
+            git.close()
+
+    return clone_path


 def _git_checkout(repo_path, rev):
-    from dvc.scm import Git
-
+    logger.debug("erepo: git checkout %s@%s", repo_path, rev)
    git = Git(repo_path)
    try:
        git.checkout(rev)
@@ -204,31 +257,6 @@ def _git_checkout(repo_path, rev):
        git.close()


-def _is_known_sha(repo_path, rev):
-    import git
-    from gitdb.exc import BadName
-
-    if not rev or not git.Repo.re_hexsha_shortened.search(rev):
-        return False
-
-    try:
-        git.Repo(repo_path).commit(rev)
-        return True
-    except BadName:
-        return False
-
-
-def _git_pull(repo_path):
-    import git
-
-    repo = git.Repo(repo_path)
-    try:
-        msg = repo.git.pull()
-        logger.debug("external repo: git pull: {}", msg)
-    finally:
-        repo.close()
-
-
 def _remove(path):
    if os.name == "nt":
        # git.exe may hang for a while not permitting to remove temp dir
@@ -71,10 +71,25 @@ class DvcIgnoreDirs(DvcIgnore):
        return self.basenames == other.basenames


+class DvcIgnoreRepo(DvcIgnore):
+    def __call__(self, root, dirs, files):
+        def is_dvc_repo(directory):
+            from dvc.repo import Repo
+
+            return os.path.isdir(os.path.join(directory, Repo.DVC_DIR))
+
+        dirs = [d for d in dirs if not is_dvc_repo(d)]
+
+        return dirs, files
+
+
 class DvcIgnoreFilter(object):
    def __init__(self, tree):
        self.tree = tree
-        self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
+        self.ignores = {
+            DvcIgnoreDirs([".git", ".hg", ".dvc"]),
+            DvcIgnoreRepo(),
+        }
        for root, dirs, files in self.tree.walk(self.tree.tree_root):
            self._update(root)
            dirs[:], files[:] = self(root, dirs, files)
@@ -1,6 +1,5 @@
 """Manages logging configuration for DVC repo."""

-import traceback
 import logging.config
 import logging.handlers

@@ -37,106 +36,65 @@ class ExcludeInfoFilter(logging.Filter):


 class ColorFormatter(logging.Formatter):
-    """Enable color support when logging to a terminal that supports it.
+    """Spit out colored text in supported terminals.

-    Color support on Windows versions that do not support ANSI color codes is
-    enabled by use of the colorama__ library.
+    colorama__ makes ANSI escape character sequences work under Windows.
    See the colorama documentation for details.

    __ https://pypi.python.org/pypi/colorama

-    For records containing `exc_info`, it will use a custom `_walk_exc` to
-    retrieve the whole traceback.
+    If record has an extra `tb_only` attribute, it will not show the
+    exception cause, just the message and the traceback.
    """

    color_code = {
        "DEBUG": colorama.Fore.BLUE,
-        "INFO": "",
        "WARNING": colorama.Fore.YELLOW,
        "ERROR": colorama.Fore.RED,
        "CRITICAL": colorama.Fore.RED,
    }

    def format(self, record):
-        msg = record.msg.format(*record.args) if record.args else record.msg
+        record.message = record.getMessage()
+        msg = self.formatMessage(record)

        if record.levelname == "INFO":
            return msg

-        if record.levelname == "ERROR" or record.levelname == "CRITICAL":
-            exception, stack_trace = self._parse_exc(record)
+        if record.exc_info:
+            if getattr(record, "tb_only", False):
+                cause = ""
+            else:
+                cause = ": ".join(_iter_causes(record.exc_info[1]))

-            return (
-                "{color}{levelname}{nc}: {description}" "{stack_trace}\n"
-            ).format(
-                color=self.color_code.get(record.levelname, ""),
-                nc=colorama.Fore.RESET,
-                levelname=record.levelname,
-                description=self._description(msg, exception),
-                msg=msg,
-                stack_trace=stack_trace,
+            msg = "{message}{separator}{cause}".format(
+                message=msg or "",
+                separator=" - " if msg and cause else "",
+                cause=cause,
            )

-        return "{color}{levelname}{nc}: {msg}".format(
-            color=self.color_code.get(record.levelname, ""),
+            if _is_verbose():
+                msg += _stack_trace(record.exc_info)
+
+        return "{asctime}{color}{levelname}{nc}: {msg}".format(
+            asctime=self.formatTime(record, self.datefmt),
+            color=self.color_code[record.levelname],
            nc=colorama.Fore.RESET,
            levelname=record.levelname,
            msg=msg,
        )

-    def _current_level(self):
-        return logging.getLogger("dvc").getEffectiveLevel()
+    def formatTime(self, record, datefmt=None):
+        # only show if current level is set to DEBUG
+        # also, skip INFO as it is used for UI
+        if not _is_verbose() or record.levelno == logging.INFO:
+            return ""

-    def _is_visible(self, record):
-        return record.levelno >= self._current_level()
-
-    def _description(self, message, exception):
-        description = ""
-
-        if exception and message:
-            description = "{message} - {exception}"
-        elif exception:
-            description = "{exception}"
-        elif message:
-            description = "{message}"
-
-        return description.format(message=message, exception=exception)
-
-    def _walk_exc(self, exc_info):
-        exc = exc_info[1]
-
-        exc_list = [str(exc)]
-
-        while hasattr(exc, "__cause__") and exc.__cause__:
-            exc_list.append(str(exc.__cause__))
-            exc = exc.__cause__
-
-        return exc_list
-
-    def _parse_exc(self, record):
-        tb_only = getattr(record, "tb_only", False)
-
-        if not record.exc_info:
-            return (None, "")
-
-        exc_list = self._walk_exc(record.exc_info)
-        tb = traceback.format_exception(*record.exc_info)
-
-        exception = None if tb_only else ": ".join(exc_list)
-
-        if self._current_level() == logging.DEBUG:
-            stack_trace = (
-                "\n" "{red}{line}{nc}\n" "{stack_trace}" "{red}{line}{nc}"
-            ).format(
-                red=colorama.Fore.RED,
-                nc=colorama.Fore.RESET,
-                line="-" * 60,
-                stack_trace="".join(tb),
-            )
-        else:
-            stack_trace = ""
-
-        return (exception, stack_trace)
+        return "{green}{date}{nc} ".format(
+            green=colorama.Fore.GREEN,
+            date=super().formatTime(record, datefmt),
+            nc=colorama.Fore.RESET,
+        )


 class LoggerHandler(logging.StreamHandler):
@@ -158,6 +116,39 @@ class LoggerHandler(logging.StreamHandler):
            self.handleError(record)


+def _is_verbose():
+    return logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG
+
+
+def _iter_causes(exc):
+    while exc:
+        yield str(exc)
+        exc = exc.__cause__
+
+
+def _stack_trace(exc_info):
+    import traceback
+
+    return (
+        "\n"
+        "{red}{line}{nc}\n"
+        "{trace}"
+        "{red}{line}{nc}".format(
+            red=colorama.Fore.RED,
+            line="-" * 60,
+            trace="".join(traceback.format_exception(*exc_info)),
+            nc=colorama.Fore.RESET,
+        )
+    )
+
+
+def disable_other_loggers():
+    root = logging.root
+    for (logger_name, logger) in root.manager.loggerDict.items():
+        if logger_name != "dvc" and not logger_name.startswith("dvc."):
+            logger.disabled = True
+
+
 def setup(level=logging.INFO):
    colorama.init()

@@ -200,22 +191,7 @@ def setup(level=logging.INFO):
                        "console_errors",
                    ],
                },
-                "paramiko": {
-                    "level": logging.CRITICAL,
-                    "handlers": [
-                        "console_info",
-                        "console_debug",
-                        "console_errors",
-                    ],
-                },
-                "flufl.lock": {
-                    "level": logging.CRITICAL,
-                    "handlers": [
-                        "console_info",
-                        "console_debug",
-                        "console_errors",
-                    ],
-                },
            },
+            "disable_existing_loggers": False,
        }
    )
@@ -9,7 +9,7 @@ from dvc.config import ConfigError
 from dvc.exceptions import DvcParserError
 from dvc.exceptions import NotDvcRepoError
 from dvc.external_repo import clean_repos
-from dvc.logger import FOOTER
+from dvc.logger import disable_other_loggers, FOOTER
 from dvc.utils import format_link
 from dvc.remote.pool import close_pools

@@ -34,6 +34,7 @@ def main(argv=None):
    """
    args = None
    cmd = None
+    disable_other_loggers()

    outerLogLevel = logger.level
    try:
@@ -1,5 +1,5 @@
 from urllib.parse import urlparse
-from voluptuous import Any, Required
+from voluptuous import Any, Required, Lower, Length, Coerce, And, SetTo

 from dvc.output.base import OutputBase
 from dvc.output.gs import OutputGS
@@ -29,6 +29,12 @@ OUTS_MAP = {
    Schemes.LOCAL: OutputLOCAL,
 }

+CHECKSUM_SCHEMA = Any(
+    None,
+    And(str, Length(max=0), SetTo(None)),
+    And(Any(str, And(int, Coerce(str))), Length(min=3), Lower),
+)
+
 # NOTE: currently there are only 3 possible checksum names:
 #
 #    1) md5 (LOCAL, SSH, GS);
@@ -37,15 +43,15 @@ OUTS_MAP = {
 #
 # so when a few types of outputs share the same name, we only need
 # specify it once.
-CHECKSUM_SCHEMA = {
-    RemoteLOCAL.PARAM_CHECKSUM: Any(str, None),
-    RemoteS3.PARAM_CHECKSUM: Any(str, None),
-    RemoteHDFS.PARAM_CHECKSUM: Any(str, None),
+CHECKSUMS_SCHEMA = {
+    RemoteLOCAL.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    RemoteS3.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    RemoteHDFS.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
 }

-TAGS_SCHEMA = {str: CHECKSUM_SCHEMA}
+TAGS_SCHEMA = {str: CHECKSUMS_SCHEMA}

-SCHEMA = CHECKSUM_SCHEMA.copy()
+SCHEMA = CHECKSUMS_SCHEMA.copy()
 SCHEMA[Required(OutputBase.PARAM_PATH)] = str
 SCHEMA[OutputBase.PARAM_CACHE] = bool
 SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA
@@ -162,11 +162,11 @@ class OutputBase(object):
    def changed_checksum(self):
        return self.checksum != self.remote.get_checksum(self.path_info)

-    def changed_cache(self):
+    def changed_cache(self, filter_info=None):
        if not self.use_cache or not self.checksum:
            return True

-        return self.cache.changed_cache(self.checksum)
+        return self.cache.changed_cache(self.checksum, filter_info=filter_info)

    def status(self):
        if self.checksum and self.use_cache and self.changed_cache():
@@ -409,12 +409,23 @@ class OutputBase(object):
            cache.external[dep.repo_pair].add(dep.def_path)
            return cache

-        if not self.info:
-            logger.warning(
-                "Output '{}'({}) is missing version info. Cache for it will "
-                "not be collected. Use `dvc repro` to get your pipeline up to "
-                "date.".format(self, self.stage)
+        if not self.checksum:
+            msg = (
+                "Output '{}'({}) is missing version info. "
+                "Cache for it will not be collected. "
+                "Use `dvc repro` to get your pipeline up to date.".format(
+                    self, self.stage
+                )
            )
+            if self.exists:
+                msg += (
+                    "\n"
+                    "You can also use `dvc commit {stage}` to associate "
+                    "existing '{out}' with '{stage}'.".format(
+                        out=self, stage=self.stage.relpath
+                    )
+                )
+            logger.warning(msg)
            return NamedCache()

        ret = NamedCache.make(self.scheme, self.checksum, str(self))
@@ -18,6 +18,12 @@ class OutputLOCAL(OutputBase):
    REMOTE = RemoteLOCAL
    sep = os.sep

+    def __init__(self, stage, path, *args, **kwargs):
+        if stage and path_isin(path, stage.repo.root_dir):
+            path = relpath(path, stage.wdir)
+
+        super().__init__(stage, path, *args, **kwargs)
+
    def _parse_path(self, remote, path):
        parsed = urlparse(path)
        if parsed.scheme == "remote":
@@ -241,3 +241,79 @@ class CloudURLInfo(URLInfo):
    @property
    def path(self):
        return self._spath.lstrip("/")
+
+
+class HTTPURLInfo(URLInfo):
+    def __init__(self, url):
+        p = urlparse(url)
+        stripped = p._replace(params=None, query=None, fragment=None)
+        super().__init__(stripped.geturl())
+        self.params = p.params
+        self.query = p.query
+        self.fragment = p.fragment
+
+    @classmethod
+    def from_parts(
+        cls,
+        scheme=None,
+        host=None,
+        user=None,
+        port=None,
+        path="",
+        netloc=None,
+        params=None,
+        query=None,
+        fragment=None,
+    ):
+        assert bool(host) ^ bool(netloc)
+
+        if netloc is not None:
+            return cls(
+                "{}://{}{}{}{}{}".format(
+                    scheme,
+                    netloc,
+                    path,
+                    (";" + params) if params else "",
+                    ("?" + query) if query else "",
+                    ("#" + fragment) if fragment else "",
+                )
+            )
+
+        obj = cls.__new__(cls)
+        obj.fill_parts(scheme, host, user, port, path)
+        obj.params = params
+        obj.query = query
+        obj.fragment = fragment
+        return obj
+
+    @property
+    def _extra_parts(self):
+        return (self.params, self.query, self.fragment)
+
+    @property
+    def parts(self):
+        return self._base_parts + self._path.parts + self._extra_parts
+
+    @cached_property
+    def url(self):
+        return "{}://{}{}{}{}{}".format(
+            self.scheme,
+            self.netloc,
+            self._spath,
+            (";" + self.params) if self.params else "",
+            ("?" + self.query) if self.query else "",
+            ("#" + self.fragment) if self.fragment else "",
+        )
+
+    def __eq__(self, other):
+        if isinstance(other, (str, bytes)):
+            other = self.__class__(other)
+        return (
+            self.__class__ == other.__class__
+            and self._base_parts == other._base_parts
+            and self._path == other._path
+            and self._extra_parts == other._extra_parts
+        )
+
+    def __hash__(self):
+        return hash(self.parts)
@@ -55,7 +55,8 @@ class Tqdm(tqdm):
        desc  : persists after `close()`
        level  : effective logging level for determining `disable`;
            used only if `disable` is unspecified
-        disable  : If (default: None), will be determined by logging level.
+        disable  : If (default: None) or False,
+            will be determined by logging level.
            May be overridden to `True` due to non-TTY status.
            Skip override by specifying env var `DVC_IGNORE_ISATTY`.
        kwargs  : anything accepted by `tqdm.tqdm()`
@@ -69,7 +70,7 @@ class Tqdm(tqdm):
            file = sys.stderr
        self.desc_persist = desc
        # auto-disable based on `logger.level`
-        if disable is None:
+        if not disable:
            disable = logger.getEffectiveLevel() > level
        # auto-disable based on TTY
        if (
@@ -1,4 +1,6 @@
-from .config import RemoteConfig
+import posixpath
+from urllib.parse import urlparse
+
 from dvc.remote.azure import RemoteAZURE
 from dvc.remote.gdrive import RemoteGDrive
 from dvc.remote.gs import RemoteGS
@@ -25,9 +27,9 @@ REMOTES = [
 ]


-def _get(config):
+def _get(remote_conf):
    for remote in REMOTES:
-        if remote.supported(config):
+        if remote.supported(remote_conf):
            return remote
    return RemoteLOCAL

@@ -35,8 +37,39 @@ def _get(config):
 def Remote(repo, **kwargs):
    name = kwargs.get("name")
    if name:
-        remote_config = RemoteConfig(repo.config)
-        settings = remote_config.get_settings(name)
+        remote_conf = repo.config["remote"][name.lower()]
    else:
-        settings = kwargs
-    return _get(settings)(repo, settings)
+        remote_conf = kwargs
+    remote_conf = _resolve_remote_refs(repo.config, remote_conf)
+    return _get(remote_conf)(repo, remote_conf)
+
+
+def _resolve_remote_refs(config, remote_conf):
+    # Support for cross referenced remotes.
+    # This will merge the settings, shadowing base ref with remote_conf.
+    # For example, having:
+    #
+    #       dvc remote add server ssh://localhost
+    #       dvc remote modify server user root
+    #       dvc remote modify server ask_password true
+    #
+    #       dvc remote add images remote://server/tmp/pictures
+    #       dvc remote modify images user alice
+    #       dvc remote modify images ask_password false
+    #       dvc remote modify images password asdf1234
+    #
+    # Results on a config dictionary like:
+    #
+    #       {
+    #           "url": "ssh://localhost/tmp/pictures",
+    #           "user": "alice",
+    #           "password": "asdf1234",
+    #           "ask_password": False,
+    #       }
+    parsed = urlparse(remote_conf["url"])
+    if parsed.scheme != "remote":
+        return remote_conf
+
+    base = config["remote"][parsed.netloc]
+    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
+    return {**base, **remote_conf, "url": url}
@@ -7,7 +7,6 @@ import threading

 from funcy import cached_property, wrap_prop

-from dvc.config import Config
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
@@ -33,7 +32,7 @@ class RemoteAZURE(RemoteBASE):
    def __init__(self, repo, config):
        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL, "azure://")
+        url = config.get("url", "azure://")

        match = re.match(self.REGEX, url)  # backward compatibility
        path = match.group("path")
@@ -44,7 +43,7 @@ class RemoteAZURE(RemoteBASE):
        )

        self.connection_string = (
-            config.get(Config.SECTION_AZURE_CONNECTION_STRING)
+            config.get("connection_string")
            or match.group("connection_string")  # backward compatibility
            or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
        )
@@ -13,8 +13,8 @@ from operator import itemgetter
 from shortuuid import uuid

 import dvc.prompt as prompt
-from dvc.config import Config
 from dvc.exceptions import (
+    CheckoutError,
    DvcException,
    ConfirmRemoveError,
    DvcIgnoreInCollectedDirError,
@@ -83,49 +83,50 @@ class RemoteBASE(object):
    DEFAULT_NO_TRAVERSE = True
    DEFAULT_VERIFY = False

+    CACHE_MODE = None
+    SHARED_MODE_MAP = {None: (None, None), "group": (None, None)}
+
    state = StateNoop()

    def __init__(self, repo, config):
        self.repo = repo

        self._check_requires(config)
-        self.checksum_jobs = self._get_checksum_jobs(config)

-        self.protected = False
-        self.no_traverse = config.get(
-            Config.SECTION_REMOTE_NO_TRAVERSE, self.DEFAULT_NO_TRAVERSE
-        )
-        self.verify = config.get(
-            Config.SECTION_REMOTE_VERIFY, self.DEFAULT_VERIFY
+        shared = config.get("shared")
+        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
+
+        self.checksum_jobs = (
+            config.get("checksum_jobs")
+            or (self.repo and self.repo.config["core"].get("checksum_jobs"))
+            or self.CHECKSUM_JOBS
        )
+        self.no_traverse = config.get("no_traverse", self.DEFAULT_NO_TRAVERSE)
+        self.verify = config.get("verify", self.DEFAULT_VERIFY)
        self._dir_info = {}

-        types = config.get(Config.SECTION_CACHE_TYPE, None)
-        if types:
-            if isinstance(types, str):
-                types = [t.strip() for t in types.split(",")]
-            self.cache_types = types
-        else:
-            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
+        self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES)
        self.cache_type_confirmed = False

-    def _check_requires(self, config):
+    @classmethod
+    def get_missing_deps(cls):
        import importlib

        missing = []
-
-        for package, module in self.REQUIRES.items():
+        for package, module in cls.REQUIRES.items():
            try:
                importlib.import_module(module)
            except ImportError:
                missing.append(package)

+        return missing
+
+    def _check_requires(self, config):
+        missing = self.get_missing_deps()
        if not missing:
            return

-        url = config.get(
-            Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
-        )
+        url = config.get("url", "{}://".format(self.scheme))
        msg = (
            "URL '{}' is supported but requires these missing "
            "dependencies: {}. If you have installed dvc using pip, "
@@ -146,19 +147,6 @@ class RemoteBASE(object):
        ).format(url, missing, " ".join(missing), self.scheme)
        raise RemoteMissingDepsError(msg)

-    def _get_checksum_jobs(self, config):
-        checksum_jobs = config.get(Config.SECTION_REMOTE_CHECKSUM_JOBS)
-        if checksum_jobs:
-            return checksum_jobs
-
-        if self.repo:
-            core = self.repo.config.config.get(Config.SECTION_CORE, {})
-            return core.get(
-                Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
-            )
-
-        return self.CHECKSUM_JOBS
-
    def __repr__(self):
        return "{class_name}: '{path_info}'".format(
            class_name=type(self).__name__,
@@ -170,7 +158,7 @@ class RemoteBASE(object):
        if isinstance(config, (str, bytes)):
            url = config
        else:
-            url = config[Config.SECTION_REMOTE_URL]
+            url = config["url"]

        # NOTE: silently skipping remote, calling code should handle that
        parsed = urlparse(url)
@@ -242,7 +230,7 @@ class RemoteBASE(object):
        new_info = self.cache.checksum_to_path_info(checksum)
        if self.cache.changed_cache_file(checksum):
            self.cache.makedirs(new_info.parent)
-            self.cache.move(tmp_info, new_info)
+            self.cache.move(tmp_info, new_info, mode=self.CACHE_MODE)

        self.state.save(path_info, checksum)
        self.state.save(new_info, checksum)
@@ -287,7 +275,7 @@ class RemoteBASE(object):

        if not isinstance(d, list):
            logger.error(
-                "dir cache file format error '{}' [skipping the file]",
+                "dir cache file format error '%s' [skipping the file]",
                path_info,
            )
            return []
@@ -355,35 +343,35 @@ class RemoteBASE(object):
        """

        logger.debug(
-            "checking if '{}'('{}') has changed.", path_info, checksum_info
+            "checking if '%s'('%s') has changed.", path_info, checksum_info
        )

        if not self.exists(path_info):
-            logger.debug("'{}' doesn't exist.", path_info)
+            logger.debug("'%s' doesn't exist.", path_info)
            return True

        checksum = checksum_info.get(self.PARAM_CHECKSUM)
        if checksum is None:
-            logger.debug("hash value for '{}' is missing.", path_info)
+            logger.debug("hash value for '%s' is missing.", path_info)
            return True

        if self.changed_cache(checksum):
            logger.debug(
-                "cache for '{}'('{}') has changed.", path_info, checksum
+                "cache for '%s'('%s') has changed.", path_info, checksum
            )
            return True

        actual = self.get_checksum(path_info)
        if checksum != actual:
            logger.debug(
-                "hash value '{}' for '{}' has changed (actual '{}').",
+                "hash value '%s' for '%s' has changed (actual '%s').",
                checksum,
                actual,
                path_info,
            )
            return True

-        logger.debug("'{}' hasn't changed.", path_info)
+        logger.debug("'%s' hasn't changed.", path_info)
        return False

    def link(self, from_info, to_info):
@@ -418,7 +406,7 @@ class RemoteBASE(object):

            except DvcException as exc:
                logger.debug(
-                    "Cache type '{}' is not supported: {}", link_types[0], exc
+                    "Cache type '%s' is not supported: %s", link_types[0], exc
                )
                del link_types[0]

@@ -430,15 +418,8 @@ class RemoteBASE(object):

        link_method(from_info, to_info)

-        if self.protected:
-            self.protect(to_info)
-
        logger.debug(
-            "Created {}'{}': {} -> {}",
-            "protected " if self.protected else "",
-            self.cache_types[0],
-            from_info,
-            to_info,
+            "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
        )

    def _save_file(self, path_info, checksum, save_link=True):
@@ -446,14 +427,11 @@ class RemoteBASE(object):

        cache_info = self.checksum_to_path_info(checksum)
        if self.changed_cache(checksum):
-            self.move(path_info, cache_info)
+            self.move(path_info, cache_info, mode=self.CACHE_MODE)
            self.link(cache_info, path_info)
        elif self.iscopy(path_info) and self._cache_is_copy(path_info):
            # Default relink procedure involves unneeded copy
-            if self.protected:
-                self.protect(path_info)
-            else:
-                self.unprotect(path_info)
+            self.unprotect(path_info)
        else:
            self.remove(path_info)
            self.link(cache_info, path_info)
@@ -543,7 +521,7 @@ class RemoteBASE(object):

    def _save(self, path_info, checksum):
        to_info = self.checksum_to_path_info(checksum)
-        logger.debug("Saving '{}' to '{}'.", path_info, to_info)
+        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
        if self.isdir(path_info):
            self._save_dir(path_info, checksum)
            return
@@ -556,7 +534,7 @@ class RemoteBASE(object):
            raise exception

        logger.exception(
-            "failed to {} '{}' to '{}'", operation, from_info, to_info
+            "failed to %s '%s' to '%s'", operation, from_info, to_info
        )
        return 1

@@ -570,7 +548,7 @@ class RemoteBASE(object):
        if from_info.scheme != "local":
            raise NotImplementedError

-        logger.debug("Uploading '{}' to '{}'", from_info, to_info)
+        logger.debug("Uploading '%s' to '%s'", from_info, to_info)

        name = name or from_info.name

@@ -649,7 +627,7 @@ class RemoteBASE(object):
    ):
        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)

-        logger.debug("Downloading '{}' to '{}'", from_info, to_info)
+        logger.debug("Downloading '%s' to '%s'", from_info, to_info)
        name = name or to_info.name

        tmp_file = tmp_fname(to_info)
@@ -677,7 +655,8 @@ class RemoteBASE(object):
    def remove(self, path_info):
        raise RemoteActionNotImplemented("remove", self.scheme)

-    def move(self, from_info, to_info):
+    def move(self, from_info, to_info, mode=None):
+        assert mode is None
        self.copy(from_info, to_info)
        self.remove(from_info)

@@ -739,6 +718,9 @@ class RemoteBASE(object):
            removed = True
        return removed

+    def is_protected(self, path_info):
+        return False
+
    def changed_cache_file(self, checksum):
        """Compare the given checksum with the (corresponding) actual one.

@@ -751,11 +733,18 @@ class RemoteBASE(object):

        - Remove the file from cache if it doesn't match the actual checksum
        """
+
        cache_info = self.checksum_to_path_info(checksum)
+        if self.is_protected(cache_info):
+            logger.debug(
+                "Assuming '%s' is unchanged since it is read-only", cache_info
+            )
+            return False
+
        actual = self.get_checksum(cache_info)

        logger.debug(
-            "cache '{}' expected '{}' actual '{}'",
+            "cache '%s' expected '%s' actual '%s'",
            cache_info,
            checksum,
            actual,
@@ -765,10 +754,13 @@ class RemoteBASE(object):
            return True

        if actual.split(".")[0] == checksum.split(".")[0]:
+            # making cache file read-only so we don't need to check it
+            # next time
+            self.protect(cache_info)
            return False

        if self.exists(cache_info):
-            logger.warning("corrupted cache file '{}'.", cache_info)
+            logger.warning("corrupted cache file '%s'.", cache_info)
            self.remove(cache_info)

        return True
@@ -873,15 +865,15 @@ class RemoteBASE(object):
        self.remove(path_info)

    def _checkout_file(
-        self, path_info, checksum, force, progress_callback=None
+        self, path_info, checksum, force, progress_callback=None, relink=False
    ):
        """The file is changed we need to checkout a new copy"""
+        added, modified = True, False
        cache_info = self.checksum_to_path_info(checksum)
        if self.exists(path_info):
-            logger.warning(
-                "data '{}' exists. Removing before checkout.", path_info
-            )
+            logger.debug("data '%s' will be replaced.", path_info)
            self.safe_remove(path_info, force=force)
+            added, modified = False, True

        self.link(cache_info, path_info)
        self.state.save_link(path_info)
@@ -889,11 +881,12 @@ class RemoteBASE(object):
        if progress_callback:
            progress_callback(str(path_info))

+        return added, modified and not relink
+
    def makedirs(self, path_info):
        """Optional: Implement only if the remote needs to create
        directories before copying/linking/moving data
        """
-        pass

    def _checkout_dir(
        self,
@@ -904,14 +897,16 @@ class RemoteBASE(object):
        relink=False,
        filter_info=None,
    ):
+        added, modified = False, False
        # Create dir separately so that dir is created
        # even if there are no files in it
        if not self.exists(path_info):
+            added = True
            self.makedirs(path_info)

        dir_info = self.get_dir_cache(checksum)

-        logger.debug("Linking directory '{}'.", path_info)
+        logger.debug("Linking directory '%s'.", path_info)

        for entry in dir_info:
            relative_path = entry[self.PARAM_RELPATH]
@@ -924,27 +919,36 @@ class RemoteBASE(object):

            entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
            if relink or self.changed(entry_info, entry_checksum_info):
+                modified = True
                self.safe_remove(entry_info, force=force)
                self.link(entry_cache_info, entry_info)
                self.state.save(entry_info, entry_checksum)
            if progress_callback:
                progress_callback(str(entry_info))

-        self._remove_redundant_files(path_info, dir_info, force)
+        modified = (
+            self._remove_redundant_files(path_info, dir_info, force)
+            or modified
+        )

        self.state.save_link(path_info)
        self.state.save(path_info, checksum)

+        # relink is not modified, assume it as nochange
+        return added, not added and modified and not relink
+
    def _remove_redundant_files(self, path_info, dir_info, force):
        existing_files = set(self.walk_files(path_info))

        needed_files = {
            path_info / entry[self.PARAM_RELPATH] for entry in dir_info
        }
-
-        for path in existing_files - needed_files:
+        redundant_files = existing_files - needed_files
+        for path in redundant_files:
            self.safe_remove(path, force)

+        return bool(redundant_files)
+
    def checkout(
        self,
        path_info,
@@ -962,21 +966,21 @@ class RemoteBASE(object):
        skip = False
        if not checksum:
            logger.warning(
-                "No file hash info found for '{}'. " "It won't be created.",
+                "No file hash info found for '%s'. " "It won't be created.",
                path_info,
            )
            self.safe_remove(path_info, force=force)
            failed = path_info

        elif not relink and not self.changed(path_info, checksum_info):
-            logger.debug("Data '{}' didn't change.", path_info)
+            logger.debug("Data '%s' didn't change.", path_info)
            skip = True

        elif self.changed_cache(
            checksum, path_info=path_info, filter_info=filter_info
        ):
            logger.warning(
-                "Cache '{}' not found. File '{}' won't be created.",
+                "Cache '%s' not found. File '%s' won't be created.",
                checksum,
                path_info,
            )
@@ -991,12 +995,14 @@ class RemoteBASE(object):
                        self.path_info, checksum, filter_info
                    ),
                )
-            return failed
+            if failed:
+                raise CheckoutError([failed])
+            return

-        logger.debug("Checking out '{}' with cache '{}'.", path_info, checksum)
+        logger.debug("Checking out '%s' with cache '%s'.", path_info, checksum)

-        self._checkout(
-            path_info, checksum, force, progress_callback, relink, filter_info
+        return self._checkout(
+            path_info, checksum, force, progress_callback, relink, filter_info,
        )

    def _checkout(
@@ -1010,8 +1016,9 @@ class RemoteBASE(object):
    ):
        if not self.is_dir_checksum(checksum):
            return self._checkout_file(
-                path_info, checksum, force, progress_callback=progress_callback
+                path_info, checksum, force, progress_callback, relink
            )
+
        return self._checkout_dir(
            path_info, checksum, force, progress_callback, relink, filter_info
        )
@@ -1,163 +0,0 @@
-import logging
-import os
-import posixpath
-from urllib.parse import urlparse
-
-from dvc.config import Config, ConfigError
-from dvc.utils import relpath
-
-
-logger = logging.getLogger(__name__)
-
-
-class RemoteConfig(object):
-    def __init__(self, config):
-        self.config = config
-
-    def get_settings(self, name):
-        """
-        Args:
-            name (str): The name of the remote that we want to retrieve
-
-        Returns:
-            dict: The content beneath the given remote name.
-
-        Example:
-            >>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
-            >>> get_settings("server")
-            {'url': 'ssh://localhost/'}
-        """
-        settings = self.config.config.get(
-            Config.SECTION_REMOTE_FMT.format(name.lower())
-        )
-
-        if settings is None:
-            raise ConfigError(
-                "Unable to find remote section '{}'".format(name)
-            )
-
-        parsed = urlparse(settings["url"])
-
-        # Support for cross referenced remotes.
-        # This will merge the settings, giving priority to the outer reference.
-        # For example, having:
-        #
-        #       dvc remote add server ssh://localhost
-        #       dvc remote modify server user root
-        #       dvc remote modify server ask_password true
-        #
-        #       dvc remote add images remote://server/tmp/pictures
-        #       dvc remote modify images user alice
-        #       dvc remote modify images ask_password false
-        #       dvc remote modify images password asdf1234
-        #
-        # Results on a config dictionary like:
-        #
-        #       {
-        #           "url": "ssh://localhost/tmp/pictures",
-        #           "user": "alice",
-        #           "password": "asdf1234",
-        #           "ask_password": False,
-        #       }
-        #
-        if parsed.scheme == "remote":
-            reference = self.get_settings(parsed.netloc)
-            url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
-            merged = reference.copy()
-            merged.update(settings)
-            merged["url"] = url
-            return merged
-
-        return settings
-
-    @staticmethod
-    def resolve_path(path, config_file):
-        """Resolve path relative to config file location.
-
-        Args:
-            path: Path to be resolved.
-            config_file: Path to config file, which `path` is specified
-                relative to.
-
-        Returns:
-            Path relative to the `config_file` location. If `path` is an
-            absolute path then it will be returned without change.
-
-        """
-        if os.path.isabs(path):
-            return path
-        return relpath(path, os.path.dirname(config_file))
-
-    def add(self, name, url, default=False, force=False, level=None):
-        from dvc.remote import _get, RemoteLOCAL
-
-        configobj = self.config.get_configobj(level)
-        remote = _get({Config.SECTION_REMOTE_URL: url})
-        if remote == RemoteLOCAL and not url.startswith("remote://"):
-            url = self.resolve_path(url, configobj.filename)
-
-        self.config.set(
-            Config.SECTION_REMOTE_FMT.format(name),
-            Config.SECTION_REMOTE_URL,
-            url,
-            force=force,
-            level=level,
-        )
-        if default:
-            self.config.set(
-                Config.SECTION_CORE,
-                Config.SECTION_CORE_REMOTE,
-                name,
-                level=level,
-            )
-
-    def remove(self, name, level=None):
-        self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level)
-
-        if level is None:
-            level = Config.LEVEL_REPO
-
-        for lev in [
-            Config.LEVEL_LOCAL,
-            Config.LEVEL_REPO,
-            Config.LEVEL_GLOBAL,
-            Config.LEVEL_SYSTEM,
-        ]:
-            self.config.unset(
-                Config.SECTION_CORE,
-                Config.SECTION_CORE_REMOTE,
-                level=lev,
-                force=True,
-            )
-            if lev == level:
-                break
-
-    def modify(self, name, option, value, level=None):
-        # try to get remote settings to verify that section exists
-        self.get_settings(name)
-
-        self.config.set(
-            Config.SECTION_REMOTE_FMT.format(name), option, value, level=level
-        )
-
-    def list(self, level=None):
-        return self.config.list_options(
-            Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level
-        )
-
-    def set_default(self, name, unset=False, level=None):
-        if unset:
-            self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE)
-            return
-        self.config.set(
-            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level
-        )
-
-    def get_default(self, level=None):
-        return self.config.get(
-            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level
-        )
-
-    def has_default(self):
-        core = self.config.config[Config.SECTION_CORE]
-        return bool(core.get(Config.SECTION_CORE_REMOTE))
@@ -1,17 +1,18 @@
+from collections import defaultdict
 import os
 import posixpath
 import logging
-import threading
 import re
+import threading
+from urllib.parse import urlparse

-from funcy import retry, compose, decorator, wrap_with
+from funcy import retry, wrap_with, wrap_prop, cached_property
 from funcy.py3 import cat

 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
 from dvc.path_info import CloudURLInfo
 from dvc.remote.base import RemoteBASE
-from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.utils import tmp_fname, format_link

@@ -19,100 +20,240 @@ logger = logging.getLogger(__name__)
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"


-class GDriveRetriableError(DvcException):
-    pass
+class GDrivePathNotFound(DvcException):
+    def __init__(self, path_info):
+        super().__init__("Google Drive path '{}' not found.".format(path_info))


 class GDriveAccessTokenRefreshError(DvcException):
-    pass
+    def __init__(self):
+        super().__init__("Google Drive access token refreshment is failed.")


 class GDriveMissedCredentialKeyError(DvcException):
-    pass
+    def __init__(self, path):
+        super().__init__(
+            "Google Drive user credentials file '{}' "
+            "misses value for key.".format(path)
+        )


-@decorator
-def _wrap_pydrive_retriable(call):
-    from pydrive2.files import ApiRequestError
+def gdrive_retry(func):
+    retry_re = re.compile(r"HttpError (403|500|502|503|504)")

-    try:
-        result = call()
-    except ApiRequestError as exception:
-        retry_codes = ["403", "500", "502", "503", "504"]
-        if any(
-            "HttpError {}".format(code) in str(exception)
-            for code in retry_codes
-        ):
-            raise GDriveRetriableError("Google API request failed")
-        raise
-    return result
+    def should_retry(exc):
+        from pydrive2.files import ApiRequestError
+
+        return isinstance(exc, ApiRequestError) and retry_re.search(str(exc))
+
+    # 15 tries, start at 0.5s, multiply by golden ratio, cap at 20s
+    return retry(
+        15,
+        timeout=lambda a: min(0.5 * 1.618 ** a, 20),
+        filter_errors=should_retry,
+    )(func)


-gdrive_retry = compose(
-    # 8 tries, start at 0.5s, multiply by golden ratio, cap at 10s
-    retry(
-        8, GDriveRetriableError, timeout=lambda a: min(0.5 * 1.618 ** a, 10)
-    ),
-    _wrap_pydrive_retriable,
-)
+class GDriveURLInfo(CloudURLInfo):
+    def __init__(self, url):
+        super().__init__(url)
+
+        # GDrive URL host part is case sensitive,
+        # we are restoring it here.
+        p = urlparse(url)
+        self.host = p.netloc
+        assert self.netloc == self.host
+
+        # Normalize path. Important since we have a cache (path to ID)
+        # and don't want to deal with different variations of path in it.
+        self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))


 class RemoteGDrive(RemoteBASE):
    scheme = Schemes.GDRIVE
-    path_cls = CloudURLInfo
+    path_cls = GDriveURLInfo
    REQUIRES = {"pydrive2": "pydrive2"}
    DEFAULT_NO_TRAVERSE = False
    DEFAULT_VERIFY = True

-    GDRIVE_USER_CREDENTIALS_DATA = "GDRIVE_USER_CREDENTIALS_DATA"
+    GDRIVE_CREDENTIALS_DATA = "GDRIVE_CREDENTIALS_DATA"
    DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"

    def __init__(self, repo, config):
        super().__init__(repo, config)
-        self.path_info = self.path_cls(config[Config.SECTION_REMOTE_URL])
+        self.path_info = self.path_cls(config["url"])

-        bucket = re.search(
-            "{}://(.*)".format(self.scheme),
-            config[Config.SECTION_REMOTE_URL],
-            re.IGNORECASE,
-        )
-        self.bucket = (
-            bucket.group(1).split("/")[0] if bucket else self.path_info.bucket
-        )
-
-        self.config = config
-        self.init_drive()
-
-    def init_drive(self):
-        self.client_id = self.config.get(Config.SECTION_GDRIVE_CLIENT_ID, None)
-        self.client_secret = self.config.get(
-            Config.SECTION_GDRIVE_CLIENT_SECRET, None
-        )
-        if not self.client_id or not self.client_secret:
+        if not self.path_info.bucket:
            raise DvcException(
-                "Please specify Google Drive's client id and "
-                "secret in DVC's config. Learn more at "
-                "{}.".format(format_link("https://man.dvc.org/remote/add"))
+                "Empty Google Drive URL '{}'. Learn more at "
+                "{}.".format(
+                    config["url"],
+                    format_link("https://man.dvc.org/remote/add"),
+                )
            )
-        self.gdrive_user_credentials_path = (
+
+        self._bucket = self.path_info.bucket
+        self._use_service_account = config.get("gdrive_use_service_account")
+        self._service_account_email = config.get(
+            "gdrive_service_account_email"
+        )
+        self._service_account_user_email = config.get(
+            "gdrive_service_account_user_email"
+        )
+        self._service_account_p12_file_path = config.get(
+            "gdrive_service_account_p12_file_path"
+        )
+        self._client_id = config.get("gdrive_client_id")
+        self._client_secret = config.get("gdrive_client_secret")
+        self._validate_config()
+        self._gdrive_user_credentials_path = (
            tmp_fname(os.path.join(self.repo.tmp_dir, ""))
-            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
-            else self.config.get(
-                Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE,
+            if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA)
+            else config.get(
+                "gdrive_user_credentials_file",
                os.path.join(
                    self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE
                ),
            )
        )
-        self.remote_drive_id = None
+
+        self._list_params = None
+
+    def _validate_config(self):
+        # Validate Service Account configuration
+        if self._use_service_account and (
+            not self._service_account_email
+            or not self._service_account_p12_file_path
+        ):
+            raise DvcException(
+                "To use service account please specify {}, {} and "
+                "{} in DVC config. Learn more at "
+                "{}.".format(
+                    "gdrive_service_account_email",
+                    "gdrive_service_account_p12_file_path",
+                    "gdrive_service_account_user_email (optional)",
+                    format_link("https://man.dvc.org/remote/modify"),
+                )
+            )
+
+        # Validate OAuth 2.0 Client ID configuration
+        if not self._use_service_account and (
+            not self._client_id or not self._client_secret
+        ):
+            raise DvcException(
+                "Please specify Google Drive's client id and "
+                "secret in DVC config. Learn more at "
+                "{}.".format(format_link("https://man.dvc.org/remote/modify"))
+            )
+
+    @wrap_prop(threading.RLock())
+    @cached_property
+    def drive(self):
+        from pydrive2.auth import RefreshError
+        from pydrive2.auth import GoogleAuth
+        from pydrive2.drive import GoogleDrive
+
+        if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA):
+            with open(
+                self._gdrive_user_credentials_path, "w"
+            ) as credentials_file:
+                credentials_file.write(
+                    os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA)
+                )
+
+        GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
+        if self._use_service_account:
+            GoogleAuth.DEFAULT_SETTINGS["service_config"] = {
+                "client_service_email": self._service_account_email,
+                "client_user_email": self._service_account_user_email,
+                "client_pkcs12_file_path": self._service_account_p12_file_path,
+            }
+        else:
+            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
+                "client_id": self._client_id,
+                "client_secret": self._client_secret,
+                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                "token_uri": "https://oauth2.googleapis.com/token",
+                "revoke_uri": "https://oauth2.googleapis.com/revoke",
+                "redirect_uri": "",
+            }
+        GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
+        GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
+        GoogleAuth.DEFAULT_SETTINGS[
+            "save_credentials_file"
+        ] = self._gdrive_user_credentials_path
+        GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
+        GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
+            "https://www.googleapis.com/auth/drive",
+            "https://www.googleapis.com/auth/drive.appdata",
+        ]
+
+        # Pass non existent settings path to force DEFAULT_SETTINGS loading
+        gauth = GoogleAuth(settings_file="")
+
+        try:
+            if self._use_service_account:
+                gauth.ServiceAuth()
+            else:
+                gauth.CommandLineAuth()
+        except RefreshError as exc:
+            raise GDriveAccessTokenRefreshError from exc
+        except KeyError as exc:
+            raise GDriveMissedCredentialKeyError(
+                self._gdrive_user_credentials_path
+            ) from exc
+        # Handle pydrive2.auth.AuthenticationError and other auth failures
+        except Exception as exc:
+            raise DvcException("Google Drive authentication failed") from exc
+        finally:
+            if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA):
+                os.remove(self._gdrive_user_credentials_path)
+
+        return GoogleDrive(gauth)
+
+    @wrap_prop(threading.RLock())
+    @cached_property
+    def cache(self):
+        cache = {"dirs": defaultdict(list), "ids": {}}
+
+        cache["root_id"] = self._get_remote_id(self.path_info)
+        cache["dirs"][self.path_info.path] = [cache["root_id"]]
+        self._cache_path(self.path_info.path, cache["root_id"], cache)
+
+        for item in self.gdrive_list_item(
+            "'{}' in parents and trashed=false".format(cache["root_id"])
+        ):
+            remote_path = (self.path_info / item["title"]).path
+            self._cache_path(remote_path, item["id"], cache)
+
+        return cache
+
+    def _cache_path(self, remote_path, remote_id, cache=None):
+        cache = cache or self.cache
+        cache["dirs"][remote_path].append(remote_id)
+        cache["ids"][remote_id] = remote_path
+
+    @cached_property
+    def list_params(self):
+        params = {"corpora": "default"}
+        if self._bucket != "root" and self._bucket != "appDataFolder":
+            drive_id = self._get_remote_drive_id(self._bucket)
+            if drive_id:
+                params["driveId"] = drive_id
+                params["corpora"] = "drive"
+        return params

    @gdrive_retry
    def gdrive_upload_file(
-        self, args, no_progress_bar=True, from_file="", progress_name=""
+        self,
+        parent_id,
+        title,
+        no_progress_bar=False,
+        from_file="",
+        progress_name="",
    ):
-        parent = {"id": args["parent_id"]}
        item = self.drive.CreateFile(
-            {"title": args["title"], "parents": [parent]}
+            {"title": title, "parents": [{"id": parent_id}]}
        )

        with open(from_file, "rb") as fobj:
@@ -139,7 +280,7 @@ class RemoteGDrive(RemoteBASE):
        # it does not create a file on the remote
        gdrive_file = self.drive.CreateFile(param)
        bar_format = (
-            "Donwloading {desc:{ncols_desc}.{ncols_desc}}... "
+            "Downloading {desc:{ncols_desc}.{ncols_desc}}... "
            + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
        )
        with Tqdm(
@@ -148,10 +289,8 @@ class RemoteGDrive(RemoteBASE):
            gdrive_file.GetContentFile(to_file)

    def gdrive_list_item(self, query):
-        param = {"q": query, "maxResults": 1000, "corpora": self.corpora}
-
-        if self.remote_drive_id:
-            param["driveId"] = self.remote_drive_id
+        param = {"q": query, "maxResults": 1000}
+        param.update(self.list_params)

        file_list = self.drive.ListFile(param)

@@ -161,113 +300,21 @@ class RemoteGDrive(RemoteBASE):
        # Fetch pages until None is received, lazily flatten the thing
        return cat(iter(get_list, None))

-    def cache_root_dirs(self):
-        cached_dirs = {}
-        cached_ids = {}
-        for dir1 in self.gdrive_list_item(
-            "'{}' in parents and trashed=false".format(self.remote_root_id)
-        ):
-            remote_path = posixpath.join(self.path_info.path, dir1["title"])
-            cached_dirs.setdefault(remote_path, []).append(dir1["id"])
-            cached_ids[dir1["id"]] = dir1["title"]
-        return cached_dirs, cached_ids
-
-    @property
-    def cached_dirs(self):
-        if not hasattr(self, "_cached_dirs"):
-            self.drive
-        return self._cached_dirs
-
-    @property
-    def cached_ids(self):
-        if not hasattr(self, "_cached_ids"):
-            self.drive
-        return self._cached_ids
-
-    @property
-    def corpora(self):
-        if not hasattr(self, "_corpora"):
-            self.drive
-        return self._corpora
-
-    @property
    @wrap_with(threading.RLock())
-    def drive(self):
-        from pydrive2.auth import RefreshError
+    def gdrive_create_dir(self, parent_id, title, remote_path):
+        cached = self.cache["dirs"].get(remote_path)
+        if cached:
+            return cached[0]

-        if not hasattr(self, "_gdrive"):
-            from pydrive2.auth import GoogleAuth
-            from pydrive2.drive import GoogleDrive
+        item = self._create_remote_dir(parent_id, title)

-            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
-                with open(
-                    self.gdrive_user_credentials_path, "w"
-                ) as credentials_file:
-                    credentials_file.write(
-                        os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
-                    )
+        if parent_id == self.cache["root_id"]:
+            self._cache_path(remote_path, item["id"])

-            GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
-            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
-                "client_id": self.client_id,
-                "client_secret": self.client_secret,
-                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-                "token_uri": "https://oauth2.googleapis.com/token",
-                "revoke_uri": "https://oauth2.googleapis.com/revoke",
-                "redirect_uri": "",
-            }
-            GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
-            GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
-            GoogleAuth.DEFAULT_SETTINGS[
-                "save_credentials_file"
-            ] = self.gdrive_user_credentials_path
-            GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
-            GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
-                "https://www.googleapis.com/auth/drive",
-                # drive.appdata grants access to appDataFolder GDrive directory
-                "https://www.googleapis.com/auth/drive.appdata",
-            ]
-
-            # Pass non existent settings path to force DEFAULT_SETTINGS loading
-            gauth = GoogleAuth(settings_file="")
-
-            try:
-                gauth.CommandLineAuth()
-            except RefreshError as exc:
-                raise GDriveAccessTokenRefreshError(
-                    "Google Drive's access token refreshment is failed"
-                ) from exc
-            except KeyError as exc:
-                raise GDriveMissedCredentialKeyError(
-                    "Google Drive's user credentials file '{}' "
-                    "misses value for key '{}'".format(
-                        self.gdrive_user_credentials_path, str(exc)
-                    )
-                )
-            # Handle pydrive2.auth.AuthenticationError and others auth failures
-            except Exception as exc:
-                raise DvcException(
-                    "Google Drive authentication failed"
-                ) from exc
-            finally:
-                if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
-                    os.remove(self.gdrive_user_credentials_path)
-
-            self._gdrive = GoogleDrive(gauth)
-
-            if self.bucket != "root" and self.bucket != "appDataFolder":
-                self.remote_drive_id = self.get_remote_drive_id(self.bucket)
-            self._corpora = "drive" if self.remote_drive_id else "default"
-            self.remote_root_id = self.get_remote_id(
-                self.path_info, create=True
-            )
-
-            self._cached_dirs, self._cached_ids = self.cache_root_dirs()
-
-        return self._gdrive
+        return item["id"]

    @gdrive_retry
-    def create_remote_dir(self, parent_id, title):
+    def _create_remote_dir(self, parent_id, title):
        parent = {"id": parent_id}
        item = self.drive.CreateFile(
            {"title": title, "parents": [parent], "mimeType": FOLDER_MIME_TYPE}
@@ -276,14 +323,14 @@ class RemoteGDrive(RemoteBASE):
        return item

    @gdrive_retry
-    def delete_remote_file(self, remote_id):
+    def _delete_remote_file(self, remote_id):
        param = {"id": remote_id}
        # it does not create a file on the remote
        item = self.drive.CreateFile(param)
        item.Delete()

    @gdrive_retry
-    def get_remote_item(self, name, parents_ids):
+    def _get_remote_item(self, name, parents_ids):
        if not parents_ids:
            return None
        query = "({})".format(
@@ -295,106 +342,91 @@ class RemoteGDrive(RemoteBASE):

        query += " and trashed=false and title='{}'".format(name)

-        param = {
-            "q": query,
-            # Remote might contain items with duplicated titles
-            "maxResults": 1,
-            "corpora": self.corpora,
-        }
-
-        if self.remote_drive_id:
-            param["driveId"] = self.remote_drive_id
+        # Remote might contain items with duplicated path (titles).
+        # We thus limit number of items.
+        param = {"q": query, "maxResults": 1}
+        param.update(self.list_params)

        # Limit found remote items count to 1 in response
        item_list = self.drive.ListFile(param).GetList()
        return next(iter(item_list), None)

    @gdrive_retry
-    def get_remote_drive_id(self, remote_id):
+    def _get_remote_drive_id(self, remote_id):
        param = {"id": remote_id}
        # it does not create a file on the remote
        item = self.drive.CreateFile(param)
        item.FetchMetadata("driveId")
        return item.get("driveId", None)

-    def resolve_remote_item_from_path(self, path_parts, create):
-        parents_ids = [self.bucket]
-        current_path = ""
-        for path_part in path_parts:
-            current_path = posixpath.join(current_path, path_part)
-            remote_ids = self.get_remote_id_from_cache(current_path)
-            if remote_ids:
-                parents_ids = remote_ids
-                continue
-            item = self.get_remote_item(path_part, parents_ids)
-            if not item and create:
-                item = self.create_remote_dir(parents_ids[0], path_part)
-            elif not item:
-                return None
-            parents_ids = [item["id"]]
-        return item
-
-    def get_remote_id_from_cache(self, remote_path):
-        if hasattr(self, "_cached_dirs"):
-            return self.cached_dirs.get(remote_path, [])
+    def _get_cached_remote_ids(self, path):
+        if not path:
+            return [self._bucket]
+        if "cache" in self.__dict__:
+            return self.cache["dirs"].get(path, [])
        return []

-    def get_remote_id(self, path_info, create=False):
-        if not path_info.path and path_info.bucket:
-            # Case sensitive base path
-            return self.bucket
-
-        remote_ids = self.get_remote_id_from_cache(path_info.path)
-
+    def _path_to_remote_ids(self, path, create):
+        remote_ids = self._get_cached_remote_ids(path)
        if remote_ids:
-            return remote_ids[0]
+            return remote_ids

-        file1 = self.resolve_remote_item_from_path(
-            path_info.path.split("/"), create
-        )
-        return file1["id"] if file1 else ""
+        parent_path, part = posixpath.split(path)
+        parent_ids = self._path_to_remote_ids(parent_path, create)
+        item = self._get_remote_item(part, parent_ids)
+
+        if not item:
+            return (
+                [self.gdrive_create_dir(parent_ids[0], part, path)]
+                if create
+                else []
+            )
+
+        return [item["id"]]
+
+    def _get_remote_id(self, path_info, create=False):
+        assert path_info.bucket == self._bucket
+
+        remote_ids = self._path_to_remote_ids(path_info.path, create)
+        if not remote_ids:
+            raise GDrivePathNotFound(path_info)
+
+        return remote_ids[0]

    def exists(self, path_info):
-        return self.get_remote_id(path_info) != ""
+        try:
+            self._get_remote_id(path_info)
+        except GDrivePathNotFound:
+            return False
+        else:
+            return True

    def _upload(self, from_file, to_info, name, no_progress_bar):
        dirname = to_info.parent
-        if dirname:
-            parent_id = self.get_remote_id(dirname, True)
-        else:
-            parent_id = to_info.bucket
+        assert dirname
+        parent_id = self._get_remote_id(dirname, True)

        self.gdrive_upload_file(
-            {"title": to_info.name, "parent_id": parent_id},
-            no_progress_bar,
-            from_file,
-            name,
+            parent_id, to_info.name, no_progress_bar, from_file, name
        )

    def _download(self, from_info, to_file, name, no_progress_bar):
-        file_id = self.get_remote_id(from_info)
+        file_id = self._get_remote_id(from_info)
        self.gdrive_download_file(file_id, to_file, name, no_progress_bar)

-    def all(self):
-        if not self.cached_ids:
+    def list_cache_paths(self):
+        if not self.cache["ids"]:
            return

-        query = "({})".format(
-            " or ".join(
-                "'{}' in parents".format(dir_id) for dir_id in self.cached_ids
-            )
+        parents_query = " or ".join(
+            "'{}' in parents".format(dir_id) for dir_id in self.cache["ids"]
        )
+        query = "({}) and trashed=false".format(parents_query)

-        query += " and trashed=false"
-        for file1 in self.gdrive_list_item(query):
-            parent_id = file1["parents"][0]["id"]
-            path = posixpath.join(self.cached_ids[parent_id], file1["title"])
-            try:
-                yield self.path_to_checksum(path)
-            except ValueError:
-                # We ignore all the non-cache looking files
-                logger.debug('Ignoring path as "non-cache looking"')
+        for item in self.gdrive_list_item(query):
+            parent_id = item["parents"][0]["id"]
+            yield posixpath.join(self.cache["ids"][parent_id], item["title"])

    def remove(self, path_info):
-        remote_id = self.get_remote_id(path_info)
-        self.delete_remote_file(remote_id)
+        remote_id = self._get_remote_id(path_info)
+        self._delete_remote_file(remote_id)
@@ -7,7 +7,6 @@ import threading

 from funcy import cached_property, wrap_prop

-from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
@@ -52,7 +51,7 @@ def _upload_to_bucket(
    to_info,
    chunk_size=None,
    name=None,
-    no_progress_bar=True,
+    no_progress_bar=False,
 ):
    blob = bucket.blob(to_info.path, chunk_size=chunk_size)
    with io.open(from_file, mode="rb") as fobj:
@@ -75,11 +74,11 @@ class RemoteGS(RemoteBASE):
    def __init__(self, repo, config):
        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL, "gs:///")
+        url = config.get("url", "gs:///")
        self.path_info = self.path_cls(url)

-        self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
-        self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
+        self.projectname = config.get("projectname", None)
+        self.credentialpath = config.get("credentialpath")

    @wrap_prop(threading.Lock())
    @cached_property
@@ -167,7 +166,7 @@ class RemoteGS(RemoteBASE):
        """
        return self.isfile(path_info) or self.isdir(path_info)

-    def _upload(self, from_file, to_info, name=None, no_progress_bar=True):
+    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
        bucket = self.gs.bucket(to_info.bucket)
        _upload_to_bucket(
            bucket,
@@ -177,7 +176,7 @@ class RemoteGS(RemoteBASE):
            no_progress_bar=no_progress_bar,
        )

-    def _download(self, from_info, to_file, name=None, no_progress_bar=True):
+    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
        bucket = self.gs.bucket(from_info.bucket)
        blob = bucket.get_blob(from_info.path)
        with io.open(to_file, mode="wb") as fobj:
@@ -10,7 +10,6 @@ from urllib.parse import urlparse

 from .base import RemoteBASE, RemoteCmdError
 from .pool import get_connection
-from dvc.config import Config
 from dvc.scheme import Schemes
 from dvc.utils import fix_env, tmp_fname

@@ -26,13 +25,12 @@ class RemoteHDFS(RemoteBASE):
    def __init__(self, repo, config):
        super().__init__(repo, config)
        self.path_info = None
-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
        if not url:
            return

        parsed = urlparse(url)
-
-        user = parsed.username or config.get(Config.SECTION_REMOTE_USER)
+        user = parsed.username or config.get("user")

        self.path_info = self.path_cls.from_parts(
            scheme=self.scheme,
@@ -1,9 +1,11 @@
 import logging
+import os.path
 import threading

-from funcy import cached_property, wrap_prop
+from funcy import cached_property, memoize, wrap_prop, wrap_with

-from dvc.config import Config
+from dvc.path_info import HTTPURLInfo
+import dvc.prompt as prompt
 from dvc.config import ConfigError
 from dvc.exceptions import DvcException, HTTPError
 from dvc.progress import Tqdm
@@ -13,8 +15,18 @@ from dvc.scheme import Schemes
 logger = logging.getLogger(__name__)


+@wrap_with(threading.Lock())
+@memoize
+def ask_password(host, user):
+    return prompt.password(
+        "Enter a password for "
+        "host '{host}' user '{user}'".format(host=host, user=user)
+    )
+
+
 class RemoteHTTP(RemoteBASE):
    scheme = Schemes.HTTP
+    path_cls = HTTPURLInfo
    SESSION_RETRIES = 5
    SESSION_BACKOFF_FACTOR = 0.1
    REQUEST_TIMEOUT = 10
@@ -24,8 +36,14 @@ class RemoteHTTP(RemoteBASE):
    def __init__(self, repo, config):
        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL)
-        self.path_info = self.path_cls(url) if url else None
+        url = config.get("url")
+        if url:
+            self.path_info = self.path_cls(url)
+            user = config.get("user", None)
+            if user:
+                self.path_info.user = user
+        else:
+            self.path_info = None

        if not self.no_traverse:
            raise ConfigError(
@@ -33,6 +51,12 @@ class RemoteHTTP(RemoteBASE):
                "files. Use: `dvc remote modify <name> no_traverse true`"
            )

+        self.auth = config.get("auth", None)
+        self.custom_auth_header = config.get("custom_auth_header", None)
+        self.password = config.get("password", None)
+        self.ask_password = config.get("ask_password", False)
+        self.headers = {}
+
    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
        response = self._request("GET", from_info.url, stream=True)
        if response.status_code != 200:
@@ -49,6 +73,28 @@ class RemoteHTTP(RemoteBASE):
                    fd.write(chunk)
                    pbar.update(len(chunk))

+    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
+        with Tqdm(
+            total=None if no_progress_bar else os.path.getsize(from_file),
+            leave=False,
+            bytes=True,
+            desc=to_info.url if name is None else name,
+            disable=no_progress_bar,
+        ) as pbar:
+
+            def chunks():
+                with open(from_file, "rb") as fd:
+                    while True:
+                        chunk = fd.read(self.CHUNK_SIZE)
+                        if not chunk:
+                            break
+                        pbar.update(len(chunk))
+                        yield chunk
+
+            response = self._request("POST", to_info.url, data=chunks())
+            if response.status_code not in (200, 201):
+                raise HTTPError(response.status_code, response.reason)
+
    def exists(self, path_info):
        return bool(self._request("HEAD", path_info.url))

@@ -75,6 +121,24 @@ class RemoteHTTP(RemoteBASE):

        return etag

+    def auth_method(self, path_info=None):
+        from requests.auth import HTTPBasicAuth, HTTPDigestAuth
+
+        if path_info is None:
+            path_info = self.path_info
+
+        if self.auth:
+            if self.ask_password and self.password is None:
+                host, user = path_info.host, path_info.user
+                self.password = ask_password(host, user)
+            if self.auth == "basic":
+                return HTTPBasicAuth(path_info.user, self.password)
+            if self.auth == "digest":
+                return HTTPDigestAuth(path_info.user, self.password)
+            if self.auth == "custom" and self.custom_auth_header:
+                self.headers.update({self.custom_auth_header: self.password})
+        return None
+
    @wrap_prop(threading.Lock())
    @cached_property
    def _session(self):
@@ -101,7 +165,13 @@ class RemoteHTTP(RemoteBASE):
        kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)

        try:
-            res = self._session.request(method, url, **kwargs)
+            res = self._session.request(
+                method,
+                url,
+                auth=self.auth_method(),
+                headers=self.headers,
+                **kwargs,
+            )

            redirect_no_location = (
                kwargs["allow_redirects"]
@@ -7,27 +7,17 @@ from functools import partial

 from shortuuid import uuid

-from dvc.config import Config
-from dvc.exceptions import DownloadError
-from dvc.exceptions import DvcException
-from dvc.exceptions import UploadError
+from dvc.compat import fspath_py35
+from dvc.exceptions import DvcException, DownloadError, UploadError
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
-from dvc.remote.base import STATUS_DELETED
-from dvc.remote.base import STATUS_MAP
-from dvc.remote.base import STATUS_MISSING
-from dvc.remote.base import STATUS_NEW
+from dvc.remote.base import RemoteBASE, STATUS_MAP
+from dvc.remote.base import STATUS_DELETED, STATUS_MISSING, STATUS_NEW
 from dvc.scheme import Schemes
 from dvc.scm.tree import is_working_tree
 from dvc.system import System
-from dvc.utils.fs import copyfile
-from dvc.utils import file_md5
-from dvc.utils import relpath
-from dvc.utils import tmp_fname
-from dvc.compat import fspath_py35
-from dvc.utils.fs import move, makedirs, remove
-from dvc.utils.fs import walk_files
+from dvc.utils import file_md5, relpath, tmp_fname
+from dvc.utils.fs import copyfile, move, makedirs, remove, walk_files

 logger = logging.getLogger(__name__)

@@ -42,26 +32,12 @@ class RemoteLOCAL(RemoteBASE):

    DEFAULT_CACHE_TYPES = ["reflink", "copy"]

+    CACHE_MODE = 0o444
    SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}

    def __init__(self, repo, config):
        super().__init__(repo, config)
-        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
-
-        shared = config.get(Config.SECTION_CACHE_SHARED)
-        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
-
-        if self.protected:
-            # cache files are set to be read-only for everyone
-            self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
-
-        cache_dir = config.get(Config.SECTION_REMOTE_URL)
-
-        if cache_dir is not None and not os.path.isabs(cache_dir):
-            cwd = config[Config.PRIVATE_CWD]
-            cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
-
-        self.cache_dir = cache_dir
+        self.cache_dir = config.get("url")
        self._dir_info = {}

    @property
@@ -108,6 +84,12 @@ class RemoteLOCAL(RemoteBASE):

        return not self.changed_cache(current_md5)

+    def _verify_link(self, path_info, link_type):
+        if link_type == "hardlink" and self.getsize(path_info) == 0:
+            return
+
+        super()._verify_link(path_info, link_type)
+
    def is_empty(self, path_info):
        path = path_info.fspath

@@ -152,16 +134,17 @@ class RemoteLOCAL(RemoteBASE):
        if self.exists(path_info):
            remove(path_info.fspath)

-    def move(self, from_info, to_info):
+    def move(self, from_info, to_info, mode=None):
        if from_info.scheme != "local" or to_info.scheme != "local":
            raise NotImplementedError

        self.makedirs(to_info.parent)

-        if self.isfile(from_info):
-            mode = self._file_mode
-        else:
-            mode = self._dir_mode
+        if mode is None:
+            if self.isfile(from_info):
+                mode = self._file_mode
+            else:
+                mode = self._dir_mode

        move(from_info, to_info, mode=mode)

@@ -169,6 +152,7 @@ class RemoteLOCAL(RemoteBASE):
        tmp_info = to_info.parent / tmp_fname(to_info.name)
        try:
            System.copy(from_info, tmp_info)
+            os.chmod(fspath_py35(tmp_info), self._file_mode)
            os.rename(fspath_py35(tmp_info), fspath_py35(to_info))
        except Exception:
            self.remove(tmp_info)
@@ -212,9 +196,13 @@ class RemoteLOCAL(RemoteBASE):
    def is_hardlink(path_info):
        return System.is_hardlink(path_info)

-    @staticmethod
-    def reflink(from_info, to_info):
-        System.reflink(from_info, to_info)
+    def reflink(self, from_info, to_info):
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        System.reflink(from_info, tmp_info)
+        # NOTE: reflink has its own separate inode, so you can set permissions
+        # that are different from the source.
+        os.chmod(fspath_py35(tmp_info), self._file_mode)
+        os.rename(fspath_py35(tmp_info), fspath_py35(to_info))

    def cache_exists(self, checksums, jobs=None, name=None):
        return [
@@ -412,8 +400,7 @@ class RemoteLOCAL(RemoteBASE):
            )
            logger.warning(msg)

-    @staticmethod
-    def _unprotect_file(path):
+    def _unprotect_file(self, path):
        if System.is_symlink(path) or System.is_hardlink(path):
            logger.debug("Unprotecting '{}'".format(path))
            tmp = os.path.join(os.path.dirname(path), "." + uuid())
@@ -433,13 +420,13 @@ class RemoteLOCAL(RemoteBASE):
                "a symlink or a hardlink.".format(path)
            )

-        os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
+        os.chmod(path, self._file_mode)

    def _unprotect_dir(self, path):
        assert is_working_tree(self.repo.tree)

        for fname in self.repo.tree.walk_files(path):
-            RemoteLOCAL._unprotect_file(fname)
+            self._unprotect_file(fname)

    def unprotect(self, path_info):
        path = path_info.fspath
@@ -451,12 +438,11 @@ class RemoteLOCAL(RemoteBASE):
        if os.path.isdir(path):
            self._unprotect_dir(path)
        else:
-            RemoteLOCAL._unprotect_file(path)
+            self._unprotect_file(path)

-    @staticmethod
-    def protect(path_info):
+    def protect(self, path_info):
        path = fspath_py35(path_info)
-        mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
+        mode = self.CACHE_MODE

        try:
            os.chmod(path, mode)
@@ -529,3 +515,11 @@ class RemoteLOCAL(RemoteBASE):
            if self.is_dir_checksum(c):
                unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
        return unpacked
+
+    def is_protected(self, path_info):
+        if not self.exists(path_info):
+            return False
+
+        mode = os.stat(fspath_py35(path_info)).st_mode
+
+        return stat.S_IMODE(mode) == self.CACHE_MODE
@@ -4,7 +4,6 @@ import threading

 from funcy import cached_property, wrap_prop

-from dvc.config import Config
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
@@ -42,21 +41,19 @@ class RemoteOSS(RemoteBASE):
    def __init__(self, repo, config):
        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
        self.path_info = self.path_cls(url) if url else None

-        self.endpoint = config.get(Config.SECTION_OSS_ENDPOINT) or os.getenv(
-            "OSS_ENDPOINT"
-        )
+        self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")

        self.key_id = (
-            config.get(Config.SECTION_OSS_ACCESS_KEY_ID)
+            config.get("oss_key_id")
            or os.getenv("OSS_ACCESS_KEY_ID")
            or "defaultId"
        )

        self.key_secret = (
-            config.get(Config.SECTION_OSS_ACCESS_KEY_SECRET)
+            config.get("oss_key_secret")
            or os.getenv("OSS_ACCESS_KEY_SECRET")
            or "defaultSecret"
        )
@@ -6,7 +6,7 @@ import threading

 from funcy import cached_property, wrap_prop

-from dvc.config import Config, ConfigError
+from dvc.config import ConfigError
 from dvc.exceptions import DvcException
 from dvc.exceptions import ETagMismatchError
 from dvc.path_info import CloudURLInfo
@@ -26,35 +26,33 @@ class RemoteS3(RemoteBASE):
    def __init__(self, repo, config):
        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL, "s3://")
+        url = config.get("url", "s3://")
        self.path_info = self.path_cls(url)

-        self.region = config.get(Config.SECTION_AWS_REGION)
+        self.region = config.get("region")
+        self.profile = config.get("profile")
+        self.endpoint_url = config.get("endpointurl")

-        self.profile = config.get(Config.SECTION_AWS_PROFILE)
-
-        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
-
-        if config.get(Config.SECTION_AWS_LIST_OBJECTS):
+        if config.get("listobjects"):
            self.list_objects_api = "list_objects"
        else:
            self.list_objects_api = "list_objects_v2"

-        self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
+        self.use_ssl = config.get("use_ssl", True)

        self.extra_args = {}

-        self.sse = config.get(Config.SECTION_AWS_SSE, "")
+        self.sse = config.get("sse")
        if self.sse:
            self.extra_args["ServerSideEncryption"] = self.sse

-        self.acl = config.get(Config.SECTION_AWS_ACL, "")
+        self.acl = config.get("acl")
        if self.acl:
            self.extra_args["ACL"] = self.acl

        self._append_aws_grants_to_extra_args(config)

-        shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
+        shared_creds = config.get("credentialpath")
        if shared_creds:
            os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)

@@ -322,14 +320,14 @@ class RemoteS3(RemoteBASE):
        """

        grants = {
-            Config.SECTION_AWS_GRANT_FULL_CONTROL: "GrantFullControl",
-            Config.SECTION_AWS_GRANT_READ: "GrantRead",
-            Config.SECTION_AWS_GRANT_READ_ACP: "GrantReadACP",
-            Config.SECTION_AWS_GRANT_WRITE_ACP: "GrantWriteACP",
+            "grant_full_control": "GrantFullControl",
+            "grant_read": "GrantRead",
+            "grant_read_acp": "GrantReadACP",
+            "grant_write_acp": "GrantWriteACP",
        }

        for grant_option, extra_args_key in grants.items():
-            if config.get(grant_option, ""):
+            if config.get(grant_option):
                if self.acl:
                    raise ConfigError(
                        "`acl` and `grant_*` AWS S3 config options "
@@ -1,12 +1,10 @@
 import logging
 import sys
 import time
+
 from functools import wraps
-
 import colorama

-from dvc.config import Config
-

 logger = logging.getLogger(__name__)
 this = sys.modules[__name__]
@@ -31,11 +29,9 @@ def slow_link_guard(f):
        if this.already_displayed:
            return f(remote, *args, **kwargs)

-        config = remote.repo.config.config.get(Config.SECTION_CACHE, {})
-        cache_type = config.get(Config.SECTION_CACHE_TYPE)
-        should_warn = config.get(Config.SECTION_CACHE_SLOW_LINK_WARNING, True)
-
-        if not should_warn or cache_type:
+        cache_conf = remote.repo.config["cache"]
+        slow_link_warning = cache_conf.get("slow_link_warning", True)
+        if not slow_link_warning or cache_conf.get("type"):
            return f(remote, *args, **kwargs)

        start = time.time()
@@ -9,10 +9,9 @@ from concurrent.futures import ThreadPoolExecutor
 from contextlib import closing, contextmanager
 from urllib.parse import urlparse

-from funcy import memoize, wrap_with
+from funcy import memoize, wrap_with, silent, first

 import dvc.prompt as prompt
-from dvc.config import Config
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.remote.pool import get_connection
@@ -50,20 +49,20 @@ class RemoteSSH(RemoteBASE):

    def __init__(self, repo, config):
        super().__init__(repo, config)
-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
        if url:
            parsed = urlparse(url)
            user_ssh_config = self._load_user_ssh_config(parsed.hostname)

            host = user_ssh_config.get("hostname", parsed.hostname)
            user = (
-                config.get(Config.SECTION_REMOTE_USER)
+                config.get("user")
                or parsed.username
                or user_ssh_config.get("user")
                or getpass.getuser()
            )
            port = (
-                config.get(Config.SECTION_REMOTE_PORT)
+                config.get("port")
                or parsed.port
                or self._try_get_ssh_config_port(user_ssh_config)
                or self.DEFAULT_PORT
@@ -80,14 +79,12 @@ class RemoteSSH(RemoteBASE):
            user_ssh_config = {}

        self.keyfile = config.get(
-            Config.SECTION_REMOTE_KEY_FILE
+            "keyfile"
        ) or self._try_get_ssh_config_keyfile(user_ssh_config)
-        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
-        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
-        self.ask_password = config.get(
-            Config.SECTION_REMOTE_ASK_PASSWORD, False
-        )
-        self.gss_auth = config.get(Config.SECTION_REMOTE_GSS_AUTH, False)
+        self.timeout = config.get("timeout", self.TIMEOUT)
+        self.password = config.get("password", None)
+        self.ask_password = config.get("ask_password", False)
+        self.gss_auth = config.get("gss_auth", False)

    @staticmethod
    def ssh_config_filename():
@@ -110,17 +107,11 @@ class RemoteSSH(RemoteBASE):

    @staticmethod
    def _try_get_ssh_config_port(user_ssh_config):
-        try:
-            return int(user_ssh_config.get("port"))
-        except (ValueError, TypeError):
-            return None
+        return silent(int)(user_ssh_config.get("port"))

    @staticmethod
    def _try_get_ssh_config_keyfile(user_ssh_config):
-        identity_file = user_ssh_config.get("identityfile")
-        if identity_file and len(identity_file) > 0:
-            return identity_file[0]
-        return None
+        return first(user_ssh_config.get("identityfile") or ())

    def ensure_credentials(self, path_info=None):
        if path_info is None:
@@ -218,7 +209,8 @@ class RemoteSSH(RemoteBASE):
        with self.ssh(path_info) as ssh:
            ssh.remove(path_info.path)

-    def move(self, from_info, to_info):
+    def move(self, from_info, to_info, mode=None):
+        assert mode is None
        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
            raise NotImplementedError

@@ -1,17 +1,16 @@
-import logging
 import os
 from contextlib import contextmanager
 from functools import wraps
-from itertools import chain

 from dvc.ignore import CleanTree
 from dvc.compat import fspath_py35

-from funcy import cached_property, cat
+from funcy import cached_property, cat, first

 from dvc.config import Config
 from dvc.exceptions import (
    FileMissingError,
+    IsADirectoryError,
    NotDvcRepoError,
    OutputNotFoundError,
 )
@@ -21,13 +20,11 @@ from dvc.utils.fs import path_isin
 from .graph import check_acyclic, get_pipeline, get_pipelines


-logger = logging.getLogger(__name__)
-
-
 def locked(f):
    @wraps(f)
    def wrapper(repo, *args, **kwargs):
        with repo.lock, repo.state:
+            repo._reset()
            ret = f(repo, *args, **kwargs)
            # Our graph cache is no longer valid after we release the repo.lock
            repo._reset()
@@ -43,6 +40,7 @@ class Repo(object):
    from dvc.repo.install import install
    from dvc.repo.add import add
    from dvc.repo.remove import remove
+    from dvc.repo.ls import ls
    from dvc.repo.lock import lock as lock_stage
    from dvc.repo.move import move
    from dvc.repo.run import run
@@ -80,14 +78,15 @@ class Repo(object):

        self.config = Config(self.dvc_dir)

-        self.scm = SCM(self.root_dir)
+        no_scm = self.config["core"].get("no_scm", False)
+        self.scm = SCM(self.root_dir, no_scm=no_scm)

        self.tree = WorkingTree(self.root_dir)

        self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
        makedirs(self.tmp_dir, exist_ok=True)

-        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        hardlink_lock = self.config["core"].get("hardlink_lock", False)
        self.lock = make_lock(
            os.path.join(self.dvc_dir, "lock"),
            tmp_dir=os.path.join(self.dvc_dir, "tmp"),
@@ -97,13 +96,7 @@ class Repo(object):

        # NOTE: storing state and link_state in the repository itself to avoid
        # any possible state corruption in 'shared cache dir' scenario.
-        self.state = State(self, self.config.config)
-
-        core = self.config.config[Config.SECTION_CORE]
-
-        level = core.get(Config.SECTION_CORE_LOGLEVEL)
-        if level:
-            logger.setLevel(level.upper())
+        self.state = State(self)

        self.cache = Cache(self)
        self.cloud = DataCloud(self)
@@ -129,19 +122,24 @@ class Repo(object):

    @classmethod
    def find_root(cls, root=None):
-        if root is None:
-            root = os.getcwd()
-        else:
-            root = os.path.abspath(os.path.realpath(root))
+        root_dir = os.path.realpath(root or os.curdir)
+
+        if not os.path.isdir(root_dir):
+            raise NotDvcRepoError("directory '{}' does not exist".format(root))

        while True:
-            dvc_dir = os.path.join(root, cls.DVC_DIR)
+            dvc_dir = os.path.join(root_dir, cls.DVC_DIR)
            if os.path.isdir(dvc_dir):
-                return root
-            if os.path.ismount(root):
+                return root_dir
+            if os.path.ismount(root_dir):
                break
-            root = os.path.dirname(root)
-        raise NotDvcRepoError(root)
+            root_dir = os.path.dirname(root_dir)
+
+        message = (
+            "you are not inside of a DVC repository "
+            "(checked up to mount point '{}')"
+        ).format(root_dir)
+        raise NotDvcRepoError(message)

    @classmethod
    def find_dvc_dir(cls, root=None):
@@ -149,10 +147,10 @@ class Repo(object):
        return os.path.join(root_dir, cls.DVC_DIR)

    @staticmethod
-    def init(root_dir=os.curdir, no_scm=False, force=False):
+    def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
        from dvc.repo.init import init

-        init(root_dir=root_dir, no_scm=no_scm, force=force)
+        init(root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir)
        return Repo(root_dir)

    def unprotect(self, target):
@@ -164,7 +162,7 @@ class Repo(object):
        updater = Updater(self.dvc_dir)

        flist = (
-            [self.config.config_local_file, updater.updater_file]
+            [self.config.files["local"], updater.updater_file]
            + [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir]
            + self.state.files
        )
@@ -182,28 +180,34 @@ class Repo(object):
        import networkx as nx
        from dvc.stage import Stage

-        G = graph or self.graph
-
        if not target:
-            return list(G)
+            return list(graph) if graph else self.stages

        target = os.path.abspath(target)

        if recursive and os.path.isdir(target):
-            stages = nx.dfs_postorder_nodes(G)
+            stages = nx.dfs_postorder_nodes(graph or self.graph)
            return [stage for stage in stages if path_isin(stage.path, target)]

        stage = Stage.load(self, target)
+
+        # Optimization: do not collect the graph for a specific target
        if not with_deps:
            return [stage]

-        pipeline = get_pipeline(get_pipelines(G), stage)
+        pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
        return list(nx.dfs_postorder_nodes(pipeline, stage))

    def collect_granular(self, target, *args, **kwargs):
+        from dvc.stage import Stage
+
        if not target:
            return [(stage, None) for stage in self.stages]

+        # Optimization: do not collect the graph for a specific .dvc target
+        if Stage.is_valid_filename(target) and not kwargs.get("with_deps"):
+            return [(Stage.load(self, target), None)]
+
        try:
            (out,) = self.find_outs_by_path(target, strict=False)
            filter_info = PathInfo(os.path.abspath(target))
@@ -231,7 +235,7 @@ class Repo(object):
        (namely, a file described as an output on a stage).

        The scope is, by default, the working directory, but you can use
-        `all_branches` or `all_tags` to expand scope.
+        `all_branches`/`all_tags`/`all_commits` to expand the scope.

        Returns:
            A dictionary with Schemes (representing output's location) as keys,
@@ -307,6 +311,7 @@ class Repo(object):
            CyclicGraphError: resulting graph has cycles
        """
        import networkx as nx
+        from pygtrie import Trie
        from dvc.exceptions import (
            OutputDuplicationError,
            StagePathAsOutputError,
@@ -316,38 +321,57 @@ class Repo(object):
        G = nx.DiGraph()
        stages = stages or self.stages
        stages = [stage for stage in stages if stage]
-        outs = {}
+        outs = Trie()  # Use trie to efficiently find overlapping outs and deps

        for stage in stages:
            for out in stage.outs:
-                if out.path_info in outs:
-                    dup_stages = [stage, outs[out.path_info].stage]
+                out_key = out.path_info.parts
+
+                # Check for dup outs
+                if out_key in outs:
+                    dup_stages = [stage, outs[out_key].stage]
                    raise OutputDuplicationError(str(out), dup_stages)
-                outs[out.path_info] = out
+
+                # Check for overlapping outs
+                if outs.has_subtrie(out_key):
+                    parent = out
+                    overlapping = first(outs.values(prefix=out_key))
+                else:
+                    parent = outs.shortest_prefix(out_key).value
+                    overlapping = out
+                if parent and overlapping:
+                    msg = (
+                        "Paths for outs:\n'{}'('{}')\n'{}'('{}')\n"
+                        "overlap. To avoid unpredictable behaviour, "
+                        "rerun command with non overlapping outs paths."
+                    ).format(
+                        str(parent),
+                        parent.stage.relpath,
+                        str(overlapping),
+                        overlapping.stage.relpath,
+                    )
+                    raise OverlappingOutputPathsError(parent, overlapping, msg)
+
+                outs[out_key] = out

        for stage in stages:
-            for out in stage.outs:
-                for p in out.path_info.parents:
-                    if p in outs:
-                        raise OverlappingOutputPathsError(outs[p], out)
+            out = outs.shortest_prefix(PathInfo(stage.path).parts).value
+            if out:
+                raise StagePathAsOutputError(stage, str(out))

+        # Building graph
+        G.add_nodes_from(stages)
        for stage in stages:
-            stage_path_info = PathInfo(stage.path)
-            for p in chain([stage_path_info], stage_path_info.parents):
-                if p in outs:
-                    raise StagePathAsOutputError(stage, str(outs[p]))
-
-        for stage in stages:
-            G.add_node(stage)
-
            for dep in stage.deps:
                if dep.path_info is None:
                    continue

-                for out_path_info, out in outs.items():
-                    if out_path_info.overlaps(dep.path_info):
-                        G.add_node(out.stage)
-                        G.add_edge(stage, out.stage)
+                dep_key = dep.path_info.parts
+                overlapping = list(n.value for n in outs.prefixes(dep_key))
+                if outs.has_subtrie(dep_key):
+                    overlapping.extend(outs.values(prefix=dep_key))
+
+                G.add_edges_from((stage, out.stage) for out in overlapping)

        check_acyclic(G)

@@ -453,7 +477,7 @@ class Repo(object):

    def _open_cached(self, out, remote=None, mode="r", encoding=None):
        if out.isdir():
-            raise ValueError("Can't open a dir")
+            raise IsADirectoryError("Can't open a dir")

        cache_file = self.cache.local.checksum_to_path_info(out.checksum)
        cache_file = fspath_py35(cache_file)
@@ -4,7 +4,10 @@ import os
 import colorama

 from . import locked
-from ..exceptions import RecursiveAddingWhileUsingFilename
+from ..exceptions import (
+    RecursiveAddingWhileUsingFilename,
+    OverlappingOutputPathsError,
+)
 from ..output.base import OutputDoesNotExistError
 from ..progress import Tqdm
 from ..repo.scm_context import scm_context
@@ -49,13 +52,28 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):

            stages = _create_stages(repo, sub_targets, fname, pbar=pbar)

-            repo.check_modified_graph(stages)
+            try:
+                repo.check_modified_graph(stages)
+            except OverlappingOutputPathsError as exc:
+                msg = (
+                    "Cannot add '{out}', because it is overlapping with other "
+                    "DVC tracked output: '{parent}'.\n"
+                    "To include '{out}' in '{parent}', run "
+                    "'dvc commit {parent_stage}'"
+                ).format(
+                    out=exc.overlapping_out.path_info,
+                    parent=exc.parent.path_info,
+                    parent_stage=exc.parent.stage.relpath,
+                )
+                raise OverlappingOutputPathsError(
+                    exc.parent, exc.overlapping_out, msg
+                )

            with Tqdm(
                total=len(stages),
                desc="Processing",
                unit="file",
-                disable=True if len(stages) == 1 else None,
+                disable=len(stages) == 1,
            ) as pbar_stages:
                for stage in stages:
                    try:
@@ -102,10 +120,13 @@ def _create_stages(repo, targets, fname, pbar=None):
    for out in Tqdm(
        targets,
        desc="Creating DVC-files",
-        disable=True if len(targets) < LARGE_DIR_SIZE else None,
+        disable=len(targets) < LARGE_DIR_SIZE,
        unit="file",
    ):
-        stage = Stage.create(repo, outs=[out], add=True, fname=fname)
+        stage = Stage.create(
+            repo, outs=[out], accompany_outs=True, fname=fname
+        )
+        repo._reset()

        if not stage:
            if pbar is not None:
@@ -9,9 +9,9 @@ def brancher(  # noqa: E302
    """Generator that iterates over specified revisions.

    Args:
-        branches (list): a list of branches to iterate over.
+        revs (list): a list of revisions to iterate over.
        all_branches (bool): iterate over all available branches.
-        tags (list): a list of tags to iterate over.
+        all_commits (bool): iterate over all commits.
        all_tags (bool): iterate over all available tags.

    Yields:
@@ -1,21 +1,31 @@
 import logging
+import os

+from dvc.compat import fspath
 from dvc.exceptions import CheckoutError
 from dvc.exceptions import CheckoutErrorSuggestGit
 from dvc.progress import Tqdm
-
+from dvc.utils import relpath

 logger = logging.getLogger(__name__)


-def _cleanup_unused_links(repo):
+def _get_unused_links(repo):
    used = [
        out.fspath
        for stage in repo.stages
        for out in stage.outs
        if out.scheme == "local"
    ]
-    repo.state.remove_unused_links(used)
+    return repo.state.get_unused_links(used)
+
+
+def _fspath_dir(path, root):
+    if not os.path.exists(str(path)):
+        return str(path)
+
+    path = relpath(fspath(path), root)
+    return os.path.join(path, "") if os.path.isdir(path) else path


 def get_all_files_numbers(pairs):
@@ -34,9 +44,19 @@ def _checkout(
 ):
    from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError

+    unused = []
+    stats = {
+        "added": [],
+        "deleted": [],
+        "modified": [],
+        "failed": [],
+    }
    if not targets:
        targets = [None]
-        _cleanup_unused_links(self)
+        unused = _get_unused_links(self)
+
+    stats["deleted"] = [_fspath_dir(u, self.root_dir) for u in unused]
+    self.state.remove_links(unused)

    pairs = set()
    for target in targets:
@@ -52,20 +72,23 @@ def _checkout(
            raise CheckoutErrorSuggestGit(target) from exc

    total = get_all_files_numbers(pairs)
-    if total == 0:
-        logger.info("Nothing to do")
-    failed = []
    with Tqdm(
        total=total, unit="file", desc="Checkout", disable=total == 0
    ) as pbar:
        for stage, filter_info in pairs:
-            failed.extend(
-                stage.checkout(
-                    force=force,
-                    progress_callback=pbar.update_desc,
-                    relink=relink,
-                    filter_info=filter_info,
-                )
+            result = stage.checkout(
+                force=force,
+                progress_callback=pbar.update_desc,
+                relink=relink,
+                filter_info=filter_info,
            )
-    if failed:
-        raise CheckoutError(failed)
+            for data in ["failed", "added", "modified"]:
+                stats[data].extend(
+                    _fspath_dir(path, self.root_dir) for path in result[data]
+                )
+
+    if stats.get("failed"):
+        raise CheckoutError(stats["failed"], stats)
+
+    del stats["failed"]
+    return stats
@@ -6,7 +6,7 @@ from dvc.scm.git import Git


@locked
-def diff(self, a_ref="HEAD", b_ref=None):
+def diff(self, a_rev="HEAD", b_rev=None):
    """
    By default, it compares the working tree with the last commit's tree.

@@ -22,33 +22,29 @@ def diff(self, a_ref="HEAD", b_ref=None):
        A dictionary of checksums addressed by relpaths collected from
        the current tree outputs.

-        Unpack directories to include their entries
-
        To help distinguish between a directory and a file output,
        the former one will come with a trailing slash in the path:

            directory: "data/"
            file:      "data"
        """
-        result = {}

-        for stage in self.stages:
-            for output in stage.outs:
-                if not output.is_dir_checksum:
-                    result.update({str(output): output.checksum})
-                    continue
+        def _to_path(output):
+            return (
+                str(output)
+                if not output.is_dir_checksum
+                else os.path.join(str(output), "")
+            )

-                result.update({os.path.join(str(output), ""): output.checksum})
-
-                for entry in output.dir_cache:
-                    path = str(output.path_info / entry["relpath"])
-                    result.update({path: entry["md5"]})
-
-        return result
+        return {
+            _to_path(output): output.checksum
+            for stage in self.stages
+            for output in stage.outs
+        }

    working_tree = self.tree
-    a_tree = self.scm.get_tree(a_ref)
-    b_tree = self.scm.get_tree(b_ref) if b_ref else working_tree
+    a_tree = self.scm.get_tree(a_rev)
+    b_tree = self.scm.get_tree(b_rev) if b_rev else working_tree

    try:
        self.tree = a_tree
@@ -66,10 +62,10 @@ def diff(self, a_ref="HEAD", b_ref=None):
    modified = sorted(set(old) & set(new))

    return {
-        "added": [{"path": path, "checksum": new[path]} for path in added],
-        "deleted": [{"path": path, "checksum": old[path]} for path in deleted],
+        "added": [{"path": path, "hash": new[path]} for path in added],
+        "deleted": [{"path": path, "hash": old[path]} for path in deleted],
        "modified": [
-            {"path": path, "checksum": {"old": old[path], "new": new[path]}}
+            {"path": path, "hash": {"old": old[path], "new": new[path]}}
            for path in modified
            if old[path] != new[path]
        ],
@@ -58,7 +58,7 @@ def _fetch(
        failed += exc.amount

    for (repo_url, repo_rev), files in used.external.items():
-        d, f = _fetch_external(self, repo_url, repo_rev, files)
+        d, f = _fetch_external(self, repo_url, repo_rev, files, jobs)
        downloaded += d
        failed += f

@@ -68,7 +68,7 @@ def _fetch(
    return downloaded


-def _fetch_external(self, repo_url, repo_rev, files):
+def _fetch_external(self, repo_url, repo_rev, files, jobs):
    from dvc.external_repo import external_repo

    failed = 0
@@ -91,7 +91,7 @@ def _fetch_external(self, repo_url, repo_rev, files):
                        cache.update(out.get_used_cache())

                try:
-                    return repo.cloud.pull(cache), failed
+                    return repo.cloud.pull(cache, jobs=jobs), failed
                except DownloadError as exc:
                    failed += exc.amount
    except CloneError:
@@ -2,6 +2,7 @@ import logging

 from . import locked
 from dvc.cache import NamedCache
+from dvc.exceptions import InvalidArgumentError


 logger = logging.getLogger(__name__)
@@ -13,6 +14,14 @@ def _do_gc(typ, func, clist):
        logger.info("No unused '{}' cache to remove.".format(typ))


+def _raise_error_if_all_disabled(**kwargs):
+    if not any(kwargs.values()):
+        raise InvalidArgumentError(
+            "Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags` "
+            "or `--all-commits` needs to be set."
+        )
+
+
@locked
 def gc(
    self,
@@ -25,7 +34,19 @@ def gc(
    force=False,
    jobs=None,
    repos=None,
+    workspace=False,
 ):
+
+    # require `workspace` to be true to come into effect.
+    # assume `workspace` to be enabled if any of `all_tags`, `all_commits`,
+    # or `all_branches` are enabled.
+    _raise_error_if_all_disabled(
+        workspace=workspace,
+        all_tags=all_tags,
+        all_commits=all_commits,
+        all_branches=all_branches,
+    )
+
    from contextlib import ExitStack
    from dvc.repo import Repo

@@ -11,7 +11,13 @@ def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
    out = resolve_output(url, out)

    stage = Stage.create(
-        self, cmd=None, deps=[url], outs=[out], fname=fname, erepo=erepo
+        self,
+        cmd=None,
+        deps=[url],
+        outs=[out],
+        fname=fname,
+        erepo=erepo,
+        accompany_outs=True,
    )

    if stage is None:
@@ -5,10 +5,10 @@ import colorama

 from dvc import analytics
 from dvc.config import Config
-from dvc.exceptions import InitError
+from dvc.exceptions import InitError, InvalidArgumentError
 from dvc.repo import Repo
-from dvc.scm import NoSCM
 from dvc.scm import SCM
+from dvc.scm.base import SCMError
 from dvc.utils import boxify
 from dvc.utils import relpath
 from dvc.utils.fs import remove
@@ -44,7 +44,7 @@ def _welcome_message():
    logger.info(msg)


-def init(root_dir=os.curdir, no_scm=False, force=False):
+def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
    """
    Creates an empty repo on the given directory -- basically a
    `.dvc` directory with subdirectories for configuration and cache.
@@ -63,15 +63,23 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
    Raises:
        KeyError: Raises an exception.
    """
+
+    if no_scm and subdir:
+        raise InvalidArgumentError(
+            "Cannot initialize repo with `--no-scm` and `--subdir`"
+        )
+
    root_dir = os.path.realpath(root_dir)
    dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
-    scm = SCM(root_dir)
-    if isinstance(scm, NoSCM) and not no_scm:
+
+    try:
+        scm = SCM(root_dir, search_parent_directories=subdir, no_scm=no_scm)
+    except SCMError:
        raise InitError(
-            "{repo} is not tracked by any supported scm tool (e.g. git). "
-            "Use `--no-scm` if you don't want to use any scm.".format(
-                repo=root_dir
-            )
+            "{repo} is not tracked by any supported SCM tool (e.g. Git). "
+            "Use `--no-scm` if you don't want to use any SCM or "
+            "`--subdir` if initializing inside a subdirectory of a parent SCM "
+            "repository.".format(repo=root_dir)
        )

    if os.path.isdir(dvc_dir):
@@ -87,9 +95,14 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
    os.mkdir(dvc_dir)

    config = Config.init(dvc_dir)
+
+    if no_scm:
+        with config.edit() as conf:
+            conf["core"]["no_scm"] = True
+
    proj = Repo(root_dir)

-    scm.add([config.config_file])
+    scm.add([config.files["repo"]])

    if scm.ignore_file:
        scm.add([os.path.join(dvc_dir, scm.ignore_file)])
@@ -1,2 +1,2 @@
-def install(self):
-    self.scm.install()
+def install(self, use_pre_commit_tool):
+    self.scm.install(use_pre_commit_tool)
@@ -0,0 +1,154 @@
+import os
+import stat
+
+from dvc.exceptions import PathMissingError, OutputNotFoundError
+
+
+@staticmethod
+def ls(
+    url, path=None, rev=None, recursive=None, outs_only=False,
+):
+    """Methods for getting files and outputs for the repo.
+
+    Args:
+        url (str): the repo url
+        path (str, optional): relative path into the repo
+        rev (str, optional): SHA commit, branch or tag name
+        recursive (bool, optional): recursively walk the repo
+        outs_only (bool, optional): show only DVC-artifacts
+
+    Returns:
+        list of `entry`
+
+    Notes:
+        `entry` is a dictionary with structure
+        {
+            "path": str,
+            "isout": bool,
+            "isdir": bool,
+            "isexec": bool,
+        }
+    """
+    from dvc.external_repo import external_repo
+    from dvc.repo import Repo
+    from dvc.utils import relpath
+
+    with external_repo(url, rev) as repo:
+        path_info = _get_path_info(repo, path)
+        fs_nodes = []
+        if isinstance(repo, Repo):
+            fs_nodes.extend(_ls_outs_repo(repo, path_info, recursive))
+
+        if not outs_only:
+            fs_nodes.extend(_ls_files_repo(path_info, recursive))
+
+        if path and not fs_nodes:
+            raise PathMissingError(path, repo, output_only=outs_only)
+
+        fs_nodes = {n["path_info"]: n for n in fs_nodes}.values()
+
+        def get_entry(fs_node):
+            node_path_info = fs_node["path_info"]
+            path = (
+                node_path_info.name
+                if node_path_info == path_info
+                else relpath(node_path_info, path_info)
+            )
+            return {
+                "path": path,
+                "isout": fs_node.get("isout", False),
+                "isdir": fs_node.get("isdir", False),
+                "isexec": fs_node.get("isexec", False),
+            }
+
+        entries = sorted(map(get_entry, fs_nodes), key=lambda f: f["path"])
+    return entries
+
+
+def _ls_files_repo(path_info, recursive=None):
+    from dvc.compat import fspath
+    from dvc.ignore import CleanTree
+    from dvc.path_info import PathInfo
+    from dvc.scm.tree import WorkingTree
+
+    if not os.path.exists(fspath(path_info)):
+        return []
+
+    files = []
+    tree = CleanTree(WorkingTree(path_info))
+    try:
+        for dirpath, dirnames, filenames in tree.walk(path_info):
+            files.extend(PathInfo(dirpath, f) for f in filenames)
+            if not recursive:
+                files.extend(PathInfo(dirpath, d) for d in dirnames)
+                break
+    except NotADirectoryError:
+        if os.path.isfile(fspath(path_info)):
+            files = [path_info]
+
+    return [_get_fs_node(f) for f in files]
+
+
+def _ls_outs_repo(repo, path_info, recursive=None):
+    from dvc.compat import fspath
+    from dvc.path_info import PathInfo
+
+    try:
+        outs = repo.find_outs_by_path(fspath(path_info), recursive=True)
+    except OutputNotFoundError:
+        return []
+
+    if recursive:
+        return [_get_fs_node(out.path_info, out) for out in outs]
+
+    def get_first_segment(out):
+        """Returns tuple with path_info and related out
+
+        path_info calculated as the first relpath segment
+        Example:
+            dir/file -> dir
+            dir/subdir/file -> dir
+            file -> file
+        """
+        relpath = out.path_info.relpath(path_info)
+        if relpath.parts:
+            out_path_info = PathInfo(path_info, relpath.parts[0])
+            isout = len(relpath.parts) == 1
+            return (out_path_info, out if isout else None)
+        return (out.path_info, out)
+
+    return [
+        _get_fs_node(p, out)
+        for (p, out) in {get_first_segment(out) for out in outs}
+    ]
+
+
+def _get_path_info(repo, path=None):
+    from dvc.path_info import PathInfo
+
+    if not path:
+        return PathInfo(repo.root_dir)
+    return PathInfo(repo.root_dir, path)
+
+
+def _get_fs_node(path_info, out=None):
+    from dvc.compat import fspath
+
+    if out:
+        isdir = out.is_dir_checksum if out.checksum else False
+        isexec = False
+    else:
+        try:
+            isdir = os.path.isdir(fspath(path_info))
+            mode = os.stat(fspath(path_info)).st_mode
+            isexec = mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+        except FileNotFoundError:
+            isdir = False
+            isexec = False
+
+    return {
+        "path_info": path_info,
+        "isout": bool(out),
+        "isdir": isdir,
+        "isexec": isexec,
+    }
@@ -83,9 +83,9 @@ def _get_metrics(repo, *args, rev=None, **kwargs):
        return {}


-def diff(repo, *args, a_ref=None, b_ref=None, **kwargs):
-    old = _get_metrics(repo, *args, **kwargs, rev=(a_ref or "HEAD"))
-    new = _get_metrics(repo, *args, **kwargs, rev=b_ref)
+def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
+    old = _get_metrics(repo, *args, **kwargs, rev=(a_rev or "HEAD"))
+    new = _get_metrics(repo, *args, **kwargs, rev=b_rev)

    paths = set(old.keys())
    paths.update(set(new.keys()))
@@ -27,7 +27,9 @@ def pull(
        with_deps=with_deps,
        recursive=recursive,
    )
-    self._checkout(
+    stats = self._checkout(
        targets=targets, with_deps=with_deps, force=force, recursive=recursive
    )
-    return processed_files_count
+
+    stats["downloaded"] = processed_files_count
+    return stats
@@ -1,6 +1,6 @@
 import logging

-from dvc.exceptions import ReproductionError
+from dvc.exceptions import ReproductionError, InvalidArgumentError
 from dvc.repo.scm_context import scm_context
 from . import locked
 from .graph import get_pipeline, get_pipelines
@@ -33,11 +33,18 @@ def _get_active_graph(G):
    for stage in G:
        if not stage.locked:
            continue
-        for st in nx.dfs_postorder_nodes(G, stage):
-            if st == stage:
-                continue
-            if st in active:
-                active.remove_node(st)
+        active.remove_edges_from(G.out_edges(stage))
+        for edge in G.out_edges(stage):
+            _, to_stage = edge
+            for node in nx.dfs_preorder_nodes(G, to_stage):
+                # NOTE: `in_degree` will return InDegreeView({}) if stage
+                # no longer exists in the `active` DAG.
+                if not active.in_degree(node):
+                    # NOTE: if some edge no longer exists `remove_edges_from`
+                    # will ignore it without error.
+                    active.remove_edges_from(G.out_edges(node))
+                    active.remove_node(node)
+
    return active


@@ -54,15 +61,13 @@ def reproduce(
    from dvc.stage import Stage

    if not target and not all_pipelines:
-        raise ValueError()
+        raise InvalidArgumentError(
+            "Neither `target` nor `--all-pipelines` are specified."
+        )

    interactive = kwargs.get("interactive", False)
    if not interactive:
-        config = self.config
-        core = config.config[config.SECTION_CORE]
-        kwargs["interactive"] = core.get(
-            config.SECTION_CORE_INTERACTIVE, False
-        )
+        kwargs["interactive"] = self.config["core"].get("interactive", False)

    active_graph = _get_active_graph(self.graph)
    active_pipelines = get_pipelines(active_graph)
@@ -3,7 +3,7 @@ from itertools import compress

 from funcy.py3 import cat

-from dvc.exceptions import DvcException
+from dvc.exceptions import InvalidArgumentError
 from . import locked


@@ -127,7 +127,7 @@ def status(
        )
    )
    if ignored:
-        msg = "the following options are meaningless for local status: {}"
-        raise DvcException(msg.format(", ".join(ignored)))
+        msg = "The following options are meaningless for local status: {}"
+        raise InvalidArgumentError(msg.format(", ".join(ignored)))

    return _local_status(self, targets, with_deps=with_deps)
@@ -2,10 +2,12 @@ from . import locked


@locked
-def update(self, target):
+def update(self, target, rev=None):
    from dvc.stage import Stage

    stage = Stage.load(self, target)
-    stage.update()
+    stage.update(rev)

    stage.dump()
+
+    return stage
@@ -4,24 +4,29 @@ from dvc.scm.base import Base
 from dvc.scm.git import Git


-# just a sugar to point that this is an actual implementation for a dvc
-# project under no SCM control
+# Syntactic sugar to signal that this is an actual implementation for a DVC
+# project under no SCM control.
 class NoSCM(Base):
    pass


-def SCM(root_dir):  # pylint: disable=invalid-name
+def SCM(
+    root_dir, search_parent_directories=True, no_scm=False
+):  # pylint: disable=invalid-name
    """Returns SCM instance that corresponds to a repo at the specified
    path.

    Args:
        root_dir (str): path to a root directory of the repo.
-        repo (dvc.repo.Repo): DVC repo instance that root_dir belongs to.
+        search_parent_directories (bool): whether to look for repo root in
+        parent directories.
+        no_scm (bool): return NoSCM if True.

    Returns:
        dvc.scm.base.Base: SCM instance.
    """
-    if Git.is_repo(root_dir) or Git.is_submodule(root_dir):
-        return Git(root_dir)

-    return NoSCM(root_dir)
+    if no_scm:
+        return NoSCM(root_dir)
+
+    return Git(root_dir, search_parent_directories=search_parent_directories)
@@ -28,7 +28,11 @@ class Base(object):
    """Base class for source control management driver implementations."""

    def __init__(self, root_dir=os.curdir):
-        self.root_dir = os.path.realpath(root_dir)
+        self._root_dir = os.path.realpath(root_dir)
+
+    @property
+    def root_dir(self):
+        return self._root_dir

    def __repr__(self):
        return "{class_name}: '{directory}'".format(
@@ -110,8 +114,13 @@ class Base(object):
        """Returns a list of commits in the repo."""
        return []

-    def install(self):
-        """Adds dvc commands to SCM hooks for the repo."""
+    def install(self, use_pre_commit_tool=False):
+        """
+        Adds dvc commands to SCM hooks for the repo.
+
+        If use_pre_commit_tool is set and pre-commit is
+        installed it will be used to install the hooks.
+        """

    def cleanup_ignores(self):
        """
@@ -2,20 +2,16 @@

 import logging
 import os
+import yaml

 from funcy import cached_property
 from pathspec.patterns import GitWildMatchPattern

 from dvc.exceptions import GitHookAlreadyExistsError
 from dvc.scm.base import Base
-from dvc.scm.base import CloneError
-from dvc.scm.base import FileNotInRepoError
-from dvc.scm.base import RevError
-from dvc.scm.base import SCMError
+from dvc.scm.base import CloneError, FileNotInRepoError, RevError, SCMError
 from dvc.scm.git.tree import GitTree
-from dvc.utils import fix_env
-from dvc.utils import is_binary
-from dvc.utils import relpath
+from dvc.utils import fix_env, is_binary, relpath
 from dvc.utils.fs import path_isin

 logger = logging.getLogger(__name__)
@@ -27,7 +23,7 @@ class Git(Base):
    GITIGNORE = ".gitignore"
    GIT_DIR = ".git"

-    def __init__(self, root_dir=os.curdir):
+    def __init__(self, root_dir=os.curdir, search_parent_directories=True):
        """Git class constructor.
        Requires `Repo` class from `git` module (from gitpython package).
        """
@@ -37,10 +33,12 @@ class Git(Base):
        from git.exc import InvalidGitRepositoryError

        try:
-            self.repo = git.Repo(self.root_dir)
+            self.repo = git.Repo(
+                root_dir, search_parent_directories=search_parent_directories
+            )
        except InvalidGitRepositoryError:
            msg = "{} is not a git repository"
-            raise SCMError(msg.format(self.root_dir))
+            raise SCMError(msg.format(root_dir))

        # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
        # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
@@ -51,6 +49,10 @@ class Git(Base):
        self.ignored_paths = []
        self.files_to_track = set()

+    @property
+    def root_dir(self):
+        return self.repo.working_tree_dir
+
    @staticmethod
    def clone(url, to_path, rev=None):
        import git
@@ -95,12 +97,10 @@ class Git(Base):
        return repo

    @staticmethod
-    def is_repo(root_dir):
-        return os.path.isdir(Git._get_git_dir(root_dir))
+    def is_sha(rev):
+        import git

-    @staticmethod
-    def is_submodule(root_dir):
-        return os.path.isfile(Git._get_git_dir(root_dir))
+        return rev and git.Repo.re_hexsha_shortened.search(rev)

    @staticmethod
    def _get_git_dir(root_dir):
@@ -211,14 +211,16 @@ class Git(Base):
            self.repo.git.checkout(branch)

    def pull(self):
-        info, = self.repo.remote().pull()
-        if info.flags & info.ERROR:
-            raise SCMError("pull failed: {}".format(info.note))
+        infos = self.repo.remote().pull()
+        for info in infos:
+            if info.flags & info.ERROR:
+                raise SCMError("pull failed: {}".format(info.note))

    def push(self):
-        info, = self.repo.remote().push()
-        if info.flags & info.ERROR:
-            raise SCMError("push failed: {}".format(info.summary))
+        infos = self.repo.remote().push()
+        for info in infos:
+            if info.flags & info.ERROR:
+                raise SCMError("push failed: {}".format(info.summary))

    def branch(self, branch):
        self.repo.git.branch(branch)
@@ -252,43 +254,57 @@ class Git(Base):
    def list_all_commits(self):
        return [c.hexsha for c in self.repo.iter_commits("--all")]

-    def _install_hook(self, name, preconditions, cmd):
-        # only run in dvc repo
-        in_dvc_repo = '[ -n "$(git ls-files --full-name .dvc)" ]'
-
-        command = "if {}; then exec dvc {}; fi".format(
-            " && ".join([in_dvc_repo] + preconditions), cmd
-        )
-
+    def _install_hook(self, name):
        hook = self._hook_path(name)
-
-        if os.path.isfile(hook):
-            with open(hook, "r+") as fobj:
-                if command not in fobj.read():
-                    fobj.write("{command}\n".format(command=command))
-        else:
-            with open(hook, "w+") as fobj:
-                fobj.write("#!/bin/sh\n" "{command}\n".format(command=command))
+        with open(hook, "w+") as fobj:
+            fobj.write("#!/bin/sh\nexec dvc git-hook {} $@\n".format(name))

        os.chmod(hook, 0o777)

-    def install(self):
-        self._verify_dvc_hooks()
+    def install(self, use_pre_commit_tool=False):
+        if not use_pre_commit_tool:
+            self._verify_dvc_hooks()
+            self._install_hook("post-checkout")
+            self._install_hook("pre-commit")
+            self._install_hook("pre-push")
+            return

-        self._install_hook(
-            "post-checkout",
-            [
-                # checking out some reference and not specific file.
-                '[ "$3" = "1" ]',
-                # make sure we are not in the middle of a rebase/merge, so we
-                # don't accidentally break it with an unsuccessful checkout.
-                # Note that git hooks are always running in repo root.
-                "[ ! -d .git/rebase-merge ]",
+        config_path = os.path.join(self.root_dir, ".pre-commit-config.yaml")
+
+        config = {}
+        if os.path.exists(config_path):
+            with open(config_path, "r") as fobj:
+                config = yaml.safe_load(fobj)
+
+        entry = {
+            "repo": "https://github.com/iterative/dvc",
+            "rev": "master",
+            "hooks": [
+                {
+                    "id": "dvc-pre-commit",
+                    "language_version": "python3",
+                    "stages": ["commit"],
+                },
+                {
+                    "id": "dvc-pre-push",
+                    "language_version": "python3",
+                    "stages": ["push"],
+                },
+                {
+                    "id": "dvc-post-checkout",
+                    "language_version": "python3",
+                    "stages": ["post-checkout"],
+                    "always_run": True,
+                },
            ],
-            "checkout",
-        )
-        self._install_hook("pre-commit", [], "status")
-        self._install_hook("pre-push", [], "push")
+        }
+
+        if entry in config["repos"]:
+            return
+
+        config["repos"].append(entry)
+        with open(config_path, "w+") as fobj:
+            yaml.dump(config, fobj)

    def cleanup_ignores(self):
        for path in self.ignored_paths:
@@ -324,15 +340,45 @@ class Git(Base):
        return GitTree(self.repo, self.resolve_rev(rev))

    def get_rev(self):
-        return self.repo.git.rev_parse("HEAD")
+        return self.repo.rev_parse("HEAD").hexsha

    def resolve_rev(self, rev):
-        from git.exc import GitCommandError
+        from git.exc import BadName, GitCommandError
+        from contextlib import suppress

+        def _resolve_rev(name):
+            with suppress(BadName, GitCommandError):
+                try:
+                    # Try python implementation of rev-parse first, it's faster
+                    return self.repo.rev_parse(name).hexsha
+                except NotImplementedError:
+                    # Fall back to `git rev-parse` for advanced features
+                    return self.repo.git.rev_parse(name)
+
+        # Resolve across local names
+        sha = _resolve_rev(rev)
+        if sha:
+            return sha
+
+        # Try all the remotes and if it resolves unambiguously then take it
+        if not Git.is_sha(rev):
+            shas = {
+                _resolve_rev("{}/{}".format(remote.name, rev))
+                for remote in self.repo.remotes
+            } - {None}
+            if len(shas) > 1:
+                raise RevError("ambiguous Git revision '{}'".format(rev))
+            if len(shas) == 1:
+                return shas.pop()
+
+        raise RevError("unknown Git revision '{}'".format(rev))
+
+    def has_rev(self, rev):
        try:
-            return self.repo.git.rev_parse(rev)
-        except GitCommandError:
-            raise RevError("unknown Git revision '{}'".format(rev))
+            self.resolve_rev(rev)
+            return True
+        except RevError:
+            return False

    def close(self):
        self.repo.close()
@@ -12,19 +12,11 @@ GIT_MODE_DIR = 0o40000
 GIT_MODE_FILE = 0o644


-def _iter_tree(tree):
-    # During usual tree iteration with ``for .. in ..`` GitPython returns each
-    # submodule instance without correctly filled ``name`` property. It raises
-    # an exception during accessing such submodule ``name`` later.
-    # The same time repo's submodules property contains correctly initialized
-    # submodules list.
-    # Here we just replace submodule instance from iterator with the
-    # corresponding instance from ``repo.submodules.``
-    submodules = {x.hexsha: x for x in tree.repo.submodules}
-    for node in tree:
-        if node.type == "submodule":
-            node = submodules[node.hexsha]
-        yield node
+def _item_basename(item):
+    # NOTE: `item.name` is not always a basename. See [1] for more details.
+    #
+    # [1] https://github.com/iterative/dvc/issues/3481#issuecomment-600693884
+    return os.path.basename(item.path)


 class GitTree(BaseTree):
@@ -87,8 +79,8 @@ class GitTree(BaseTree):
            return False
        # see https://github.com/gitpython-developers/GitPython/issues/851
        # `return (i in tree)` doesn't work so here is a workaround:
-        for i in _iter_tree(obj):
-            if i.name == path:
+        for item in obj:
+            if _item_basename(item) == path:
                return True
        return False

@@ -118,11 +110,12 @@ class GitTree(BaseTree):

    def _walk(self, tree, topdown=True):
        dirs, nondirs = [], []
-        for i in _iter_tree(tree):
-            if i.mode == GIT_MODE_DIR:
-                dirs.append(i.name)
+        for item in tree:
+            name = _item_basename(item)
+            if item.mode == GIT_MODE_DIR:
+                dirs.append(name)
            else:
-                nondirs.append(i.name)
+                nondirs.append(name)

        if topdown:
            yield os.path.normpath(tree.abspath), dirs, nondirs
@@ -15,7 +15,7 @@ from voluptuous import Any, Schema, MultipleInvalid
 import dvc.dependency as dependency
 import dvc.output as output
 import dvc.prompt as prompt
-from dvc.exceptions import DvcException
+from dvc.exceptions import CheckoutError, DvcException
 from dvc.utils import dict_md5
 from dvc.utils import fix_env
 from dvc.utils import relpath
@@ -191,7 +191,7 @@ class Stage(object):
    PARAM_ALWAYS_CHANGED = "always_changed"

    SCHEMA = {
-        PARAM_MD5: Any(str, None),
+        PARAM_MD5: output.CHECKSUM_SCHEMA,
        PARAM_CMD: Any(str, None),
        PARAM_WDIR: Any(str, None),
        PARAM_DEPS: Any([dependency.SCHEMA], None),
@@ -405,11 +405,11 @@ class Stage(object):

        return self

-    def update(self):
+    def update(self, rev=None):
        if not self.is_repo_import and not self.is_import:
            raise StageUpdateError(self.relpath)

-        self.deps[0].update()
+        self.deps[0].update(rev=rev)
        locked = self.locked
        self.locked = False
        try:
@@ -502,12 +502,11 @@ class Stage(object):
        return True

    @staticmethod
-    def create(repo, **kwargs):
+    def create(repo, accompany_outs=False, **kwargs):

        wdir = kwargs.get("wdir", None)
        cwd = kwargs.get("cwd", None)
        fname = kwargs.get("fname", None)
-        add = kwargs.get("add", False)

        # Backward compatibility for `cwd` option
        if wdir is None and cwd is not None:
@@ -539,12 +538,12 @@ class Stage(object):
        stage._check_duplicated_arguments()

        if not fname:
-            fname = Stage._stage_fname(stage.outs, add)
+            fname = Stage._stage_fname(stage.outs, accompany_outs)
        stage._check_dvc_filename(fname)

        # Autodetecting wdir for add, we need to create outs first to do that,
        # so we start with wdir = . and remap out paths later.
-        if add and kwargs.get("wdir") is None and cwd is None:
+        if accompany_outs and kwargs.get("wdir") is None and cwd is None:
            wdir = os.path.dirname(fname)

            for out in chain(stage.outs, stage.deps):
@@ -983,18 +982,25 @@ class Stage(object):
        relink=False,
        filter_info=None,
    ):
-        failed_checkouts = []
+        checkouts = {"failed": [], "added": [], "modified": []}
        for out in self._filter_outs(filter_info):
-            failed = out.checkout(
-                force=force,
-                tag=self.tag,
-                progress_callback=progress_callback,
-                relink=relink,
-                filter_info=filter_info,
-            )
-            if failed:
-                failed_checkouts.append(failed)
-        return failed_checkouts
+            try:
+                result = out.checkout(
+                    force=force,
+                    tag=self.tag,
+                    progress_callback=progress_callback,
+                    relink=relink,
+                    filter_info=filter_info,
+                )
+                added, modified = result or (None, None)
+                if modified:
+                    checkouts["modified"].append(out.path_info)
+                elif added:
+                    checkouts["added"].append(out.path_info)
+            except CheckoutError as exc:
+                checkouts["failed"].extend(exc.target_infos)
+
+        return checkouts

    @staticmethod
    def _status(entries):
@@ -6,7 +6,6 @@ import re
 import sqlite3
 from urllib.parse import urlunparse, urlencode

-from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.utils import current_timestamp
 from dvc.utils import relpath
@@ -95,18 +94,15 @@ class State(object):  # pylint: disable=too-many-instance-attributes
    MAX_INT = 2 ** 63 - 1
    MAX_UINT = 2 ** 64 - 2

-    def __init__(self, repo, config):
+    def __init__(self, repo):
        self.repo = repo
        self.dvc_dir = repo.dvc_dir
        self.root_dir = repo.root_dir

-        state_config = config.get(Config.SECTION_STATE, {})
-        self.row_limit = state_config.get(
-            Config.SECTION_STATE_ROW_LIMIT, self.STATE_ROW_LIMIT
-        )
+        state_config = repo.config.get("state", {})
+        self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
        self.row_cleanup_quota = state_config.get(
-            Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
-            self.STATE_ROW_CLEANUP_QUOTA,
+            "row_cleanup_quota", self.STATE_ROW_CLEANUP_QUOTA
        )

        if not self.dvc_dir:
@@ -141,7 +137,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes

    def _fetchall(self):
        ret = self.cursor.fetchall()
-        logger.debug("fetched: {}", ret)
+        logger.debug("fetched: %s", ret)
        return ret

    def _to_sqlite(self, num):
@@ -182,8 +178,8 @@ class State(object):  # pylint: disable=too-many-instance-attributes
                )
            elif version < self.VERSION:
                logger.warning(
-                    "State file version '{}' is too old. "
-                    "Reformatting to the current version '{}'.",
+                    "State file version '%d' is too old. "
+                    "Reformatting to the current version '%d'.",
                    version,
                    self.VERSION,
                )
@@ -443,7 +439,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
        )
        self._execute(cmd, (relative_path, self._to_sqlite(inode), mtime))

-    def remove_unused_links(self, used):
+    def get_unused_links(self, used):
        """Removes all saved links except the ones that are used.

        Args:
@@ -457,20 +453,22 @@ class State(object):  # pylint: disable=too-many-instance-attributes
            inode = self._from_sqlite(inode)
            path = os.path.join(self.root_dir, relpath)

-            if path in used:
-                continue
-
-            if not os.path.exists(path):
+            if path in used or not os.path.exists(path):
                continue

            actual_inode = get_inode(path)
            actual_mtime, _ = get_mtime_and_size(path, self.repo.tree)

-            if inode == actual_inode and mtime == actual_mtime:
-                logger.debug("Removing '{}' as unused link.", path)
-                remove(path)
+            if (inode, mtime) == (actual_inode, actual_mtime):
+                logger.debug("Removing '%s' as unused link.", path)
                unused.append(relpath)

+        return unused
+
+    def remove_links(self, unused):
+        for path in unused:
+            remove(path)
+
        for chunk_unused in to_chunks(
            unused, chunk_size=SQLITE_MAX_VARIABLES_NUMBER
        ):
@@ -3,28 +3,20 @@ import logging
 import os
 import platform
 import shutil
+import sys

 from dvc.compat import fspath
 from dvc.exceptions import DvcException

 logger = logging.getLogger(__name__)

-if platform.system() == "Windows":
+if platform.system() == "Windows" and sys.version_info < (3, 8):
    try:
        import speedcopy

        speedcopy.patch_copyfile()
    except ImportError:
        pass
-else:
-    import sys
-
-    if sys.version_info < (3, 8):
-        try:
-            # Importing the module monkey-patches shutil.copyfile
-            import pyfastcopy  # noqa: F401
-        except ImportError:
-            pass


 class System(object):
@@ -29,7 +29,7 @@ def fs_copy(src, dst):

 def get_inode(path):
    inode = System.inode(path)
-    logger.debug("Path {} inode {}", path, inode)
+    logger.debug("Path '%s' inode '%d'", path, inode)
    return inode


@@ -97,8 +97,8 @@ def move(src, dst, mode=None):
    of data is happening.
    """

-    src = fspath_py35(src)
-    dst = fspath_py35(dst)
+    src = fspath(src)
+    dst = fspath(dst)

    dst = os.path.abspath(dst)
    tmp = "{}.{}".format(dst, uuid())
@@ -130,7 +130,7 @@ def _chmod(func, p, excinfo):


 def remove(path):
-    logger.debug("Removing '{}'", path)
+    logger.debug("Removing '%s'", path)

    path = fspath_py35(path)
    try:
@@ -7,7 +7,7 @@ import os
 import subprocess


-_BASE_VERSION = "0.82.10"
+_BASE_VERSION = "0.90.2"


 def _generate_version(base_version):
@@ -44,8 +44,8 @@ if [[ "$TRAVIS_BUILD_STAGE_NAME" == "Test" ]]; then
 fi

 if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
-  $scriptdir/retry.sh choco install python --version 3.7.5
-  echo 'PATH="/c/Python37:/c/Python37/Scripts:$PATH"' >>env.sh
+  $scriptdir/retry.sh choco install python --version $PYTHON_VERSION
+  echo "PATH='$PATH'" >>env.sh
 elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
  ln -s -f /usr/local/bin/python3 /usr/local/bin/python
  ln -s -f /usr/local/bin/pip3 /usr/local/bin/pip
@@ -16,7 +16,7 @@ git config --global user.name "DVC Tester"
 if [[ "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
  aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
  aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
-  aws configure set region us-east-2
+  aws configure set region us-west-2

  openssl enc -d -aes-256-cbc -md md5 -k $GCP_CREDS -in scripts/ci/gcp-creds.json.enc -out scripts/ci/gcp-creds.json
 fi
@@ -11,7 +11,7 @@ sudo apt-get install openjdk-7-jdk
 java -version

 pushd /usr/local
-sudo wget https://s3-us-west-2.amazonaws.com/dvc-share/test/hadoop-2.6.5.tar.gz
+sudo wget https://s3-us-east-2.amazonaws.com/dvc-public/travis-test/hadoop-2.6.5.tar.gz
 sudo tar xzf hadoop-2.6.5.tar.gz
 sudo mkdir hadoop
 sudo mv hadoop-2.6.5/* hadoop/
@@ -23,6 +23,14 @@ echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >>env.sh
 echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >>env.sh
 echo "export YARN_HOME=/usr/local/hadoop" >>env.sh
 echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >>env.sh
+
+# PyArrow==0.16.0 regression https://issues.apache.org/jira/browse/ARROW-7841
+# retrieves native library from $HADOOP_HOME directory instead of
+# `$HADOOP_HOME/lib/native`.
+# Fix: force search for `libhdfs.so` inside `$HADOOP_HOME/lib/native`.
+# Note: not needed for PyArrow==0.17.0.
+echo "export ARROW_LIBHDFS_DIR=/usr/local/hadoop/lib/native" >> env.sh
+
 echo "export JAVA_HOME=/usr/" >>env.sh
 echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >>env.sh

@@ -1,82 +1,127 @@
 #!/usr/bin/env bash
-
-#----------------------------------------------------------
-# Repository:  https://github.com/iterative/dvc
-#
 # References:
 #   - https://www.gnu.org/software/bash/manual/html_node/Programmable-Completion.html
 #   - https://opensource.com/article/18/3/creating-bash-completion-script
-#----------------------------------------------------------
+#   - https://stackoverflow.com/questions/12933362

 _dvc_commands='add cache checkout commit config destroy diff fetch get-url get gc \
-  import-url import init install lock metrics move pipeline pull push \
+  import-url import init install lock list metrics move pipeline pull push \
  remote remove repro root run status unlock unprotect update version'

 _dvc_options='-h --help -V --version'
 _dvc_global_options='-h --help -q --quiet -v --verbose'

-_dvc_add='-R --recursive -f --file --no-commit $(compgen -G *)'
+_dvc_add='-R --recursive -f --file --no-commit'
+_dvc_add_COMPGEN=_dvc_compgen_files
 _dvc_cache='dir'
 _dvc_cache_dir=' --global --system --local -u --unset'
-_dvc_checkout='-d --with-deps -R --recursive -f --force --relink $(compgen -G *.dvc)'
-_dvc_commit='-f --force -d --with-deps -R --recursive $(compgen -G *.dvc)'
+_dvc_checkout='-d --with-deps -R --recursive -f --force --relink --summary'
+_dvc_checkout_COMPGEN=_dvc_compgen_DVCFiles
+_dvc_commit='-f --force -d --with-deps -R --recursive'
+_dvc_commit_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_config='-u --unset --local --system --global'
 _dvc_destroy='-f --force'
-_dvc_diff='-t --show-json --checksums'
-_dvc_fetch='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive $(compgen -G *.dvc)'
-_dvc_get_url=''
+_dvc_diff='-t --show-json --show-hash'
+_dvc_fetch='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
+_dvc_fetch_COMPGEN=_dvc_compgen_DVCFiles
+_dvc_gc='-a --all-branches --all-commits -T --all-tags -w --workspace -c --cloud -r --remote -f --force -p --projects -j --jobs'
 _dvc_get='-o --out --rev --show-url'
-_dvc_gc='-a --all-branches -T --all-tags -c --cloud -r --remote -f --force -p --projects -j --jobs'
-_dvc_import_url='-f --file'
+_dvc_get_url=''
 _dvc_import='-o --out --rev'
+_dvc_import_url='-f --file'
 _dvc_init='--no-scm -f --force'
 _dvc_install=''
-_dvc_lock='$(compgen -G *.dvc)'
-_dvc_metrics='add modify rmeove show'
-_dvc_metrics_add='-t --type -x --xpath $(compgen -G *)'
-_dvc_metrics_show='$(-t --type -x --xpath -a --all-branches -T --all-tags -R --recursive $(compgen -G *)'
+_dvc_list='-R --recursive --outs-only --rev'
+_dvc_list_COMPGEN=_dvc_compgen_files
+_dvc_lock=_dvc_compgen_DVCFiles
+_dvc_metrics='add diff modify remove show'
+_dvc_metrics_add='-t --type -x --xpath'
+_dvc_metrics_add_COMPGEN=_dvc_compgen_files
 _dvc_metrics_diff='--targets -t --type -x --xpath -R --show-json'
-_dvc_metrics_modify='-t --type -x --xpath $(compgen -G *)'
-_dvc_metrics_remove='$(compgen -G *)'
-_dvc_move='$(compgen -G *)'
+_dvc_metrics_modify='-t --type -x --xpath'
+_dvc_metrics_modify_COMPGEN=_dvc_compgen_files
+_dvc_metrics_remove=_dvc_compgen_files
+_dvc_metrics_show='-t --type -x --xpath -a --all-branches -T --all-tags -R --recursive'
+_dvc_metrics_show_COMPGEN=_dvc_compgen_files
+_dvc_move=_dvc_compgen_files
 _dvc_pipeline='list show'
 _dvc_pipeline_list=''
-_dvc_pipeline_show='-c --commands -o --outs --ascii --dot --tree -l --locked $(compgen -G *.dvc)'
-_dvc_pull='-j --jobs -r --remote -a --all-branches -T --all-tags -f --force -d --with-deps -R --recursive $(compgen -G *.dvc)'
-_dvc_push='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive $(compgen -G *.dvc)'
+_dvc_pipeline_show='-c --commands -o --outs --ascii --dot --tree -l --locked'
+_dvc_pipeline_show_COMPGEN=_dvc_compgen_DVCFiles
+_dvc_pull='-j --jobs -r --remote -a --all-branches -T --all-tags -f --force -d --with-deps -R --recursive'
+_dvc_pull_COMPGEN=_dvc_compgen_DVCFiles
+_dvc_push='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
+_dvc_push_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_remote='add default list modify remove'
 _dvc_remote_add='--global --system --local -d --default -f --force'
 _dvc_remote_default='--global --system --local -u --unset'
 _dvc_remote_list='--global --system --local'
 _dvc_remote_modify='--global --system --local -u --unset'
 _dvc_remote_remove='--global --system --local'
-_dvc_remove='-o --outs -p --purge -f --force $(compgen -G *.dvc)'
-_dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --ignore-build-cache --no-commit -R --recursive --downstream $(compgen -G *.dvc)'
+_dvc_remove='-o --outs -p --purge -f --force'
+_dvc_remove_COMPGEN=_dvc_compgen_DVCFiles
+_dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --ignore-build-cache --no-commit -R --recursive --downstream'
+_dvc_repro_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_root=''
-_dvc_run='--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache -y --yes --overwrite-dvcfile --ignore-build-cache --remove-outs --no-commit -w --wdir'
-_dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud $(compgen -G *.dvc)'
-_dvc_unlock='$(compgen -G *.dvc)'
-_dvc_unprotect='$(compgen -G *)'
-_dvc_update='$(compgen -G *.dvc)'
+_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --ignore-build-cache --no-commit -w --wdir'
+_dvc_run_COMPGEN=_dvc_compgen_DVCFiles
+_dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud'
+_dvc_status_COMPGEN=_dvc_compgen_DVCFiles
+_dvc_unlock_COMPGEN=_dvc_compgen_DVCFiles
+_dvc_unprotect_COMPGEN=_dvc_compgen_files
+_dvc_update='--rev'
+_dvc_update_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_version=''

+# $1=COMP_WORDS[1]
+_dvc_compgen_DVCFiles() {
+  compgen -f -X '!*?.dvc' -- $1
+  compgen -d -S '/' -- $1  # recurse into subdirs
+  # Note that the recurse into dirs is only for looking for DVC-files.
+  # Since dirs themselves are not required, we need `-o nospace` at the bottom
+  # unfortunately :(
+}
+
+# $1=COMP_WORDS[1]
+_dvc_compgen_files() {
+  compgen -f -- $1
+  compgen -d -S '/' -- $1  # recurse into subdirs
+}
+
+# $1=COMP_WORDS[1]
+_dvc_replace_hyphen() {
+  echo $1 | sed 's/-/_/g'
+}
+
+# $1=COMP_WORDS[1]
+_dvc_compgen_command() {
+  local flags_list="_dvc_$(_dvc_replace_hyphen $1)"
+  local args_gen="${flags_list}_COMPGEN"
+  COMPREPLY=( $(compgen -W "$_dvc_global_options ${!flags_list}" -- "$word"; [ -n "${!args_gen}" ] && ${!args_gen} "$word") )
+}
+
+# $1=COMP_WORDS[1]
+# $2=COMP_WORDS[2]
+_dvc_compgen_subcommand() {
+  local flags_list="_dvc_$(_dvc_replace_hyphen $1)_$(_dvc_replace_hyphen $2)"
+  local args_gen="${flags_list}_COMPGEN"
+  [ -n "${!args_gen}" ] && local opts_more="$(${!args_gen} "$word")"
+  local opts="${!flags_list}"
+  if [ -z "$opts$opts_more" ]; then
+    _dvc_compgen_command $1
+  else
+    COMPREPLY=( $(compgen -W "$_dvc_global_options $opts" -- "$word"; [ -n "$opts_more" ] && echo "$opts_more") )
+  fi
+}
+
 # Notes:
-#
 # `COMPREPLY` contains what will be rendered after completion is triggered
-#
 # `word` refers to the current typed word
-#
 # `${!var}` is to evaluate the content of `var` and expand its content as a variable
-#
 #       hello="world"
 #       x="hello"
 #       ${!x} ->  ${hello} ->  "world"
-#
 _dvc() {
-  replace_hyphen() {
-    echo $(echo $1 | sed 's/-/_/g')
-  }
-
  local word="${COMP_WORDS[COMP_CWORD]}"

  COMPREPLY=()
@@ -87,16 +132,12 @@ _dvc() {
      *) COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
    esac
  elif [ "${COMP_CWORD}" -eq 2 ]; then
-    local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})"
-
-    COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
-  elif [ "${COMP_CWORD}" -eq 3 ]; then
-    local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})_$(replace_hyphen ${COMP_WORDS[2]})"
-
-    COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
+    _dvc_compgen_command ${COMP_WORDS[1]}
+  elif [ "${COMP_CWORD}" -ge 3 ]; then
+    _dvc_compgen_subcommand ${COMP_WORDS[1]} ${COMP_WORDS[2]}
  fi

  return 0
 }

-complete -F _dvc dvc
+complete -o nospace -F _dvc dvc
@@ -12,14 +12,14 @@

 _dvc_commands() {
  local _commands=(
-    "add:Take data files or directories under DVC control."
+    "add:Track data files or directories with DVC."
    "cache:Manage cache settings."
    "checkout:Checkout data files from cache."
    "commit:Save changed data to cache and update DVC-files."
    "config:Get or set config settings."
    "destroy:Remove DVC-files, local DVC config and data cache."
-    "diff:Show a diff of a DVC controlled data file or a directory."
-    "fetch:Fetch data files from a DVC remote storage."
+    "diff:Show changes between commits in the DVC repository, or between a commit and the workspace."
+    "fetch:Get files or directories tracked by DVC from remote storage into the cache."
    "get-url:Download or copy files from URL."
    "get:Download data from DVC repository."
    "gc:Garbage collect unused objects from cache or remote storage."
@@ -27,6 +27,7 @@ _dvc_commands() {
    "import:Download data from DVC repository and take it under DVC control."
    "init:Initialize DVC in the current directory."
    "install:Install DVC git hooks into the repository."
+    "list:List repository contents, including files and directories tracked by DVC and by Git."
    "lock:Lock DVC-file."
    "metrics:Commands to add, manage, collect and display metrics."
    "move:Rename or move a DVC controlled data file or a directory."
@@ -75,6 +76,7 @@ _dvc_checkout=(
  {-R,--recursive}"[Checkout all subdirectories of the specified directory.]"
  {-f,--force}"[Do not prompt when removing working directory files.]"
  "--relink[Recreate links or copies from cache to workspace.]"
+  "--summary[Show summary of the changes.]"
  "1:Stages:_files -g '(*.dvc|Dvcfile)'"
 )

@@ -98,9 +100,9 @@ _dvc_destroy=(

 _dvc_diff=(
  "--show-json[Format the output into a JSON]"
-  "--checksums[Display checksums for each entry]"
-  "1:Git reference to the older version:"
-  "2:Git reference to the newer version:"
+  "--show-hash[Display hash value for each entry]"
+  "1:Old Git commit to compare (defaults to HEAD):"
+  "2:New Git commit to compare (defaults to the current workspace):"
 )

 _dvc_fetch=(
@@ -127,8 +129,10 @@ _dvc_get=(
 )

 _dvc_gc=(
-  {-a,--all-branches}"[Keep data files for the tips of all git branches.]"
-  {-T,--all-tags}"[Keep data files for all git tags.]"
+  {-w,--workspace}"[Keep data files used in the current workspace (working tree).]"
+  {-a,--all-branches}"[Keep data files for the tips of all Git branches.]"
+  "--all-commits[Keep data files for all Git commits.]"
+  {-T,--all-tags}"[Keep data files for all Git tags.]"
  {-c,--cloud}"[Collect garbage in remote repository.]"
  {-r,--remote}"[Remote storage to collect garbage in.]:Remote repository:"
  {-f,--force}"[Force garbage collection - automatically agree to all prompts.]:Repos:_files"
@@ -160,6 +164,14 @@ _dvc_lock=(
  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
 )

+_dvc_list=(
+  "--rev[Git revision (e.g. branch, tag, SHA)]:Revision:"
+  {-R,--recursive}"[Recursively add each file under the directory.]"
+  "--outs-only[Only outputs DVC-outs.]"
+  "1:URL:"
+  "2:Path:"
+)
+
 _dvc_metrics=(
  "1:Sub command:(add show diff modify remove)"
 )
@@ -245,7 +257,6 @@ _dvc_run=(

 _dvc_status=(
  {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
-  "--show-checksums[Show checksums instead of file names.]"
  {-q,--quiet}"[Suppresses all output. Exit with 0 if pipelines are up to date, otherwise 1.]"
  {-c,--cloud}"[Show status of a local cache compared to a remote repository.]"
  {-r,--remote}"[Remote repository to compare local cache to.]:Remote repository:"
@@ -264,6 +275,7 @@ _dvc_unprotect=(
 )

 _dvc_update=(
+  "--rev[Git revision (e.g. SHA, branch, tag)]:Revision:"
  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
 )

@@ -292,6 +304,7 @@ case $words[1] in
  init) _arguments $_dvc_global_options $_dvc_init ;;
  install) _arguments $_dvc_global_options $_dvc_install ;;
  lock) _arguments $_dvc_global_options $_dvc_lock ;;
+  list) _arguments $_dvc_global_options $_dvc_list ;;
  metrics) _arguments $_dvc_global_options $_dvc_metrics ;;
  move) _arguments $_dvc_global_options $_dvc_move ;;
  pipeline) _arguments $_dvc_global_options $_dvc_pipeline ;;
@@ -8,7 +8,10 @@ from setuptools.command.build_py import build_py as _build_py
 # Prevents pkg_resources import in entry point script,
 # see https://github.com/ninjaaron/fast-entry_points.
 # This saves about 200 ms on startup time for non-wheel installs.
-import fastentrypoints  # noqa: F401
+try:
+    import fastentrypoints  # noqa: F401
+except ImportError:
+    pass  # not able to import when installing through pre-commit


 # Read package meta-data from version.py
@@ -48,13 +51,12 @@ install_requires = [
    "ply>=3.9",  # See https://github.com/pyinstaller/pyinstaller/issues/1945
    "colorama>=0.3.9",
    "configobj>=5.0.6",
-    "gitpython>=2.1.8",
-    "gitdb2>=2.0.6",  # See https://github.com/iterative/dvc/issues/1880
+    "gitpython>3",
    "setuptools>=34.0.0",
    "nanotime>=0.5.2",
    "pyasn1>=0.4.1",
    "voluptuous>=0.11.7",
-    "jsonpath-ng>=1.4.3",
+    "jsonpath-ng>=1.5.1",
    "requests>=2.22.0",
    "grandalf==0.6",
    "distro>=1.3.0",
@@ -75,22 +77,27 @@ install_requires = [
    "pywin32>=225; sys_platform == 'win32'",
    "networkx>=2.1,<2.4",
    "pydot>=1.2.4",
-    "speedcopy>=2.0.1",
-    "pyfastcopy>=1.0.3",
+    "speedcopy>=2.0.1; python_version < '3.8' and sys_platform == 'win32'",
    "flatten_json>=0.1.6",
    "texttable>=0.5.2",
+    "pygtrie==2.3.2",
 ]


 # Extra dependencies for remote integrations

 gs = ["google-cloud-storage==1.19.0"]
-gdrive = ["pydrive2>=1.4.5"]
+gdrive = ["pydrive2>=1.4.6"]
 s3 = ["boto3>=1.9.201"]
 azure = ["azure-storage-blob==2.1.0"]
 oss = ["oss2==2.6.1"]
 ssh = ["paramiko>=2.5.0"]
-hdfs = ["pyarrow==0.15.1"]
+hdfs = [
+    # pyarrow-0.16.0 import fails on 3.5 and 3.7 (works on 3.6 though)
+    # due to: https://issues.apache.org/jira/browse/ARROW-7852
+    "pyarrow==0.15.1; python_version < '3.8'",
+    "pyarrow==0.16.0; python_version == '3.8'",
+]
 # gssapi should not be included in all_remotes, because it doesn't have wheels
 # for linux and mac, so it will fail to compile if user doesn't have all the
 # requirements, including kerberos itself. Once all the wheels are available,
@@ -125,7 +132,7 @@ tests_requirements = [
 ]

 if (sys.version_info) >= (3, 6):
-    tests_requirements.append("black==19.3b0")
+    tests_requirements.append("black==19.10b0")

 setup(
    name="dvc",
@@ -149,7 +156,8 @@ setup(
        "hdfs": hdfs,
        "tests": tests_requirements,
    },
-    keywords="data science, data version control, machine learning",
+    keywords="data-science data-version-control machine-learning git"
+    " developer-tools reproducibility collaboration ai",
    python_requires=">=3.5",
    classifiers=[
        "Development Status :: 4 - Beta",
@@ -157,6 +165,7 @@ setup(
        "Programming Language :: Python :: 3.5",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
    ],
    packages=find_packages(exclude=["tests"]),
    include_package_data=True,
@@ -20,6 +20,8 @@ parts:
    plugin: python
    python-packages: [wheel]
    source: .
+    build-snaps:
+    - snapd
    build-packages:
    - git
    - libc-dev
@@ -4,6 +4,7 @@ import mockssh
 import pytest

 from dvc.remote.ssh.connection import SSHConnection
+from tests.utils.httpd import PushRequestHandler, StaticFileServer
 from .dir_helpers import *  # noqa


@@ -57,3 +58,9 @@ def _close_pools():

    yield
    close_pools()
+
+
+@pytest.fixture
+def http_server(tmp_dir):
+    with StaticFileServer(handler_class=PushRequestHandler) as httpd:
+        yield httpd
@@ -45,12 +45,12 @@ from global repo template to creating everything inplace, which:

 import os
 import pathlib
-import logging
 from contextlib import contextmanager

 import pytest
 from funcy import lmap, retry

+from dvc.logger import disable_other_loggers
 from dvc.utils.fs import makedirs
 from dvc.compat import fspath, fspath_py35

@@ -67,8 +67,7 @@ __all__ = [


 # see https://github.com/iterative/dvc/issues/3167
-git_logger = logging.getLogger("git")
-git_logger.setLevel(logging.CRITICAL)
+disable_other_loggers()


 class TmpDir(pathlib.Path):
@@ -99,7 +98,9 @@ class TmpDir(pathlib.Path):
        if scm:
            _git_init(str_path)
        if dvc:
-            self.dvc = Repo.init(str_path, no_scm=True)
+            self.dvc = Repo.init(
+                str_path, no_scm=not scm and not hasattr(self, "scm")
+            )
        if scm:
            self.scm = self.dvc.scm if hasattr(self, "dvc") else Git(str_path)
        if dvc and hasattr(self, "scm"):
@@ -155,7 +156,18 @@ class TmpDir(pathlib.Path):
        stages = self.dvc.add(filenames)
        if commit:
            stage_paths = [s.path for s in stages]
-            self.scm_add(stage_paths, commit=commit)
+
+            def to_gitignore(stage_path):
+                from dvc.scm import Git
+
+                return os.path.join(os.path.dirname(stage_path), Git.GITIGNORE)
+
+            gitignores = [
+                to_gitignore(s)
+                for s in stage_paths
+                if os.path.exists(to_gitignore(s))
+            ]
+            self.scm_add(stage_paths + gitignores, commit=commit)

        return stages

@@ -264,15 +276,15 @@ def run_copy(tmp_dir, dvc):

@pytest.fixture
 def erepo_dir(make_tmp_dir):
-    from dvc.remote.config import RemoteConfig
-
    path = make_tmp_dir("erepo", scm=True, dvc=True)

    # Chdir for git and dvc to work locally
    with path.chdir():
-        rconfig = RemoteConfig(path.dvc.config)
-        rconfig.add("upstream", path.dvc.cache.local.cache_dir, default=True)
-        path.scm_add([path.dvc.config.config_file], commit="add remote")
+        with path.dvc.config.edit() as conf:
+            cache_dir = path.dvc.cache.local.cache_dir
+            conf["remote"]["upstream"] = {"url": cache_dir}
+            conf["core"]["remote"] = "upstream"
+        path.scm_add([path.dvc.config.files["repo"]], commit="add remote")

    return path

@@ -8,7 +8,7 @@ from tests.utils import trees_equal


 def test_dont_fail_on_unpacked_create_fail(tmp_dir, dvc):
-    stage, = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
+    (stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})

    with mock.patch.object(
        RemoteLOCAL, "_create_unpacked_dir", side_effect=DvcException("msg")
@@ -18,7 +18,7 @@ def test_dont_fail_on_unpacked_create_fail(tmp_dir, dvc):


 def test_remove_unpacked_on_create_fail(tmp_dir, dvc):
-    stage, = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
+    (stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
    unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX

    # artificial unpacked dir for test purpose
@@ -34,7 +34,7 @@ def test_remove_unpacked_on_create_fail(tmp_dir, dvc):


 def test_create_unpacked_on_status(tmp_dir, dvc):
-    stage, = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
+    (stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
    unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX
    assert not os.path.exists(unpacked_dir)

@@ -45,7 +45,7 @@ def test_create_unpacked_on_status(tmp_dir, dvc):


 def test_dir_cache_changed_on_single_cache_file_modification(tmp_dir, dvc):
-    stage, = tmp_dir.dvc_gen(
+    (stage,) = tmp_dir.dvc_gen(
        {"dir": {"file1": "file1 content", "file2": "file2 content"}}
    )
    unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX
@@ -57,6 +57,7 @@ def test_dir_cache_changed_on_single_cache_file_modification(tmp_dir, dvc):
    assert os.path.exists(unpacked_dir)

    cache_file_path = dvc.cache.local.get(file_md5)
+    os.chmod(cache_file_path, 0o644)
    with open(cache_file_path, "a") as fobj:
        fobj.write("modification")

@@ -7,11 +7,11 @@ import time

 import colorama
 import pytest
-from mock import patch
+from mock import patch, call

 import dvc as dvc_module
 from dvc.cache import Cache
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, OverlappingOutputPathsError
 from dvc.exceptions import RecursiveAddingWhileUsingFilename
 from dvc.exceptions import StageFileCorruptedError
 from dvc.main import main
@@ -31,7 +31,7 @@ from tests.utils import get_gitignore_content


 def test_add(tmp_dir, dvc):
-    stage, = tmp_dir.dvc_gen({"foo": "foo"})
+    (stage,) = tmp_dir.dvc_gen({"foo": "foo"})
    md5, _ = file_md5("foo")

    assert stage is not None
@@ -49,7 +49,7 @@ def test_add_unicode(tmp_dir, dvc):
    with open("\xe1", "wb") as fd:
        fd.write("something".encode("utf-8"))

-    stage, = dvc.add("\xe1")
+    (stage,) = dvc.add("\xe1")

    assert os.path.isfile(stage.path)

@@ -60,7 +60,7 @@ def test_add_unsupported_file(dvc):


 def test_add_directory(tmp_dir, dvc):
-    stage, = tmp_dir.dvc_gen({"dir": {"file": "file"}})
+    (stage,) = tmp_dir.dvc_gen({"dir": {"file": "file"}})

    assert stage is not None
    assert len(stage.deps) == 0
@@ -161,7 +161,7 @@ def test_add_file_in_dir(tmp_dir, dvc):
    tmp_dir.gen({"dir": {"subdir": {"subdata": "subdata content"}}})
    subdir_path = os.path.join("dir", "subdir", "subdata")

-    stage, = dvc.add(subdir_path)
+    (stage,) = dvc.add(subdir_path)

    assert stage is not None
    assert len(stage.deps) == 0
@@ -216,7 +216,7 @@ class TestAddLocalRemoteFile(TestDvc):
        self.assertEqual(ret, 0)

        d = load_stage_file("bar.dvc")
-        self.assertEqual(d["outs"][0]["path"], bar)
+        self.assertEqual(d["outs"][0]["path"], self.BAR)


 class TestCmdAdd(TestDvc):
@@ -489,12 +489,12 @@ class TestAddUnprotected(TestDvc):
        ret = main(["config", "cache.type", "hardlink"])
        self.assertEqual(ret, 0)

-        ret = main(["config", "cache.protected", "true"])
-        self.assertEqual(ret, 0)
-
        ret = main(["add", self.FOO])
        self.assertEqual(ret, 0)

+        self.assertFalse(os.access(self.FOO, os.W_OK))
+        self.assertTrue(System.is_hardlink(self.FOO))
+
        ret = main(["unprotect", self.FOO])
        self.assertEqual(ret, 0)

@@ -546,10 +546,10 @@ def temporary_windows_drive(tmp_path_factory):
 def test_windows_should_add_when_cache_on_different_drive(
    tmp_dir, dvc, temporary_windows_drive
 ):
-    dvc.config.set("cache", "dir", temporary_windows_drive)
+    dvc.config["cache"]["dir"] = temporary_windows_drive
    dvc.cache = Cache(dvc)

-    stage, = tmp_dir.dvc_gen({"file": "file"})
+    (stage,) = tmp_dir.dvc_gen({"file": "file"})
    cache_path = stage.outs[0].cache_path

    assert path_isin(cache_path, temporary_windows_drive)
@@ -561,7 +561,6 @@ def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
    tmp_dir.gen("dir/data", "data")

    dvc.cache.local.cache_types = ["symlink"]
-    dvc.cache.local.protected = True

    dvc.add("dir")
    tmp_dir.gen("dir/new_file", "new_file_content")
@@ -601,7 +600,7 @@ def test_should_relink_on_repeated_add(
 ):
    from dvc.path_info import PathInfo

-    dvc.config.set("cache", "type", link)
+    dvc.config["cache"]["type"] = link

    tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})

@@ -618,7 +617,6 @@ def test_should_relink_on_repeated_add(
@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
 def test_should_protect_on_repeated_add(link, tmp_dir, dvc):
    dvc.cache.local.cache_types = [link]
-    dvc.cache.local.protected = True

    tmp_dir.dvc_gen({"foo": "foo"})

@@ -626,7 +624,16 @@ def test_should_protect_on_repeated_add(link, tmp_dir, dvc):

    dvc.add("foo")

-    assert not os.access("foo", os.W_OK)
+    assert not os.access(
+        os.path.join(".dvc", "cache", "ac", "bd18db4cc2f85cedef654fccc4a4d8"),
+        os.W_OK,
+    )
+
+    # NOTE: Windows symlink perms don't propagate to the target
+    if link == "copy" or (link == "symlink" and os.name == "nt"):
+        assert os.access("foo", os.W_OK)
+    else:
+        assert not os.access("foo", os.W_OK)


 def test_escape_gitignore_entries(tmp_dir, scm, dvc):
@@ -641,3 +648,57 @@ def test_escape_gitignore_entries(tmp_dir, scm, dvc):

    tmp_dir.dvc_gen(fname, "...")
    assert ignored_fname in get_gitignore_content()
+
+
+def test_add_from_data_dir(tmp_dir, scm, dvc):
+    tmp_dir.dvc_gen({"dir": {"file1": "file1 content"}})
+
+    tmp_dir.gen({"dir": {"file2": "file2 content"}})
+
+    with pytest.raises(OverlappingOutputPathsError) as e:
+        dvc.add(os.path.join("dir", "file2"))
+    assert str(e.value) == (
+        "Cannot add '{out}', because it is overlapping with other DVC "
+        "tracked output: 'dir'.\n"
+        "To include '{out}' in 'dir', run 'dvc commit dir.dvc'"
+    ).format(out=os.path.join("dir", "file2"))
+
+
+def test_not_raises_on_re_add(tmp_dir, dvc):
+    tmp_dir.dvc_gen("file", "file content")
+
+    tmp_dir.gen({"file2": "file2 content", "file": "modified file"})
+    dvc.add(["file2", "file"])
+
+
+@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
+def test_add_empty_files(tmp_dir, dvc, link):
+    file = "foo"
+    dvc.cache.local.cache_types = [link]
+    stages = tmp_dir.dvc_gen(file, "")
+
+    assert (tmp_dir / file).exists()
+    assert (tmp_dir / (file + Stage.STAGE_FILE_SUFFIX)).exists()
+    assert os.path.exists(stages[0].outs[0].cache_path)
+
+
+def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker):
+    dvc.cache.local.cache_types = ["hardlink"]
+    tmp_dir.gen({"foo": "", "bar": "", "lorem": "lorem", "ipsum": "ipsum"})
+    m = mocker.spy(RemoteLOCAL, "is_hardlink")
+    stages = dvc.add(["foo", "bar", "lorem", "ipsum"])
+
+    assert m.call_count == 1
+    assert m.call_args != call(tmp_dir / "foo")
+    assert m.call_args != call(tmp_dir / "bar")
+
+    for stage in stages[:2]:
+        # hardlinks are not created for empty files
+        assert not System.is_hardlink(stage.outs[0].path_info)
+
+    for stage in stages[2:]:
+        assert System.is_hardlink(stage.outs[0].path_info)
+
+    for stage in stages:
+        assert os.path.exists(stage.path)
+        assert os.path.exists(stage.outs[0].cache_path)
@@ -1,7 +1,9 @@
 import mock

+from dvc.analytics import _scm_in_use
 from dvc.main import main
 from dvc.compat import fspath
+from dvc.repo import Repo


@mock.patch("dvc.analytics.send")
@@ -19,3 +21,24 @@ def test_main_analytics(mock_is_enabled, mock_report, tmp_dir, dvc):
    assert 0 == main(["add", "foo"])
    assert mock_is_enabled.called
    assert mock_report.called
+
+
+def test_scm_dvc_only(tmp_dir, dvc):
+    scm = _scm_in_use()
+    assert scm == "NoSCM"
+
+
+def test_scm_git(tmp_dir, scm, dvc):
+    scm = _scm_in_use()
+    assert scm == "Git"
+
+
+def test_scm_subrepo(tmp_dir, scm):
+    subdir = tmp_dir / "subdir"
+    subdir.mkdir()
+
+    with subdir.chdir():
+        Repo.init(subdir=True)
+        scm = _scm_in_use()
+
+    assert scm == "Git"
@@ -1,5 +1,4 @@
 import os
-import shutil

 import pytest

@@ -9,7 +8,8 @@ from dvc.compat import fspath
 from dvc.exceptions import FileMissingError
 from dvc.main import main
 from dvc.path_info import URLInfo
-from dvc.remote.config import RemoteConfig
+from dvc.utils.fs import remove
+
 from tests.remotes import Azure, GCP, HDFS, Local, OSS, S3, SSH


@@ -66,7 +66,7 @@ def test_open(remote_url, tmp_dir, dvc):
    run_dvc("push")

    # Remove cache to force download
-    shutil.rmtree(dvc.cache.local.cache_dir)
+    remove(dvc.cache.local.cache_dir)

    with api.open("foo") as fd:
        assert fd.read() == "foo-text"
@@ -86,7 +86,7 @@ def test_open_external(remote_url, erepo_dir):
    erepo_dir.dvc.push(all_branches=True)

    # Remove cache to force download
-    shutil.rmtree(erepo_dir.dvc.cache.local.cache_dir)
+    remove(erepo_dir.dvc.cache.local.cache_dir)

    # Using file url to force clone to tmp repo
    repo_url = "file://{}".format(erepo_dir)
@@ -102,16 +102,16 @@ def test_missing(remote_url, tmp_dir, dvc):
    run_dvc("remote", "add", "-d", "upstream", remote_url)

    # Remove cache to make foo missing
-    shutil.rmtree(dvc.cache.local.cache_dir)
+    remove(dvc.cache.local.cache_dir)

    with pytest.raises(FileMissingError):
        api.read("foo")


 def _set_remote_url_and_commit(repo, remote_url):
-    rconfig = RemoteConfig(repo.config)
-    rconfig.modify("upstream", "url", remote_url)
-    repo.scm.add([repo.config.config_file])
+    with repo.config.edit() as conf:
+        conf["remote"]["upstream"]["url"] = remote_url
+    repo.scm.add([repo.config.files["repo"]])
    repo.scm.commit("modify remote")


@@ -154,7 +154,7 @@ class TestCmdCacheDir(TestDvc):
        ret = main(["cache", "dir", dname])
        self.assertEqual(ret, 0)

-        config = configobj.ConfigObj(self.dvc.config.config_file)
+        config = configobj.ConfigObj(self.dvc.config.files["repo"])
        self.assertEqual(config["cache"]["dir"], dname)

    def test_relative_path(self):
@@ -166,7 +166,7 @@ class TestCmdCacheDir(TestDvc):
        # NOTE: we are in the repo's root and config is in .dvc/, so
        # dir path written to config should be just one level above.
        rel = os.path.join("..", dname)
-        config = configobj.ConfigObj(self.dvc.config.config_file)
+        config = configobj.ConfigObj(self.dvc.config.files["repo"])
        self.assertEqual(config["cache"]["dir"], rel)

        ret = main(["add", self.FOO])
@@ -178,19 +178,18 @@ class TestCmdCacheDir(TestDvc):
        self.assertEqual(len(files), 1)


-class TestShouldCacheBeReflinkOrCopyByDefault(TestDvc):
-    def test(self):
-        self.assertEqual(self.dvc.cache.local.cache_types, ["reflink", "copy"])
+def test_default_cache_type(dvc):
+    assert dvc.cache.local.cache_types == ["reflink", "copy"]


@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
@pytest.mark.parametrize(
-    "protected,dir_mode,file_mode",
-    [(False, 0o775, 0o664), (True, 0o775, 0o444)],
+    "group, dir_mode", [(False, 0o755), (True, 0o775)],
 )
-def test_shared_cache(tmp_dir, dvc, protected, dir_mode, file_mode):
-    dvc.config.set("cache", "shared", "group")
-    dvc.config.set("cache", "protected", str(protected))
+def test_shared_cache(tmp_dir, dvc, group, dir_mode):
+    if group:
+        with dvc.config.edit() as conf:
+            conf["cache"].update({"shared": "group"})
    dvc.cache = Cache(dvc)

    tmp_dir.dvc_gen(
@@ -204,4 +203,4 @@ def test_shared_cache(tmp_dir, dvc, protected, dir_mode, file_mode):

        for fname in fnames:
            path = os.path.join(root, fname)
-            assert stat.S_IMODE(os.stat(path).st_mode) == file_mode
+            assert stat.S_IMODE(os.stat(path).st_mode) == 0o444
--- a/Show More
+++ b/Show More