dvc: bump to 0.91.2

snap: deploy: fix authentication (#3551 )
Fixes #3550
2026-11-02 15:28:30 +01:00 · 2020-03-30 02:10:26 +03:00 · 2020-03-29 22:45:11 +03:00 · 2020-03-29 20:54:32 +03:00 · 2020-03-29 16:33:51 +03:00 · 2020-03-28 12:51:42 +09:00
257 changed files with 10828 additions and 7226 deletions
@@ -1,8 +1,7 @@
-* [ ] ❗ Have you followed the guidelines in the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) list?
+* [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.

-* [ ] 📖 Check this box if this PR **does not** require [documentation](https://dvc.org/doc) updates, or if it does **and** you have created a separate PR in [dvc.org](https://github.com/iterative/dvc.org) with such updates (or at least opened an issue about it in that repo). Please link below to your PR (or issue) in the [dvc.org](https://github.com/iterative/dvc.org) repo.
+* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here. If the CLI API is changed, I have updated [tab completion scripts](https://github.com/iterative/dvc/tree/master/scripts/completion).

-* [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect everything to be addressed. Please review them carefully and fix those that actually improve code or fix bugs.
+* [ ] ❌ I will check DeepSource, CodeClimate, and other sanity checks below. (We consider them recommendatory and don't expect everything to be addressed. Please fix things that actually improve code or fix bugs.)

 Thank you for the contribution - we'll try to review it as soon as possible. 🙏
-
@@ -26,7 +26,7 @@ innosetup/config.ini
 .coverage
 .coverage.*

-*.swp
+*.sw?

 pip-wheel-metadata/
 .vscode/
@@ -1,11 +1,35 @@
 repos:
-   repo: https://github.com/ambv/black
-    rev: 19.3b0
-    hooks:
-    - id: black
-      language_version: python3
-   repo: https://gitlab.com/pycqa/flake8
-    rev: master
-    hooks:
-    - id: flake8
-      language_version: python3
+- hooks:
+  - id: black
+    language_version: python3
+  repo: https://github.com/ambv/black
+  rev: 19.10b0
+- hooks:
+  - id: flake8
+    language_version: python3
+  repo: https://gitlab.com/pycqa/flake8
+  rev: master
+- hooks:
+  - args:
+    - -i
+    - '2'
+    id: beautysh
+    language_version: python3
+  repo: https://github.com/lovesegfault/beautysh
+  rev: master
+- hooks:
+  - id: dvc-pre-commit
+    language_version: python3
+    stages:
+    - commit
+  - id: dvc-pre-push
+    language_version: python3
+    stages:
+    - push
+  - always_run: true
+    id: dvc-post-checkout
+    language_version: python3
+    stages:
+    - post-checkout
+  repo: https://github.com/iterative/dvc
+  rev: master
@@ -0,0 +1,32 @@
+- args:
+  - git-hook
+  - pre-commit
+  entry: dvc
+  id: dvc-pre-commit
+  language: python
+  language_version: python3
+  name: DVC pre-commit
+  stages:
+  - commit
+- args:
+  - git-hook
+  - pre-push
+  entry: dvc
+  id: dvc-pre-push
+  language: python
+  language_version: python3
+  name: DVC pre-push
+  stages:
+  - push
+- always_run: true
+  args:
+  - git-hook
+  - post-checkout
+  entry: dvc
+  id: dvc-post-checkout
+  language: python
+  language_version: python3
+  minimum_pre_commit_version: 2.2.0
+  name: DVC post-checkout
+  stages:
+  - post-checkout
@@ -1,7 +1,7 @@
 ---
 restylers:
  - name: black
-    image: restyled/restyler-black:v19.3b0
+    image: restyled/restyler-black:v19.10b0
    command:
    - black
    arguments: []
@@ -13,8 +13,6 @@ stages:
  - check
  - test
  - build
-  - name: deploy
-    if: (branch = master OR tag is present) AND repo = iterative/dvc
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
@@ -35,20 +33,19 @@ jobs:
    install:
    script: ./scripts/ci/check_patch.sh
  # test jobs
-  - name: "2.7 on Windows"
+  - name: "3.7 on Windows"
    stage: test
    os: windows           # Windows 10.0.17134 N/A Build 17134
    language: shell       # 'language: python' is an error on Travis CI Windows
    env:
-    - PY2=1
-  - name: "3.7 on Windows"
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
+  - name: "3.8 on Windows"
    os: windows           # Windows 10.0.17134 N/A Build 17134
    language: shell       # 'language: python' is an error on Travis CI Windows
-  - name: "2.7 on Linux"
-    language: python
-    python: 2.7
    env:
-    - PY2=1
+      - PYTHON_VERSION=3.8.0
+      - PATH=/c/Python38:/c/Python38/Scripts:$PATH
  - name: "3.5 on Linux"
    language: python
    python: 3.5
@@ -58,14 +55,12 @@ jobs:
  - name: "3.7 on Linux"
    language: python
    python: 3.7
-  - name: "2.7 on Mac"
+  - name: "3.8 on Linux"
+    language: python
+    python: 3.8
+  - name: "3.7 on Mac"
    os: osx
-    osx_image: xcode9.4
-    env:
-    - PY2=1
-  - name: "3.6 on Mac"
-    os: osx
-    osx_image: xcode9.4
+    osx_image: xcode11.3
  # build jobs
  - name: "Mac pkg"
    stage: build
@@ -80,57 +75,41 @@ jobs:
    language: shell
    install:
    script: ./scripts/build_windows.cmd
+    env:
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
  - name: "Linux pkgs"
    language: python
    python: 3.7
    before_install:
    install:
    script: ./scripts/build_posix.sh
-  - name: "Pypi pkgs"
+  - name: "PyPI pkgs"
    language: python
-    python: 3.7
+    python: 3.8
    script: ./scripts/build_package.sh
  - name: Snapcraft snap
+    language: python
+    python: 3.8
    addons:
     snaps:
     - name: snapcraft
       channel: stable
       confinement: classic
+     - name: review-tools
+       channel: stable
     - name: lxd
       channel: stable
    env:
    - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}'
-    before_install:
+    install:
    - sudo /snap/bin/lxd.migrate -yes
    - sudo /snap/bin/lxd waitready
    - sudo /snap/bin/lxd init --auto
-    install:
    script:
-    - sudo snapcraft --use-lxd
+    - ./scripts/build_snap.sh
    after_failure:
    - sudo journalctl -u snapd
-  # deploy jobs
-  - name: "Pypi pkgs"
-    if: tag is present
-    stage: deploy
-    language: python
-    python: 3.7
-    script: ./scripts/build_package.sh
-    deploy:
-    - provider: pypi
-      user:
-        secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
-      password:
-        secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
-      skip_existing: true
-      file_glob: true
-      file:
-        - dist/dvc*.whl
-        - dist/dvc*.tar.gz
-      skip_cleanup: true
-      on:
-        tags: true
-        repo: iterative/dvc

 before_install:
  - bash ./scripts/ci/before_install.sh
@@ -142,7 +121,6 @@ script:
  - bash ./scripts/ci/script.sh
 deploy:
  - provider: releases
-    condition: "[[ $(./scripts/ci/deploy_condition.sh) == 'true' ]]"
    api_key:
      secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=
    file_glob: true
@@ -151,9 +129,33 @@ deploy:
      - dvc*.deb
      - dvc*.pkg
      - dvc*.exe
-      - dvc*.snap
    skip_cleanup: true
    on:
+      condition: "$(./scripts/ci/deploy_condition.sh dvc*.rpm dvc*.deb dvc*.pkg dvc*.exe)"
      tags: true
      repo: iterative/dvc
      stage: build
+  - provider: pypi
+    user:
+      secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
+    password:
+      secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
+    skip_existing: true
+    file_glob: true
+    file:
+      - dist/dvc*.whl
+      - dist/dvc*.tar.gz
+    skip_cleanup: true
+    on:
+      condition: "$(./scripts/ci/deploy_condition.sh dist/dvc*.whl dist/dvc*.tar.gz)"
+      tags: true
+      repo: iterative/dvc
+      stage: build
+  - provider: script
+    skip_cleanup: true
+    script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
+    on:
+      all_branches: true
+      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG)"
+      repo: iterative/dvc
+      stage: build
@@ -0,0 +1,8 @@
+{
+  "title": "DVC: Data Version Control - Git for Data & Models",
+  "keywords": [
+    "data-science", "data-version-control", "machine-learning", "git",
+    "developer-tools", "reproducibility", "collaboration", "ai", "python"],
+  "contributors": [
+    {"name": "DVC team", "type": "Other", "affiliation": "Iterative"}]
+}
@@ -1,6 +1,4 @@
-.. image:: https://dvc.org/static/img/logo-github-readme.png
-  :target: https://dvc.org
-  :alt: DVC logo
+|Banner|

 `Website <https://dvc.org>`_
 • `Docs <https://dvc.org/doc>`_
@@ -10,29 +8,9 @@
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_

-.. image:: https://travis-ci.com/iterative/dvc.svg?branch=master
-  :target: https://travis-ci.com/iterative/dvc
-  :alt: Travis
+|Release| |CI| |Maintainability| |Coverage| |Donate| |DOI|

-.. image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
-  :target: https://codeclimate.com/github/iterative/dvc
-  :alt: Code Climate
-
-.. image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
-  :target: https://codecov.io/gh/iterative/dvc
-  :alt: Codecov
-
-.. image:: https://img.shields.io/badge/patreon-donate-green.svg
-  :target: https://www.patreon.com/DVCorg/overview
-  :alt: Donate
-
-.. image:: https://anaconda.org/conda-forge/dvc/badges/version.svg
-  :target: https://anaconda.org/conda-forge/dvc
-  :alt: Conda-forge
-
-.. image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
-  :target: https://snapcraft.io/dvc
-  :alt: Snapcraft
+|Snap| |Choco| |Brew| |Conda| |PyPI| |Packages|

 |

@@ -75,9 +53,7 @@ to store data and model files seamlessly out of Git, while preserving almost the
 were stored in Git itself. To store and share the data cache, DVC supports multiple remotes - any cloud (S3, Azure,
 Google Cloud, etc) or any on-premise network storage (via SSH, for example).

-.. image:: https://dvc.org/static/img/flow.gif
-   :target: https://dvc.org/static/img/flow.gif
-   :alt: how_dvc_works
+|Flowchart|

 The DVC pipelines (computational graph) feature connects code and data together. It is possible to explicitly
 specify all steps required to produce a model: input dependencies including data, commands to run,
@@ -115,9 +91,52 @@ Installation
 There are four options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package.
 Full instructions are `available here <https://dvc.org/doc/get-started/install>`_.

+Snap (Snapcraft/Linux)
+----------------------
+
+|Snap|
+
+.. code-block:: bash
+
+   snap install dvc --classic
+
+This corresponds to the latest tagged release.
+Add ``--edge`` for the latest ``master`` version.
+
+Choco (Chocolatey/Windows)
+--------------------------
+
+|Choco|
+
+.. code-block:: bash
+
+   choco install dvc
+
+Brew (Homebrew/Mac OS)
+----------------------
+
+|Brew|
+
+.. code-block:: bash
+
+   brew install dvc
+
+Conda (Anaconda)
+----------------
+
+|Conda|
+
+.. code-block:: bash
+
+   conda install -c conda-forge dvc
+
+Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
+
 pip (PyPI)
 ----------

+|PyPI|
+
 .. code-block:: bash

   pip install dvc
@@ -133,41 +152,11 @@ To install the development version, run:

   pip install git+git://github.com/iterative/dvc

-Homebrew
--------
-
-.. code-block:: bash
-
-   brew install dvc
-
-
-Conda (Anaconda)
----------------
-
-.. code-block:: bash
-
-   conda install -c conda-forge dvc
-
-Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
-
-Snap (Snapcraft)
----------------
-
-Download the latest ``dvc_*.snap`` from the
-GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
-
-.. code-block:: bash
-
-   snap install dvc_*.snap --dangerous --classic
-
-Once ``dvc`` is released on the snap store
-(`pending request <https://forum.snapcraft.io/t/classic-confinement-request-for-dvc/14124>`_)
-there will be no need to download ``dvc_*.snap`` or use ``--dangerous``
-(``snap install dvc --classic`` would suffice).
-
 Package
 -------

+|Packages|
+
 Self-contained packages for Linux, Windows, and Mac are available. The latest version of the packages
 can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.

@@ -207,40 +196,43 @@ Comparison to related technologies

 Contributing
 ============
+
+|Maintainability| |Donate|
+
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
-  :alt: 0
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+   :alt: 0

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
-  :alt: 1
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+   :alt: 1

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
-  :alt: 2
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+   :alt: 2

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
-  :alt: 3
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
+   :alt: 3

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
-  :alt: 4
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+   :alt: 4

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
-  :alt: 5
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+   :alt: 5

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
-  :alt: 6
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+   :alt: 6

 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
-  :alt: 7
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+   :alt: 7

 Mailing List
 ============
@@ -254,3 +246,67 @@ This project is distributed under the Apache license version 2.0 (see the LICENS

 By submitting a pull request to this project, you agree to license your contribution under the Apache license version
 2.0 to this project.
+
+Citation
+========
+
+|DOI|
+
+Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
+`DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
+
+.. |Banner| image:: https://dvc.org/img/logo-github-readme.png
+   :target: https://dvc.org
+   :alt: DVC logo
+
+.. |Release| image:: https://img.shields.io/badge/release-ok-brightgreen
+   :target: https://travis-ci.com/iterative/dvc/branches
+   :alt: Release
+
+.. |CI| image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev&logo=travis
+   :target: https://travis-ci.com/iterative/dvc/builds
+   :alt: Travis dev branch
+
+.. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
+   :target: https://codeclimate.com/github/iterative/dvc
+   :alt: Code Climate
+
+.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/iterative/dvc
+   :alt: Codecov
+
+.. |Donate| image:: https://img.shields.io/badge/patreon-donate-green.svg?logo=patreon
+   :target: https://www.patreon.com/DVCorg/overview
+   :alt: Donate
+
+.. |Snap| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
+   :target: https://snapcraft.io/dvc
+   :alt: Snapcraft
+
+.. |Choco| image:: https://img.shields.io/chocolatey/v/dvc?label=choco
+   :target: https://chocolatey.org/packages/dvc
+   :alt: Chocolatey
+
+.. |Brew| image:: https://img.shields.io/homebrew/v/dvc?label=brew
+   :target: https://formulae.brew.sh/formula/dvc
+   :alt: Homebrew
+
+.. |Conda| image:: https://img.shields.io/conda/v/conda-forge/dvc.svg?label=conda&logo=conda-forge
+   :target: https://anaconda.org/conda-forge/dvc
+   :alt: Conda-forge
+
+.. |PyPI| image:: https://img.shields.io/pypi/v/dvc.svg?label=pip&logo=PyPI&logoColor=white
+   :target: https://pypi.org/project/dvc
+   :alt: PyPI
+
+.. |Packages| image:: https://img.shields.io/github/v/release/iterative/dvc?label=deb|pkg|rpm|exe&logo=GitHub
+   :target: https://github.com/iterative/dvc/releases/latest
+   :alt: deb|pkg|rpm|exe
+
+.. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.3677553-blue.svg
+   :target: https://doi.org/10.5281/zenodo.3677553
+   :alt: DOI
+
+.. |Flowchart| image:: https://dvc.org/img/flow.gif
+   :target: https://dvc.org/img/flow.gif
+   :alt: how_dvc_works
@@ -2,4 +2,4 @@ dvc (0.75.b1) experimental; urgency=low

  * First native Debian package version.

- -- Dmitry Petrov <dmitry@dataversioncontrol.com>  Sat, 07 Dec 2019 09:49:28 +0300
+ -- Dmitry Petrov <dmitry@dvc.org>  Sat, 07 Dec 2019 09:49:28 +0300
@@ -1,5 +1,5 @@
 Source: dvc
-Maintainer: Dmitry Petrov <dmitry@dataversioncontrol.com>
+Maintainer: Dmitry Petrov <dmitry@dvc.org>
 Section: python
 Priority: optional
 Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
@@ -1,10 +1,10 @@
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: DVC
-Upstream-Contact: Dmitry Petrov <dmitry@dataversioncontrol.com>
+Upstream-Contact: Dmitry Petrov <dmitry@dvc.org>
 Source: https://github.com/iterative/dvc

 Files: *
-Copyright: Dmitry Petrov <dmitry@dataversioncontrol.com>
+Copyright: Dmitry Petrov <dmitry@dvc.org>
 License: Apache 2.0
 On Debian systems, the full text of the Apache 2.0
 License can be found in the file
@@ -3,8 +3,6 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-from __future__ import unicode_literals
-
 import warnings

 import dvc.logger
@@ -1,6 +1,4 @@
-"""Main entry point for dvc command line tool."""
-from __future__ import unicode_literals
-
+"""Main entry point for DVC command line tool."""
 import sys

 from dvc.main import main
@@ -15,9 +15,10 @@ from dvc.daemon import daemon
 from dvc.exceptions import NotDvcRepoError
 from dvc.lock import Lock, LockError
 from dvc.repo import Repo
-from dvc.scm import SCM
-from dvc.utils import env2bool, is_binary, makedirs
-from dvc.utils.compat import str, FileNotFoundError
+from dvc.scm import SCM, NoSCM
+from dvc.scm.base import SCMError
+from dvc.utils import env2bool, is_binary
+from dvc.utils.fs import makedirs


 logger = logging.getLogger(__name__)
@@ -54,11 +55,8 @@ def is_enabled():
        return False

    enabled = to_bool(
-        Config(validate=False)
-        .config.get(Config.SECTION_CORE, {})
-        .get(Config.SECTION_CORE_ANALYTICS, "true")
+        Config(validate=False).get("core", {}).get("analytics", "true")
    )
-
    logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))

    return enabled
@@ -76,7 +74,10 @@ def send(report):
    headers = {"content-type": "application/json"}

    with open(report, "rb") as fobj:
-        requests.post(url, data=fobj, headers=headers, timeout=5)
+        try:
+            requests.post(url, data=fobj, headers=headers, timeout=5)
+        except requests.exceptions.RequestException:
+            logger.debug("failed to send analytics report", exc_info=True)

    os.remove(report)

@@ -85,6 +86,8 @@ def _scm_in_use():
    try:
        scm = SCM(root_dir=Repo.find_root())
        return type(scm).__name__
+    except SCMError:
+        return NoSCM.__name__
    except NotDvcRepoError:
        pass

@@ -141,7 +144,7 @@ def _find_or_create_user_id():

    IDs are generated randomly with UUID.
    """
-    config_dir = Config.get_global_config_dir()
+    config_dir = Config.get_dir("global")
    fname = os.path.join(config_dir, "user_id")
    lockfile = os.path.join(config_dir, "user_id.lock")

@@ -164,4 +167,6 @@ def _find_or_create_user_id():
            return user_id

    except LockError:
-        logger.debug("Failed to acquire {lockfile}".format(lockfile=lockfile))
+        logger.debug(
+            "Failed to acquire '{lockfile}'".format(lockfile=lockfile)
+        )
@@ -1,27 +1,50 @@
 import os
-from contextlib import contextmanager
+from contextlib import contextmanager, _GeneratorContextManager as GCM

-try:
-    from contextlib import _GeneratorContextManager as GCM
-except ImportError:
-    from contextlib import GeneratorContextManager as GCM
-
-from dvc.utils.compat import urlparse
 from dvc.repo import Repo
+from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.external_repo import external_repo


+class UrlNotDvcRepoError(DvcException):
+    """Thrown if the given URL is not a DVC repository."""
+
+    def __init__(self, url):
+        super().__init__("'{}' is not a DVC repository.".format(url))
+
+
 def get_url(path, repo=None, rev=None, remote=None):
-    """Returns an url of a resource specified by path in repo"""
+    """
+    Returns the URL to the storage location of a data file or directory tracked
+    in a DVC repo. For Git repos, HEAD is used unless a rev argument is
+    supplied. The default remote is tried unless a remote argument is supplied.
+
+    Raises UrlNotDvcRepoError if repo is not a DVC project.
+
+    NOTE: This function does not check for the actual existence of the file or
+    directory in the remote storage.
+    """
    with _make_repo(repo, rev=rev) as _repo:
-        abspath = os.path.join(_repo.root_dir, path)
-        out, = _repo.find_outs_by_path(abspath)
+        if not isinstance(_repo, Repo):
+            raise UrlNotDvcRepoError(_repo.url)
+        out = _repo.find_out_by_relpath(path)
        remote_obj = _repo.cloud.get_remote(remote)
        return str(remote_obj.checksum_to_path_info(out.checksum))


 def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Opens a specified resource as a file descriptor"""
+    """
+    Open file in the supplied path tracked in a repo (both DVC projects and
+    plain Git repos are supported). For Git repos, HEAD is used unless a rev
+    argument is supplied. The default remote is tried unless a remote argument
+    is supplied. It may only be used as a context manager:
+
+        with dvc.api.open(
+                'path/to/file',
+                repo='https://example.com/url/to/repo'
+                ) as fd:
+            # ... Handle file object fd
+    """
    args = (path,)
    kwargs = {
        "repo": repo,
@@ -40,21 +63,24 @@ class _OpenContextManager(GCM):

    def __getattr__(self, name):
        raise AttributeError(
-            "dvc.api.open() should be used in a with statement"
+            "dvc.api.open() should be used in a with statement."
        )


 def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
    with _make_repo(repo, rev=rev) as _repo:
-        abspath = os.path.join(_repo.root_dir, path)
-        with _repo.open(
-            abspath, remote=remote, mode=mode, encoding=encoding
+        with _repo.open_by_relpath(
+            path, remote=remote, mode=mode, encoding=encoding
        ) as fd:
            yield fd


 def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Read a specified resource into string"""
+    """
+    Returns the contents of a tracked file (by DVC or Git). For Git repos, HEAD
+    is used unless a rev argument is supplied. The default remote is tried
+    unless a remote argument is supplied.
+    """
    with open(
        path, repo=repo, rev=rev, remote=remote, mode=mode, encoding=encoding
    ) as fd:
@@ -62,10 +88,13 @@ def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):


@contextmanager
-def _make_repo(repo_url, rev=None):
-    if not repo_url or urlparse(repo_url).scheme == "":
-        assert rev is None, "Custom revision is not supported for local repo"
-        yield Repo(repo_url)
-    else:
-        with external_repo(url=repo_url, rev=rev) as repo:
-            yield repo
+def _make_repo(repo_url=None, rev=None):
+    repo_url = repo_url or os.getcwd()
+    if rev is None and os.path.exists(repo_url):
+        try:
+            yield Repo(repo_url)
+            return
+        except NotDvcRepoError:
+            pass  # fallthrough to external_repo
+    with external_repo(url=repo_url, rev=rev) as repo:
+        yield repo
@@ -1,28 +1,8 @@
-"""Manages cache of a dvc repo."""
-from __future__ import unicode_literals
-
-import os
+"""Manages cache of a DVC repo."""
 from collections import defaultdict

 from funcy import cached_property

-from dvc.config import Config
-from dvc.utils.compat import builtin_str
-
-
-class CacheConfig(object):
-    def __init__(self, config):
-        self.config = config
-
-    def set_dir(self, dname, level=None):
-        from dvc.remote.config import RemoteConfig
-
-        configobj = self.config.get_configobj(level)
-        path = RemoteConfig.resolve_path(dname, configobj.filename)
-        self.config.set(
-            Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level
-        )
-

 def _make_remote_property(name):
    """
@@ -56,12 +36,12 @@ def _make_remote_property(name):

        return Remote(self.repo, name=remote)

-    getter.__name__ = builtin_str(name)
+    getter.__name__ = name
    return cached_property(getter)


 class Cache(object):
-    """Class that manages cache locations of a dvc repo.
+    """Class that manages cache locations of a DVC repo.

    Args:
        repo (dvc.repo.Repo): repo instance that this cache belongs to.
@@ -73,37 +53,22 @@ class Cache(object):
        from dvc.remote import Remote

        self.repo = repo
+        self.config = config = repo.config["cache"]

-        self.config = config = repo.config.config[Config.SECTION_CACHE]
-        local = config.get(Config.SECTION_CACHE_LOCAL)
+        local = config.get("local")

        if local:
-            name = Config.SECTION_REMOTE_FMT.format(local)
-            settings = repo.config.config[name]
+            settings = {"name": local}
        else:
-            default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
-            cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
-            cache_type = config.get(Config.SECTION_CACHE_TYPE)
-            protected = config.get(Config.SECTION_CACHE_PROTECTED)
-            shared = config.get(Config.SECTION_CACHE_SHARED)
-
-            settings = {
-                Config.PRIVATE_CWD: config.get(
-                    Config.PRIVATE_CWD, repo.dvc_dir
-                ),
-                Config.SECTION_REMOTE_URL: cache_dir,
-                Config.SECTION_CACHE_TYPE: cache_type,
-                Config.SECTION_CACHE_PROTECTED: protected,
-                Config.SECTION_CACHE_SHARED: shared,
-            }
+            settings = {**config, "url": config["dir"]}

        self.local = Remote(repo, **settings)

-    s3 = _make_remote_property(Config.SECTION_CACHE_S3)
-    gs = _make_remote_property(Config.SECTION_CACHE_GS)
-    ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
-    hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
-    azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
+    s3 = _make_remote_property("s3")
+    gs = _make_remote_property("gs")
+    ssh = _make_remote_property("ssh")
+    hdfs = _make_remote_property("hdfs")
+    azure = _make_remote_property("azure")


 class NamedCache(object):
@@ -1,42 +1,43 @@
-"""Command line interface for dvc."""
-from __future__ import print_function
-from __future__ import unicode_literals
-
+"""DVC command line interface"""
 import argparse
 import logging
 import sys

-import dvc.command.add as add
-import dvc.command.cache as cache
-import dvc.command.checkout as checkout
-import dvc.command.commit as commit
-import dvc.command.config as config
-import dvc.command.daemon as daemon
-import dvc.command.data_sync as data_sync
-import dvc.command.destroy as destroy
-import dvc.command.diff as diff
-import dvc.command.gc as gc
-import dvc.command.get as get
-import dvc.command.get_url as get_url
-import dvc.command.imp as imp
-import dvc.command.imp_url as imp_url
-import dvc.command.init as init
-import dvc.command.install as install
-import dvc.command.lock as lock
-import dvc.command.metrics as metrics
-import dvc.command.move as move
-import dvc.command.pipeline as pipeline
-import dvc.command.remote as remote
-import dvc.command.remove as remove
-import dvc.command.repro as repro
-import dvc.command.root as root
-import dvc.command.run as run
-import dvc.command.tag as tag
-import dvc.command.unprotect as unprotect
-import dvc.command.update as update
-import dvc.command.version as version
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import DvcParserError
+from .command import (
+    add,
+    cache,
+    checkout,
+    commit,
+    config,
+    daemon,
+    data_sync,
+    destroy,
+    diff,
+    gc,
+    get,
+    get_url,
+    imp,
+    imp_url,
+    init,
+    install,
+    ls,
+    lock,
+    metrics,
+    move,
+    pipeline,
+    remote,
+    remove,
+    repro,
+    root,
+    run,
+    tag,
+    unprotect,
+    update,
+    version,
+    git_hook,
+)
+from .command.base import fix_subparsers
+from .exceptions import DvcParserError


 logger = logging.getLogger(__name__)
@@ -63,6 +64,7 @@ COMMANDS = [
    metrics,
    install,
    root,
+    ls,
    lock,
    pipeline,
    daemon,
@@ -71,6 +73,7 @@ COMMANDS = [
    diff,
    version,
    update,
+    git_hook,
 ]


@@ -92,7 +95,7 @@ class DvcParser(argparse.ArgumentParser):

 class VersionAction(argparse.Action):  # pragma: no cover
    # pylint: disable=too-few-public-methods
-    """Shows dvc version and exits."""
+    """Shows DVC version and exits."""

    def __call__(self, parser, namespace, values, option_string=None):
        from dvc import __version__
@@ -173,7 +176,7 @@ def parse_args(argv=None):
        title="Available Commands",
        metavar="COMMAND",
        dest="cmd",
-        help="Use dvc COMMAND --help for command-specific help.",
+        help="Use `dvc COMMAND --help` for command-specific help.",
    )

    fix_subparsers(subparsers)
@@ -1,13 +1,8 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.exceptions import DvcException
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
-
+from dvc.command.base import append_doc_link, CmdBase
+from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename

 logger = logging.getLogger(__name__)

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import logging


@@ -14,11 +12,8 @@ def fix_subparsers(subparsers):
        Args:
            subparsers: subparsers to fix.
    """
-    from dvc.utils.compat import is_py3
-
-    if is_py3:  # pragma: no cover
-        subparsers.required = True
-        subparsers.dest = "cmd"
+    subparsers.required = True
+    subparsers.dest = "cmd"


 def append_doc_link(help_message, path):
@@ -40,7 +35,7 @@ class CmdBase(object):
        self.repo = Repo()
        self.config = self.repo.config
        self.args = args
-        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        hardlink_lock = self.config["core"].get("hardlink_lock", False)
        updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
        updater.check()

@@ -1,20 +1,13 @@
-from __future__ import unicode_literals
-
 import argparse

-from dvc.cache import CacheConfig
-from dvc.command.base import append_doc_link
-from dvc.command.base import fix_subparsers
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig


 class CmdCacheDir(CmdConfig):
-    def __init__(self, args):
-        super(CmdCacheDir, self).__init__(args)
-        self.cache_config = CacheConfig(self.config)
-
    def run(self):
-        self.cache_config.set_dir(self.args.value, level=self.args.level)
+        with self.config.edit(level=self.args.level) as edit:
+            edit["cache"]["dir"] = self.args.value
        return 0


@@ -32,7 +25,8 @@ def add_parser(subparsers, parent_parser):
    )

    cache_subparsers = cache_parser.add_subparsers(
-        dest="cmd", help="Use dvc cache CMD --help for command-specific help."
+        dest="cmd",
+        help="Use `dvc cache CMD --help` for command-specific " "help.",
    )

    fix_subparsers(cache_subparsers)
@@ -1,20 +1,86 @@
-from __future__ import unicode_literals
-
 import argparse
+import logging
+
+import colorama

 from dvc.command.base import append_doc_link
 from dvc.command.base import CmdBase
+from dvc.exceptions import CheckoutError
+
+logger = logging.getLogger(__name__)
+
+
+def _human_join(words):
+    words = list(words)
+    if not words:
+        return ""
+
+    return (
+        "{before} and {after}".format(
+            before=", ".join(words[:-1]), after=words[-1],
+        )
+        if len(words) > 1
+        else words[0]
+    )
+
+
+def log_summary(stats):
+    states = ("added", "deleted", "modified")
+    summary = (
+        "{} {}".format(len(stats[state]), state)
+        for state in states
+        if stats.get(state)
+    )
+    logger.info(_human_join(summary) or "No changes.")
+
+
+def log_changes(stats):
+    colors = [
+        ("modified", colorama.Fore.YELLOW,),
+        ("added", colorama.Fore.GREEN),
+        ("deleted", colorama.Fore.RED,),
+    ]
+
+    for state, color in colors:
+        entries = stats.get(state)
+
+        if not entries:
+            continue
+
+        for entry in entries:
+            logger.info(
+                "{color}{state}{nc}{spacing}{entry}".format(
+                    color=color,
+                    state=state[0].upper(),
+                    nc=colorama.Fore.RESET,
+                    spacing="\t",
+                    entry=entry,
+                )
+            )


 class CmdCheckout(CmdBase):
    def run(self):
-        self.repo.checkout(
-            targets=self.args.targets,
-            with_deps=self.args.with_deps,
-            force=self.args.force,
-            relink=self.args.relink,
-            recursive=self.args.recursive,
-        )
+        stats, exc = None, None
+        try:
+            stats = self.repo.checkout(
+                targets=self.args.targets,
+                with_deps=self.args.with_deps,
+                force=self.args.force,
+                relink=self.args.relink,
+                recursive=self.args.recursive,
+            )
+        except CheckoutError as _exc:
+            exc = _exc
+            stats = exc.stats
+
+        if self.args.summary:
+            log_summary(stats)
+        else:
+            log_changes(stats)
+
+        if exc:
+            raise exc
        return 0


@@ -29,17 +95,10 @@ def add_parser(subparsers, parent_parser):
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    checkout_parser.add_argument(
-        "-f",
-        "--force",
+        "--summary",
        action="store_true",
        default=False,
-        help="Do not prompt when removing working directory files.",
-    )
-    checkout_parser.add_argument(
-        "--relink",
-        action="store_true",
-        default=False,
-        help="Recreate links or copies from cache to workspace.",
+        help="Show summary of the changes.",
    )
    checkout_parser.add_argument(
        "-d",
@@ -55,6 +114,19 @@ def add_parser(subparsers, parent_parser):
        default=False,
        help="Checkout all subdirectories of the specified directory.",
    )
+    checkout_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Do not prompt when removing working directory files.",
+    )
+    checkout_parser.add_argument(
+        "--relink",
+        action="store_true",
+        default=False,
+        help="Recreate links or copies from cache to workspace.",
+    )
    checkout_parser.add_argument(
        "targets",
        nargs="*",
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -49,7 +47,7 @@ def add_parser(subparsers, parent_parser):
        "--force",
        action="store_true",
        default=False,
-        help="Commit even if checksums for dependencies/outputs changed.",
+        help="Commit even if hash value for dependencies/outputs changed.",
    )
    commit_parser.add_argument(
        "-d",
@@ -1,36 +1,36 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
-from dvc.config import Config
-
+from dvc.command.base import append_doc_link, CmdBaseNoRepo
+from dvc.config import Config, ConfigError

 logger = logging.getLogger(__name__)


 class CmdConfig(CmdBaseNoRepo):
    def __init__(self, args):
-        super(CmdConfig, self).__init__(args)
+        super().__init__(args)

        self.config = Config(validate=False)

    def run(self):
        section, opt = self.args.name.lower().strip().split(".", 1)

-        if self.args.unset:
-            self.config.unset(section, opt, level=self.args.level)
-        elif self.args.value is None:
-            logger.info(self.config.get(section, opt, level=self.args.level))
-        else:
-            self.config.set(
-                section, opt, self.args.value, level=self.args.level
-            )
+        if self.args.value is None and not self.args.unset:
+            conf = self.config.load_one(self.args.level)
+            self._check(conf, section, opt)
+            logger.info(conf[section][opt])
+            return 0

-        is_write = self.args.unset or self.args.value is not None
-        if is_write and self.args.name == "cache.type":
+        with self.config.edit(self.args.level) as conf:
+            if self.args.unset:
+                self._check(conf, section, opt)
+                del conf[section][opt]
+            else:
+                self._check(conf, section)
+                conf[section][opt] = self.args.value
+
+        if self.args.name == "cache.type":
            logger.warning(
                "You have changed the 'cache.type' option. This doesn't update"
                " any existing workspace file links, but it can be done with:"
@@ -39,30 +39,39 @@ class CmdConfig(CmdBaseNoRepo):

        return 0

+    def _check(self, conf, section, opt=None):
+        if section not in conf:
+            msg = "section {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+
+        if opt and opt not in conf[section]:
+            msg = "option {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+

 parent_config_parser = argparse.ArgumentParser(add_help=False)
 parent_config_parser.add_argument(
    "--global",
    dest="level",
    action="store_const",
-    const=Config.LEVEL_GLOBAL,
+    const="global",
    help="Use global config.",
 )
 parent_config_parser.add_argument(
    "--system",
    dest="level",
    action="store_const",
-    const=Config.LEVEL_SYSTEM,
+    const="system",
    help="Use system config.",
 )
 parent_config_parser.add_argument(
    "--local",
    dest="level",
    action="store_const",
-    const=Config.LEVEL_LOCAL,
+    const="local",
    help="Use local config.",
 )
-parent_config_parser.set_defaults(level=Config.LEVEL_REPO)
+parent_config_parser.set_defaults(level="repo")


 def add_parser(subparsers, parent_parser):
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.command.base import CmdBaseNoRepo
 from dvc.command.base import fix_subparsers

@@ -18,9 +16,7 @@ class CmdDaemonUpdater(CmdDaemonBase):
        root_dir = Repo.find_root()
        dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
        config = Config(dvc_dir, validate=False)
-        hardlink_lock = config.config.get("core", {}).get(
-            "hardlink_lock", False
-        )
+        hardlink_lock = config.get("core", {}).get("hardlink_lock", False)
        updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
        updater.fetch(detach=False)

@@ -46,7 +42,8 @@ def add_parser(subparsers, parent_parser):
    )

    daemon_subparsers = daemon_parser.add_subparsers(
-        dest="cmd", help="Use dvc daemon CMD --help for command-specific help."
+        dest="cmd",
+        help="Use `dvc daemon CMD --help` for command-specific " "help.",
    )

    fix_subparsers(daemon_subparsers)
@@ -1,11 +1,10 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

 from dvc.command.base import append_doc_link
 from dvc.command.base import CmdBase
-from dvc.exceptions import DvcException
+from dvc.command.checkout import log_summary
+from dvc.exceptions import DvcException, CheckoutError


 logger = logging.getLogger(__name__)
@@ -21,7 +20,7 @@ class CmdDataBase(CmdBase):
 class CmdDataPull(CmdDataBase):
    def run(self):
        try:
-            processed_files_count = self.repo.pull(
+            stats = self.repo.pull(
                targets=self.args.targets,
                jobs=self.args.jobs,
                remote=self.args.remote,
@@ -31,10 +30,13 @@ class CmdDataPull(CmdDataBase):
                force=self.args.force,
                recursive=self.args.recursive,
            )
-        except DvcException:
+            log_summary(stats)
+        except (CheckoutError, DvcException) as exc:
+            log_summary(getattr(exc, "stats", {}))
            logger.exception("failed to pull data from the cloud")
            return 1
-        self.check_up_to_date(processed_files_count)
+
+        self.check_up_to_date(stats["downloaded"])
        return 0


@@ -100,7 +102,7 @@ def add_parser(subparsers, _parent_parser):
    from dvc.command.status import CmdDataStatus

    # Pull
-    PULL_HELP = "Pull data files from a DVC remote storage."
+    PULL_HELP = "Download tracked files or directories from remote storage."

    pull_parser = subparsers.add_parser(
        "pull",
@@ -150,7 +152,7 @@ def add_parser(subparsers, _parent_parser):
    pull_parser.set_defaults(func=CmdDataPull)

    # Push
-    PUSH_HELP = "Push data files to a DVC remote storage."
+    PUSH_HELP = "Upload tracked files or directories to remote storage."

    push_parser = subparsers.add_parser(
        "push",
@@ -193,7 +195,8 @@ def add_parser(subparsers, _parent_parser):
    push_parser.set_defaults(func=CmdDataPush)

    # Fetch
-    FETCH_HELP = "Fetch data files from a DVC remote storage."
+    FETCH_HELP = "Get tracked files or directories from remote storage"
+    " into the cache."

    fetch_parser = subparsers.add_parser(
        "fetch",
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -1,14 +1,11 @@
-from __future__ import unicode_literals
-
 import argparse
+import json
 import logging
+import os

-import humanize
-import inflect
-from funcy import compact
+import colorama

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException


@@ -17,156 +14,144 @@ logger = logging.getLogger(__name__)

 class CmdDiff(CmdBase):
    @staticmethod
-    def _print_size(size):
-        if size < 0:
-            change = "decreased by {}"
-        elif size > 0:
-            change = "increased by {}"
-        else:
-            change = "not changed"
-        natur_size = humanize.naturalsize(abs(size))
-        return change.format(natur_size)
+    def _format(diff):
+        """
+        Given a diff structure, generate a string of paths separated
+        by new lines and grouped together by their state.

-    @staticmethod
-    def _get_md5_string(sign, file_name, checksum):
-        sample_msg = ""
-        if file_name:
-            sample_msg = "{}{} with md5 {}\n"
-            sample_msg = sample_msg.format(sign, file_name, checksum)
-        return sample_msg
+        A group's header is colored and its entries are sorted to enhance
+        readability, for example:

-    @classmethod
-    def _get_dir_changes(cls, dct):
-        import dvc.repo.diff as diff
+            Added:
+                another_file.txt
+                backup.tar
+                dir/
+                dir/1

-        engine = inflect.engine()
-        changes_msg = (
-            "{} {} untouched, {} {} modified, {} {} added, "
-            "{} {} deleted, size was {}"
+        An example of a diff formatted when entries contain hash:
+
+            Added:
+                d3b07384 foo
+
+            Modified:
+                c157a790..f98bf6f1 bar
+
+        If a group has no entries, it won't be included in the result.
+
+        At the bottom, include a summary with the number of files per state.
+        """
+
+        def _digest(checksum):
+            if type(checksum) is str:
+                return checksum[0:8]
+            return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
+
+        colors = {
+            "added": colorama.Fore.GREEN,
+            "modified": colorama.Fore.YELLOW,
+            "deleted": colorama.Fore.RED,
+        }
+
+        summary = {}
+        groups = []
+
+        for state in ["added", "deleted", "modified"]:
+            summary[state] = 0
+            entries = diff[state]
+
+            if not entries:
+                continue
+
+            content = []
+
+            for entry in entries:
+                path = entry["path"]
+                checksum = entry.get("hash")
+                summary[state] += 1 if not path.endswith(os.sep) else 0
+                content.append(
+                    "{space}{checksum}{separator}{path}".format(
+                        space="    ",
+                        checksum=_digest(checksum) if checksum else "",
+                        separator="  " if checksum else "",
+                        path=entry["path"],
+                    )
+                )
+
+            groups.append(
+                "{color}{header}{nc}:\n{content}".format(
+                    color=colors[state],
+                    header=state.capitalize(),
+                    nc=colorama.Fore.RESET,
+                    content="\n".join(content),
+                )
+            )
+
+        groups.append(
+            "files summary: {added} added, {deleted} deleted,"
+            " {modified} modified".format_map(summary)
        )
-        changes_msg = changes_msg.format(
-            dct[diff.DIFF_IDENT],
-            engine.plural("file", dct[diff.DIFF_IDENT]),
-            dct[diff.DIFF_CHANGE],
-            engine.plural("file", dct[diff.DIFF_CHANGE]),
-            dct[diff.DIFF_NEW],
-            engine.plural("file", dct[diff.DIFF_NEW]),
-            dct[diff.DIFF_DEL],
-            engine.plural("file", dct[diff.DIFF_DEL]),
-            cls._print_size(dct[diff.DIFF_SIZE]),
-        )
-        return changes_msg

-    @classmethod
-    def _get_file_changes(cls, dct):
-        import dvc.repo.diff as diff
-
-        if (
-            dct.get(diff.DIFF_OLD_FILE)
-            and dct.get(diff.DIFF_NEW_FILE)
-            and dct[diff.DIFF_SIZE] == 0
-        ):
-            msg = "file size was not changed"
-        elif dct.get(diff.DIFF_NEW_FILE):
-            msg = "added file with size {}".format(
-                humanize.naturalsize(dct[diff.DIFF_SIZE])
-            )
-        elif dct.get(diff.DIFF_OLD_FILE):
-            msg = "deleted file with size {}".format(
-                humanize.naturalsize(abs(dct[diff.DIFF_SIZE]))
-            )
-        else:
-            msg = "file was modified, file size {}".format(
-                cls._print_size(dct[diff.DIFF_SIZE])
-            )
-        return msg
-
-    @classmethod
-    def _get_royal_changes(cls, dct):
-        import dvc.repo.diff as diff
-
-        if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
-            if dct.get("is_dir"):
-                return cls._get_dir_changes(dct)
-            else:
-                return cls._get_file_changes(dct)
-        return "size is ?"
-
-    @classmethod
-    def _show(cls, diff_dct):
-        import dvc.repo.diff as diff
-
-        msg = "dvc diff from {} to {}".format(
-            diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
-        )
-        if diff_dct.get(diff.DIFF_EQUAL):
-            logger.info(msg)
-            return
-        for dct in diff_dct[diff.DIFF_LIST]:
-            msg += "\n\ndiff for '{}'\n".format(dct[diff.DIFF_TARGET])
-            msg += cls._get_md5_string(
-                "-",
-                dct.get(diff.DIFF_OLD_FILE),
-                dct.get(diff.DIFF_OLD_CHECKSUM),
-            )
-            msg += cls._get_md5_string(
-                "+",
-                dct.get(diff.DIFF_NEW_FILE),
-                dct.get(diff.DIFF_NEW_CHECKSUM),
-            )
-            msg += "\n"
-            msg += cls._get_royal_changes(dct)
-        logger.info(msg)
-        return msg
+        return "\n\n".join(groups)

    def run(self):
        try:
-            msg = self.repo.diff(
-                self.args.a_ref, target=self.args.target, b_ref=self.args.b_ref
-            )
-            self._show(msg)
+            diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
+
+            if not any(diff.values()):
+                return 0
+
+            if not self.args.show_hash:
+                for _, entries in diff.items():
+                    for entry in entries:
+                        del entry["hash"]
+
+            if self.args.show_json:
+                res = json.dumps(diff)
+            else:
+                res = self._format(diff)
+
+            logger.info(res)
+
        except DvcException:
-            msg = "failed to get 'diff {}'"
-            args = " ".join(
-                compact([self.args.target, self.args.a_ref, self.args.b_ref])
-            )
-            msg = msg.format(args)
-            logger.exception(msg)
+            logger.exception("failed to get diff")
            return 1
        return 0


 def add_parser(subparsers, parent_parser):
    DIFF_DESCRIPTION = (
-        "Show diff of a data file or a directory that is under DVC control.\n"
-        "Some basic statistics summary, how many files were deleted/changed."
+        "Show changes between commits in the DVC repository,"
+        " or between a commit and the workspace."
+        "Prints the list of paths added, modified, or deleted."
    )
-    DIFF_HELP = "Show a diff of a DVC controlled data file or a directory."
    diff_parser = subparsers.add_parser(
        "diff",
        parents=[parent_parser],
        description=append_doc_link(DIFF_DESCRIPTION, "diff"),
-        help=DIFF_HELP,
+        help=DIFF_DESCRIPTION,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    diff_parser.add_argument(
-        "-t",
-        "--target",
-        help=(
-            "Source path to a data file or directory. Default None. "
-            "If not specified, compares all files and directories "
-            "that are under DVC control in the current working space."
-        ),
+        "a_rev",
+        help="Old Git commit to compare (defaults to HEAD)",
+        nargs="?",
+        default="HEAD",
    )
    diff_parser.add_argument(
-        "a_ref", help="Git reference from which diff calculates"
-    )
-    diff_parser.add_argument(
-        "b_ref",
-        help=(
-            "Git reference until which diff calculates, if omitted "
-            "diff shows the difference between current HEAD and a_ref"
-        ),
+        "b_rev",
+        help=("New Git commit to compare (defaults to the current workspace)"),
        nargs="?",
    )
+    diff_parser.add_argument(
+        "--show-json",
+        help="Format the output into a JSON",
+        action="store_true",
+        default=False,
+    )
+    diff_parser.add_argument(
+        "--show-hash",
+        help="Display hash value for each entry",
+        action="store_true",
+        default=False,
+    )
    diff_parser.set_defaults(func=CmdDiff)
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 import os
@@ -14,6 +12,15 @@ logger = logging.getLogger(__name__)

 class CmdGC(CmdBase):
    def run(self):
+        from dvc.repo.gc import _raise_error_if_all_disabled
+
+        _raise_error_if_all_disabled(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+        )
+
        msg = "This will remove all cache except items used in "

        msg += "the working tree"
@@ -49,15 +56,16 @@ class CmdGC(CmdBase):
            force=self.args.force,
            jobs=self.args.jobs,
            repos=self.args.repos,
+            workspace=self.args.workspace,
        )
        return 0


 def add_parser(subparsers, parent_parser):
-    GC_HELP = "Collect unused data from DVC cache or a remote storage."
+    GC_HELP = "Garbage collect unused objects from cache or remote storage."
    GC_DESCRIPTION = (
-        "Deletes all files in the cache or a remote which are not in\n"
-        "use by the specified git references (defaults to just HEAD)."
+        "Removes all files in the cache or a remote which are not in\n"
+        "use by the specified Git revisions (defaults to just HEAD)."
    )
    gc_parser = subparsers.add_parser(
        "gc",
@@ -66,25 +74,32 @@ def add_parser(subparsers, parent_parser):
        help=GC_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
+    gc_parser.add_argument(
+        "-w",
+        "--workspace",
+        action="store_true",
+        default=False,
+        help="Keep data files used in the current workspace (working tree).",
+    )
    gc_parser.add_argument(
        "-a",
        "--all-branches",
        action="store_true",
        default=False,
-        help="Keep data files for the tips of all git branches.",
+        help="Keep data files for the tips of all Git branches.",
    )
    gc_parser.add_argument(
        "-T",
        "--all-tags",
        action="store_true",
        default=False,
-        help="Keep data files for all git tags.",
+        help="Keep data files for all Git tags.",
    )
    gc_parser.add_argument(
        "--all-commits",
        action="store_true",
        default=False,
-        help=argparse.SUPPRESS,
+        help="Keep data files for all Git commits.",
    )
    gc_parser.add_argument(
        "-c",
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -12,7 +10,27 @@ logger = logging.getLogger(__name__)


 class CmdGet(CmdBaseNoRepo):
+    def _show_url(self):
+        from dvc.api import get_url
+
+        try:
+            url = get_url(
+                self.args.path, repo=self.args.url, rev=self.args.rev
+            )
+            logger.info(url)
+        except DvcException:
+            logger.exception("failed to show URL")
+            return 1
+
+        return 0
+
    def run(self):
+        if self.args.show_url:
+            return self._show_url()
+
+        return self._get_file_from_repo()
+
+    def _get_file_from_repo(self):
        from dvc.repo import Repo

        try:
@@ -33,7 +51,7 @@ class CmdGet(CmdBaseNoRepo):


 def add_parser(subparsers, parent_parser):
-    GET_HELP = "Download/copy files or directories from DVC repository."
+    GET_HELP = "Download file or directory tracked by DVC or by Git."
    get_parser = subparsers.add_parser(
        "get",
        parents=[parent_parser],
@@ -42,18 +60,21 @@ def add_parser(subparsers, parent_parser):
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    get_parser.add_argument(
-        "url", help="URL of Git repository with DVC project to download from."
+        "url", help="Location of DVC or Git repository to download from"
    )
    get_parser.add_argument(
-        "path", help="Path to a file or directory within a DVC repository."
+        "path", help="Path to a file or directory within the repository"
    )
    get_parser.add_argument(
-        "-o",
-        "--out",
-        nargs="?",
-        help="Destination path to copy/download files to.",
+        "-o", "--out", nargs="?", help="Destination path to download files to"
    )
    get_parser.add_argument(
-        "--rev", nargs="?", help="DVC repository git revision."
+        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+    )
+    get_parser.add_argument(
+        "--show-url",
+        action="store_true",
+        help="Print the storage location (URL) the target data would be "
+        "downloaded from, and exit.",
    )
    get_parser.set_defaults(func=CmdGet)
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -0,0 +1,112 @@
+import logging
+import os
+
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.exceptions import NotDvcRepoError
+
+logger = logging.getLogger(__name__)
+
+
+class CmdHookBase(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            repo = Repo()
+            repo.close()
+        except NotDvcRepoError:
+            return 0
+
+        return self._run()
+
+
+class CmdPreCommit(CmdHookBase):
+    def _run(self):
+        from dvc.main import main
+
+        return main(["status"])
+
+
+class CmdPostCheckout(CmdHookBase):
+    def run(self):
+        # when we are running from pre-commit tool, it doesn't provide CLI
+        # flags, but instead provides respective env vars that we could use.
+        flag = os.environ.get("PRE_COMMIT_CHECKOUT_TYPE")
+        if flag is None and len(self.args.args) >= 3:
+            # see https://git-scm.com/docs/githooks#_post_checkout
+            flag = self.args.args[2]
+
+        # checking out some reference and not specific file.
+        if flag != "1":
+            return 0
+
+        # make sure we are not in the middle of a rebase/merge, so we
+        # don't accidentally break it with an unsuccessful checkout.
+        # Note that git hooks are always running in repo root.
+        if os.path.isdir(os.path.join(".git", "rebase-merge")):
+            return 0
+
+        from dvc.main import main
+
+        return main(["checkout"])
+
+
+class CmdPrePush(CmdHookBase):
+    def run(self):
+        from dvc.main import main
+
+        return main(["push"])
+
+
+def add_parser(subparsers, parent_parser):
+    GIT_HOOK_HELP = "Run GIT hook."
+
+    git_hook_parser = subparsers.add_parser(
+        "git-hook",
+        parents=[parent_parser],
+        description=GIT_HOOK_HELP,
+        add_help=False,
+    )
+
+    git_hook_subparsers = git_hook_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc daemon CMD --help` for command-specific help.",
+    )
+
+    fix_subparsers(git_hook_subparsers)
+
+    PRE_COMMIT_HELP = "Run pre-commit GIT hook."
+    pre_commit_parser = git_hook_subparsers.add_parser(
+        "pre-commit",
+        parents=[parent_parser],
+        description=PRE_COMMIT_HELP,
+        help=PRE_COMMIT_HELP,
+    )
+    pre_commit_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    pre_commit_parser.set_defaults(func=CmdPreCommit)
+
+    POST_CHECKOUT_HELP = "Run post-checkout GIT hook."
+    post_checkout_parser = git_hook_subparsers.add_parser(
+        "post-checkout",
+        parents=[parent_parser],
+        description=POST_CHECKOUT_HELP,
+        help=POST_CHECKOUT_HELP,
+    )
+    post_checkout_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    post_checkout_parser.set_defaults(func=CmdPostCheckout)
+
+    PRE_PUSH_HELP = "Run pre-push GIT hook."
+    pre_push_parser = git_hook_subparsers.add_parser(
+        "pre-push",
+        parents=[parent_parser],
+        description=PRE_PUSH_HELP,
+        help=PRE_PUSH_HELP,
+    )
+    pre_push_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    pre_push_parser.set_defaults(func=CmdPrePush)
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -32,7 +30,8 @@ class CmdImport(CmdBase):

 def add_parser(subparsers, parent_parser):
    IMPORT_HELP = (
-        "Download data from DVC repository and take it under DVC control."
+        "Download file or directory tracked by DVC or by Git "
+        "into the workspace, and track it."
    )

    import_parser = subparsers.add_parser(
@@ -43,15 +42,15 @@ def add_parser(subparsers, parent_parser):
        formatter_class=argparse.RawTextHelpFormatter,
    )
    import_parser.add_argument(
-        "url", help="URL of Git repository with DVC project to download from."
+        "url", help="Location of DVC or Git repository to download from"
    )
    import_parser.add_argument(
-        "path", help="Path to data within DVC repository."
+        "path", help="Path to a file or directory within the repository"
    )
    import_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to put data to."
+        "-o", "--out", nargs="?", help="Destination path to download files to"
    )
    import_parser.add_argument(
-        "--rev", nargs="?", help="DVC repository git revision."
+        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
    )
    import_parser.set_defaults(func=CmdImport)
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -17,11 +15,14 @@ class CmdInit(CmdBaseNoRepo):

        try:
            self.repo = Repo.init(
-                ".", no_scm=self.args.no_scm, force=self.args.force
+                ".",
+                no_scm=self.args.no_scm,
+                force=self.args.force,
+                subdir=self.args.subdir,
            )
            self.config = self.repo.config
        except InitError:
-            logger.exception("failed to initiate dvc")
+            logger.exception("failed to initiate DVC")
            return 1
        return 0

@@ -45,7 +46,7 @@ def add_parser(subparsers, parent_parser):
        "--no-scm",
        action="store_true",
        default=False,
-        help="Initiate dvc in directory that is "
+        help="Initiate DVC in directory that is "
        "not tracked by any scm tool (e.g. git).",
    )
    init_parser.add_argument(
@@ -58,4 +59,13 @@ def add_parser(subparsers, parent_parser):
            "This operation removes local cache."
        ),
    )
+    init_parser.add_argument(
+        "--subdir",
+        action="store_true",
+        default=False,
+        help=(
+            "Necessary for running this command inside a subdirectory of a "
+            "parent SCM repository."
+        ),
+    )
    init_parser.set_defaults(func=CmdInit)
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -13,9 +11,9 @@ logger = logging.getLogger(__name__)
 class CmdInstall(CmdBase):
    def run(self):
        try:
-            self.repo.install()
+            self.repo.install(self.args.use_pre_commit_tool)
        except Exception:
-            logger.exception("failed to install dvc hooks")
+            logger.exception("failed to install DVC Git hooks")
            return 1
        return 0

@@ -29,4 +27,11 @@ def add_parser(subparsers, parent_parser):
        help=INSTALL_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
+    install_parser.add_argument(
+        "--use-pre-commit-tool",
+        action="store_true",
+        default=False,
+        help="Install DVC hooks using pre-commit "
+        "(https://pre-commit.com) if it is installed.",
+    )
    install_parser.set_defaults(func=CmdInstall)
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -0,0 +1,77 @@
+import argparse
+import logging
+import sys
+
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
+from dvc.command.ls.ls_colors import LsColors
+from dvc.exceptions import DvcException
+
+
+logger = logging.getLogger(__name__)
+
+
+def _prettify(entries, with_color=False):
+    if with_color:
+        ls_colors = LsColors()
+        fmt = ls_colors.format
+    else:
+
+        def fmt(entry):
+            return entry["path"]
+
+    return [fmt(entry) for entry in entries]
+
+
+class CmdList(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            entries = Repo.ls(
+                self.args.url,
+                self.args.path,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+                outs_only=self.args.outs_only,
+            )
+            if entries:
+                entries = _prettify(entries, sys.stdout.isatty())
+                logger.info("\n".join(entries))
+            return 0
+        except DvcException:
+            logger.exception("failed to list '{}'".format(self.args.url))
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    LIST_HELP = (
+        "List repository contents, including files"
+        " and directories tracked by DVC and by Git."
+    )
+    list_parser = subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(LIST_HELP, "list"),
+        help=LIST_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    list_parser.add_argument("url", help="Location of DVC repository to list")
+    list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        help="Recursively list files.",
+    )
+    list_parser.add_argument(
+        "--outs-only", action="store_true", help="Show only DVC outputs."
+    )
+    list_parser.add_argument(
+        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
+    )
+    list_parser.add_argument(
+        "path",
+        nargs="?",
+        help="Path to directory within the repository to list outputs for",
+    )
+    list_parser.set_defaults(func=CmdList)
@@ -0,0 +1,48 @@
+import os
+
+
+class LsColors(object):
+    default = "rs=0:di=01;34:ex=01;32"
+
+    def __init__(self, lscolors=None):
+        self._extensions = {}
+        self._codes = {}
+        self._load(lscolors or os.environ.get("LS_COLORS") or LsColors.default)
+
+    def _load(self, lscolors):
+        for item in lscolors.split(":"):
+            try:
+                code, color = item.split("=", 1)
+            except ValueError:
+                continue
+            if code.startswith("*."):
+                self._extensions[code[1:]] = color
+            else:
+                self._codes[code] = color
+
+    def format(self, entry):
+        text = entry["path"]
+
+        if entry.get("isout", False) and "out" in self._codes:
+            return self._format(text, code="out")
+
+        if entry.get("isdir", False):
+            return self._format(text, code="di")
+
+        if entry.get("isexec", False):
+            return self._format(text, code="ex")
+
+        _, ext = os.path.splitext(text)
+        return self._format(text, ext=ext)
+
+    def _format(self, text, code=None, ext=None):
+        val = None
+        if ext:
+            val = self._extensions.get(ext, None)
+        if code:
+            val = self._codes.get(code, None)
+
+        if not val:
+            return text
+        rs = self._codes.get("rs", 0)
+        return "\033[{}m{}\033[{}m".format(val, text, rs)
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -28,7 +26,12 @@ def show_metrics(metrics, all_branches=False, all_tags=False):
            logger.info("{branch}:".format(branch=branch))

        for fname, metric in val.items():
-            lines = metric if type(metric) is list else metric.splitlines()
+            if isinstance(metric, dict):
+                lines = list(metric.values())
+            elif isinstance(metric, list):
+                lines = metric
+            else:
+                lines = metric.splitlines()

            if len(lines) > 1:
                logger.info("\t{fname}:".format(fname=fname))
@@ -102,8 +105,61 @@ class CmdMetricsRemove(CmdBase):
        return 0


+def _show_diff(diff):
+    from texttable import Texttable
+
+    if not diff:
+        return "No changes."
+
+    table = Texttable()
+
+    # remove borders to make it easier for users to copy stuff
+    table.set_chars(("", "", "", ""))
+    table.set_deco(0)
+
+    rows = [["Path", "Metric", "Value", "Change"]]
+    for fname, mdiff in diff.items():
+        for metric, change in mdiff.items():
+            rows.append(
+                [
+                    fname,
+                    metric,
+                    change["new"],
+                    change.get("diff", "diff not supported"),
+                ]
+            )
+    table.add_rows(rows)
+    return table.draw()
+
+
+class CmdMetricsDiff(CmdBase):
+    def run(self):
+        try:
+            diff = self.repo.metrics.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                targets=self.args.targets,
+                typ=self.args.type,
+                xpath=self.args.xpath,
+                recursive=self.args.recursive,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                logger.info(_show_diff(diff))
+
+        except DvcException:
+            logger.exception("failed to show metrics diff")
+            return 1
+
+        return 0
+
+
 def add_parser(subparsers, parent_parser):
-    METRICS_HELP = "Commands to add, manage, collect and display metrics."
+    METRICS_HELP = "Commands to add, manage, collect, and display metrics."

    metrics_parser = subparsers.add_parser(
        "metrics",
@@ -115,12 +171,12 @@ def add_parser(subparsers, parent_parser):

    metrics_subparsers = metrics_parser.add_subparsers(
        dest="cmd",
-        help="Use dvc metrics CMD --help to display command-specific help.",
+        help="Use `dvc metrics CMD --help` to display command-specific help.",
    )

    fix_subparsers(metrics_subparsers)

-    METRICS_SHOW_HELP = "Output metric values."
+    METRICS_SHOW_HELP = "Print metrics, with optional formatting."
    metrics_show_parser = metrics_subparsers.add_parser(
        "show",
        parents=[parent_parser],
@@ -131,8 +187,7 @@ def add_parser(subparsers, parent_parser):
    metrics_show_parser.add_argument(
        "targets",
        nargs="*",
-        help="Metric files or directories (see -R) to show "
-        "(leave empty to display all)",
+        help="Metric files or directories (see -R) to show",
    )
    metrics_show_parser.add_argument(
        "-t",
@@ -172,7 +227,7 @@ def add_parser(subparsers, parent_parser):
    )
    metrics_show_parser.set_defaults(func=CmdMetricsShow)

-    METRICS_ADD_HELP = "Tag file as a metric file."
+    METRICS_ADD_HELP = "Mark a DVC-tracked file as a metric."
    metrics_add_parser = metrics_subparsers.add_parser(
        "add",
        parents=[parent_parser],
@@ -189,7 +244,7 @@ def add_parser(subparsers, parent_parser):
    metrics_add_parser.add_argument("path", help="Path to a metric file.")
    metrics_add_parser.set_defaults(func=CmdMetricsAdd)

-    METRICS_MODIFY_HELP = "Modify metric file options."
+    METRICS_MODIFY_HELP = "Modify metric default formatting."
    metrics_modify_parser = metrics_subparsers.add_parser(
        "modify",
        parents=[parent_parser],
@@ -206,7 +261,7 @@ def add_parser(subparsers, parent_parser):
    metrics_modify_parser.add_argument("path", help="Path to a metric file.")
    metrics_modify_parser.set_defaults(func=CmdMetricsModify)

-    METRICS_REMOVE_HELP = "Remove files's metric tag."
+    METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
    metrics_remove_parser = metrics_subparsers.add_parser(
        "remove",
        parents=[parent_parser],
@@ -216,3 +271,58 @@ def add_parser(subparsers, parent_parser):
    )
    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
+
+    METRICS_DIFF_HELP = "Show changes in metrics between commits"
+    " in the DVC repository, or between a commit and the workspace."
+    metrics_diff_parser = metrics_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_DIFF_HELP, "metrics/diff"),
+        help=METRICS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_diff_parser.add_argument(
+        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
+    )
+    metrics_diff_parser.add_argument(
+        "b_rev",
+        nargs="?",
+        help=("New Git commit to compare (defaults to the current workspace)"),
+    )
+    metrics_diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help=(
+            "Metric files or directories (see -R) to show diff for. "
+            "Shows diff for all metric files by default."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "-t",
+        "--type",
+        help=(
+            "Type of metrics (json/tsv/htsv/csv/hcsv). "
+            "It can be detected by the file extension automatically. "
+            "Unsupported types will be treated as raw."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
+    )
+    metrics_diff_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help=(
+            "If any target is a directory, recursively search and process "
+            "metric files."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    metrics_diff_parser.set_defaults(func=CmdMetricsDiff)
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -1,14 +1,8 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.command.base import fix_subparsers
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.utils import relpath
-from dvc.utils.compat import str


 logger = logging.getLogger(__name__)
@@ -21,52 +15,44 @@ class CmdPipelineShow(CmdBase):

        stage = Stage.load(self.repo, target)
        G = self.repo.graph
-        stages = networkx.get_node_attributes(G, "stage")
-        node = relpath(stage.path, self.repo.root_dir)
-        nodes = networkx.dfs_postorder_nodes(G, node)
+        stages = networkx.dfs_postorder_nodes(G, stage)

        if locked:
-            nodes = [n for n in nodes if stages[n].locked]
+            stages = [s for s in stages if s.locked]

-        for n in nodes:
+        for stage in stages:
            if commands:
-                if stages[n].cmd is None:
+                if stage.cmd is None:
                    continue
-                logger.info(stages[n].cmd)
+                logger.info(stage.cmd)
            elif outs:
-                for out in stages[n].outs:
+                for out in stage.outs:
                    logger.info(str(out))
            else:
-                logger.info(n)
+                logger.info(stage.path_in_repo)

-    def __build_graph(self, target, commands, outs):
+    def _build_graph(self, target, commands, outs):
        import networkx
        from dvc.stage import Stage
        from dvc.repo.graph import get_pipeline

-        stage = Stage.load(self.repo, target)
-        node = relpath(stage.path, self.repo.root_dir)
+        target_stage = Stage.load(self.repo, target)
+        G = get_pipeline(self.repo.pipelines, target_stage)

-        G = get_pipeline(self.repo.pipelines, node)
-        stages = networkx.get_node_attributes(G, "stage")
-
-        nodes = []
-        for n in G:
-            stage = stages[n]
+        nodes = set()
+        for stage in networkx.dfs_preorder_nodes(G, target_stage):
            if commands:
                if stage.cmd is None:
                    continue
-                nodes.append(stage.cmd)
+                nodes.add(stage.cmd)
            elif outs:
                for out in stage.outs:
-                    nodes.append(str(out))
+                    nodes.add(str(out))
            else:
-                nodes.append(stage.relpath)
+                nodes.add(stage.relpath)

        edges = []
-        for e in G.edges():
-            from_stage = stages[e[0]]
-            to_stage = stages[e[1]]
+        for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
            if commands:
                if to_stage.cmd is None:
                    continue
@@ -78,12 +64,12 @@ class CmdPipelineShow(CmdBase):
            else:
                edges.append((from_stage.relpath, to_stage.relpath))

-        return nodes, edges, networkx.is_tree(G)
+        return list(nodes), edges, networkx.is_tree(G)

    def _show_ascii(self, target, commands, outs):
        from dvc.dagascii import draw

-        nodes, edges, _ = self.__build_graph(target, commands, outs)
+        nodes, edges, _ = self._build_graph(target, commands, outs)

        if not nodes:
            return
@@ -93,7 +79,7 @@ class CmdPipelineShow(CmdBase):
    def _show_dependencies_tree(self, target, commands, outs):
        from treelib import Tree

-        nodes, edges, is_tree = self.__build_graph(target, commands, outs)
+        nodes, edges, is_tree = self._build_graph(target, commands, outs)
        if not nodes:
            return
        if not is_tree:
@@ -114,18 +100,18 @@ class CmdPipelineShow(CmdBase):
            observe_list.pop(0)
        tree.show()

-    def __write_dot(self, target, commands, outs):
-        from dvc.utils.compat import StringIO
+    def _write_dot(self, target, commands, outs):
+        import io
        import networkx
        from networkx.drawing.nx_pydot import write_dot

-        _, edges, _ = self.__build_graph(target, commands, outs)
+        _, edges, _ = self._build_graph(target, commands, outs)
        edges = [edge[::-1] for edge in edges]

        simple_g = networkx.DiGraph()
        simple_g.add_edges_from(edges)

-        dot_file = StringIO()
+        dot_file = io.StringIO()
        write_dot(simple_g, dot_file)
        logger.info(dot_file.getvalue())

@@ -140,9 +126,7 @@ class CmdPipelineShow(CmdBase):
                        target, self.args.commands, self.args.outs
                    )
                elif self.args.dot:
-                    self.__write_dot(
-                        target, self.args.commands, self.args.outs
-                    )
+                    self._write_dot(target, self.args.commands, self.args.outs)
                elif self.args.tree:
                    self._show_dependencies_tree(
                        target, self.args.commands, self.args.outs
@@ -163,14 +147,11 @@ class CmdPipelineShow(CmdBase):

 class CmdPipelineList(CmdBase):
    def run(self):
-        import networkx
-
        pipelines = self.repo.pipelines
-        for p in pipelines:
-            stages = networkx.get_node_attributes(p, "stage")
-            for stage in stages:
-                logger.info(stage)
-            if len(stages) != 0:
+        for pipeline in pipelines:
+            for stage in pipeline:
+                logger.info(stage.relpath)
+            if len(pipeline) != 0:
                logger.info("=" * 80)
        logger.info("{} pipelines total".format(len(pipelines)))

@@ -189,12 +170,12 @@ def add_parser(subparsers, parent_parser):

    pipeline_subparsers = pipeline_parser.add_subparsers(
        dest="cmd",
-        help="Use dvc pipeline CMD --help for command-specific help.",
+        help="Use `dvc pipeline CMD --help` for command-specific help.",
    )

    fix_subparsers(pipeline_subparsers)

-    PIPELINE_SHOW_HELP = "Show pipelines."
+    PIPELINE_SHOW_HELP = "Show stages in a pipeline."
    pipeline_show_parser = pipeline_subparsers.add_parser(
        "show",
        parents=[parent_parser],
@@ -250,7 +231,7 @@ def add_parser(subparsers, parent_parser):
    )
    pipeline_show_parser.set_defaults(func=CmdPipelineShow)

-    PIPELINE_LIST_HELP = "List pipelines."
+    PIPELINE_LIST_HELP = "List connected groups of stages (pipelines)."
    pipeline_list_parser = pipeline_subparsers.add_parser(
        "list",
        parents=[parent_parser],
@@ -1,80 +1,106 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

-from dvc.command.base import append_doc_link
-from dvc.command.base import fix_subparsers
+from dvc.config import ConfigError
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.remote.config import RemoteConfig
+from dvc.utils import format_link

 logger = logging.getLogger(__name__)


-class CmdRemoteConfig(CmdConfig):
+class CmdRemote(CmdConfig):
    def __init__(self, args):
-        super(CmdRemoteConfig, self).__init__(args)
-        self.remote_config = RemoteConfig(self.config)
+        super().__init__(args)
+
+        if getattr(self.args, "name", None):
+            self.args.name = self.args.name.lower()
+
+    def _check_exists(self, conf):
+        if self.args.name not in conf["remote"]:
+            raise ConfigError(
+                "remote '{}' doesn't exists.".format(self.args.name)
+            )


-class CmdRemoteAdd(CmdRemoteConfig):
+class CmdRemoteAdd(CmdRemote):
    def run(self):
        if self.args.default:
            logger.info(
                "Setting '{}' as a default remote.".format(self.args.name)
            )
-        self.remote_config.add(
-            self.args.name,
-            self.args.url,
-            force=self.args.force,
-            default=self.args.default,
-            level=self.args.level,
-        )
+
+        with self.config.edit(self.args.level) as conf:
+            if self.args.name in conf["remote"] and not self.args.force:
+                raise ConfigError(
+                    "remote '{}' already exists. Use `-f|--force` to "
+                    "overwrite it.".format(self.args.name)
+                )
+
+            conf["remote"][self.args.name] = {"url": self.args.url}
+            if self.args.default:
+                conf["core"]["remote"] = self.args.name
+
        return 0


-class CmdRemoteRemove(CmdRemoteConfig):
+class CmdRemoteRemove(CmdRemote):
    def run(self):
-        self.remote_config.remove(self.args.name, level=self.args.level)
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            del conf["remote"][self.args.name]
+
+        # Remove core.remote refs to this remote in any shadowing configs
+        for level in reversed(self.config.LEVELS):
+            with self.config.edit(level) as conf:
+                if conf["core"].get("remote") == self.args.name:
+                    del conf["core"]["remote"]
+
+            if level == self.args.level:
+                break
+
        return 0


-class CmdRemoteModify(CmdRemoteConfig):
+class CmdRemoteModify(CmdRemote):
    def run(self):
-        self.remote_config.modify(
-            self.args.name,
-            self.args.option,
-            self.args.value,
-            level=self.args.level,
-        )
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            conf["remote"][self.args.name][self.args.option] = self.args.value
        return 0


-class CmdRemoteDefault(CmdRemoteConfig):
+class CmdRemoteDefault(CmdRemote):
    def run(self):
+
        if self.args.name is None and not self.args.unset:
-            name = self.remote_config.get_default(level=self.args.level)
-            print(name)
+            conf = self.config.load_one(self.args.level)
+            try:
+                print(conf["core"]["remote"])
+            except KeyError:
+                logger.info("No default remote set")
+                return 1
        else:
-            self.remote_config.set_default(
-                self.args.name, unset=self.args.unset, level=self.args.level
-            )
+            with self.config.edit(self.args.level) as conf:
+                if self.args.unset:
+                    conf["core"].pop("remote", None)
+                else:
+                    conf["core"]["remote"] = self.args.name
        return 0


-class CmdRemoteList(CmdRemoteConfig):
+class CmdRemoteList(CmdRemote):
    def run(self):
-        for name, url in self.remote_config.list(
-            level=self.args.level
-        ).items():
-            logger.info("{}\t{}".format(name, url))
+        conf = self.config.load_one(self.args.level)
+        for name, conf in conf["remote"].items():
+            logger.info("{}\t{}".format(name, conf["url"]))
        return 0


 def add_parser(subparsers, parent_parser):
    from dvc.command.config import parent_config_parser

-    REMOTE_HELP = "Manage remote storage configuration."
+    REMOTE_HELP = "Set up and manage data remotes."
    remote_parser = subparsers.add_parser(
        "remote",
        parents=[parent_parser],
@@ -85,12 +111,12 @@ def add_parser(subparsers, parent_parser):

    remote_subparsers = remote_parser.add_subparsers(
        dest="cmd",
-        help="Use dvc remote CMD --help for " "command-specific help.",
+        help="Use `dvc remote CMD --help` for " "command-specific help.",
    )

    fix_subparsers(remote_subparsers)

-    REMOTE_ADD_HELP = "Add remote."
+    REMOTE_ADD_HELP = "Add a new data remote."
    remote_add_parser = remote_subparsers.add_parser(
        "add",
        parents=[parent_config_parser, parent_parser],
@@ -101,7 +127,9 @@ def add_parser(subparsers, parent_parser):
    remote_add_parser.add_argument("name", help="Name of the remote")
    remote_add_parser.add_argument(
        "url",
-        help="URL. See full list of supported urls at " "man.dvc.org/remote",
+        help="URL. See full list of supported urls at {}".format(
+            format_link("https://man.dvc.org/remote")
+        ),
    )
    remote_add_parser.add_argument(
        "-d",
@@ -119,7 +147,7 @@ def add_parser(subparsers, parent_parser):
    )
    remote_add_parser.set_defaults(func=CmdRemoteAdd)

-    REMOTE_DEFAULT_HELP = "Set/unset default remote."
+    REMOTE_DEFAULT_HELP = "Set/unset the default data remote."
    remote_default_parser = remote_subparsers.add_parser(
        "default",
        parents=[parent_config_parser, parent_parser],
@@ -139,7 +167,7 @@ def add_parser(subparsers, parent_parser):
    )
    remote_default_parser.set_defaults(func=CmdRemoteDefault)

-    REMOTE_REMOVE_HELP = "Remove remote."
+    REMOTE_REMOVE_HELP = "Remove a data remote."
    remote_remove_parser = remote_subparsers.add_parser(
        "remove",
        parents=[parent_config_parser, parent_parser],
@@ -152,7 +180,7 @@ def add_parser(subparsers, parent_parser):
    )
    remote_remove_parser.set_defaults(func=CmdRemoteRemove)

-    REMOTE_MODIFY_HELP = "Modify remote."
+    REMOTE_MODIFY_HELP = "Modify the configuration of a data remote."
    remote_modify_parser = remote_subparsers.add_parser(
        "modify",
        parents=[parent_config_parser, parent_parser],
@@ -176,7 +204,7 @@ def add_parser(subparsers, parent_parser):
    )
    remote_modify_parser.set_defaults(func=CmdRemoteModify)

-    REMOTE_LIST_HELP = "List available remotes."
+    REMOTE_LIST_HELP = "List all available data remotes."
    remote_list_parser = remote_subparsers.add_parser(
        "list",
        parents=[parent_config_parser, parent_parser],
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -20,7 +18,7 @@ class CmdRemove(CmdBase):
        if self.args.force:
            return False

-        msg = "Are you sure you want to remove {} with its outputs?".format(
+        msg = "Are you sure you want to remove '{}' with its outputs?".format(
            target
        )

@@ -38,13 +36,13 @@ class CmdRemove(CmdBase):
                outs_only = self._is_outs_only(target)
                self.repo.remove(target, outs_only=outs_only)
            except DvcException:
-                logger.exception("failed to remove {}".format(target))
+                logger.exception("failed to remove '{}'".format(target))
                return 1
        return 0


 def add_parser(subparsers, parent_parser):
-    REMOVE_HELP = "Remove DVC-file outputs."
+    REMOVE_HELP = "Remove DVC-tracked files or directories."
    remove_parser = subparsers.add_parser(
        "remove",
        parents=[parent_parser],
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 import os
@@ -59,7 +57,8 @@ class CmdRepro(CmdBase):


 def add_parser(subparsers, parent_parser):
-    REPRO_HELP = "Check for changes and reproduce stages and dependencies."
+    REPRO_HELP = "Reproduce complete or partial pipelines"
+    " by executing their stages."
    repro_parser = subparsers.add_parser(
        "repro",
        parents=[parent_parser],
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -19,7 +17,7 @@ class CmdRoot(CmdBaseNoRepo):


 def add_parser(subparsers, parent_parser):
-    ROOT_HELP = "Relative path to the repository's directory."
+    ROOT_HELP = "Return the relative path to the root of the DVC project."
    root_parser = subparsers.add_parser(
        "root",
        parents=[parent_parser],
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -24,13 +22,14 @@ class CmdRun(CmdBase):
                self.args.metrics_no_cache,
                self.args.outs_persist,
                self.args.outs_persist_no_cache,
+                self.args.params,
                self.args.command,
            ]
        ):  # pragma: no cover
            logger.error(
                "too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
-                "`-m`, `-M`, `--outs-persist`, `--outs-persist-no-cache`, "
-                "`command`."
+                "`-m`, `-M`, `-p`, `--outs-persist`, "
+                "`--outs-persist-no-cache`, `command`."
            )
            return 1

@@ -42,6 +41,7 @@ class CmdRun(CmdBase):
                metrics=self.args.metrics,
                metrics_no_cache=self.args.metrics_no_cache,
                deps=self.args.deps,
+                params=self.args.params,
                fname=self.args.file,
                cwd=self.args.cwd,
                wdir=self.args.wdir,
@@ -113,6 +113,13 @@ def add_parser(subparsers, parent_parser):
        help="Declare output file or directory "
        "(do not put into DVC cache).",
    )
+    run_parser.add_argument(
+        "-p",
+        "--params",
+        action="append",
+        default=[],
+        help="Declare parameter to use as additional dependency.",
+    )
    run_parser.add_argument(
        "-m",
        "--metrics",
@@ -1,9 +1,6 @@
-from __future__ import unicode_literals
-
 import logging

 from dvc.command.data_sync import CmdDataBase
-from dvc.utils.compat import str


 logger = logging.getLogger(__name__)
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -24,7 +22,10 @@ class CmdUnprotect(CmdBase):


 def add_parser(subparsers, parent_parser):
-    UNPROTECT_HELP = "Unprotect data files or directories."
+    UNPROTECT_HELP = (
+        "Unprotect tracked files or directories (when hardlinks or symlinks "
+        "have been enabled with `dvc config cache.type`)"
+    )
    unprotect_parser = subparsers.add_parser(
        "unprotect",
        parents=[parent_parser],
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging

@@ -16,7 +14,7 @@ class CmdUpdate(CmdBase):
        ret = 0
        for target in self.args.targets:
            try:
-                self.repo.update(target)
+                self.repo.update(target, self.args.rev)
            except DvcException:
                logger.exception("failed to update '{}'.".format(target))
                ret = 1
@@ -35,4 +33,7 @@ def add_parser(subparsers, parent_parser):
    update_parser.add_argument(
        "targets", nargs="+", help="DVC-files to update."
    )
+    update_parser.add_argument(
+        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+    )
    update_parser.set_defaults(func=CmdUpdate)
@@ -1,11 +1,10 @@
-from __future__ import unicode_literals
-
 import argparse
 import itertools
 import logging
 import os
 import platform
 import uuid
+import pathlib

 try:
    import psutil
@@ -13,10 +12,10 @@ except ImportError:
    psutil = None

 from dvc.utils import is_binary, relpath
-from dvc.utils.compat import pathlib
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
 from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.scm.base import SCMError
 from dvc.system import System
 from dvc.utils.pkg import PKG

@@ -27,23 +26,14 @@ class CmdVersion(CmdBaseNoRepo):
    def run(self):
        from dvc.repo import Repo

-        dvc_version = __version__
-        python_version = platform.python_version()
-        platform_type = platform.platform()
-        binary = is_binary()
-        info = (
-            "DVC version: {dvc_version}\n"
-            "Python version: {python_version}\n"
-            "Platform: {platform_type}\n"
-            "Binary: {binary}\n"
-            "Package: {package}\n"
-        ).format(
-            dvc_version=dvc_version,
-            python_version=python_version,
-            platform_type=platform_type,
-            binary=binary,
-            package=PKG,
-        )
+        info = [
+            "DVC version: {}".format(__version__),
+            "Python version: {}".format(platform.python_version()),
+            "Platform: {}".format(platform.platform()),
+            "Binary: {}".format(is_binary()),
+            "Package: {}".format(PKG),
+            "Supported remotes: {}".format(self.get_supported_remotes()),
+        ]

        try:
            repo = Repo()
@@ -54,9 +44,14 @@ class CmdVersion(CmdBaseNoRepo):
            # later decides to enable shared cache mode with
            # `dvc config cache.shared group`.
            if os.path.exists(repo.cache.local.cache_dir):
-                info += "Cache: {cache}\n".format(
-                    cache=self.get_linktype_support_info(repo)
+                info.append(
+                    "Cache: {}".format(self.get_linktype_support_info(repo))
                )
+                if psutil:
+                    fs_type = self.get_fs_type(repo.cache.local.cache_dir)
+                    info.append(
+                        "Filesystem type (cache directory): {}".format(fs_type)
+                    )
            else:
                logger.warning(
                    "Unable to detect supported link types, as cache "
@@ -66,19 +61,19 @@ class CmdVersion(CmdBaseNoRepo):
                    "check.".format(relpath(repo.cache.local.cache_dir))
                )

-            if psutil:
-                info += (
-                    "Filesystem type (cache directory): {fs_cache}\n"
-                ).format(fs_cache=self.get_fs_type(repo.cache.local.cache_dir))
        except NotDvcRepoError:
            root_directory = os.getcwd()
+        except SCMError:
+            root_directory = os.getcwd()
+            info.append("Repo: dvc, git (broken)")
+        else:
+            info.append("Repo: {}".format(_get_dvc_repo_info(repo)))

        if psutil:
-            info += ("Filesystem type (workspace): {fs_root}").format(
-                fs_root=self.get_fs_type(os.path.abspath(root_directory))
-            )
+            fs_root = self.get_fs_type(os.path.abspath(root_directory))
+            info.append("Filesystem type (workspace): {}".format(fs_root))

-        logger.info(info)
+        logger.info("\n".join(info))
        return 0

    @staticmethod
@@ -98,9 +93,9 @@ class CmdVersion(CmdBaseNoRepo):
    @staticmethod
    def get_linktype_support_info(repo):
        links = {
-            "reflink": System.reflink,
-            "hardlink": System.hardlink,
-            "symlink": System.symlink,
+            "reflink": (System.reflink, None),
+            "hardlink": (System.hardlink, System.is_hardlink),
+            "symlink": (System.symlink, System.is_symlink),
        }

        fname = "." + str(uuid.uuid4())
@@ -110,25 +105,45 @@ class CmdVersion(CmdBaseNoRepo):

        cache = []

-        for name, link in links.items():
+        for name, (link, is_link) in links.items():
            try:
                link(src, dst)
+                status = "supported"
+                if is_link and not is_link(dst):
+                    status = "broken"
                os.unlink(dst)
-                supported = True
            except DvcException:
-                supported = False
-            cache.append(
-                "{name} - {supported}".format(
-                    name=name, supported=True if supported else False
-                )
-            )
+                status = "not supported"
+            cache.append("{name} - {status}".format(name=name, status=status))
        os.remove(src)

        return ", ".join(cache)

+    @staticmethod
+    def get_supported_remotes():
+        from dvc.remote import REMOTES
+
+        supported_remotes = []
+        for remote in REMOTES:
+            if not remote.get_missing_deps():
+                supported_remotes.append(remote.scheme)
+
+        return ", ".join(supported_remotes)
+
+
+def _get_dvc_repo_info(repo):
+    if repo.config.get("core", {}).get("no_scm", False):
+        return "dvc (no_scm)"
+
+    if repo.root_dir != repo.scm.root_dir:
+        return "dvc (subdir), git"
+
+    return "dvc, git"
+

 def add_parser(subparsers, parent_parser):
-    VERSION_HELP = "Show DVC version and system/environment information."
+    VERSION_HELP = "Display the DVC version"
+    " and system/environment information."

    version_parser = subparsers.add_parser(
        "version",
@@ -0,0 +1,42 @@
+"""Python version compatibility code"""
+
+# Backport os.fspath() from Python 3.6
+try:
+    from os import fspath  # noqa: F821
+
+    fspath_py35 = lambda s: s  # noqa: E731
+except ImportError:
+
+    def fspath(path):
+        """Return the path representation of a path-like object.
+
+        If str or bytes is passed in, it is returned unchanged. Otherwise the
+        os.PathLike interface is used to get the path representation. If the
+        path representation is not str or bytes, TypeError is raised. If the
+        provided path is not str, bytes, or os.PathLike, TypeError is raised.
+        """
+        if isinstance(path, (str, bytes)):
+            return path
+
+        # Work from the object's type to match method resolution of other magic
+        # methods.
+        path_type = type(path)
+        try:
+            path_repr = path_type.__fspath__(path)
+        except AttributeError:
+            if hasattr(path_type, "__fspath__"):
+                raise
+            else:
+                raise TypeError(
+                    "expected str, bytes or os.PathLike object, "
+                    "not " + path_type.__name__
+                )
+        if isinstance(path_repr, (str, bytes)):
+            return path_repr
+        else:
+            raise TypeError(
+                "expected {}.__fspath__() to return str or bytes, "
+                "not {}".format(path_type.__name__, type(path_repr).__name__)
+            )
+
+    fspath_py35 = fspath
@@ -1,47 +1,30 @@
 """DVC config objects."""
-from __future__ import unicode_literals
-
-import copy
-import errno
+from contextlib import contextmanager
 import logging
 import os
 import re
+from urllib.parse import urlparse

+from funcy import cached_property, re_find, walk_values, compact
 import configobj
-from voluptuous import Schema, Required, Optional, Invalid
-from voluptuous import All, Any, Lower, Range, Coerce, Match
+from voluptuous import Schema, Optional, Invalid, ALLOW_EXTRA
+from voluptuous import All, Any, Lower, Range, Coerce

-from dvc.exceptions import DvcException
-from dvc.exceptions import NotDvcRepoError
-from dvc.utils.compat import open, str
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.utils import relpath

 logger = logging.getLogger(__name__)


 class ConfigError(DvcException):
-    """DVC config exception.
+    """DVC config exception."""

-    Args:
-        msg (str): error message.
-        cause (Exception): optional exception that has caused this error.
-    """
-
-    def __init__(self, msg, cause=None):
-        super(ConfigError, self).__init__(
-            "config file error: {}".format(msg), cause=cause
-        )
+    def __init__(self, msg):
+        super().__init__("config file error: {}".format(msg))


 class NoRemoteError(ConfigError):
-    def __init__(self, command, cause=None):
-        msg = (
-            "no remote specified. Setup default remote with\n"
-            "    dvc config core.remote <name>\n"
-            "or use:\n"
-            "    dvc {} -r <name>\n".format(command)
-        )
-
-        super(NoRemoteError, self).__init__(msg, cause=cause)
+    pass


 def supported_cache_type(types):
@@ -65,13 +48,12 @@ def supported_cache_type(types):


 # Checks that value is either true or false and converts it to bool
-Bool = All(
+to_bool = Bool = All(
    Lower,
    Any("true", "false"),
    lambda v: v == "true",
    msg="expected true or false",
 )
-to_bool = Schema(Bool)


 def Choices(*choices):
@@ -81,11 +63,141 @@ def Choices(*choices):
        *choices: pass allowed values as arguments, or pass a list or
            tuple as a single argument
    """
-    return Any(*choices, msg="expected one of {}".format(",".join(choices)))
+    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))


-class Config(object):  # pylint: disable=too-many-instance-attributes
-    """Class that manages configuration files for a dvc repo.
+def ByUrl(mapping):
+    schemas = walk_values(Schema, mapping)
+
+    def validate(data):
+        if "url" not in data:
+            raise Invalid("expected 'url'")
+
+        parsed = urlparse(data["url"])
+        # Windows absolute paths should really have scheme == "" (local)
+        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+            return schemas[""](data)
+        if parsed.scheme not in schemas:
+            raise Invalid("Unsupported URL type {}://".format(parsed.scheme))
+
+        return schemas[parsed.scheme](data)
+
+    return validate
+
+
+class RelPath(str):
+    pass
+
+
+REMOTE_COMMON = {
+    "url": str,
+    "checksum_jobs": All(Coerce(int), Range(1)),
+    Optional("no_traverse"): Bool,  # obsoleted
+    "verify": Bool,
+}
+LOCAL_COMMON = {
+    "type": supported_cache_type,
+    Optional("protected", default=False): Bool,  # obsoleted
+    "shared": All(Lower, Choices("group")),
+    Optional("slow_link_warning", default=True): Bool,
+}
+HTTP_COMMON = {
+    "auth": All(Lower, Choices("basic", "digest", "custom")),
+    "custom_auth_header": str,
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+}
+SCHEMA = {
+    "core": {
+        "remote": Lower,
+        "checksum_jobs": All(Coerce(int), Range(1)),
+        Optional("interactive", default=False): Bool,
+        Optional("analytics", default=True): Bool,
+        Optional("hardlink_lock", default=False): Bool,
+        Optional("no_scm", default=False): Bool,
+    },
+    "cache": {
+        "local": str,
+        "s3": str,
+        "gs": str,
+        "hdfs": str,
+        "ssh": str,
+        "azure": str,
+        # This is for default local cache
+        "dir": str,
+        **LOCAL_COMMON,
+    },
+    "remote": {
+        str: ByUrl(
+            {
+                "": {**LOCAL_COMMON, **REMOTE_COMMON},
+                "s3": {
+                    "region": str,
+                    "profile": str,
+                    "credentialpath": str,
+                    "endpointurl": str,
+                    Optional("listobjects", default=False): Bool,
+                    Optional("use_ssl", default=True): Bool,
+                    "sse": str,
+                    "acl": str,
+                    "grant_read": str,
+                    "grant_read_acp": str,
+                    "grant_write_acp": str,
+                    "grant_full_control": str,
+                    **REMOTE_COMMON,
+                },
+                "gs": {
+                    "projectname": str,
+                    "credentialpath": str,
+                    **REMOTE_COMMON,
+                },
+                "ssh": {
+                    "type": supported_cache_type,
+                    "port": Coerce(int),
+                    "user": str,
+                    "password": str,
+                    "ask_password": Bool,
+                    "keyfile": str,
+                    "timeout": Coerce(int),
+                    "gss_auth": Bool,
+                    **REMOTE_COMMON,
+                },
+                "hdfs": {"user": str, **REMOTE_COMMON},
+                "azure": {"connection_string": str, **REMOTE_COMMON},
+                "oss": {
+                    "oss_key_id": str,
+                    "oss_key_secret": str,
+                    "oss_endpoint": str,
+                    **REMOTE_COMMON,
+                },
+                "gdrive": {
+                    "gdrive_use_service_account": Bool,
+                    "gdrive_client_id": str,
+                    "gdrive_client_secret": str,
+                    "gdrive_user_credentials_file": str,
+                    "gdrive_service_account_email": str,
+                    "gdrive_service_account_user_email": str,
+                    "gdrive_service_account_p12_file_path": str,
+                    Optional("gdrive_trash_only", default=False): Bool,
+                    **REMOTE_COMMON,
+                },
+                "http": {**HTTP_COMMON, **REMOTE_COMMON},
+                "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "remote": {str: object},  # Any of the above options are valid
+            }
+        )
+    },
+    "state": {
+        "row_limit": All(Coerce(int), Range(1)),
+        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
+    },
+}
+COMPILED_SCHEMA = Schema(SCHEMA)
+
+
+class Config(dict):
+    """Class that manages configuration files for a DVC repo.

    Args:
        dvc_dir (str): optional path to `.dvc` directory, that is used to
@@ -94,172 +206,21 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
        validate (bool): optional flag to tell dvc if it should validate the
            config or just load it as is. 'True' by default.

-
    Raises:
-        ConfigError: thrown when config has an invalid format.
+        ConfigError: thrown if config has an invalid format.
    """

    APPNAME = "dvc"
    APPAUTHOR = "iterative"

-    # NOTE: used internally in RemoteLOCAL to know config
-    # location, that url should resolved relative to.
-    PRIVATE_CWD = "_cwd"
+    # In the order they shadow each other
+    LEVELS = ("system", "global", "repo", "local")

    CONFIG = "config"
    CONFIG_LOCAL = "config.local"

-    CREDENTIALPATH = "credentialpath"
-
-    LEVEL_LOCAL = 0
-    LEVEL_REPO = 1
-    LEVEL_GLOBAL = 2
-    LEVEL_SYSTEM = 3
-
-    SECTION_CORE = "core"
-    SECTION_CORE_LOGLEVEL = "loglevel"
-    SECTION_CORE_LOGLEVEL_SCHEMA = All(
-        Lower, Choices("info", "debug", "warning", "error")
-    )
-    SECTION_CORE_REMOTE = "remote"
-    SECTION_CORE_INTERACTIVE = "interactive"
-    SECTION_CORE_ANALYTICS = "analytics"
-    SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_CORE_HARDLINK_LOCK = "hardlink_lock"
-
-    SECTION_CACHE = "cache"
-    SECTION_CACHE_DIR = "dir"
-    SECTION_CACHE_TYPE = "type"
-    SECTION_CACHE_PROTECTED = "protected"
-    SECTION_CACHE_SHARED = "shared"
-    SECTION_CACHE_SHARED_SCHEMA = All(Lower, Choices("group"))
-    SECTION_CACHE_LOCAL = "local"
-    SECTION_CACHE_S3 = "s3"
-    SECTION_CACHE_GS = "gs"
-    SECTION_CACHE_SSH = "ssh"
-    SECTION_CACHE_HDFS = "hdfs"
-    SECTION_CACHE_AZURE = "azure"
-    SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
-    SECTION_CACHE_SCHEMA = {
-        SECTION_CACHE_LOCAL: str,
-        SECTION_CACHE_S3: str,
-        SECTION_CACHE_GS: str,
-        SECTION_CACHE_HDFS: str,
-        SECTION_CACHE_SSH: str,
-        SECTION_CACHE_AZURE: str,
-        SECTION_CACHE_DIR: str,
-        SECTION_CACHE_TYPE: supported_cache_type,
-        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
-        SECTION_CACHE_SHARED: SECTION_CACHE_SHARED_SCHEMA,
-        PRIVATE_CWD: str,
-        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): Bool,
-    }
-
-    SECTION_CORE_SCHEMA = {
-        SECTION_CORE_LOGLEVEL: SECTION_CORE_LOGLEVEL_SCHEMA,
-        SECTION_CORE_REMOTE: Lower,
-        Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
-        Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
-        SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
-        Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool,
-    }
-
-    # aws specific options
-    SECTION_AWS_CREDENTIALPATH = CREDENTIALPATH
-    SECTION_AWS_ENDPOINT_URL = "endpointurl"
-    SECTION_AWS_LIST_OBJECTS = "listobjects"
-    SECTION_AWS_REGION = "region"
-    SECTION_AWS_PROFILE = "profile"
-    SECTION_AWS_USE_SSL = "use_ssl"
-    SECTION_AWS_SSE = "sse"
-    SECTION_AWS_ACL = "acl"
-    SECTION_AWS_GRANT_READ = "grant_read"
-    SECTION_AWS_GRANT_READ_ACP = "grant_read_acp"
-    SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp"
-    SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control"
-
-    # gcp specific options
-    SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH
-    SECTION_GCP_PROJECTNAME = "projectname"
-
-    # azure specific option
-    SECTION_AZURE_CONNECTION_STRING = "connection_string"
-
-    # Alibabacloud oss options
-    SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
-    SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
-    SECTION_OSS_ENDPOINT = "oss_endpoint"
-
-    # GDrive options
-    SECTION_GDRIVE_CLIENT_ID = "gdrive_client_id"
-    SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret"
-    SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file"
-
-    SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
-    SECTION_REMOTE_FMT = 'remote "{}"'
-    SECTION_REMOTE_URL = "url"
-    SECTION_REMOTE_USER = "user"
-    SECTION_REMOTE_PORT = "port"
-    SECTION_REMOTE_KEY_FILE = "keyfile"
-    SECTION_REMOTE_TIMEOUT = "timeout"
-    SECTION_REMOTE_PASSWORD = "password"
-    SECTION_REMOTE_ASK_PASSWORD = "ask_password"
-    SECTION_REMOTE_GSS_AUTH = "gss_auth"
-    SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
-    SECTION_REMOTE_SCHEMA = {
-        Required(SECTION_REMOTE_URL): str,
-        SECTION_AWS_REGION: str,
-        SECTION_AWS_PROFILE: str,
-        SECTION_AWS_CREDENTIALPATH: str,
-        SECTION_AWS_ENDPOINT_URL: str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): Bool,
-        Optional(SECTION_AWS_USE_SSL, default=True): Bool,
-        SECTION_AWS_SSE: str,
-        SECTION_AWS_ACL: str,
-        SECTION_AWS_GRANT_READ: str,
-        SECTION_AWS_GRANT_READ_ACP: str,
-        SECTION_AWS_GRANT_WRITE_ACP: str,
-        SECTION_AWS_GRANT_FULL_CONTROL: str,
-        SECTION_GCP_PROJECTNAME: str,
-        SECTION_CACHE_TYPE: supported_cache_type,
-        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
-        SECTION_REMOTE_USER: str,
-        SECTION_REMOTE_PORT: Coerce(int),
-        SECTION_REMOTE_KEY_FILE: str,
-        SECTION_REMOTE_TIMEOUT: Coerce(int),
-        SECTION_REMOTE_PASSWORD: str,
-        SECTION_REMOTE_ASK_PASSWORD: Bool,
-        SECTION_REMOTE_GSS_AUTH: Bool,
-        SECTION_AZURE_CONNECTION_STRING: str,
-        SECTION_OSS_ACCESS_KEY_ID: str,
-        SECTION_OSS_ACCESS_KEY_SECRET: str,
-        SECTION_OSS_ENDPOINT: str,
-        SECTION_GDRIVE_CLIENT_ID: str,
-        SECTION_GDRIVE_CLIENT_SECRET: str,
-        SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
-        PRIVATE_CWD: str,
-        Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): Bool,
-    }
-
-    SECTION_STATE = "state"
-    SECTION_STATE_ROW_LIMIT = "row_limit"
-    SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
-    SECTION_STATE_SCHEMA = {
-        SECTION_STATE_ROW_LIMIT: All(Coerce(int), Range(1)),
-        SECTION_STATE_ROW_CLEANUP_QUOTA: All(Coerce(int), Range(0, 100)),
-    }
-
-    SCHEMA = {
-        Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
-        Match(SECTION_REMOTE_REGEX): SECTION_REMOTE_SCHEMA,
-        Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
-        Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
-    }
-    COMPILED_SCHEMA = Schema(SCHEMA)
-
    def __init__(self, dvc_dir=None, validate=True):
        self.dvc_dir = dvc_dir
-        self.validate = validate

        if not dvc_dir:
            try:
@@ -271,33 +232,31 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
        else:
            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))

-        self.load()
+        self.load(validate=validate)

-    @staticmethod
-    def get_global_config_dir():
-        """Returns global config location. E.g. ~/.config/dvc/config.
+    @classmethod
+    def get_dir(cls, level):
+        from appdirs import user_config_dir, site_config_dir

-        Returns:
-            str: path to the global config directory.
-        """
-        from appdirs import user_config_dir
+        assert level in ("global", "system")

-        return user_config_dir(
-            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
-        )
+        if level == "global":
+            return user_config_dir(cls.APPNAME, cls.APPAUTHOR)
+        if level == "system":
+            return site_config_dir(cls.APPNAME, cls.APPAUTHOR)

-    @staticmethod
-    def get_system_config_dir():
-        """Returns system config location. E.g. /etc/dvc.conf.
+    @cached_property
+    def files(self):
+        files = {
+            level: os.path.join(self.get_dir(level), self.CONFIG)
+            for level in ("system", "global")
+        }

-        Returns:
-            str: path to the system config directory.
-        """
-        from appdirs import site_config_dir
+        if self.dvc_dir is not None:
+            files["repo"] = os.path.join(self.dvc_dir, self.CONFIG)
+            files["local"] = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)

-        return site_config_dir(
-            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
-        )
+        return files

    @staticmethod
    def init(dvc_dir):
@@ -313,243 +272,137 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
        open(config_file, "w+").close()
        return Config(dvc_dir)

-    def _resolve_cache_path(self, config):
-        cache = config.get(self.SECTION_CACHE)
-        if cache is None:
-            return
-
-        cache_dir = cache.get(self.SECTION_CACHE_DIR)
-        if cache_dir is None:
-            return
-
-        cache[self.PRIVATE_CWD] = os.path.dirname(config.filename)
-
-    def _resolve_paths(self, config):
-        if config.filename is None:
-            return config
-
-        ret = copy.deepcopy(config)
-        self._resolve_cache_path(ret)
-
-        for section in ret.values():
-            if self.SECTION_REMOTE_URL not in section.keys():
-                continue
-
-            section[self.PRIVATE_CWD] = os.path.dirname(ret.filename)
-
-        return ret
-
-    def _load_configs(self):
-        system_config_file = os.path.join(
-            self.get_system_config_dir(), self.CONFIG
-        )
-
-        global_config_file = os.path.join(
-            self.get_global_config_dir(), self.CONFIG
-        )
-
-        self._system_config = configobj.ConfigObj(system_config_file)
-        self._global_config = configobj.ConfigObj(global_config_file)
-        self._repo_config = configobj.ConfigObj()
-        self._local_config = configobj.ConfigObj()
-
-        if not self.dvc_dir:
-            return
-
-        config_file = os.path.join(self.dvc_dir, self.CONFIG)
-        config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
-
-        self._repo_config = configobj.ConfigObj(config_file)
-        self._local_config = configobj.ConfigObj(config_local_file)
-
-    @property
-    def config_local_file(self):
-        return self._local_config.filename
-
-    @property
-    def config_file(self):
-        return self._repo_config.filename
-
-    def load(self):
+    def load(self, validate=True):
        """Loads config from all the config files.

        Raises:
-            dvc.config.ConfigError: thrown if config has invalid format.
+            ConfigError: thrown if config has an invalid format.
        """
-        self._load_configs()
+        conf = {}
+        for level in self.LEVELS:
+            if level in self.files:
+                _merge(conf, self.load_one(level))

-        self.config = configobj.ConfigObj()
-        for c in [
-            self._system_config,
-            self._global_config,
-            self._repo_config,
-            self._local_config,
-        ]:
-            c = self._resolve_paths(c)
-            c = self._lower(c)
-            self.config.merge(c)
+        if validate:
+            conf = self.validate(conf)

-        if not self.validate:
-            return
+        self.clear()
+        self.update(conf)

-        d = self.config.dict()
-        try:
-            d = self.COMPILED_SCHEMA(d)
-        except Invalid as exc:
-            raise ConfigError(str(exc), cause=exc)
-        self.config = configobj.ConfigObj(d, write_empty_values=True)
+        # Add resolved default cache.dir
+        if not self["cache"].get("dir") and self.dvc_dir:
+            self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")

-    def save(self, config=None):
-        """Saves config to config files.
+    def load_one(self, level):
+        conf = _load_config(self.files[level])
+        conf = self._load_paths(conf, self.files[level])

-        Raises:
-            dvc.config.ConfigError: thrown if failed to write config file.
-        """
-        if config is not None:
-            clist = [config]
-        else:
-            clist = [
-                self._system_config,
-                self._global_config,
-                self._repo_config,
-                self._local_config,
-            ]
+        # Autovivify sections
+        for key in COMPILED_SCHEMA.schema:
+            conf.setdefault(key, {})

-        for conf in clist:
-            self._save(conf)
+        return conf

+    @staticmethod
+    def _load_paths(conf, filename):
+        abs_conf_dir = os.path.abspath(os.path.dirname(filename))
+
+        def resolve(path):
+            if os.path.isabs(path) or re.match(r"\w+://", path):
+                return path
+            return RelPath(os.path.join(abs_conf_dir, path))
+
+        return Config._map_dirs(conf, resolve)
+
+    @staticmethod
+    def _save_paths(conf, filename):
+        conf_dir = os.path.dirname(filename)
+
+        def rel(path):
+            if re.match(r"\w+://", path):
+                return path
+
+            if isinstance(path, RelPath) or not os.path.isabs(path):
+                return relpath(path, conf_dir)
+            return path
+
+        return Config._map_dirs(conf, rel)
+
+    @staticmethod
+    def _map_dirs(conf, func):
+        dirs_schema = {"cache": {"dir": func}, "remote": {str: {"url": func}}}
+        return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
+
+    @contextmanager
+    def edit(self, level="repo"):
+        if level in {"repo", "local"} and self.dvc_dir is None:
+            raise ConfigError("Not inside a DVC repo")
+
+        conf = self.load_one(level)
+        yield conf
+
+        conf = self._save_paths(conf, self.files[level])
+        _save_config(self.files[level], conf)
        self.load()

    @staticmethod
-    def _save(config):
-        if config.filename is None:
-            return
-
-        logger.debug("Writing '{}'.".format(config.filename))
-        dname = os.path.dirname(os.path.abspath(config.filename))
+    def validate(data):
        try:
-            os.makedirs(dname)
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                raise
-        config.write()
+            return COMPILED_SCHEMA(data)
+        except Invalid as exc:
+            raise ConfigError(str(exc)) from None

-    def unset(self, section, opt=None, level=None, force=False):
-        """Unsets specified option and/or section in the config.

-        Args:
-            section (str): section name.
-            opt (str): optional option name.
-            level (int): config level to use.
-            force (bool): don't error-out even if section doesn't exist. False
-                by default.
+def _load_config(filename):
+    conf_obj = configobj.ConfigObj(filename)
+    return _parse_remotes(_lower_keys(conf_obj.dict()))

-        Raises:
-            dvc.config.ConfigError: thrown if section doesn't exist and
-                `force != True`.
-        """
-        config = self.get_configobj(level)

-        if section not in config.keys():
-            if force:
-                return
-            raise ConfigError("section '{}' doesn't exist".format(section))
+def _save_config(filename, conf_dict):
+    logger.debug("Writing '{}'.".format(filename))
+    os.makedirs(os.path.dirname(filename), exist_ok=True)

-        if opt:
-            if opt not in config[section].keys():
-                if force:
-                    return
-                raise ConfigError(
-                    "option '{}.{}' doesn't exist".format(section, opt)
-                )
-            del config[section][opt]
+    config = configobj.ConfigObj(_pack_remotes(conf_dict))
+    config.filename = filename
+    config.write()

-            if not config[section]:
-                del config[section]
+
+def _parse_remotes(conf):
+    result = {"remote": {}}
+
+    for section, val in conf.items():
+        name = re_find(r'^\s*remote\s*"(.*)"\s*$', section)
+        if name:
+            result["remote"][name] = val
        else:
-            del config[section]
+            result[section] = val

-        self.save(config)
+    return result

-    def set(self, section, opt, value, level=None, force=True):
-        """Sets specified option in the config.

-        Args:
-            section (str): section name.
-            opt (str): option name.
-            value: value to set option to.
-            level (int): config level to use.
-            force (bool): set option even if section already exists. True by
-                default.
+def _pack_remotes(conf):
+    # Drop empty sections
+    result = compact(conf)

-        Raises:
-            dvc.config.ConfigError: thrown if section already exists and
-                `force != True`.
+    # Transform remote.name -> 'remote "name"'
+    for name, val in conf["remote"].items():
+        result['remote "{}"'.format(name)] = val
+    result.pop("remote", None)

-        """
-        config = self.get_configobj(level)
+    return result

-        if section not in config.keys():
-            config[section] = {}
-        elif not force:
-            raise ConfigError(
-                "Section '{}' already exists. Use `-f|--force` to overwrite "
-                "section with new value.".format(section)
-            )

-        config[section][opt] = value
-        self.save(config)
+def _merge(into, update):
+    """Merges second dict into first recursively"""
+    for key, val in update.items():
+        if isinstance(into.get(key), dict) and isinstance(val, dict):
+            _merge(into[key], val)
+        else:
+            into[key] = val

-    def get(self, section, opt=None, level=None):
-        """Return option value from the config.

-        Args:
-            section (str): section name.
-            opt (str): option name.
-            level (int): config level to use.
-
-        Returns:
-            value (str, int): option value.
-        """
-        config = self.get_configobj(level)
-
-        if section not in config.keys():
-            raise ConfigError("section '{}' doesn't exist".format(section))
-
-        if opt not in config[section].keys():
-            raise ConfigError(
-                "option '{}.{}' doesn't exist".format(section, opt)
-            )
-
-        return config[section][opt]
-
-    @staticmethod
-    def _lower(config):
-        new_config = configobj.ConfigObj()
-        for s_key, s_value in config.items():
-            new_s = {}
-            for key, value in s_value.items():
-                new_s[key.lower()] = str(value)
-            new_config[s_key.lower()] = new_s
-        return new_config
-
-    def get_configobj(self, level):
-        configs = {
-            self.LEVEL_LOCAL: self._local_config,
-            self.LEVEL_REPO: self._repo_config,
-            self.LEVEL_GLOBAL: self._global_config,
-            self.LEVEL_SYSTEM: self._system_config,
-        }
-
-        return configs.get(level, self._repo_config)
-
-    def list_options(self, section_regex, option, level=None):
-        ret = {}
-        config = self.get_configobj(level)
-        for section in config.keys():
-            r = re.match(section_regex, section)
-            if r:
-                name = r.group("name")
-                value = config[section].get(option, "")
-                ret[name] = value
-        return ret
+def _lower_keys(data):
+    return {
+        k.lower(): _lower_keys(v) if isinstance(v, dict) else v
+        for k, v in data.items()
+    }
@@ -1,5 +1,4 @@
 """Launch `dvc daemon` command in a separate detached process."""
-from __future__ import unicode_literals

 import inspect
 import logging
@@ -10,7 +9,6 @@ from subprocess import Popen
 from dvc.env import DVC_DAEMON
 from dvc.utils import fix_env
 from dvc.utils import is_binary
-from dvc.utils.compat import cast_bytes_py2


 logger = logging.getLogger(__name__)
@@ -99,9 +97,7 @@ def daemon(args):

    env = fix_env()
    file_path = os.path.abspath(inspect.stack()[0][1])
-    env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2(
-        os.path.dirname(os.path.dirname(file_path))
-    )
-    env[cast_bytes_py2(DVC_DAEMON)] = cast_bytes_py2("1")
+    env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
+    env[DVC_DAEMON] = "1"

    _spawn(cmd, env)
@@ -1,6 +1,4 @@
 """Draws DAG in ASCII."""
-from __future__ import print_function
-from __future__ import unicode_literals

 import logging
 import math
@@ -16,7 +14,7 @@ from grandalf.routing import EdgeViewer
 from grandalf.routing import route_with_lines

 from dvc.env import DVC_PAGER
-
+from dvc.utils import format_link

 logger = logging.getLogger(__name__)

@@ -47,7 +45,9 @@ def find_pager():

    logger.warning(
        "Unable to find `less` in the PATH. Check out "
-        "man.dvc.org/doc/command-reference/pipeline/show for more info."
+        "{} for more info.".format(
+            format_link("https://man.dvc.org/pipeline/show")
+        )
    )
    return pydoc.plainpager

@@ -1,9 +1,7 @@
 """Manages dvc remotes that user can use with push/pull/status commands."""
-from __future__ import unicode_literals

 import logging

-from dvc.config import Config
 from dvc.config import NoRemoteError
 from dvc.remote import Remote

@@ -25,22 +23,27 @@ class DataCloud(object):
    def __init__(self, repo):
        self.repo = repo

-    @property
-    def _config(self):
-        return self.repo.config.config
-
-    @property
-    def _core(self):
-        return self._config.get(Config.SECTION_CORE, {})
-
    def get_remote(self, remote=None, command="<command>"):
        if not remote:
-            remote = self._core.get(Config.SECTION_CORE_REMOTE)
+            remote = self.repo.config["core"].get("remote")

        if remote:
            return self._init_remote(remote)

-        raise NoRemoteError(command)
+        if bool(self.repo.config["remote"]):
+            error_msg = (
+                "no remote specified. Setup default remote with\n"
+                "    dvc remote default <remote name>\n"
+                "or use:\n"
+                "    dvc {} -r <remote name>".format(command)
+            )
+        else:
+            error_msg = (
+                "no remote specified. Create a default remote with\n"
+                "    dvc remote add -d <remote name> <remote url>"
+            )
+
+        raise NoRemoteError(error_msg)

    def _init_remote(self, remote):
        return Remote(self.repo, name=remote)
@@ -74,13 +77,25 @@ class DataCloud(object):
            show_checksums (bool): show checksums instead of file names in
                information messages.
        """
-        return self.repo.cache.local.pull(
-            cache,
-            jobs=jobs,
-            remote=self.get_remote(remote, "pull"),
-            show_checksums=show_checksums,
+        remote = self.get_remote(remote, "pull")
+        downloaded_items_num = self.repo.cache.local.pull(
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
        )

+        if not remote.verify:
+            self._save_pulled_checksums(cache)
+
+        return downloaded_items_num
+
+    def _save_pulled_checksums(self, cache):
+        for checksum in cache["local"].keys():
+            cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
+            if self.repo.cache.local.exists(cache_file):
+                # We can safely save here, as existing corrupted files will be
+                # removed upon status, while files corrupted during download
+                # will not be moved from tmp_file (see `RemoteBASE.download()`)
+                self.repo.state.save(cache_file, checksum)
+
    def status(self, cache, jobs=None, remote=None, show_checksums=False):
        """Check status of data items in a cloud-agnostic way.

@@ -1,4 +1,5 @@
-from __future__ import unicode_literals
+from urllib.parse import urlparse
+from collections import defaultdict

 import dvc.output as output
 from dvc.dependency.gs import DependencyGS
@@ -8,6 +9,7 @@ from dvc.dependency.https import DependencyHTTPS
 from dvc.dependency.local import DependencyLOCAL
 from dvc.dependency.s3 import DependencyS3
 from dvc.dependency.ssh import DependencySSH
+from dvc.dependency.param import DependencyPARAMS
 from dvc.output.base import OutputBase
 from dvc.remote import Remote
 from dvc.scheme import Schemes
@@ -42,15 +44,13 @@ DEP_MAP = {
 SCHEMA = output.SCHEMA.copy()
 del SCHEMA[OutputBase.PARAM_CACHE]
 del SCHEMA[OutputBase.PARAM_METRIC]
-SCHEMA[DependencyREPO.PARAM_REPO] = DependencyREPO.REPO_SCHEMA
+SCHEMA.update(DependencyREPO.REPO_SCHEMA)
+SCHEMA.update(DependencyPARAMS.PARAM_SCHEMA)


 def _get(stage, p, info):
-    from dvc.utils.compat import urlparse
-
-    parsed = urlparse(p)
-
-    if parsed.scheme == "remote":
+    parsed = urlparse(p) if p else None
+    if parsed and parsed.scheme == "remote":
        remote = Remote(stage.repo, name=parsed.netloc)
        return DEP_MAP[remote.scheme](stage, p, info, remote=remote)

@@ -58,6 +58,10 @@ def _get(stage, p, info):
        repo = info.pop(DependencyREPO.PARAM_REPO)
        return DependencyREPO(repo, stage, p, info)

+    if info and info.get(DependencyPARAMS.PARAM_PARAMS):
+        params = info.pop(DependencyPARAMS.PARAM_PARAMS)
+        return DependencyPARAMS(stage, p, params)
+
    for d in DEPS:
        if d.supported(p):
            return d(stage, p, info)
@@ -67,7 +71,7 @@ def _get(stage, p, info):
 def loadd_from(stage, d_list):
    ret = []
    for d in d_list:
-        p = d.pop(OutputBase.PARAM_PATH)
+        p = d.pop(OutputBase.PARAM_PATH, None)
        ret.append(_get(stage, p, d))
    return ret

@@ -78,3 +82,28 @@ def loads_from(stage, s_list, erepo=None):
        info = {DependencyREPO.PARAM_REPO: erepo} if erepo else {}
        ret.append(_get(stage, s, info))
    return ret
+
+
+def _parse_params(path_params):
+    path, _, params_str = path_params.rpartition(":")
+    params = params_str.split(",")
+    return path, params
+
+
+def loads_params(stage, s_list):
+    # Creates an object for each unique file that is referenced in the list
+    params_by_path = defaultdict(list)
+    for s in s_list:
+        path, params = _parse_params(s)
+        params_by_path[path].extend(params)
+
+    d_list = []
+    for path, params in params_by_path.items():
+        d_list.append(
+            {
+                OutputBase.PARAM_PATH: path,
+                DependencyPARAMS.PARAM_PARAMS: params,
+            }
+        )
+
+    return loadd_from(stage, d_list)
@@ -1,23 +1,21 @@
-from __future__ import unicode_literals
-
 from dvc.exceptions import DvcException


 class DependencyDoesNotExistError(DvcException):
    def __init__(self, path):
        msg = "dependency '{}' does not exist".format(path)
-        super(DependencyDoesNotExistError, self).__init__(msg)
+        super().__init__(msg)


 class DependencyIsNotFileOrDirError(DvcException):
    def __init__(self, path):
        msg = "dependency '{}' is not a file or directory".format(path)
-        super(DependencyIsNotFileOrDirError, self).__init__(msg)
+        super().__init__(msg)


 class DependencyIsStageFileError(DvcException):
    def __init__(self, path):
-        super(DependencyIsStageFileError, self).__init__(
+        super().__init__(
            "Stage file '{}' cannot be a dependency.".format(path)
        )

@@ -29,5 +27,5 @@ class DependencyBase(object):
    IsNotFileOrDirError = DependencyIsNotFileOrDirError
    IsStageFileError = DependencyIsStageFileError

-    def update(self):
+    def update(self, rev=None):
        pass
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.gs import OutputGS

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.hdfs import OutputHDFS

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from .http import DependencyHTTP
 from dvc.remote.https import RemoteHTTPS

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.local import OutputLOCAL

@@ -0,0 +1,105 @@
+import os
+import yaml
+from collections import defaultdict
+
+import dpath.util
+from voluptuous import Any
+
+from dvc.compat import fspath_py35
+from dvc.dependency.local import DependencyLOCAL
+from dvc.exceptions import DvcException
+
+
+class MissingParamsError(DvcException):
+    pass
+
+
+class BadParamFileError(DvcException):
+    pass
+
+
+class DependencyPARAMS(DependencyLOCAL):
+    PARAM_PARAMS = "params"
+    PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
+    DEFAULT_PARAMS_FILE = "params.yaml"
+
+    def __init__(self, stage, path, params):
+        info = {}
+        self.params = []
+        if params:
+            if isinstance(params, list):
+                self.params = params
+            else:
+                assert isinstance(params, dict)
+                self.params = list(params.keys())
+                info = params
+
+        super().__init__(
+            stage,
+            path
+            or os.path.join(stage.repo.root_dir, self.DEFAULT_PARAMS_FILE),
+            info=info,
+        )
+
+    def save(self):
+        super().save()
+        self.info = self.save_info()
+
+    def status(self):
+        status = super().status()
+
+        if status[str(self)] == "deleted":
+            return status
+
+        status = defaultdict(dict)
+        info = self._get_info()
+        for param in self.params:
+            if param not in info.keys():
+                st = "deleted"
+            elif param not in self.info:
+                st = "new"
+            elif info[param] != self.info[param]:
+                st = "modified"
+            else:
+                assert info[param] == self.info[param]
+                continue
+
+            status[str(self)][param] = st
+
+        return status
+
+    def dumpd(self):
+        return {
+            self.PARAM_PATH: self.def_path,
+            self.PARAM_PARAMS: self.info or self.params,
+        }
+
+    def _get_info(self):
+        if not self.exists:
+            return {}
+
+        with open(fspath_py35(self.path_info), "r") as fobj:
+            try:
+                config = yaml.safe_load(fobj)
+            except yaml.YAMLError as exc:
+                raise BadParamFileError(
+                    "Unable to read parameters from '{}'".format(self)
+                ) from exc
+
+        ret = {}
+        for param in self.params:
+            ret[param] = dpath.util.get(config, param, separator=".")
+        return ret
+
+    def save_info(self):
+        info = self._get_info()
+
+        missing_params = set(self.params) - set(info.keys())
+        if missing_params:
+            raise MissingParamsError(
+                "Parameters '{}' are missing from '{}'.".format(
+                    ", ".join(missing_params), self,
+                )
+            )
+
+        return info
@@ -1,13 +1,10 @@
-from __future__ import unicode_literals
+import os

-import copy
-from contextlib import contextmanager
-
-from funcy import merge
+from voluptuous import Required

 from .local import DependencyLOCAL
-from dvc.external_repo import external_repo
-from dvc.utils.compat import str
+from dvc.exceptions import OutputNotFoundError
+from dvc.path_info import PathInfo


 class DependencyREPO(DependencyLOCAL):
@@ -16,11 +13,17 @@ class DependencyREPO(DependencyLOCAL):
    PARAM_REV = "rev"
    PARAM_REV_LOCK = "rev_lock"

-    REPO_SCHEMA = {PARAM_URL: str, PARAM_REV: str, PARAM_REV_LOCK: str}
+    REPO_SCHEMA = {
+        PARAM_REPO: {
+            Required(PARAM_URL): str,
+            PARAM_REV: str,
+            PARAM_REV_LOCK: str,
+        }
+    }

    def __init__(self, def_repo, stage, *args, **kwargs):
        self.def_repo = def_repo
-        super(DependencyREPO, self).__init__(stage, *args, **kwargs)
+        super().__init__(stage, *args, **kwargs)

    def _parse_path(self, remote, path):
        return None
@@ -32,24 +35,33 @@ class DependencyREPO(DependencyLOCAL):
    @property
    def repo_pair(self):
        d = self.def_repo
-        return d[self.PARAM_URL], d[self.PARAM_REV_LOCK] or d[self.PARAM_REV]
+        rev = d.get(self.PARAM_REV_LOCK) or d.get(self.PARAM_REV)
+        return d[self.PARAM_URL], rev

    def __str__(self):
        return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])

-    @contextmanager
-    def _make_repo(self, **overrides):
-        with external_repo(**merge(self.def_repo, overrides)) as repo:
-            yield repo
+    def _make_repo(self, *, locked=True):
+        from dvc.external_repo import external_repo
+
+        d = self.def_repo
+        rev = (d.get("rev_lock") if locked else None) or d.get("rev")
+        return external_repo(d["url"], rev=rev)
+
+    def _get_checksum(self, locked=True):
+        with self._make_repo(locked=locked) as repo:
+            try:
+                return repo.find_out_by_relpath(self.def_path).info["md5"]
+            except OutputNotFoundError:
+                path = PathInfo(os.path.join(repo.root_dir, self.def_path))
+                # We are polluting our repo cache with some dir listing here
+                return self.repo.cache.local.get_checksum(path)

    def status(self):
-        with self._make_repo() as repo:
-            current = repo.find_out_by_relpath(self.def_path).info
+        current_checksum = self._get_checksum(locked=True)
+        updated_checksum = self._get_checksum(locked=False)

-        with self._make_repo(rev_lock=None) as repo:
-            updated = repo.find_out_by_relpath(self.def_path).info
-
-        if current != updated:
+        if current_checksum != updated_checksum:
            return {str(self): "update available"}

        return {}
@@ -60,23 +72,19 @@ class DependencyREPO(DependencyLOCAL):
    def dumpd(self):
        return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}

-    def fetch(self):
-        with self._make_repo(
-            cache_dir=self.repo.cache.local.cache_dir
-        ) as repo:
-            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
-
-            out = repo.find_out_by_relpath(self.def_path)
-            with repo.state:
-                repo.cloud.pull(out.get_used_cache())
-
-        return out
-
    def download(self, to):
-        out = self.fetch()
-        to.info = copy.copy(out.info)
-        to.checkout()
+        with self._make_repo() as repo:
+            if self.def_repo.get(self.PARAM_REV_LOCK) is None:
+                self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()

-    def update(self):
-        with self._make_repo(rev_lock=None) as repo:
+            if hasattr(repo, "cache"):
+                repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
+
+            repo.pull_to(self.def_path, to.path_info)
+
+    def update(self, rev=None):
+        if rev:
+            self.def_repo[self.PARAM_REV] = rev
+
+        with self._make_repo(locked=False) as repo:
            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.s3 import OutputS3

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.ssh import OutputSSH

@@ -1,32 +1,14 @@
 """Exceptions raised by the dvc."""
-from __future__ import unicode_literals

-import traceback
-
-from dvc.utils import relpath
-from dvc.utils.compat import builtin_str
-from dvc.utils.compat import str
+from dvc.utils import relpath, format_link


 class DvcException(Exception):
-    """Base class for all dvc exceptions.
+    """Base class for all dvc exceptions."""

-    Args:
-        msg (unicode): message for this exception.
-        cause (Exception): optional cause exception.
-    """

-    def __init__(self, msg, cause=None):
-        # NOTE: unlike python 3, python 2 doesn't have built-in support
-        # for chained exceptions, so we are using our own implementation.
-        self.cause = cause
-        self.cause_tb = None
-        if cause:
-            try:
-                self.cause_tb = traceback.format_exc()
-            except AttributeError:  # pragma: no cover
-                pass
-        super(DvcException, self).__init__(msg)
+class InvalidArgumentError(ValueError, DvcException):
+    """Thrown if arguments are invalid."""


 class OutputDuplicationError(DvcException):
@@ -39,18 +21,14 @@ class OutputDuplicationError(DvcException):
    """

    def __init__(self, output, stages):
-        assert isinstance(output, str) or isinstance(output, builtin_str)
-        assert isinstance(stages, list)
-        assert all(
-            isinstance(stage, str) or isinstance(stage, builtin_str)
-            for stage in stages
-        )
+        assert isinstance(output, str)
+        assert all(hasattr(stage, "relpath") for stage in stages)
        msg = (
            "file/directory '{}' is specified as an output in more than one "
            "stage: {}\n"
            "This is not allowed. Consider using a different output name."
-        ).format(output, "\n    ".join(stages))
-        super(OutputDuplicationError, self).__init__(msg)
+        ).format(output, "\n    ".join(s.relpath for s in stages))
+        super().__init__(msg)


 class OutputNotFoundError(DvcException):
@@ -63,8 +41,8 @@ class OutputNotFoundError(DvcException):
    def __init__(self, output, repo=None):
        self.output = output
        self.repo = repo
-        super(OutputNotFoundError, self).__init__(
-            "unable to find DVC-file with output '{path}'".format(
+        super().__init__(
+            "Unable to find DVC-file with output '{path}'".format(
                path=relpath(self.output)
            )
        )
@@ -75,21 +53,17 @@ class StagePathAsOutputError(DvcException):
    an output of another stage.

    Args:
-        cwd (str): path to the directory.
-        fname (str): path to the DVC-file that has cwd specified as an
-            output.
+        stage (Stage): a stage that is in some other stages output
+        output (str): an output covering the stage above
    """

-    def __init__(self, wdir, fname):
-        assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
-        assert isinstance(fname, str) or isinstance(fname, builtin_str)
-        msg = (
-            "current working directory '{cwd}' is specified as an output in "
-            "'{fname}'. Use another CWD to prevent any data removal.".format(
-                cwd=wdir, fname=fname
+    def __init__(self, stage, output):
+        assert isinstance(output, str)
+        super().__init__(
+            "'{stage}' is within an output '{output}' of another stage".format(
+                stage=stage.relpath, output=output
            )
        )
-        super(StagePathAsOutputError, self).__init__(msg)


 class CircularDependencyError(DvcException):
@@ -101,13 +75,13 @@ class CircularDependencyError(DvcException):
    """

    def __init__(self, dependency):
-        assert isinstance(dependency, (str, builtin_str))
+        assert isinstance(dependency, str)

        msg = (
            "file/directory '{}' is specified as an output and as a "
            "dependency."
        )
-        super(CircularDependencyError, self).__init__(msg.format(dependency))
+        super().__init__(msg.format(dependency))


 class ArgumentDuplicationError(DvcException):
@@ -119,9 +93,8 @@ class ArgumentDuplicationError(DvcException):
    """

    def __init__(self, path):
-        assert isinstance(path, (str, builtin_str))
-        msg = "file '{}' is specified more than once."
-        super(ArgumentDuplicationError, self).__init__(msg.format(path))
+        assert isinstance(path, str)
+        super().__init__("file '{}' is specified more than once.".format(path))


 class MoveNotDataSourceError(DvcException):
@@ -139,68 +112,55 @@ class MoveNotDataSourceError(DvcException):
            "it by hand, or remove '{path}' and create a new one at the "
            "desired location."
        )
-        super(MoveNotDataSourceError, self).__init__(msg.format(path=path))
+        super().__init__(msg.format(path=path))


 class NotDvcRepoError(DvcException):
-    """Thrown if a directory is not a dvc repo.
-
-    Args:
-        root (str): path to the directory.
-    """
-
-    def __init__(self, root):
-        msg = (
-            "you are not inside of a dvc repository "
-            "(checked up to mount point '{}')"
-        )
-        super(NotDvcRepoError, self).__init__(msg.format(root))
+    """Thrown if a directory is not a DVC repo"""


 class DvcParserError(DvcException):
    """Base class for CLI parser errors."""

    def __init__(self):
-        super(DvcParserError, self).__init__("parser error")
+        super().__init__("parser error")


 class CyclicGraphError(DvcException):
    def __init__(self, stages):
        assert isinstance(stages, list)
-        stages = "\n".join("\t- {}".format(stage) for stage in stages)
+        stages = "\n".join("\t- {}".format(stage.relpath) for stage in stages)
        msg = (
            "you've introduced a cycle in your pipeline that involves "
            "the following stages:"
            "\n"
            "{stages}".format(stages=stages)
        )
-        super(CyclicGraphError, self).__init__(msg)
+        super().__init__(msg)


 class ConfirmRemoveError(DvcException):
    def __init__(self, path):
-        super(ConfirmRemoveError, self).__init__(
+        super().__init__(
            "unable to remove '{}' without a confirmation from the user. Use "
            "`-f` to force.".format(path)
        )


 class InitError(DvcException):
-    def __init__(self, msg):
-        super(InitError, self).__init__(msg)
+    pass


 class ReproductionError(DvcException):
-    def __init__(self, dvc_file_name, ex):
+    def __init__(self, dvc_file_name):
        self.path = dvc_file_name
-        msg = "failed to reproduce '{}'".format(dvc_file_name)
-        super(ReproductionError, self).__init__(msg, cause=ex)
+        super().__init__("failed to reproduce '{}'".format(dvc_file_name))


 class BadMetricError(DvcException):
    def __init__(self, paths):
-        super(BadMetricError, self).__init__(
-            "the following metrics do not exists, "
+        super().__init__(
+            "the following metrics do not exist, "
            "are not metric files or are malformed: {paths}".format(
                paths=", ".join("'{}'".format(path) for path in paths)
            )
@@ -209,87 +169,71 @@ class BadMetricError(DvcException):

 class NoMetricsError(DvcException):
    def __init__(self):
-        super(NoMetricsError, self).__init__(
+        super().__init__(
            "no metric files in this repository. "
            "Use `dvc metrics add` to add a metric file to track."
        )


 class StageFileCorruptedError(DvcException):
-    def __init__(self, path, cause=None):
+    def __init__(self, path):
        path = relpath(path)
-        super(StageFileCorruptedError, self).__init__(
+        super().__init__(
            "unable to read DVC-file: {} "
-            "YAML file structure is corrupted".format(path),
-            cause=cause,
+            "YAML file structure is corrupted".format(path)
        )


 class RecursiveAddingWhileUsingFilename(DvcException):
    def __init__(self):
-        super(RecursiveAddingWhileUsingFilename, self).__init__(
+        super().__init__(
            "cannot use `fname` with multiple targets or `-R|--recursive`"
        )


 class OverlappingOutputPathsError(DvcException):
-    def __init__(self, out_1, out_2):
-        super(OverlappingOutputPathsError, self).__init__(
-            "Paths for outs:\n'{}'('{}')\n'{}'('{}')\noverlap. To avoid "
-            "unpredictable behaviour, rerun command with non overlapping outs "
-            "paths.".format(
-                str(out_1),
-                out_1.stage.relpath,
-                str(out_2),
-                out_2.stage.relpath,
-            )
-        )
+    def __init__(self, parent, overlapping_out, message):
+        self.parent = parent
+        self.overlapping_out = overlapping_out
+        super().__init__(message)


 class CheckoutErrorSuggestGit(DvcException):
-    def __init__(self, target, cause):
-        super(CheckoutErrorSuggestGit, self).__init__(
-            "Did you mean `git checkout {}`?".format(target), cause=cause
-        )
+    def __init__(self, target):
+        super().__init__("Did you mean `git checkout {}`?".format(target))


 class ETagMismatchError(DvcException):
    def __init__(self, etag, cached_etag):
-        super(ETagMismatchError, self).__init__(
+        super().__init__(
            "ETag mismatch detected when copying file to cache! "
            "(expected: '{}', actual: '{}')".format(etag, cached_etag)
        )


 class FileMissingError(DvcException):
-    def __init__(self, path, cause=None):
-        super(FileMissingError, self).__init__(
-            "Can't find '{}' neither locally nor on remote".format(path),
-            cause=cause,
+    def __init__(self, path):
+        self.path = path
+        super().__init__(
+            "Can't find '{}' neither locally nor on remote".format(path)
        )


 class DvcIgnoreInCollectedDirError(DvcException):
    def __init__(self, ignore_dirname):
-        super(DvcIgnoreInCollectedDirError, self).__init__(
+        super().__init__(
            ".dvcignore file should not be in collected dir path: "
            "'{}'".format(ignore_dirname)
        )


-class UrlNotDvcRepoError(DvcException):
-    def __init__(self, url):
-        super(UrlNotDvcRepoError, self).__init__(
-            "URL '{}' is not a dvc repository.".format(url)
-        )
-
-
 class GitHookAlreadyExistsError(DvcException):
    def __init__(self, hook_name):
-        super(GitHookAlreadyExistsError, self).__init__(
-            "Hook '{}' already exists. Please refer to "
-            "https://man.dvc.org/install "
-            "for more info.".format(hook_name)
+        super().__init__(
+            "Hook '{}' already exists. Please refer to {} for more "
+            "info.".format(
+                hook_name, format_link("https://man.dvc.org/install")
+            )
        )


@@ -297,7 +241,7 @@ class DownloadError(DvcException):
    def __init__(self, amount):
        self.amount = amount

-        super(DownloadError, self).__init__(
+        super().__init__(
            "{amount} files failed to download".format(amount=amount)
        )

@@ -306,19 +250,21 @@ class UploadError(DvcException):
    def __init__(self, amount):
        self.amount = amount

-        super(UploadError, self).__init__(
+        super().__init__(
            "{amount} files failed to upload".format(amount=amount)
        )


 class CheckoutError(DvcException):
-    def __init__(self, target_infos):
+    def __init__(self, target_infos, stats=None):
+        self.target_infos = target_infos
+        self.stats = stats
        targets = [str(t) for t in target_infos]
        m = (
-            "Checkout failed for following targets:\n {}\nDid you "
+            "Checkout failed for following targets:\n{}\nDid you "
            "forget to fetch?".format("\n".join(targets))
        )
-        super(CheckoutError, self).__init__(m)
+        super().__init__(m)


 class CollectCacheError(DvcException):
@@ -327,14 +273,14 @@ class CollectCacheError(DvcException):

 class NoRemoteInExternalRepoError(DvcException):
    def __init__(self, url):
-        super(NoRemoteInExternalRepoError, self).__init__(
+        super().__init__(
            "No DVC remote is specified in target repository '{}'.".format(url)
        )


 class NoOutputInExternalRepoError(DvcException):
    def __init__(self, path, external_repo_path, external_repo_url):
-        super(NoOutputInExternalRepoError, self).__init__(
+        super().__init__(
            "Output '{}' not found in target repository '{}'".format(
                relpath(path, external_repo_path), external_repo_url
            )
@@ -343,4 +289,43 @@ class NoOutputInExternalRepoError(DvcException):

 class HTTPError(DvcException):
    def __init__(self, code, reason):
-        super(HTTPError, self).__init__("'{} {}'".format(code, reason))
+        super().__init__("'{} {}'".format(code, reason))
+
+
+class PathMissingError(DvcException):
+    default_msg = (
+        "The path '{}' does not exist in the target repository '{}'"
+        " neither as a DVC output nor as a Git-tracked file."
+    )
+    default_msg_output_only = (
+        "The path '{}' does not exist in the target repository '{}'"
+        " as an DVC output."
+    )
+
+    def __init__(self, path, repo, output_only=False):
+        msg = (
+            self.default_msg
+            if not output_only
+            else self.default_msg_output_only
+        )
+        super().__init__(msg.format(path, repo))
+        self.output_only = output_only
+
+
+class RemoteCacheRequiredError(DvcException):
+    def __init__(self, path_info):
+        super().__init__(
+            (
+                "Current operation was unsuccessful because '{}' requires "
+                "existing cache on '{}' remote. See {} for information on how "
+                "to set up remote cache."
+            ).format(
+                path_info,
+                path_info.scheme,
+                format_link("https://man.dvc.org/config#cache"),
+            )
+        )
+
+
+class IsADirectoryError(DvcException):
+    """Raised when a file operation is requested on a directory."""
@@ -1,29 +1,36 @@
-from __future__ import unicode_literals
-
+import logging
 import os
 import tempfile
 from contextlib import contextmanager
 from distutils.dir_util import copy_tree
+import threading

-from funcy import retry
+from funcy import retry, suppress, wrap_with, cached_property

-from dvc.config import NoRemoteError, ConfigError
-from dvc.exceptions import NoRemoteInExternalRepoError
-from dvc.remote import RemoteConfig
-from dvc.exceptions import NoOutputInExternalRepoError
-from dvc.exceptions import OutputNotFoundError
-from dvc.utils.fs import remove
+from dvc.path_info import PathInfo
+from dvc.compat import fspath
+from dvc.repo import Repo
+from dvc.config import NoRemoteError, NotDvcRepoError
+from dvc.exceptions import NoRemoteInExternalRepoError, CheckoutError
+from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
+from dvc.exceptions import FileMissingError, PathMissingError
+from dvc.utils.fs import remove, fs_copy, move
+from dvc.utils import tmp_fname
+from dvc.scm.git import Git


-REPO_CACHE = {}
+logger = logging.getLogger(__name__)


@contextmanager
-def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
-    from dvc.repo import Repo
+def external_repo(url, rev=None, for_write=False):
+    logger.debug("Creating external repo %s@%s", url, rev)
+    path = _cached_clone(url, rev, for_write=for_write)
+    try:
+        repo = ExternalRepo(path, url)
+    except NotDvcRepoError:
+        repo = ExternalGitRepo(path, url)

-    path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
-    repo = Repo(path)
    try:
        yield repo
    except NoRemoteError:
@@ -32,90 +39,233 @@ def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
        if exc.repo is repo:
            raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url)
        raise
-    repo.close()
-
-
-def _external_repo(url=None, rev=None, cache_dir=None):
-    from dvc.config import Config
-    from dvc.cache import CacheConfig
-    from dvc.repo import Repo
-
-    key = (url, rev, cache_dir)
-    if key in REPO_CACHE:
-        return REPO_CACHE[key]
-
-    new_path = tempfile.mkdtemp("dvc-erepo")
-
-    # Copy and adjust existing clone
-    if (url, None, None) in REPO_CACHE:
-        old_path = REPO_CACHE[url, None, None]
-
-        # This one unlike shutil.copytree() works with an existing dir
-        copy_tree(old_path, new_path)
-    else:
-        # Create a new clone
-        _clone_repo(url, new_path)
-
-        # Save clean clone dir so that we will have access to a default branch
-        clean_clone_path = tempfile.mkdtemp("dvc-erepo")
-        copy_tree(new_path, clean_clone_path)
-        REPO_CACHE[url, None, None] = clean_clone_path
-
-    # Adjust new clone/copy to fit rev and cache_dir
-
-    # Checkout needs to be done first because current branch might not be
-    # DVC repository
-    if rev is not None:
-        _git_checkout(new_path, rev)
-
-    repo = Repo(new_path)
-    try:
-        # check if the URL is local and no default remote is present
-        # add default remote pointing to the original repo's cache location
-        if os.path.isdir(url):
-            rconfig = RemoteConfig(repo.config)
-            if not _default_remote_set(rconfig):
-                original_repo = Repo(url)
-                try:
-                    rconfig.add(
-                        "auto-generated-upstream",
-                        original_repo.cache.local.cache_dir,
-                        default=True,
-                        level=Config.LEVEL_LOCAL,
-                    )
-                finally:
-                    original_repo.close()
-
-        if cache_dir is not None:
-            cache_config = CacheConfig(repo.config)
-            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
+    except FileMissingError as exc:
+        raise PathMissingError(exc.path, url)
    finally:
-        # Need to close/reopen repo to force config reread
        repo.close()
-
-    REPO_CACHE[key] = new_path
-    return new_path
+        if for_write:
+            _remove(path)


-def _git_checkout(repo_path, revision):
-    from dvc.scm import Git
-
-    git = Git(repo_path)
-    try:
-        git.checkout(revision)
-    finally:
-        git.close()
+CLONES = {}
+CACHE_DIRS = {}


 def clean_repos():
    # Outside code should not see cache while we are removing
-    repo_paths = list(REPO_CACHE.values())
-    REPO_CACHE.clear()
+    paths = list(CLONES.values()) + list(CACHE_DIRS.values())
+    CLONES.clear()
+    CACHE_DIRS.clear()

-    for path in repo_paths:
+    for path in paths:
        _remove(path)


+class ExternalRepo(Repo):
+    def __init__(self, root_dir, url):
+        super().__init__(root_dir)
+        self.url = url
+        self._set_cache_dir()
+        self._fix_upstream()
+
+    def pull_to(self, path, to_info):
+        """
+        Pull the corresponding file or directory specified by `path` and
+        checkout it into `to_info`.
+
+        It works with files tracked by Git and DVC, and also local files
+        outside the repository.
+        """
+        out = None
+        path_info = PathInfo(self.root_dir) / path
+
+        with suppress(OutputNotFoundError):
+            (out,) = self.find_outs_by_path(fspath(path_info), strict=False)
+
+        try:
+            if out and out.use_cache:
+                self._pull_cached(out, path_info, to_info)
+                return
+
+            # Check if it is handled by Git (it can't have an absolute path)
+            if os.path.isabs(path):
+                raise FileNotFoundError
+
+            fs_copy(fspath(path_info), fspath(to_info))
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+    def _pull_cached(self, out, path_info, dest):
+        with self.state:
+            tmp = PathInfo(tmp_fname(dest))
+            src = tmp / path_info.relative_to(out.path_info)
+
+            out.path_info = tmp
+
+            # Only pull unless all needed cache is present
+            if out.changed_cache(filter_info=src):
+                self.cloud.pull(out.get_used_cache(filter_info=src))
+
+            try:
+                out.checkout(filter_info=src)
+            except CheckoutError:
+                raise FileNotFoundError
+
+            move(src, dest)
+            remove(tmp)
+
+    @wrap_with(threading.Lock())
+    def _set_cache_dir(self):
+        try:
+            cache_dir = CACHE_DIRS[self.url]
+        except KeyError:
+            cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
+
+        self.cache.local.cache_dir = cache_dir
+
+    def _fix_upstream(self):
+        if not os.path.isdir(self.url):
+            return
+
+        try:
+            src_repo = Repo(self.url)
+        except NotDvcRepoError:
+            # If ExternalRepo does not throw NotDvcRepoError and Repo does,
+            # the self.url might be a bare git repo.
+            # NOTE: This will fail to resolve remote with relative path,
+            # same as if it was a remote DVC repo.
+            return
+
+        try:
+            remote_name = self.config["core"].get("remote")
+            if remote_name:
+                self._fix_local_remote(src_repo, remote_name)
+            else:
+                self._add_upstream(src_repo)
+        finally:
+            src_repo.close()
+
+    def _fix_local_remote(self, src_repo, remote_name):
+        # If a remote URL is relative to the source repo,
+        # it will have changed upon config load and made
+        # relative to this new repo. Restore the old one here.
+        new_remote = self.config["remote"][remote_name]
+        old_remote = src_repo.config["remote"][remote_name]
+        if new_remote["url"] != old_remote["url"]:
+            new_remote["url"] = old_remote["url"]
+
+    def _add_upstream(self, src_repo):
+        # Fill the empty upstream entry with a new remote pointing to the
+        # original repo's cache location.
+        cache_dir = src_repo.cache.local.cache_dir
+        self.config["remote"]["auto-generated-upstream"] = {"url": cache_dir}
+        self.config["core"]["remote"] = "auto-generated-upstream"
+
+
+class ExternalGitRepo:
+    def __init__(self, root_dir, url):
+        self.root_dir = root_dir
+        self.url = url
+
+    @cached_property
+    def scm(self):
+        return Git(self.root_dir)
+
+    def close(self):
+        if "scm" in self.__dict__:
+            self.scm.close()
+
+    def find_out_by_relpath(self, path):
+        raise OutputNotFoundError(path, self)
+
+    def pull_to(self, path, to_info):
+        try:
+            # Git handled files can't have absolute path
+            if os.path.isabs(path):
+                raise FileNotFoundError
+
+            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+    @contextmanager
+    def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
+        """Opens a specified resource as a file object."""
+        try:
+            abs_path = os.path.join(self.root_dir, path)
+            with open(abs_path, mode, encoding=encoding) as fd:
+                yield fd
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+
+def _cached_clone(url, rev, for_write=False):
+    """Clone an external git repo to a temporary directory.
+
+    Returns the path to a local temporary directory with the specified
+    revision checked out. If for_write is set prevents reusing this dir via
+    cache.
+    """
+    if not for_write and Git.is_sha(rev) and (url, rev) in CLONES:
+        return CLONES[url, rev]
+
+    clone_path = _clone_default_branch(url, rev)
+    rev_sha = Git(clone_path).resolve_rev(rev or "HEAD")
+
+    if not for_write and (url, rev_sha) in CLONES:
+        return CLONES[url, rev_sha]
+
+    # Copy to a new dir to keep the clone clean
+    repo_path = tempfile.mkdtemp("dvc-erepo")
+    logger.debug("erepo: making a copy of %s clone", url)
+    copy_tree(clone_path, repo_path)
+
+    # Check out the specified revision
+    if rev is not None:
+        _git_checkout(repo_path, rev)
+
+    if not for_write:
+        CLONES[url, rev_sha] = repo_path
+    return repo_path
+
+
+@wrap_with(threading.Lock())
+def _clone_default_branch(url, rev):
+    """Get or create a clean clone of the url.
+
+    The cloned is reactualized with git pull unless rev is a known sha.
+    """
+    clone_path = CLONES.get(url)
+
+    git = None
+    try:
+        if clone_path:
+            git = Git(clone_path)
+            # Do not pull for known shas, branches and tags might move
+            if not Git.is_sha(rev) or not git.has_rev(rev):
+                logger.debug("erepo: git pull %s", url)
+                git.pull()
+        else:
+            logger.debug("erepo: git clone %s to a temporary dir", url)
+            clone_path = tempfile.mkdtemp("dvc-clone")
+            git = Git.clone(url, clone_path)
+            CLONES[url] = clone_path
+    finally:
+        if git:
+            git.close()
+
+    return clone_path
+
+
+def _git_checkout(repo_path, rev):
+    logger.debug("erepo: git checkout %s@%s", repo_path, rev)
+    git = Git(repo_path)
+    try:
+        git.checkout(rev)
+    finally:
+        git.close()
+
+
 def _remove(path):
    if os.name == "nt":
        # git.exe may hang for a while not permitting to remove temp dir
@@ -123,26 +273,3 @@ def _remove(path):
        os_retry(remove)(path)
    else:
        remove(path)
-
-
-def _clone_repo(url, path):
-    from dvc.scm.git import Git
-
-    git = Git.clone(url, path)
-    git.close()
-
-
-def _default_remote_set(rconfig):
-    """
-    Checks if default remote config is present.
-    Args:
-        rconfig: a remote config
-
-    Returns:
-        True if the default remote config is set, else False
-    """
-    try:
-        rconfig.get_default()
-        return True
-    except ConfigError:
-        return False
@@ -1,11 +1,11 @@
-from __future__ import unicode_literals
-
 import logging
 import os

+from funcy import cached_property
 from pathspec import PathSpec
 from pathspec.patterns import GitWildMatchPattern

+from dvc.scm.tree import BaseTree
 from dvc.utils import relpath

 logger = logging.getLogger(__name__)
@@ -71,14 +71,28 @@ class DvcIgnoreDirs(DvcIgnore):
        return self.basenames == other.basenames


+class DvcIgnoreRepo(DvcIgnore):
+    def __call__(self, root, dirs, files):
+        def is_dvc_repo(directory):
+            from dvc.repo import Repo
+
+            return os.path.isdir(os.path.join(directory, Repo.DVC_DIR))
+
+        dirs = [d for d in dirs if not is_dvc_repo(d)]
+
+        return dirs, files
+
+
 class DvcIgnoreFilter(object):
-    def __init__(self, root_dir, tree):
+    def __init__(self, tree):
        self.tree = tree
-        self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
-        self._update(root_dir)
-        for root, dirs, _ in self.tree.walk(root_dir, dvcignore=self):
-            for d in dirs:
-                self._update(os.path.join(root, d))
+        self.ignores = {
+            DvcIgnoreDirs([".git", ".hg", ".dvc"]),
+            DvcIgnoreRepo(),
+        }
+        for root, dirs, files in self.tree.walk(self.tree.tree_root):
+            self._update(root)
+            dirs[:], files[:] = self(root, dirs, files)

    def _update(self, dirname):
        ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
@@ -90,3 +104,39 @@ class DvcIgnoreFilter(object):
            dirs, files = ignore(root, dirs, files)

        return dirs, files
+
+
+class CleanTree(BaseTree):
+    def __init__(self, tree):
+        self.tree = tree
+
+    @cached_property
+    def dvcignore(self):
+        return DvcIgnoreFilter(self.tree)
+
+    @property
+    def tree_root(self):
+        return self.tree.tree_root
+
+    def open(self, path, mode="r", encoding="utf-8"):
+        return self.tree.open(path, mode, encoding)
+
+    def exists(self, path):
+        return self.tree.exists(path)
+
+    def isdir(self, path):
+        return self.tree.isdir(path)
+
+    def isfile(self, path):
+        return self.tree.isfile(path)
+
+    def walk(self, top, topdown=True):
+        for root, dirs, files in self.tree.walk(top, topdown):
+            dirs[:], files[:] = self.dvcignore(root, dirs, files)
+
+            yield root, dirs, files
+
+    def walk_files(self, top):
+        for root, _, files in self.walk(top):
+            for file in files:
+                yield os.path.join(root, file)
@@ -1,24 +1,10 @@
 """Use heuristics to guess if it is a text file or a binary file."""
-from __future__ import unicode_literals
-
-from dvc.utils.compat import is_py3
-from dvc.utils.compat import open

 # Based on https://eli.thegreenplace.net/2011/10/19/
 # perls-guess-if-file-is-text-or-binary-implemented-in-python


-# A function that takes an integer in the 8-bit range and returns
-# a single-character byte object in py3 / a single-character string
-# in py2.
-#
-def _int2byte(i):
-    if is_py3:
-        return bytes((i,))
-    return chr(i)
-
-
-TEXT_CHARS = b"".join(_int2byte(i) for i in range(32, 127)) + b"\n\r\t\f\b"
+TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"


 def istextfile(fname, blocksize=512):
@@ -1,5 +1,4 @@
 """Manages dvc lock file."""
-from __future__ import unicode_literals

 import hashlib
 import os
@@ -7,10 +6,10 @@ import time
 from datetime import timedelta

 import zc.lockfile
+import flufl.lock

 from dvc.exceptions import DvcException
 from dvc.utils import format_link
-from dvc.utils.compat import is_py3, is_py2
 from dvc.progress import Tqdm

 DEFAULT_TIMEOUT = 5
@@ -23,11 +22,11 @@ FAILED_TO_LOCK_MESSAGE = (


 class LockError(DvcException):
-    """Thrown when unable to acquire the lock for dvc repo."""
+    """Thrown when unable to acquire the lock for DVC repo."""


 class Lock(object):
-    """Class for dvc repo lock.
+    """Class for DVC repo lock.

    Uses zc.lockfile as backend.
    """
@@ -69,6 +68,10 @@ class Lock(object):
        self._lock.close()
        self._lock = None

+    @property
+    def is_locked(self):
+        return bool(self._lock)
+
    def __enter__(self):
        self.lock()

@@ -76,83 +79,70 @@ class Lock(object):
        self.unlock()


-if is_py3:
-    import flufl.lock
+class HardlinkLock(flufl.lock.Lock):
+    """Class for DVC repo lock.

-    class HardlinkLock(flufl.lock.Lock):
-        """Class for dvc repo lock.
+    Args:
+        lockfile (str): the lock filename
+            in.
+        tmp_dir (str): a directory to store claim files.
+    """

-        Args:
-            lockfile (str): the lock filename
-                in.
-            tmp_dir (str): a directory to store claim files.
-        """
+    def __init__(self, lockfile, tmp_dir=None, **kwargs):
+        import socket

-        def __init__(self, lockfile, tmp_dir=None, **kwargs):
-            import socket
+        self._tmp_dir = tmp_dir

-            self._tmp_dir = tmp_dir
+        # NOTE: this is basically Lock.__init__ copy-paste, except that
+        # instead of using `socket.getfqdn()` we use `socket.gethostname()`
+        # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
+        # anything, which is way too slow. `gethostname()` is actually a
+        # fallback for `getfqdn()` when it is not able to resolve a
+        # canonical hostname through network. The claimfile that uses
+        # `self._hostname` is still usable, as it uses `pid` and random
+        # number to generate the resulting lock file name, which is unique
+        # enough for our application.
+        #
+        # [1] https://github.com/iterative/dvc/issues/2582
+        self._hostname = socket.gethostname()

-            # NOTE: this is basically Lock.__init__ copy-paste, except that
-            # instead of using `socket.getfqdn()` we use `socket.gethostname()`
-            # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
-            # anything, which is way too slow. `gethostname()` is actually a
-            # fallback for `getfqdn()` when it is not able to resolve a
-            # canonical hostname through network. The claimfile that uses
-            # `self._hostname` is still usable, as it uses `pid` and random
-            # number to generate the resulting lock file name, which is unique
-            # enough for our application.
-            #
-            # [1] https://github.com/iterative/dvc/issues/2582
-            self._hostname = socket.gethostname()
+        self._lockfile = lockfile
+        self._lifetime = timedelta(days=365)  # Lock for good by default
+        self._separator = flufl.lock.SEP
+        self._set_claimfile()
+        self._owned = True
+        self._retry_errnos = []

-            self._lockfile = lockfile
-            self._lifetime = timedelta(days=365)  # Lock for good by default
-            self._separator = flufl.lock.SEP
-            self._set_claimfile()
-            self._owned = True
-            self._retry_errnos = []
+    @property
+    def lockfile(self):
+        return self._lockfile

-        @property
-        def lockfile(self):
-            return self._lockfile
+    def lock(self):
+        try:
+            super().lock(timedelta(seconds=DEFAULT_TIMEOUT))
+        except flufl.lock.TimeOutError:
+            raise LockError(FAILED_TO_LOCK_MESSAGE)

-        def lock(self):
-            try:
-                super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
-            except flufl.lock.TimeOutError:
-                raise LockError(FAILED_TO_LOCK_MESSAGE)
+    def _set_claimfile(self, pid=None):
+        super()._set_claimfile(pid)

-        def _set_claimfile(self, pid=None):
-            super(Lock, self)._set_claimfile(pid)
+        if self._tmp_dir is not None:
+            # Under Windows file path length is limited so we hash it
+            filename = hashlib.md5(self._claimfile.encode()).hexdigest()
+            self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")

-            if self._tmp_dir is not None:
-                # Under Windows file path length is limited so we hash it
-                filename = hashlib.md5(self._claimfile.encode()).hexdigest()
-                self._claimfile = os.path.join(
-                    self._tmp_dir, filename + ".lock"
-                )
-
-        # Fix for __del__ bug in flufl.lock [1] which is causing errors on
-        # Python shutdown [2].
-        # [1] https://gitlab.com/warsaw/flufl.lock/issues/7
-        # [2] https://github.com/iterative/dvc/issues/2573
-        def __del__(self):
-            try:
-                if self._owned:
-                    self.finalize()
-            except ImportError:
-                pass
+    # Fix for __del__ bug in flufl.lock [1] which is causing errors on
+    # Python shutdown [2].
+    # [1] https://gitlab.com/warsaw/flufl.lock/issues/7
+    # [2] https://github.com/iterative/dvc/issues/2573
+    def __del__(self):
+        try:
+            if self._owned:
+                self.finalize()
+        except ImportError:
+            pass


 def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False):
-    if hardlink_lock and is_py2:
-        raise DvcException(
-            "Hardlink locks are not supported on Python <3.5. "
-            "See `hardlink_lock` in {}".format(
-                format_link("man.dvc.org/config#core")
-            )
-        )
-
    cls = HardlinkLock if hardlink_lock else Lock
    return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly)
@@ -1,5 +1,4 @@
-"""Manages logging configuration for dvc repo."""
-from __future__ import unicode_literals
+"""Manages logging configuration for DVC repo."""

 import logging.config
 import logging.handlers
@@ -7,9 +6,6 @@ import logging.handlers
 import colorama

 from dvc.progress import Tqdm
-from dvc.utils.compat import RecursionError
-from dvc.utils.compat import str
-from dvc.utils.compat import StringIO


 FOOTER = (
@@ -26,7 +22,7 @@ FOOTER = (
 class LoggingException(Exception):
    def __init__(self, record):
        msg = "failed to log {}".format(str(record))
-        super(LoggingException, self).__init__(msg)
+        super().__init__(msg)


 class ExcludeErrorsFilter(logging.Filter):
@@ -40,117 +36,70 @@ class ExcludeInfoFilter(logging.Filter):


 class ColorFormatter(logging.Formatter):
-    """Enable color support when logging to a terminal that supports it.
+    """Spit out colored text in supported terminals.

-    Color support on Windows versions that do not support ANSI color codes is
-    enabled by use of the colorama__ library.
+    colorama__ makes ANSI escape character sequences work under Windows.
    See the colorama documentation for details.

    __ https://pypi.python.org/pypi/colorama

-    For records containing `exc_info`, it will use a custom `_walk_exc` to
-    retrieve the whole traceback.
+    If record has an extra `tb_only` attribute, it will not show the
+    exception cause, just the message and the traceback.
    """

    color_code = {
        "DEBUG": colorama.Fore.BLUE,
-        "INFO": "",
        "WARNING": colorama.Fore.YELLOW,
        "ERROR": colorama.Fore.RED,
        "CRITICAL": colorama.Fore.RED,
    }

    def format(self, record):
+        record.message = record.getMessage()
+        msg = self.formatMessage(record)
+
        if record.levelname == "INFO":
-            return record.msg
+            return msg

-        if record.levelname == "ERROR" or record.levelname == "CRITICAL":
-            exception, stack_trace = self._parse_exc(record.exc_info)
+        if record.exc_info:
+            if getattr(record, "tb_only", False):
+                cause = ""
+            else:
+                cause = ": ".join(_iter_causes(record.exc_info[1]))

-            return (
-                "{color}{levelname}{nc}: {description}" "{stack_trace}\n"
-            ).format(
-                color=self.color_code.get(record.levelname, ""),
-                nc=colorama.Fore.RESET,
-                levelname=record.levelname,
-                description=self._description(record.msg, exception),
-                msg=record.msg,
-                stack_trace=stack_trace,
+            msg = "{message}{separator}{cause}".format(
+                message=msg or "",
+                separator=" - " if msg and cause else "",
+                cause=cause,
            )

-        return "{color}{levelname}{nc}: {msg}".format(
-            color=self.color_code.get(record.levelname, ""),
+            if _is_verbose():
+                msg += _stack_trace(record.exc_info)
+
+        return "{asctime}{color}{levelname}{nc}: {msg}".format(
+            asctime=self.formatTime(record, self.datefmt),
+            color=self.color_code[record.levelname],
            nc=colorama.Fore.RESET,
            levelname=record.levelname,
-            msg=record.msg,
+            msg=msg,
        )

-    def _current_level(self):
-        return logging.getLogger("dvc").getEffectiveLevel()
+    def formatTime(self, record, datefmt=None):
+        # only show if current level is set to DEBUG
+        # also, skip INFO as it is used for UI
+        if not _is_verbose() or record.levelno == logging.INFO:
+            return ""

-    def _is_visible(self, record):
-        return record.levelno >= self._current_level()
-
-    def _description(self, message, exception):
-        description = ""
-
-        if exception and message:
-            description = "{message} - {exception}"
-        elif exception:
-            description = "{exception}"
-        elif message:
-            description = "{message}"
-
-        return description.format(message=message, exception=exception)
-
-    def _walk_exc(self, exc_info):
-        import traceback
-
-        buffer = StringIO()
-
-        traceback.print_exception(*exc_info, file=buffer)
-
-        exc = exc_info[1]
-        tb = buffer.getvalue()
-
-        exc_list = [str(exc)]
-        tb_list = [tb]
-
-        # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info
-        while hasattr(exc, "cause") and exc.cause:
-            exc_list.append(str(exc.cause))
-            if hasattr(exc, "cause_tb") and exc.cause_tb:
-                tb_list.insert(0, str(exc.cause_tb))
-            exc = exc.cause
-
-        return exc_list, tb_list
-
-    def _parse_exc(self, exc_info):
-        if not exc_info:
-            return (None, "")
-
-        exc_list, tb_list = self._walk_exc(exc_info)
-
-        exception = ": ".join(exc_list)
-
-        if self._current_level() == logging.DEBUG:
-            stack_trace = (
-                "\n" "{red}{line}{nc}\n" "{stack_trace}" "{red}{line}{nc}"
-            ).format(
-                red=colorama.Fore.RED,
-                nc=colorama.Fore.RESET,
-                line="-" * 60,
-                stack_trace="\n".join(tb_list),
-            )
-        else:
-            stack_trace = ""
-
-        return (exception, stack_trace)
+        return "{green}{date}{nc} ".format(
+            green=colorama.Fore.GREEN,
+            date=super().formatTime(record, datefmt),
+            nc=colorama.Fore.RESET,
+        )


 class LoggerHandler(logging.StreamHandler):
    def handleError(self, record):
-        super(LoggerHandler, self).handleError(record)
+        super().handleError(record)
        raise LoggingException(record)

    def emit(self, record):
@@ -167,6 +116,39 @@ class LoggerHandler(logging.StreamHandler):
            self.handleError(record)


+def _is_verbose():
+    return logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG
+
+
+def _iter_causes(exc):
+    while exc:
+        yield str(exc)
+        exc = exc.__cause__
+
+
+def _stack_trace(exc_info):
+    import traceback
+
+    return (
+        "\n"
+        "{red}{line}{nc}\n"
+        "{trace}"
+        "{red}{line}{nc}".format(
+            red=colorama.Fore.RED,
+            line="-" * 60,
+            trace="".join(traceback.format_exception(*exc_info)),
+            nc=colorama.Fore.RESET,
+        )
+    )
+
+
+def disable_other_loggers():
+    root = logging.root
+    for (logger_name, logger) in root.manager.loggerDict.items():
+        if logger_name != "dvc" and not logger_name.startswith("dvc."):
+            logger.disabled = True
+
+
 def setup(level=logging.INFO):
    colorama.init()

@@ -209,22 +191,7 @@ def setup(level=logging.INFO):
                        "console_errors",
                    ],
                },
-                "paramiko": {
-                    "level": logging.CRITICAL,
-                    "handlers": [
-                        "console_info",
-                        "console_debug",
-                        "console_errors",
-                    ],
-                },
-                "flufl.lock": {
-                    "level": logging.CRITICAL,
-                    "handlers": [
-                        "console_info",
-                        "console_debug",
-                        "console_errors",
-                    ],
-                },
            },
+            "disable_existing_loggers": False,
        }
    )
@@ -1,6 +1,6 @@
 """Main entry point for dvc CLI."""
-from __future__ import unicode_literals

+import errno
 import logging

 from dvc import analytics
@@ -9,14 +9,14 @@ from dvc.config import ConfigError
 from dvc.exceptions import DvcParserError
 from dvc.exceptions import NotDvcRepoError
 from dvc.external_repo import clean_repos
-from dvc.logger import FOOTER
+from dvc.logger import disable_other_loggers, FOOTER
+from dvc.utils import format_link
 from dvc.remote.pool import close_pools
-from dvc.utils.compat import is_py2
-

 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
 # [2] https://bugs.python.org/issue29288
+
 "".encode("idna")


@@ -34,6 +34,7 @@ def main(argv=None):
    """
    args = None
    cmd = None
+    disable_other_loggers()

    outerLogLevel = logger.level
    try:
@@ -59,10 +60,14 @@ def main(argv=None):
    except DvcParserError:
        ret = 254
    except Exception as exc:  # pylint: disable=broad-except
-        if isinstance(exc, UnicodeError) and is_py2:
+        if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
            logger.exception(
-                "unicode is not supported in DVC for Python 2 "
-                "(end-of-life January 1, 2020), please upgrade to Python 3"
+                "too many open files, please visit "
+                "{} to see how to handle this "
+                "problem".format(
+                    format_link("https://error.dvc.org/many-files")
+                ),
+                extra={"tb_only": True},
            )
        else:
            logger.exception("unexpected error")
@@ -70,8 +75,9 @@ def main(argv=None):
    finally:
        logger.setLevel(outerLogLevel)

-        # Python 2 fails to close these clean occasionally and users see
-        # weird error messages, so we do it manually
+        # Closing pools by-hand to prevent wierd messages when closing SSH
+        # connections. See https://github.com/iterative/dvc/issues/3248 for
+        # more info.
        close_pools()

        # Remove cached repos in the end of the call, these are anonymous
@@ -1,6 +1,5 @@
-from __future__ import unicode_literals
-
-from voluptuous import Any, Required
+from urllib.parse import urlparse
+from voluptuous import Any, Required, Lower, Length, Coerce, And, SetTo

 from dvc.output.base import OutputBase
 from dvc.output.gs import OutputGS
@@ -13,8 +12,6 @@ from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.local import RemoteLOCAL
 from dvc.remote.s3 import RemoteS3
 from dvc.scheme import Schemes
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse

 OUTS = [
    OutputHDFS,
@@ -32,6 +29,12 @@ OUTS_MAP = {
    Schemes.LOCAL: OutputLOCAL,
 }

+CHECKSUM_SCHEMA = Any(
+    None,
+    And(str, Length(max=0), SetTo(None)),
+    And(Any(str, And(int, Coerce(str))), Length(min=3), Lower),
+)
+
 # NOTE: currently there are only 3 possible checksum names:
 #
 #    1) md5 (LOCAL, SSH, GS);
@@ -40,15 +43,15 @@ OUTS_MAP = {
 #
 # so when a few types of outputs share the same name, we only need
 # specify it once.
-CHECKSUM_SCHEMA = {
-    RemoteLOCAL.PARAM_CHECKSUM: Any(str, None),
-    RemoteS3.PARAM_CHECKSUM: Any(str, None),
-    RemoteHDFS.PARAM_CHECKSUM: Any(str, None),
+CHECKSUMS_SCHEMA = {
+    RemoteLOCAL.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    RemoteS3.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    RemoteHDFS.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
 }

-TAGS_SCHEMA = {str: CHECKSUM_SCHEMA}
+TAGS_SCHEMA = {str: CHECKSUMS_SCHEMA}

-SCHEMA = CHECKSUM_SCHEMA.copy()
+SCHEMA = CHECKSUMS_SCHEMA.copy()
 SCHEMA[Required(OutputBase.PARAM_PATH)] = str
 SCHEMA[OutputBase.PARAM_CACHE] = bool
 SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA
@@ -1,17 +1,14 @@
-from __future__ import unicode_literals
-
 import logging
+from urllib.parse import urlparse
 from copy import copy

 from voluptuous import Any

 import dvc.prompt as prompt
 from dvc.cache import NamedCache
-from dvc.exceptions import CollectCacheError
+from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError
 from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteBASE
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse


 logger = logging.getLogger(__name__)
@@ -20,26 +17,24 @@ logger = logging.getLogger(__name__)
 class OutputDoesNotExistError(DvcException):
    def __init__(self, path):
        msg = "output '{}' does not exist".format(path)
-        super(OutputDoesNotExistError, self).__init__(msg)
+        super().__init__(msg)


 class OutputIsNotFileOrDirError(DvcException):
    def __init__(self, path):
        msg = "output '{}' is not a file or directory".format(path)
-        super(OutputIsNotFileOrDirError, self).__init__(msg)
+        super().__init__(msg)


 class OutputAlreadyTrackedError(DvcException):
    def __init__(self, path):
        msg = "output '{}' is already tracked by scm (e.g. git)".format(path)
-        super(OutputAlreadyTrackedError, self).__init__(msg)
+        super().__init__(msg)


 class OutputIsStageFileError(DvcException):
    def __init__(self, path):
-        super(OutputIsStageFileError, self).__init__(
-            "Stage file '{}' cannot be an output.".format(path)
-        )
+        super().__init__("Stage file '{}' cannot be an output.".format(path))


 class OutputBase(object):
@@ -103,14 +98,9 @@ class OutputBase(object):
        self.persist = persist
        self.tags = None if self.IS_DEPENDENCY else (tags or {})

-        if self.use_cache and self.cache is None:
-            raise DvcException(
-                "no cache location setup for '{}' outputs.".format(
-                    self.REMOTE.scheme
-                )
-            )
-
        self.path_info = self._parse_path(remote, path)
+        if self.use_cache and self.cache is None:
+            raise RemoteCacheRequiredError(self.path_info)

    def _parse_path(self, remote, path):
        if remote:
@@ -169,14 +159,17 @@ class OutputBase(object):
    def exists(self):
        return self.remote.exists(self.path_info)

+    def save_info(self):
+        return self.remote.save_info(self.path_info)
+
    def changed_checksum(self):
        return self.checksum != self.remote.get_checksum(self.path_info)

-    def changed_cache(self):
+    def changed_cache(self, filter_info=None):
        if not self.use_cache or not self.checksum:
            return True

-        return self.cache.changed_cache(self.checksum)
+        return self.cache.changed_cache(self.checksum, filter_info=filter_info)

    def status(self):
        if self.checksum and self.use_cache and self.changed_cache():
@@ -225,7 +218,7 @@ class OutputBase(object):
            self.repo.scm.ignore(self.fspath)

        if not self.use_cache:
-            self.info = self.remote.save_info(self.path_info)
+            self.info = self.save_info()
            if self.metric:
                self.verify_metric()
            if not self.IS_DEPENDENCY:
@@ -244,7 +237,7 @@ class OutputBase(object):
            )
            return

-        self.info = self.remote.save_info(self.path_info)
+        self.info = self.save_info()

    def commit(self):
        if self.use_cache:
@@ -283,11 +276,18 @@ class OutputBase(object):
        self.remote.download(self.path_info, to.path_info)

    def checkout(
-        self, force=False, progress_callback=None, tag=None, relink=False
+        self,
+        force=False,
+        progress_callback=None,
+        tag=None,
+        relink=False,
+        filter_info=None,
    ):
        if not self.use_cache:
            if progress_callback:
-                progress_callback(str(self.path_info), self.get_files_number())
+                progress_callback(
+                    str(self.path_info), self.get_files_number(filter_info)
+                )
            return None

        if tag:
@@ -301,6 +301,7 @@ class OutputBase(object):
            force=force,
            progress_callback=progress_callback,
            relink=relink,
+            filter_info=filter_info,
        )

    def remove(self, ignore_remove=False):
@@ -324,17 +325,21 @@ class OutputBase(object):
        if self.scheme == "local" and self.use_scm_ignore:
            self.repo.scm.ignore(self.fspath)

-    def get_files_number(self):
+    def get_files_number(self, filter_info=None):
        if not self.use_cache:
            return 0

-        return self.cache.get_files_number(self.checksum)
+        return self.cache.get_files_number(
+            self.path_info, self.checksum, filter_info
+        )

    def unprotect(self):
        if self.exists:
            self.remote.unprotect(self.path_info)

-    def _collect_used_dir_cache(self, remote=None, force=False, jobs=None):
+    def _collect_used_dir_cache(
+        self, remote=None, force=False, jobs=None, filter_info=None
+    ):
        """Get a list of `info`s related to the given directory.

        - Pull the directory entry from the remote cache if it was changed.
@@ -383,8 +388,9 @@ class OutputBase(object):

        for entry in self.dir_cache:
            checksum = entry[self.remote.PARAM_CHECKSUM]
-            path_info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            cache.add(self.scheme, checksum, str(path_info))
+            info = self.path_info / entry[self.remote.PARAM_RELPATH]
+            if not filter_info or info.isin_or_eq(filter_info):
+                cache.add(self.scheme, checksum, str(info))

        return cache

@@ -400,12 +406,29 @@ class OutputBase(object):
        if not self.use_cache:
            return NamedCache()

-        if not self.info:
-            logger.warning(
-                "Output '{}'({}) is missing version info. Cache for it will "
-                "not be collected. Use dvc repro to get your pipeline up to "
-                "date.".format(self, self.stage)
+        if self.stage.is_repo_import:
+            cache = NamedCache()
+            (dep,) = self.stage.deps
+            cache.external[dep.repo_pair].add(dep.def_path)
+            return cache
+
+        if not self.checksum:
+            msg = (
+                "Output '{}'({}) is missing version info. "
+                "Cache for it will not be collected. "
+                "Use `dvc repro` to get your pipeline up to date.".format(
+                    self, self.stage
+                )
            )
+            if self.exists:
+                msg += (
+                    "\n"
+                    "You can also use `dvc commit {stage}` to associate "
+                    "existing '{out}' with '{stage}'.".format(
+                        out=self, stage=self.stage.relpath
+                    )
+                )
+            logger.warning(msg)
            return NamedCache()

        ret = NamedCache.make(self.scheme, self.checksum, str(self))
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.s3 import OutputS3
 from dvc.remote.gs import RemoteGS

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.base import OutputBase
 from dvc.remote.hdfs import RemoteHDFS

@@ -1,16 +1,13 @@
-from __future__ import unicode_literals
-
 import logging
 import os
+from urllib.parse import urlparse

 from dvc.exceptions import DvcException
 from dvc.istextfile import istextfile
 from dvc.output.base import OutputBase
 from dvc.remote.local import RemoteLOCAL
 from dvc.utils import relpath
-from dvc.utils.compat import fspath_py35
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
+from dvc.compat import fspath_py35
 from dvc.utils.fs import path_isin


@@ -21,6 +18,12 @@ class OutputLOCAL(OutputBase):
    REMOTE = RemoteLOCAL
    sep = os.sep

+    def __init__(self, stage, path, *args, **kwargs):
+        if stage and path_isin(path, stage.repo.root_dir):
+            path = relpath(path, stage.wdir)
+
+        super().__init__(stage, path, *args, **kwargs)
+
    def _parse_path(self, remote, path):
        parsed = urlparse(path)
        if parsed.scheme == "remote":
@@ -59,7 +62,7 @@ class OutputLOCAL(OutputBase):
        return def_scheme != "remote" and not os.path.isabs(self.def_path)

    def dumpd(self):
-        ret = super(OutputLOCAL, self).dumpd()
+        ret = super().dumpd()
        if self.is_in_repo:
            path = self.path_info.relpath(self.stage.wdir).as_posix()
        else:
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.base import OutputBase
 from dvc.remote.s3 import RemoteS3

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.base import OutputBase
 from dvc.remote.ssh import RemoteSSH

@@ -1,35 +1,23 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 import os
 import posixpath
-import sys
+import pathlib
+from urllib.parse import urlparse

 from funcy import cached_property

 from dvc.utils import relpath
-from dvc.utils.compat import basestring
-from dvc.utils.compat import builtin_str
-from dvc.utils.compat import is_py2
-from dvc.utils.compat import pathlib
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
-
-
-# On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
-# which is lossy, thus we can't use that,
-# see https://github.com/mcmtroffaes/pathlib2/issues/56.
-if is_py2:
-    fs_encoding = "utf-8"


 class _BasePath(object):
    def overlaps(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
            other = self.__class__(other)
        elif self.__class__ != other.__class__:
            return False
-        return self == other or self.isin(other) or other.isin(self)
+        return self.isin_or_eq(other) or other.isin(self)
+
+    def isin_or_eq(self, other):
+        return self == other or self.isin(other)


 class PathInfo(pathlib.PurePath, _BasePath):
@@ -63,7 +51,7 @@ class PathInfo(pathlib.PurePath, _BasePath):
        return relpath(path)

    def __repr__(self):
-        return builtin_str("{}: '{}'").format(type(self).__name__, self)
+        return "{}: '{}'".format(type(self).__name__, self)

    # This permits passing it to file utils directly in Python 3.6+
    # With Python 2.7, Python 3.5+ we are stuck with path_info.fspath for now
@@ -80,7 +68,7 @@ class PathInfo(pathlib.PurePath, _BasePath):
        return self.__class__(relpath(self, other))

    def isin(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
            other = self.__class__(other)
        elif self.__class__ != other.__class__:
            return False
@@ -88,34 +76,6 @@ class PathInfo(pathlib.PurePath, _BasePath):
        n = len(other._cparts)
        return len(self._cparts) > n and self._cparts[:n] == other._cparts

-    # pathlib2 uses bytes internally in Python 2, and we use unicode everywhere
-    # for paths in both pythons, thus we need this glue.
-    if is_py2:
-        __unicode__ = __str__
-
-        def __str__(self):
-            return self.__unicode__().encode(sys.getfilesystemencoding())
-
-        @classmethod
-        def _parse_args(cls, args):
-            args = [
-                a.encode(fs_encoding)
-                if isinstance(a, unicode)  # noqa: F821
-                else a
-                for a in args
-            ]
-            return super(PathInfo, cls)._parse_args(args)
-
-        @property
-        def name(self):
-            return super(PathInfo, self).name.decode(fs_encoding)
-
-        def __fspath__(self):  # noqa: F811
-            return pathlib.PurePath.__str__(self).decode(fs_encoding)
-
-        def with_name(self, name):
-            return pathlib.PurePath.with_name(self, name.encode(fs_encoding))
-

 class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
    pass
@@ -155,7 +115,7 @@ class URLInfo(_BasePath):
        assert not p.query and not p.params and not p.fragment
        assert p.password is None

-        self.fill_parts(p.scheme, p.hostname, p.username, p.port, p.path)
+        self._fill_parts(p.scheme, p.hostname, p.username, p.port, p.path)

    @classmethod
    def from_parts(
@@ -167,18 +127,18 @@ class URLInfo(_BasePath):
            return cls("{}://{}{}".format(scheme, netloc, path))

        obj = cls.__new__(cls)
-        obj.fill_parts(scheme, host, user, port, path)
+        obj._fill_parts(scheme, host, user, port, path)
        return obj

-    def fill_parts(self, scheme, host, user, port, path):
+    def _fill_parts(self, scheme, host, user, port, path):
        assert scheme != "remote"
-        assert isinstance(path, (basestring, _URLPathInfo))
+        assert isinstance(path, (str, bytes, _URLPathInfo))

        self.scheme, self.host, self.user = scheme, host, user
        self.port = int(port) if port else self.DEFAULT_PORTS.get(self.scheme)

        if isinstance(path, _URLPathInfo):
-            self._spath = builtin_str(path)
+            self._spath = str(path)
            self._path = path
        else:
            if path and path[0] != "/":
@@ -207,7 +167,7 @@ class URLInfo(_BasePath):
        return "{}: '{}'".format(type(self).__name__, self)

    def __eq__(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
            other = self.__class__(other)
        return (
            self.__class__ == other.__class__
@@ -257,7 +217,7 @@ class URLInfo(_BasePath):
        return _URLPathParents(self)

    def relative_to(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
            other = self.__class__(other)
        if self.__class__ != other.__class__:
            msg = "'{}' has incompatible class with '{}'".format(self, other)
@@ -268,7 +228,7 @@ class URLInfo(_BasePath):
        return self._path.relative_to(other._path)

    def isin(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
            other = self.__class__(other)
        elif self.__class__ != other.__class__:
            return False
@@ -281,3 +241,78 @@ class CloudURLInfo(URLInfo):
    @property
    def path(self):
        return self._spath.lstrip("/")
+
+
+class HTTPURLInfo(URLInfo):
+    __hash__ = URLInfo.__hash__
+
+    def __init__(self, url):
+        p = urlparse(url)
+        stripped = p._replace(params=None, query=None, fragment=None)
+        super().__init__(stripped.geturl())
+        self.params = p.params
+        self.query = p.query
+        self.fragment = p.fragment
+
+    @classmethod
+    def from_parts(
+        cls,
+        scheme=None,
+        host=None,
+        user=None,
+        port=None,
+        path="",
+        netloc=None,
+        params=None,
+        query=None,
+        fragment=None,
+    ):
+        assert bool(host) ^ bool(netloc)
+
+        if netloc is not None:
+            return cls(
+                "{}://{}{}{}{}{}".format(
+                    scheme,
+                    netloc,
+                    path,
+                    (";" + params) if params else "",
+                    ("?" + query) if query else "",
+                    ("#" + fragment) if fragment else "",
+                )
+            )
+
+        obj = cls.__new__(cls)
+        obj._fill_parts(scheme, host, user, port, path)
+        obj.params = params
+        obj.query = query
+        obj.fragment = fragment
+        return obj
+
+    @property
+    def _extra_parts(self):
+        return (self.params, self.query, self.fragment)
+
+    @property
+    def parts(self):
+        return self._base_parts + self._path.parts + self._extra_parts
+
+    @cached_property
+    def url(self):
+        return "{}://{}{}{}{}{}".format(
+            self.scheme,
+            self.netloc,
+            self._spath,
+            (";" + self.params) if self.params else "",
+            ("?" + self.query) if self.query else "",
+            ("#" + self.fragment) if self.fragment else "",
+        )
+
+    def __eq__(self, other):
+        if isinstance(other, (str, bytes)):
+            other = self.__class__(other)
+        return (
+            self.__class__ == other.__class__
+            and self._base_parts == other._base_parts
+            and self._path == other._path
+            and self._extra_parts == other._extra_parts
+        )
@@ -1,11 +1,9 @@
-"""Manages progress bars for dvc repo."""
-from __future__ import print_function
+"""Manages progress bars for DVC repo."""

 import logging
 import sys
 from threading import RLock

-from funcy import merge
 from tqdm import tqdm

 from dvc.utils import env2bool
@@ -32,7 +30,10 @@ class Tqdm(tqdm):
    )
    BAR_FMT_NOTOTAL = (
        "{desc:{ncols_desc}.{ncols_desc}}{n_fmt}"
-        " [{elapsed}<??:??, {rate_fmt:>11}{postfix}]"
+        " [{elapsed}, {rate_fmt:>11}{postfix}]"
+    )
+    BYTES_DEFAULTS = dict(
+        unit="B", unit_scale=True, unit_divisor=1024, miniters=1
    )

    def __init__(
@@ -45,6 +46,7 @@ class Tqdm(tqdm):
        bar_format=None,
        bytes=False,  # pylint: disable=W0622
        file=None,
+        total=None,
        **kwargs
    ):
        """
@@ -53,23 +55,22 @@ class Tqdm(tqdm):
        desc  : persists after `close()`
        level  : effective logging level for determining `disable`;
            used only if `disable` is unspecified
-        disable  : If (default: None), will be determined by logging level.
+        disable  : If (default: None) or False,
+            will be determined by logging level.
            May be overridden to `True` due to non-TTY status.
            Skip override by specifying env var `DVC_IGNORE_ISATTY`.
        kwargs  : anything accepted by `tqdm.tqdm()`
        """
        kwargs = kwargs.copy()
-        kwargs.setdefault("unit_scale", True)
        if bytes:
-            bytes_defaults = dict(
-                unit="B", unit_scale=True, unit_divisor=1024, miniters=1
-            )
-            kwargs = merge(bytes_defaults, kwargs)
+            kwargs = {**self.BYTES_DEFAULTS, **kwargs}
+        else:
+            kwargs.setdefault("unit_scale", total > 999 if total else True)
        if file is None:
            file = sys.stderr
        self.desc_persist = desc
        # auto-disable based on `logger.level`
-        if disable is None:
+        if not disable:
            disable = logger.getEffectiveLevel() > level
        # auto-disable based on TTY
        if (
@@ -78,13 +79,14 @@ class Tqdm(tqdm):
            and hasattr(file, "isatty")
        ):
            disable = not file.isatty()
-        super(Tqdm, self).__init__(
+        super().__init__(
            iterable=iterable,
            disable=disable,
            leave=leave,
            desc=desc,
            bar_format="!",
            lock_args=(False,),
+            total=total,
            **kwargs
        )
        if bar_format is None:
@@ -112,15 +114,34 @@ class Tqdm(tqdm):
            self.total = total  # pylint: disable=W0613,W0201
        self.update(current - self.n)

+    def wrap_fn(self, fn, callback=None):
+        """
+        Returns a wrapped `fn` which calls `callback()` on each call.
+        `callback` is `self.update` by default.
+        """
+        if callback is None:
+            callback = self.update
+
+        def wrapped(*args, **kwargs):
+            res = fn(*args, **kwargs)
+            callback()
+            return res
+
+        return wrapped
+
    def close(self):
        if self.desc_persist is not None:
            self.set_description_str(self.desc_persist, refresh=False)
-        super(Tqdm, self).close()
+        # unknown/zero ETA
+        self.bar_format = self.bar_format.replace("<{remaining}", "")
+        # remove completed bar
+        self.bar_format = self.bar_format.replace("|{bar:10}|", " ")
+        super().close()

    @property
    def format_dict(self):
        """inject `ncols_desc` to fill the display width (`ncols`)"""
-        d = super(Tqdm, self).format_dict
+        d = super().format_dict
        ncols = d["ncols"] or 80
        ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1
        ncols_desc = max(ncols_desc, 0)
@@ -1,14 +1,9 @@
 """Manages user prompts."""
-from __future__ import print_function
-from __future__ import unicode_literals

 import logging
 import sys
 from getpass import getpass

-from dvc.utils.compat import input
-
-
 logger = logging.getLogger(__name__)


@@ -1,6 +1,6 @@
-from __future__ import unicode_literals
+import posixpath
+from urllib.parse import urlparse

-from .config import RemoteConfig
 from dvc.remote.azure import RemoteAZURE
 from dvc.remote.gdrive import RemoteGDrive
 from dvc.remote.gs import RemoteGS
@@ -27,9 +27,9 @@ REMOTES = [
 ]


-def _get(config):
+def _get(remote_conf):
    for remote in REMOTES:
-        if remote.supported(config):
+        if remote.supported(remote_conf):
            return remote
    return RemoteLOCAL

@@ -37,8 +37,39 @@ def _get(config):
 def Remote(repo, **kwargs):
    name = kwargs.get("name")
    if name:
-        remote_config = RemoteConfig(repo.config)
-        settings = remote_config.get_settings(name)
+        remote_conf = repo.config["remote"][name.lower()]
    else:
-        settings = kwargs
-    return _get(settings)(repo, settings)
+        remote_conf = kwargs
+    remote_conf = _resolve_remote_refs(repo.config, remote_conf)
+    return _get(remote_conf)(repo, remote_conf)
+
+
+def _resolve_remote_refs(config, remote_conf):
+    # Support for cross referenced remotes.
+    # This will merge the settings, shadowing base ref with remote_conf.
+    # For example, having:
+    #
+    #       dvc remote add server ssh://localhost
+    #       dvc remote modify server user root
+    #       dvc remote modify server ask_password true
+    #
+    #       dvc remote add images remote://server/tmp/pictures
+    #       dvc remote modify images user alice
+    #       dvc remote modify images ask_password false
+    #       dvc remote modify images password asdf1234
+    #
+    # Results on a config dictionary like:
+    #
+    #       {
+    #           "url": "ssh://localhost/tmp/pictures",
+    #           "user": "alice",
+    #           "password": "asdf1234",
+    #           "ask_password": False,
+    #       }
+    parsed = urlparse(remote_conf["url"])
+    if parsed.scheme != "remote":
+        return remote_conf
+
+    base = config["remote"][parsed.netloc]
+    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
+    return {**base, **remote_conf, "url": url}
@@ -1,20 +1,17 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 import logging
 import os
+import posixpath
 import re
 from datetime import datetime, timedelta
+from urllib.parse import urlparse
 import threading

 from funcy import cached_property, wrap_prop

-from dvc.config import Config
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
-from dvc.utils.compat import urlparse


 logger = logging.getLogger(__name__)
@@ -32,11 +29,12 @@ class RemoteAZURE(RemoteBASE):
    REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
    PARAM_CHECKSUM = "etag"
    COPY_POLL_SECONDS = 5
+    LIST_OBJECT_PAGE_SIZE = 5000

    def __init__(self, repo, config):
-        super(RemoteAZURE, self).__init__(repo, config)
+        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL, "azure://")
+        url = config.get("url", "azure://")

        match = re.match(self.REGEX, url)  # backward compatibility
        path = match.group("path")
@@ -47,7 +45,7 @@ class RemoteAZURE(RemoteBASE):
        )

        self.connection_string = (
-            config.get(Config.SECTION_AZURE_CONNECTION_STRING)
+            config.get("connection_string")
            or match.group("connection_string")  # backward compatibility
            or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
        )
@@ -91,7 +89,7 @@ class RemoteAZURE(RemoteBASE):
        logger.debug("Removing {}".format(path_info))
        self.blob_service.delete_blob(path_info.bucket, path_info.path)

-    def _list_paths(self, bucket, prefix):
+    def _list_paths(self, bucket, prefix, progress_callback=None):
        blob_service = self.blob_service
        next_marker = None
        while True:
@@ -100,6 +98,8 @@ class RemoteAZURE(RemoteBASE):
            )

            for blob in blobs:
+                if progress_callback:
+                    progress_callback()
                yield blob.name

            if not blobs.next_marker:
@@ -107,8 +107,16 @@ class RemoteAZURE(RemoteBASE):

            next_marker = blobs.next_marker

-    def list_cache_paths(self):
-        return self._list_paths(self.path_info.bucket, self.path_info.path)
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            prefix = posixpath.join(
+                self.path_info.path, prefix[:2], prefix[2:]
+            )
+        else:
+            prefix = self.path_info.path
+        return self._list_paths(
+            self.path_info.bucket, prefix, progress_callback
+        )

    def _upload(
        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
@@ -1,10 +1,9 @@
-from __future__ import unicode_literals
-from dvc.utils.compat import basestring, FileNotFoundError, str, urlparse
-
+import errno
 import itertools
 import json
 import logging
 import tempfile
+from urllib.parse import urlparse
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
 from functools import partial
@@ -14,19 +13,20 @@ from operator import itemgetter
 from shortuuid import uuid

 import dvc.prompt as prompt
-from dvc.config import Config
 from dvc.exceptions import (
+    CheckoutError,
    DvcException,
    ConfirmRemoveError,
    DvcIgnoreInCollectedDirError,
+    RemoteCacheRequiredError,
 )
 from dvc.ignore import DvcIgnore
 from dvc.path_info import PathInfo, URLInfo
 from dvc.progress import Tqdm
 from dvc.remote.slow_link_detection import slow_link_guard
 from dvc.state import StateNoop
-from dvc.utils import makedirs, relpath, tmp_fname
-from dvc.utils.fs import move
+from dvc.utils import tmp_fname
+from dvc.utils.fs import move, makedirs
 from dvc.utils.http import open_url

 logger = logging.getLogger(__name__)
@@ -47,7 +47,7 @@ STATUS_MAP = {

 class RemoteCmdError(DvcException):
    def __init__(self, remote, cmd, ret, err):
-        super(RemoteCmdError, self).__init__(
+        super().__init__(
            "{remote} command '{cmd}' finished with non-zero return code"
            " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
        )
@@ -55,8 +55,8 @@ class RemoteCmdError(DvcException):

 class RemoteActionNotImplemented(DvcException):
    def __init__(self, action, scheme):
-        m = "{} is not supported by {} remote".format(action, scheme)
-        super(RemoteActionNotImplemented, self).__init__(m)
+        m = "{} is not supported for {} remotes".format(action, scheme)
+        super().__init__(m)


 class RemoteMissingDepsError(DvcException):
@@ -64,10 +64,9 @@ class RemoteMissingDepsError(DvcException):


 class DirCacheError(DvcException):
-    def __init__(self, checksum, cause=None):
-        super(DirCacheError, self).__init__(
-            "Failed to load dir cache for checksum: '{}'.".format(checksum),
-            cause=cause,
+    def __init__(self, checksum):
+        super().__init__(
+            "Failed to load dir cache for hash value: '{}'.".format(checksum)
        )


@@ -81,6 +80,15 @@ class RemoteBASE(object):
    CHECKSUM_DIR_SUFFIX = ".dir"
    CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
    DEFAULT_CACHE_TYPES = ["copy"]
+    DEFAULT_VERIFY = False
+    LIST_OBJECT_PAGE_SIZE = 1000
+    TRAVERSE_WEIGHT_MULTIPLIER = 20
+    TRAVERSE_PREFIX_LEN = 3
+    TRAVERSE_THRESHOLD_SIZE = 500000
+    CAN_TRAVERSE = True
+
+    CACHE_MODE = None
+    SHARED_MODE_MAP = {None: (None, None), "group": (None, None)}

    state = StateNoop()

@@ -89,41 +97,39 @@ class RemoteBASE(object):

        self._check_requires(config)

-        core = config.get(Config.SECTION_CORE, {})
-        self.checksum_jobs = core.get(
-            Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
-        )
+        shared = config.get("shared")
+        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]

-        self.protected = False
-        self.no_traverse = config.get(Config.SECTION_REMOTE_NO_TRAVERSE, True)
+        self.checksum_jobs = (
+            config.get("checksum_jobs")
+            or (self.repo and self.repo.config["core"].get("checksum_jobs"))
+            or self.CHECKSUM_JOBS
+        )
+        self.verify = config.get("verify", self.DEFAULT_VERIFY)
        self._dir_info = {}

-        types = config.get(Config.SECTION_CACHE_TYPE, None)
-        if types:
-            if isinstance(types, str):
-                types = [t.strip() for t in types.split(",")]
-            self.cache_types = types
-        else:
-            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
+        self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES)
        self.cache_type_confirmed = False

-    def _check_requires(self, config):
+    @classmethod
+    def get_missing_deps(cls):
        import importlib

        missing = []
-
-        for package, module in self.REQUIRES.items():
+        for package, module in cls.REQUIRES.items():
            try:
                importlib.import_module(module)
            except ImportError:
                missing.append(package)

+        return missing
+
+    def _check_requires(self, config):
+        missing = self.get_missing_deps()
        if not missing:
            return

-        url = config.get(
-            Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
-        )
+        url = config.get("url", "{}://".format(self.scheme))
        msg = (
            "URL '{}' is supported but requires these missing "
            "dependencies: {}. If you have installed dvc using pip, "
@@ -152,10 +158,10 @@ class RemoteBASE(object):

    @classmethod
    def supported(cls, config):
-        if isinstance(config, basestring):
+        if isinstance(config, (str, bytes)):
            url = config
        else:
-            url = config[Config.SECTION_REMOTE_URL]
+            url = config["url"]

        # NOTE: silently skipping remote, calling code should handle that
        parsed = urlparse(url)
@@ -170,14 +176,16 @@ class RemoteBASE(object):

    def _calculate_checksums(self, file_infos):
        file_infos = list(file_infos)
-        with ThreadPoolExecutor(max_workers=self.checksum_jobs) as executor:
-            tasks = executor.map(self.get_file_checksum, file_infos)
-            with Tqdm(
-                tasks,
-                total=len(file_infos),
-                unit="md5",
-                desc="Computing hashes (only done once)",
-            ) as tasks:
+        with Tqdm(
+            total=len(file_infos),
+            unit="md5",
+            desc="Computing file/dir hashes (only done once)",
+        ) as pbar:
+            worker = pbar.wrap_fn(self.get_file_checksum)
+            with ThreadPoolExecutor(
+                max_workers=self.checksum_jobs
+            ) as executor:
+                tasks = executor.map(worker, file_infos)
                checksums = dict(zip(file_infos, tasks))
        return checksums

@@ -219,12 +227,15 @@ class RemoteBASE(object):
        return sorted(result, key=itemgetter(self.PARAM_RELPATH))

    def get_dir_checksum(self, path_info):
+        if not self.cache:
+            raise RemoteCacheRequiredError(path_info)
+
        dir_info = self._collect_dir(path_info)
        checksum, tmp_info = self._get_dir_info_checksum(dir_info)
        new_info = self.cache.checksum_to_path_info(checksum)
        if self.cache.changed_cache_file(checksum):
            self.cache.makedirs(new_info.parent)
-            self.cache.move(tmp_info, new_info)
+            self.cache.move(tmp_info, new_info, mode=self.CACHE_MODE)

        self.state.save(path_info, checksum)
        self.state.save(new_info, checksum)
@@ -265,11 +276,13 @@ class RemoteBASE(object):
            with self.cache.open(path_info, "r") as fobj:
                d = json.load(fobj)
        except (ValueError, FileNotFoundError) as exc:
-            raise DirCacheError(checksum, cause=exc)
+            raise DirCacheError(checksum) from exc

        if not isinstance(d, list):
-            msg = "dir cache file format error '{}' [skipping the file]"
-            logger.error(msg.format(relpath(path_info)))
+            logger.error(
+                "dir cache file format error '%s' [skipping the file]",
+                path_info,
+            )
            return []

        for info in d:
@@ -322,49 +335,48 @@ class RemoteBASE(object):

        A file is considered changed if:
            - It doesn't exist on the working directory (was unlinked)
-            - Checksum is not computed (saving a new file)
-            - The checkusm stored in the State is different from the given one
+            - Hash value is not computed (saving a new file)
+            - The hash value stored is different from the given one
            - There's no file in the cache

        Args:
            path_info: dict with path information.
-            checksum: expected checksum for this data.
+            checksum: expected hash value for this data.

        Returns:
            bool: True if data has changed, False otherwise.
        """

        logger.debug(
-            "checking if '{}'('{}') has changed.".format(
-                path_info, checksum_info
-            )
+            "checking if '%s'('%s') has changed.", path_info, checksum_info
        )

        if not self.exists(path_info):
-            logger.debug("'{}' doesn't exist.".format(path_info))
+            logger.debug("'%s' doesn't exist.", path_info)
            return True

        checksum = checksum_info.get(self.PARAM_CHECKSUM)
        if checksum is None:
-            logger.debug("checksum for '{}' is missing.".format(path_info))
+            logger.debug("hash value for '%s' is missing.", path_info)
            return True

        if self.changed_cache(checksum):
            logger.debug(
-                "cache for '{}'('{}') has changed.".format(path_info, checksum)
+                "cache for '%s'('%s') has changed.", path_info, checksum
            )
            return True

        actual = self.get_checksum(path_info)
        if checksum != actual:
            logger.debug(
-                "checksum '{}'(actual '{}') for '{}' has changed.".format(
-                    checksum, actual, path_info
-                )
+                "hash value '%s' for '%s' has changed (actual '%s').",
+                checksum,
+                actual,
+                path_info,
            )
            return True

-        logger.debug("'{}' hasn't changed.".format(path_info))
+        logger.debug("'%s' hasn't changed.", path_info)
        return False

    def link(self, from_info, to_info):
@@ -377,18 +389,30 @@ class RemoteBASE(object):

        self._try_links(from_info, to_info, link_types)

+    def _verify_link(self, path_info, link_type):
+        if self.cache_type_confirmed:
+            return
+
+        is_link = getattr(self, "is_{}".format(link_type), None)
+        if is_link and not is_link(path_info):
+            self.remove(path_info)
+            raise DvcException("failed to verify {}".format(link_type))
+
+        self.cache_type_confirmed = True
+
    @slow_link_guard
    def _try_links(self, from_info, to_info, link_types):
        while link_types:
            link_method = getattr(self, link_types[0])
            try:
                self._do_link(from_info, to_info, link_method)
-                self.cache_type_confirmed = True
+                self._verify_link(to_info, link_types[0])
                return

            except DvcException as exc:
-                msg = "Cache type '{}' is not supported: {}"
-                logger.debug(msg.format(link_types[0], str(exc)))
+                logger.debug(
+                    "Cache type '%s' is not supported: %s", link_types[0], exc
+                )
                del link_types[0]

        raise DvcException("no possible cache types left to try out.")
@@ -399,30 +423,20 @@ class RemoteBASE(object):

        link_method(from_info, to_info)

-        if self.protected:
-            self.protect(to_info)
-
-        msg = "Created {}'{}': {} -> {}".format(
-            "protected " if self.protected else "",
-            self.cache_types[0],
-            from_info,
-            to_info,
+        logger.debug(
+            "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
        )
-        logger.debug(msg)

    def _save_file(self, path_info, checksum, save_link=True):
        assert checksum

        cache_info = self.checksum_to_path_info(checksum)
        if self.changed_cache(checksum):
-            self.move(path_info, cache_info)
+            self.move(path_info, cache_info, mode=self.CACHE_MODE)
            self.link(cache_info, path_info)
        elif self.iscopy(path_info) and self._cache_is_copy(path_info):
            # Default relink procedure involves unneeded copy
-            if self.protected:
-                self.protect(path_info)
-            else:
-                self.unprotect(path_info)
+            self.unprotect(path_info)
        else:
            self.remove(path_info)
            self.link(cache_info, path_info)
@@ -458,16 +472,20 @@ class RemoteBASE(object):
        self.cache_type_confirmed = True
        return self.cache_types[0] == "copy"

-    def _save_dir(self, path_info, checksum):
+    def _save_dir(self, path_info, checksum, save_link=True):
        cache_info = self.checksum_to_path_info(checksum)
        dir_info = self.get_dir_cache(checksum)

-        for entry in dir_info:
+        for entry in Tqdm(
+            dir_info, desc="Saving " + path_info.name, unit="file"
+        ):
            entry_info = path_info / entry[self.PARAM_RELPATH]
            entry_checksum = entry[self.PARAM_CHECKSUM]
            self._save_file(entry_info, entry_checksum, save_link=False)

-        self.state.save_link(path_info)
+        if save_link:
+            self.state.save_link(path_info)
+
        self.state.save(cache_info, checksum)
        self.state.save(path_info, checksum)

@@ -498,7 +516,7 @@ class RemoteBASE(object):
    def protect(path_info):
        pass

-    def save(self, path_info, checksum_info):
+    def save(self, path_info, checksum_info, save_link=True):
        if path_info.scheme != self.scheme:
            raise RemoteActionNotImplemented(
                "save {} -> {}".format(path_info.scheme, self.scheme),
@@ -506,15 +524,26 @@ class RemoteBASE(object):
            )

        checksum = checksum_info[self.PARAM_CHECKSUM]
-        self._save(path_info, checksum)
+        self._save(path_info, checksum, save_link)

-    def _save(self, path_info, checksum):
+    def _save(self, path_info, checksum, save_link=True):
        to_info = self.checksum_to_path_info(checksum)
-        logger.debug("Saving '{}' to '{}'.".format(path_info, to_info))
+        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
        if self.isdir(path_info):
-            self._save_dir(path_info, checksum)
+            self._save_dir(path_info, checksum, save_link)
            return
-        self._save_file(path_info, checksum)
+        self._save_file(path_info, checksum, save_link)
+
+    def _handle_transfer_exception(
+        self, from_info, to_info, exception, operation
+    ):
+        if isinstance(exception, OSError) and exception.errno == errno.EMFILE:
+            raise exception
+
+        logger.exception(
+            "failed to %s '%s' to '%s'", operation, from_info, to_info
+        )
+        return 1

    def upload(self, from_info, to_info, name=None, no_progress_bar=False):
        if not hasattr(self, "_upload"):
@@ -526,7 +555,7 @@ class RemoteBASE(object):
        if from_info.scheme != "local":
            raise NotImplementedError

-        logger.debug("Uploading '{}' to '{}'".format(from_info, to_info))
+        logger.debug("Uploading '%s' to '%s'", from_info, to_info)

        name = name or from_info.name

@@ -537,10 +566,10 @@ class RemoteBASE(object):
                name=name,
                no_progress_bar=no_progress_bar,
            )
-        except Exception:
-            msg = "failed to upload '{}' to '{}'"
-            logger.exception(msg.format(from_info, to_info))
-            return 1  # 1 fail
+        except Exception as e:
+            return self._handle_transfer_exception(
+                from_info, to_info, e, "upload"
+            )

        return 0

@@ -582,22 +611,23 @@ class RemoteBASE(object):
            to_info / info.relative_to(from_info) for info in from_infos
        )

-        with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
-            download_files = partial(
-                self._download_file,
-                name=name,
-                no_progress_bar=True,
-                file_mode=file_mode,
-                dir_mode=dir_mode,
+        with Tqdm(
+            total=len(from_infos),
+            desc="Downloading directory",
+            unit="Files",
+            disable=no_progress_bar,
+        ) as pbar:
+            download_files = pbar.wrap_fn(
+                partial(
+                    self._download_file,
+                    name=name,
+                    no_progress_bar=True,
+                    file_mode=file_mode,
+                    dir_mode=dir_mode,
+                )
            )
-            futures = executor.map(download_files, from_infos, to_infos)
-            with Tqdm(
-                futures,
-                total=len(from_infos),
-                desc="Downloading directory",
-                unit="Files",
-                disable=no_progress_bar,
-            ) as futures:
+            with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
+                futures = executor.map(download_files, from_infos, to_infos)
                return sum(futures)

    def _download_file(
@@ -605,7 +635,7 @@ class RemoteBASE(object):
    ):
        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)

-        logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
+        logger.debug("Downloading '%s' to '%s'", from_info, to_info)
        name = name or to_info.name

        tmp_file = tmp_fname(to_info)
@@ -614,10 +644,10 @@ class RemoteBASE(object):
            self._download(
                from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
            )
-        except Exception:
-            msg = "failed to download '{}' to '{}'"
-            logger.exception(msg.format(from_info, to_info))
-            return 1  # 1 fail
+        except Exception as e:
+            return self._handle_transfer_exception(
+                from_info, to_info, e, "download"
+            )

        move(tmp_file, to_info, mode=file_mode)

@@ -633,7 +663,8 @@ class RemoteBASE(object):
    def remove(self, path_info):
        raise RemoteActionNotImplemented("remove", self.scheme)

-    def move(self, from_info, to_info):
+    def move(self, from_info, to_info, mode=None):
+        assert mode is None
        self.copy(from_info, to_info)
        self.remove(from_info)

@@ -663,7 +694,7 @@ class RemoteBASE(object):
    def checksum_to_path_info(self, checksum):
        return self.path_info / checksum[0:2] / checksum[2:]

-    def list_cache_paths(self):
+    def list_cache_paths(self, prefix=None, progress_callback=None):
        raise NotImplementedError

    def all(self):
@@ -689,10 +720,15 @@ class RemoteBASE(object):
            if checksum in used:
                continue
            path_info = self.checksum_to_path_info(checksum)
+            if self.is_dir_checksum(checksum):
+                self._remove_unpacked_dir(checksum)
            self.remove(path_info)
            removed = True
        return removed

+    def is_protected(self, path_info):
+        return False
+
    def changed_cache_file(self, checksum):
        """Compare the given checksum with the (corresponding) actual one.

@@ -705,45 +741,68 @@ class RemoteBASE(object):

        - Remove the file from cache if it doesn't match the actual checksum
        """
+
        cache_info = self.checksum_to_path_info(checksum)
+        if self.is_protected(cache_info):
+            logger.debug(
+                "Assuming '%s' is unchanged since it is read-only", cache_info
+            )
+            return False
+
        actual = self.get_checksum(cache_info)

        logger.debug(
-            "cache '{}' expected '{}' actual '{}'".format(
-                str(cache_info), checksum, actual
-            )
+            "cache '%s' expected '%s' actual '%s'",
+            cache_info,
+            checksum,
+            actual,
        )

        if not checksum or not actual:
            return True

        if actual.split(".")[0] == checksum.split(".")[0]:
+            # making cache file read-only so we don't need to check it
+            # next time
+            self.protect(cache_info)
            return False

        if self.exists(cache_info):
-            logger.warning("corrupted cache file '{}'.".format(cache_info))
+            logger.warning("corrupted cache file '%s'.", cache_info)
            self.remove(cache_info)

        return True

-    def _changed_dir_cache(self, checksum):
+    def _changed_dir_cache(self, checksum, path_info=None, filter_info=None):
        if self.changed_cache_file(checksum):
            return True

-        if not self._changed_unpacked_dir(checksum):
+        if not (path_info and filter_info) and not self._changed_unpacked_dir(
+            checksum
+        ):
            return False

        for entry in self.get_dir_cache(checksum):
            entry_checksum = entry[self.PARAM_CHECKSUM]
+
+            if path_info and filter_info:
+                entry_info = path_info / entry[self.PARAM_RELPATH]
+                if not entry_info.isin_or_eq(filter_info):
+                    continue
+
            if self.changed_cache_file(entry_checksum):
                return True

-        self._update_unpacked_dir(checksum)
+        if not (path_info and filter_info):
+            self._update_unpacked_dir(checksum)
+
        return False

-    def changed_cache(self, checksum):
+    def changed_cache(self, checksum, path_info=None, filter_info=None):
        if self.is_dir_checksum(checksum):
-            return self._changed_dir_cache(checksum)
+            return self._changed_dir_cache(
+                checksum, path_info=path_info, filter_info=filter_info
+            )
        return self.changed_cache_file(checksum)

    def cache_exists(self, checksums, jobs=None, name=None):
@@ -751,11 +810,13 @@ class RemoteBASE(object):

        There are two ways of performing this check:

-        - Traverse: Get a list of all the files in the remote
+        - Traverse method: Get a list of all the files in the remote
            (traversing the cache directory) and compare it with
-            the given checksums.
+            the given checksums. Cache entries will be retrieved in parallel
+            threads according to prefix (i.e. entries starting with, "00...",
+            "01...", and so on) and a progress bar will be displayed.

-        - No traverse: For each given checksum, run the `exists`
+        - Exists method: For each given checksum, run the `exists`
            method and filter the checksums that aren't on the remote.
            This is done in parallel threads.
            It also shows a progress bar when performing the check.
@@ -766,12 +827,128 @@ class RemoteBASE(object):
        check if particular file exists much quicker, use their own
        implementation of cache_exists (see ssh, local).

+        Which method to use will be automatically determined after estimating
+        the size of the remote cache, and comparing the estimated size with
+        len(checksums). To estimate the size of the remote cache, we fetch
+        a small subset of cache entries (i.e. entries starting with "00...").
+        Based on the number of entries in that subset, the size of the full
+        cache can be estimated, since the cache is evenly distributed according
+        to checksum.
+
        Returns:
            A list with checksums that were found in the remote
        """
-        if not self.no_traverse:
-            return list(set(checksums) & set(self.all()))
+        # Remotes which do not use traverse prefix should override
+        # cache_exists() (see ssh, local)
+        assert self.TRAVERSE_PREFIX_LEN >= 2

+        if len(checksums) == 1 or not self.CAN_TRAVERSE:
+            return self._cache_object_exists(checksums, jobs, name)
+
+        # Fetch cache entries beginning with "00..." prefix for estimating the
+        # size of entire remote cache
+        checksums = frozenset(checksums)
+        prefix = "0" * self.TRAVERSE_PREFIX_LEN
+        total_prefixes = pow(16, self.TRAVERSE_PREFIX_LEN)
+        with Tqdm(
+            desc="Estimating size of "
+            + ("cache in '{}'".format(name) if name else "remote cache"),
+            unit="file",
+        ) as pbar:
+
+            def update(n=1):
+                pbar.update(n * total_prefixes)
+
+            paths = self.list_cache_paths(
+                prefix=prefix, progress_callback=update
+            )
+            remote_checksums = set(map(self.path_to_checksum, paths))
+
+        if remote_checksums:
+            remote_size = total_prefixes * len(remote_checksums)
+        else:
+            remote_size = total_prefixes
+        logger.debug("Estimated remote size: {} files".format(remote_size))
+
+        traverse_pages = remote_size / self.LIST_OBJECT_PAGE_SIZE
+        # For sufficiently large remotes, traverse must be weighted to account
+        # for performance overhead from large lists/sets.
+        # From testing with S3, for remotes with 1M+ files, object_exists is
+        # faster until len(checksums) is at least 10k~100k
+        if remote_size > self.TRAVERSE_THRESHOLD_SIZE:
+            traverse_weight = traverse_pages * self.TRAVERSE_WEIGHT_MULTIPLIER
+        else:
+            traverse_weight = traverse_pages
+        if len(checksums) < traverse_weight:
+            logger.debug(
+                "Large remote ('{}' checksums < '{}' traverse weight), "
+                "using object_exists for remaining checksums".format(
+                    len(checksums), traverse_weight
+                )
+            )
+            return list(
+                checksums & remote_checksums
+            ) + self._cache_object_exists(
+                checksums - remote_checksums, jobs, name
+            )
+
+        if traverse_pages < 256 / self.JOBS:
+            # Threaded traverse will require making at least 255 more requests
+            # to the remote, so for small enough remotes, fetching the entire
+            # list at once will require fewer requests (but also take into
+            # account that this must be done sequentially rather than in
+            # parallel)
+            logger.debug(
+                "Querying {} checksums via default traverse".format(
+                    len(checksums)
+                )
+            )
+            return list(checksums & set(self.all()))
+
+        return self._cache_exists_traverse(
+            checksums, remote_checksums, remote_size, jobs, name
+        )
+
+    def _cache_exists_traverse(
+        self, checksums, remote_checksums, remote_size, jobs=None, name=None
+    ):
+        logger.debug(
+            "Querying {} checksums via threaded traverse".format(
+                len(checksums)
+            )
+        )
+
+        traverse_prefixes = ["{:02x}".format(i) for i in range(1, 256)]
+        if self.TRAVERSE_PREFIX_LEN > 2:
+            traverse_prefixes += [
+                "{0:0{1}x}".format(i, self.TRAVERSE_PREFIX_LEN)
+                for i in range(1, pow(16, self.TRAVERSE_PREFIX_LEN - 2))
+            ]
+        with Tqdm(
+            desc="Querying "
+            + ("cache in '{}'".format(name) if name else "remote cache"),
+            total=remote_size,
+            initial=len(remote_checksums),
+            unit="objects",
+        ) as pbar:
+
+            def list_with_update(prefix):
+                paths = self.list_cache_paths(
+                    prefix=prefix, progress_callback=pbar.update
+                )
+                return map(self.path_to_checksum, list(paths))
+
+            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
+                in_remote = executor.map(list_with_update, traverse_prefixes,)
+                remote_checksums.update(
+                    itertools.chain.from_iterable(in_remote)
+                )
+            return list(checksums & remote_checksums)
+
+    def _cache_object_exists(self, checksums, jobs=None, name=None):
+        logger.debug(
+            "Querying {} checksums via object_exists".format(len(checksums))
+        )
        with Tqdm(
            desc="Querying "
            + ("cache in " + name if name else "remote cache"),
@@ -814,14 +991,15 @@ class RemoteBASE(object):
        self.remove(path_info)

    def _checkout_file(
-        self, path_info, checksum, force, progress_callback=None
+        self, path_info, checksum, force, progress_callback=None, relink=False
    ):
        """The file is changed we need to checkout a new copy"""
+        added, modified = True, False
        cache_info = self.checksum_to_path_info(checksum)
        if self.exists(path_info):
-            msg = "data '{}' exists. Removing before checkout."
-            logger.warning(msg.format(str(path_info)))
+            logger.debug("data '%s' will be replaced.", path_info)
            self.safe_remove(path_info, force=force)
+            added, modified = False, True

        self.link(cache_info, path_info)
        self.state.save_link(path_info)
@@ -829,23 +1007,32 @@ class RemoteBASE(object):
        if progress_callback:
            progress_callback(str(path_info))

+        return added, modified and not relink
+
    def makedirs(self, path_info):
        """Optional: Implement only if the remote needs to create
        directories before copying/linking/moving data
        """
-        pass

    def _checkout_dir(
-        self, path_info, checksum, force, progress_callback=None, relink=False
+        self,
+        path_info,
+        checksum,
+        force,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
    ):
+        added, modified = False, False
        # Create dir separately so that dir is created
        # even if there are no files in it
        if not self.exists(path_info):
+            added = True
            self.makedirs(path_info)

        dir_info = self.get_dir_cache(checksum)

-        logger.debug("Linking directory '{}'.".format(path_info))
+        logger.debug("Linking directory '%s'.", path_info)

        for entry in dir_info:
            relative_path = entry[self.PARAM_RELPATH]
@@ -853,29 +1040,41 @@ class RemoteBASE(object):
            entry_cache_info = self.checksum_to_path_info(entry_checksum)
            entry_info = path_info / relative_path

+            if filter_info and not entry_info.isin_or_eq(filter_info):
+                continue
+
            entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
            if relink or self.changed(entry_info, entry_checksum_info):
+                modified = True
                self.safe_remove(entry_info, force=force)
                self.link(entry_cache_info, entry_info)
                self.state.save(entry_info, entry_checksum)
            if progress_callback:
                progress_callback(str(entry_info))

-        self._remove_redundant_files(path_info, dir_info, force)
+        modified = (
+            self._remove_redundant_files(path_info, dir_info, force)
+            or modified
+        )

        self.state.save_link(path_info)
        self.state.save(path_info, checksum)

+        # relink is not modified, assume it as nochange
+        return added, not added and modified and not relink
+
    def _remove_redundant_files(self, path_info, dir_info, force):
        existing_files = set(self.walk_files(path_info))

        needed_files = {
            path_info / entry[self.PARAM_RELPATH] for entry in dir_info
        }
-
-        for path in existing_files - needed_files:
+        redundant_files = existing_files - needed_files
+        for path in redundant_files:
            self.safe_remove(path, force)

+        return bool(redundant_files)
+
    def checkout(
        self,
        path_info,
@@ -883,6 +1082,7 @@ class RemoteBASE(object):
        force=False,
        progress_callback=None,
        relink=False,
+        filter_info=None,
    ):
        if path_info.scheme not in ["local", self.scheme]:
            raise NotImplementedError
@@ -892,35 +1092,44 @@ class RemoteBASE(object):
        skip = False
        if not checksum:
            logger.warning(
-                "No checksum info found for '{}'. "
-                "It won't be created.".format(str(path_info))
+                "No file hash info found for '%s'. " "It won't be created.",
+                path_info,
            )
            self.safe_remove(path_info, force=force)
            failed = path_info

        elif not relink and not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            logger.debug(msg.format(str(path_info)))
+            logger.debug("Data '%s' didn't change.", path_info)
            skip = True

-        elif self.changed_cache(checksum):
-            msg = "Cache '{}' not found. File '{}' won't be created."
-            logger.warning(msg.format(checksum, str(path_info)))
+        elif self.changed_cache(
+            checksum, path_info=path_info, filter_info=filter_info
+        ):
+            logger.warning(
+                "Cache '%s' not found. File '%s' won't be created.",
+                checksum,
+                path_info,
+            )
            self.safe_remove(path_info, force=force)
            failed = path_info

        if failed or skip:
            if progress_callback:
                progress_callback(
-                    str(path_info), self.get_files_number(checksum)
+                    str(path_info),
+                    self.get_files_number(
+                        self.path_info, checksum, filter_info
+                    ),
                )
-            return failed
+            if failed:
+                raise CheckoutError([failed])
+            return

-        msg = "Checking out '{}' with cache '{}'."
-        logger.debug(msg.format(str(path_info), checksum))
+        logger.debug("Checking out '%s' with cache '%s'.", path_info, checksum)

-        self._checkout(path_info, checksum, force, progress_callback, relink)
-        return None
+        return self._checkout(
+            path_info, checksum, force, progress_callback, relink, filter_info,
+        )

    def _checkout(
        self,
@@ -929,23 +1138,33 @@ class RemoteBASE(object):
        force=False,
        progress_callback=None,
        relink=False,
+        filter_info=None,
    ):
        if not self.is_dir_checksum(checksum):
            return self._checkout_file(
-                path_info, checksum, force, progress_callback=progress_callback
+                path_info, checksum, force, progress_callback, relink
            )
+
        return self._checkout_dir(
-            path_info, checksum, force, progress_callback, relink
+            path_info, checksum, force, progress_callback, relink, filter_info
        )

-    def get_files_number(self, checksum):
+    def get_files_number(self, path_info, checksum, filter_info):
+        from funcy.py3 import ilen
+
        if not checksum:
            return 0

-        if self.is_dir_checksum(checksum):
+        if not self.is_dir_checksum(checksum):
+            return 1
+
+        if not filter_info:
            return len(self.get_dir_cache(checksum))

-        return 1
+        return ilen(
+            filter_info.isin_or_eq(path_info / entry[self.PARAM_CHECKSUM])
+            for entry in self.get_dir_cache(checksum)
+        )

    @staticmethod
    def unprotect(path_info):
@@ -964,3 +1183,6 @@ class RemoteBASE(object):

    def _update_unpacked_dir(self, checksum):
        pass
+
+    def _remove_unpacked_dir(self, checksum):
+        pass
@@ -1,157 +0,0 @@
-import logging
-import os
-import posixpath
-
-from dvc.config import Config
-from dvc.config import ConfigError
-from dvc.utils import relpath
-from dvc.utils.compat import urlparse
-
-
-logger = logging.getLogger(__name__)
-
-
-class RemoteConfig(object):
-    def __init__(self, config):
-        self.config = config
-
-    def get_settings(self, name):
-        """
-        Args:
-            name (str): The name of the remote that we want to retrieve
-
-        Returns:
-            dict: The content beneath the given remote name.
-
-        Example:
-            >>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
-            >>> get_settings("server")
-            {'url': 'ssh://localhost/'}
-        """
-        settings = self.config.config.get(
-            Config.SECTION_REMOTE_FMT.format(name.lower())
-        )
-
-        if settings is None:
-            raise ConfigError(
-                "unable to find remote section '{}'".format(name)
-            )
-
-        parsed = urlparse(settings["url"])
-
-        # Support for cross referenced remotes.
-        # This will merge the settings, giving priority to the outer reference.
-        # For example, having:
-        #
-        #       dvc remote add server ssh://localhost
-        #       dvc remote modify server user root
-        #       dvc remote modify server ask_password true
-        #
-        #       dvc remote add images remote://server/tmp/pictures
-        #       dvc remote modify images user alice
-        #       dvc remote modify images ask_password false
-        #       dvc remote modify images password asdf1234
-        #
-        # Results on a config dictionary like:
-        #
-        #       {
-        #           "url": "ssh://localhost/tmp/pictures",
-        #           "user": "alice",
-        #           "password": "asdf1234",
-        #           "ask_password": False,
-        #       }
-        #
-        if parsed.scheme == "remote":
-            reference = self.get_settings(parsed.netloc)
-            url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
-            merged = reference.copy()
-            merged.update(settings)
-            merged["url"] = url
-            return merged
-
-        return settings
-
-    @staticmethod
-    def resolve_path(path, config_file):
-        """Resolve path relative to config file location.
-
-        Args:
-            path: Path to be resolved.
-            config_file: Path to config file, which `path` is specified
-                relative to.
-
-        Returns:
-            Path relative to the `config_file` location. If `path` is an
-            absolute path then it will be returned without change.
-
-        """
-        if os.path.isabs(path):
-            return path
-        return relpath(path, os.path.dirname(config_file))
-
-    def add(self, name, url, default=False, force=False, level=None):
-        from dvc.remote import _get, RemoteLOCAL
-
-        configobj = self.config.get_configobj(level)
-        remote = _get({Config.SECTION_REMOTE_URL: url})
-        if remote == RemoteLOCAL and not url.startswith("remote://"):
-            url = self.resolve_path(url, configobj.filename)
-
-        self.config.set(
-            Config.SECTION_REMOTE_FMT.format(name),
-            Config.SECTION_REMOTE_URL,
-            url,
-            force=force,
-            level=level,
-        )
-        if default:
-            self.config.set(
-                Config.SECTION_CORE,
-                Config.SECTION_CORE_REMOTE,
-                name,
-                level=level,
-            )
-
-    def remove(self, name, level=None):
-        self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level)
-
-        if level is None:
-            level = Config.LEVEL_REPO
-
-        for lev in [
-            Config.LEVEL_LOCAL,
-            Config.LEVEL_REPO,
-            Config.LEVEL_GLOBAL,
-            Config.LEVEL_SYSTEM,
-        ]:
-            self.config.unset(
-                Config.SECTION_CORE,
-                Config.SECTION_CORE_REMOTE,
-                level=lev,
-                force=True,
-            )
-            if lev == level:
-                break
-
-    def modify(self, name, option, value, level=None):
-        self.config.set(
-            Config.SECTION_REMOTE_FMT.format(name), option, value, level=level
-        )
-
-    def list(self, level=None):
-        return self.config.list_options(
-            Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level
-        )
-
-    def set_default(self, name, unset=False, level=None):
-        if unset:
-            self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE)
-            return
-        self.config.set(
-            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level
-        )
-
-    def get_default(self, level=None):
-        return self.config.get(
-            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level
-        )
@@ -1,97 +1,283 @@
-from __future__ import unicode_literals
-
+from collections import defaultdict
 import os
 import posixpath
 import logging
+import re
 import threading
+from urllib.parse import urlparse

-from funcy import retry, compose, decorator, wrap_with
+from funcy import retry, wrap_with, wrap_prop, cached_property
 from funcy.py3 import cat

 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
 from dvc.path_info import CloudURLInfo
 from dvc.remote.base import RemoteBASE
-from dvc.config import Config
 from dvc.exceptions import DvcException
-from dvc.utils import tmp_fname
+from dvc.utils import tmp_fname, format_link

 logger = logging.getLogger(__name__)
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"


-class GDriveRetriableError(DvcException):
-    def __init__(self, msg):
-        super(GDriveRetriableError, self).__init__(msg)
+class GDrivePathNotFound(DvcException):
+    def __init__(self, path_info):
+        super().__init__("Google Drive path '{}' not found.".format(path_info))


-@decorator
-def _wrap_pydrive_retriable(call):
-    from apiclient import errors
-    from pydrive.files import ApiRequestError
-
-    try:
-        result = call()
-    except (ApiRequestError, errors.HttpError) as exception:
-        retry_codes = ["403", "500", "502", "503", "504"]
-        if any(
-            "HttpError {}".format(code) in str(exception)
-            for code in retry_codes
-        ):
-            raise GDriveRetriableError(msg="Google API request failed")
-        raise
-    return result
+class GDriveAccessTokenRefreshError(DvcException):
+    def __init__(self):
+        super().__init__("Google Drive access token refreshment is failed.")


-gdrive_retry = compose(
-    # 8 tries, start at 0.5s, multiply by golden ratio, cap at 10s
-    retry(
-        8, GDriveRetriableError, timeout=lambda a: min(0.5 * 1.618 ** a, 10)
-    ),
-    _wrap_pydrive_retriable,
-)
+class GDriveMissedCredentialKeyError(DvcException):
+    def __init__(self, path):
+        super().__init__(
+            "Google Drive user credentials file '{}' "
+            "misses value for key.".format(path)
+        )
+
+
+def gdrive_retry(func):
+    def should_retry(exc):
+        from pydrive2.files import ApiRequestError
+
+        if not isinstance(exc, ApiRequestError):
+            return False
+
+        retry_codes = [403, 500, 502, 503, 504]
+        result = exc.error.get("code", 0) in retry_codes
+        if result:
+            logger.debug("Retry GDrive API call failed with {}.".format(exc))
+        return result
+
+    # 15 tries, start at 0.5s, multiply by golden ratio, cap at 20s
+    return retry(
+        15,
+        timeout=lambda a: min(0.5 * 1.618 ** a, 20),
+        filter_errors=should_retry,
+    )(func)
+
+
+def _location(exc):
+    from pydrive2.files import ApiRequestError
+
+    assert isinstance(exc, ApiRequestError)
+
+    # https://cloud.google.com/storage/docs/json_api/v1/status-codes#errorformat
+    return (
+        exc.error["errors"][0].get("location", "")
+        if exc.error.get("errors", [])
+        else ""
+    )
+
+
+class GDriveURLInfo(CloudURLInfo):
+    def __init__(self, url):
+        super().__init__(url)
+
+        # GDrive URL host part is case sensitive,
+        # we are restoring it here.
+        p = urlparse(url)
+        self.host = p.netloc
+        assert self.netloc == self.host
+
+        # Normalize path. Important since we have a cache (path to ID)
+        # and don't want to deal with different variations of path in it.
+        self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))


 class RemoteGDrive(RemoteBASE):
    scheme = Schemes.GDRIVE
-    path_cls = CloudURLInfo
-    REQUIRES = {"pydrive": "pydrive"}
-    GDRIVE_USER_CREDENTIALS_DATA = "GDRIVE_USER_CREDENTIALS_DATA"
-    DEFAULT_USER_CREDENTIALS_FILE = ".dvc/tmp/gdrive-user-credentials.json"
+    path_cls = GDriveURLInfo
+    REQUIRES = {"pydrive2": "pydrive2"}
+    DEFAULT_VERIFY = True
+    # Always prefer traverse for GDrive since API usage quotas are a concern.
+    TRAVERSE_WEIGHT_MULTIPLIER = 1
+    TRAVERSE_PREFIX_LEN = 2
+
+    GDRIVE_CREDENTIALS_DATA = "GDRIVE_CREDENTIALS_DATA"
+    DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"
+
+    DEFAULT_GDRIVE_CLIENT_ID = "710796635688-iivsgbgsb6uv1fap6635dhvuei09o66c.apps.googleusercontent.com"  # noqa: E501
+    DEFAULT_GDRIVE_CLIENT_SECRET = "a1Fz59uTpVNeG_VGuSKDLJXv"

    def __init__(self, repo, config):
-        super(RemoteGDrive, self).__init__(repo, config)
-        self.no_traverse = False
-        self.path_info = self.path_cls(config[Config.SECTION_REMOTE_URL])
-        self.config = config
-        self.init_drive()
+        super().__init__(repo, config)
+        self.path_info = self.path_cls(config["url"])

-    def init_drive(self):
-        self.client_id = self.config.get(Config.SECTION_GDRIVE_CLIENT_ID, None)
-        self.client_secret = self.config.get(
-            Config.SECTION_GDRIVE_CLIENT_SECRET, None
-        )
-        if not self.client_id or not self.client_secret:
+        if not self.path_info.bucket:
            raise DvcException(
-                "Please specify Google Drive's client id and "
-                "secret in DVC's config. Learn more at "
-                "https://man.dvc.org/remote/add."
+                "Empty Google Drive URL '{}'. Learn more at "
+                "{}.".format(
+                    config["url"],
+                    format_link("https://man.dvc.org/remote/add"),
+                )
            )
-        self.gdrive_user_credentials_path = (
-            tmp_fname(".dvc/tmp/")
-            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
-            else self.config.get(
-                Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE,
-                self.DEFAULT_USER_CREDENTIALS_FILE,
+
+        self._bucket = self.path_info.bucket
+        self._trash_only = config.get("gdrive_trash_only")
+        self._use_service_account = config.get("gdrive_use_service_account")
+        self._service_account_email = config.get(
+            "gdrive_service_account_email"
+        )
+        self._service_account_user_email = config.get(
+            "gdrive_service_account_user_email"
+        )
+        self._service_account_p12_file_path = config.get(
+            "gdrive_service_account_p12_file_path"
+        )
+        self._client_id = config.get("gdrive_client_id")
+        self._client_secret = config.get("gdrive_client_secret")
+        self._validate_config()
+        self._gdrive_user_credentials_path = (
+            tmp_fname(os.path.join(self.repo.tmp_dir, ""))
+            if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA)
+            else config.get(
+                "gdrive_user_credentials_file",
+                os.path.join(
+                    self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE
+                ),
            )
        )

+    def _validate_config(self):
+        # Validate Service Account configuration
+        if self._use_service_account and (
+            not self._service_account_email
+            or not self._service_account_p12_file_path
+        ):
+            raise DvcException(
+                "To use service account please specify {}, {} and "
+                "{} in DVC config. Learn more at "
+                "{}.".format(
+                    "gdrive_service_account_email",
+                    "gdrive_service_account_p12_file_path",
+                    "gdrive_service_account_user_email (optional)",
+                    format_link("https://man.dvc.org/remote/modify"),
+                )
+            )
+
+        # Validate OAuth 2.0 Client ID configuration
+        if not self._use_service_account:
+            if bool(self._client_id) != bool(self._client_secret):
+                raise DvcException(
+                    "Please specify Google Drive's client id and secret in "
+                    "DVC config or omit both to use defaults. Learn more at "
+                    "{}.".format(
+                        format_link("https://man.dvc.org/remote/modify")
+                    )
+                )
+
+    @wrap_prop(threading.RLock())
+    @cached_property
+    def drive(self):
+        from pydrive2.auth import RefreshError
+        from pydrive2.auth import GoogleAuth
+        from pydrive2.drive import GoogleDrive
+
+        if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA):
+            with open(
+                self._gdrive_user_credentials_path, "w"
+            ) as credentials_file:
+                credentials_file.write(
+                    os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA)
+                )
+
+        GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
+        if self._use_service_account:
+            GoogleAuth.DEFAULT_SETTINGS["service_config"] = {
+                "client_service_email": self._service_account_email,
+                "client_user_email": self._service_account_user_email,
+                "client_pkcs12_file_path": self._service_account_p12_file_path,
+            }
+        else:
+            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
+                "client_id": self._client_id or self.DEFAULT_GDRIVE_CLIENT_ID,
+                "client_secret": self._client_secret
+                or self.DEFAULT_GDRIVE_CLIENT_SECRET,
+                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                "token_uri": "https://oauth2.googleapis.com/token",
+                "revoke_uri": "https://oauth2.googleapis.com/revoke",
+                "redirect_uri": "",
+            }
+        GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
+        GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
+        GoogleAuth.DEFAULT_SETTINGS[
+            "save_credentials_file"
+        ] = self._gdrive_user_credentials_path
+        GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
+        GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
+            "https://www.googleapis.com/auth/drive",
+            "https://www.googleapis.com/auth/drive.appdata",
+        ]
+
+        # Pass non existent settings path to force DEFAULT_SETTINGS loading
+        gauth = GoogleAuth(settings_file="")
+
+        try:
+            if self._use_service_account:
+                gauth.ServiceAuth()
+            else:
+                gauth.CommandLineAuth()
+        except RefreshError as exc:
+            raise GDriveAccessTokenRefreshError from exc
+        except KeyError as exc:
+            raise GDriveMissedCredentialKeyError(
+                self._gdrive_user_credentials_path
+            ) from exc
+        # Handle pydrive2.auth.AuthenticationError and other auth failures
+        except Exception as exc:
+            raise DvcException("Google Drive authentication failed") from exc
+        finally:
+            if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA):
+                os.remove(self._gdrive_user_credentials_path)
+
+        return GoogleDrive(gauth)
+
+    @wrap_prop(threading.RLock())
+    @cached_property
+    def cache(self):
+        cache = {"dirs": defaultdict(list), "ids": {}}
+
+        cache["root_id"] = self._get_remote_id(self.path_info)
+        cache["dirs"][self.path_info.path] = [cache["root_id"]]
+        self._cache_path(self.path_info.path, cache["root_id"], cache)
+
+        for item in self.gdrive_list_item(
+            "'{}' in parents and trashed=false".format(cache["root_id"])
+        ):
+            remote_path = (self.path_info / item["title"]).path
+            self._cache_path(remote_path, item["id"], cache)
+
+        return cache
+
+    def _cache_path(self, remote_path, remote_id, cache=None):
+        cache = cache or self.cache
+        cache["dirs"][remote_path].append(remote_id)
+        cache["ids"][remote_id] = remote_path
+
+    @cached_property
+    def list_params(self):
+        params = {"corpora": "default"}
+        if self._bucket != "root" and self._bucket != "appDataFolder":
+            drive_id = self._get_remote_drive_id(self._bucket)
+            if drive_id:
+                params["driveId"] = drive_id
+                params["corpora"] = "drive"
+        return params
+
    @gdrive_retry
    def gdrive_upload_file(
-        self, args, no_progress_bar=True, from_file="", progress_name=""
+        self,
+        parent_id,
+        title,
+        no_progress_bar=False,
+        from_file="",
+        progress_name="",
    ):
        item = self.drive.CreateFile(
-            {"title": args["title"], "parents": [{"id": args["parent_id"]}]}
+            {"title": title, "parents": [{"id": parent_id}]}
        )

        with open(from_file, "rb") as fobj:
@@ -114,9 +300,11 @@ class RemoteGDrive(RemoteBASE):
    def gdrive_download_file(
        self, file_id, to_file, progress_name, no_progress_bar
    ):
-        gdrive_file = self.drive.CreateFile({"id": file_id})
+        param = {"id": file_id}
+        # it does not create a file on the remote
+        gdrive_file = self.drive.CreateFile(param)
        bar_format = (
-            "Donwloading {desc:{ncols_desc}.{ncols_desc}}... "
+            "Downloading {desc:{ncols_desc}.{ncols_desc}}... "
            + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
        )
        with Tqdm(
@@ -125,7 +313,10 @@ class RemoteGDrive(RemoteBASE):
            gdrive_file.GetContentFile(to_file)

    def gdrive_list_item(self, query):
-        file_list = self.drive.ListFile({"q": query, "maxResults": 1000})
+        param = {"q": query, "maxResults": 1000}
+        param.update(self.list_params)
+
+        file_list = self.drive.ListFile(param)

        # Isolate and decorate fetching of remote drive items in pages
        get_list = gdrive_retry(lambda: next(file_list, None))
@@ -133,174 +324,166 @@ class RemoteGDrive(RemoteBASE):
        # Fetch pages until None is received, lazily flatten the thing
        return cat(iter(get_list, None))

-    def cache_root_dirs(self):
-        cached_dirs = {}
-        cached_ids = {}
-        for dir1 in self.gdrive_list_item(
-            "'{}' in parents and trashed=false".format(self.root_id)
-        ):
-            remote_path = posixpath.join(self.path_info.path, dir1["title"])
-            cached_dirs.setdefault(remote_path, []).append(dir1["id"])
-            cached_ids[dir1["id"]] = dir1["title"]
-        return cached_dirs, cached_ids
-
-    @property
-    def cached_dirs(self):
-        if not hasattr(self, "_cached_dirs"):
-            self.drive
-        return self._cached_dirs
-
-    @property
-    def cached_ids(self):
-        if not hasattr(self, "_cached_ids"):
-            self.drive
-        return self._cached_ids
-
-    @property
    @wrap_with(threading.RLock())
-    def drive(self):
-        if not hasattr(self, "_gdrive"):
-            from pydrive.auth import GoogleAuth
-            from pydrive.drive import GoogleDrive
+    def gdrive_create_dir(self, parent_id, title, remote_path):
+        cached = self.cache["dirs"].get(remote_path)
+        if cached:
+            return cached[0]

-            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
-                with open(
-                    self.gdrive_user_credentials_path, "w"
-                ) as credentials_file:
-                    credentials_file.write(
-                        os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
-                    )
+        item = self._create_remote_dir(parent_id, title)

-            GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
-            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
-                "client_id": self.client_id,
-                "client_secret": self.client_secret,
-                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-                "token_uri": "https://oauth2.googleapis.com/token",
-                "revoke_uri": "https://oauth2.googleapis.com/revoke",
-                "redirect_uri": "",
-            }
-            GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
-            GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
-            GoogleAuth.DEFAULT_SETTINGS[
-                "save_credentials_file"
-            ] = self.gdrive_user_credentials_path
-            GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
-            GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
-                "https://www.googleapis.com/auth/drive",
-                "https://www.googleapis.com/auth/drive.appdata",
-            ]
+        if parent_id == self.cache["root_id"]:
+            self._cache_path(remote_path, item["id"])

-            # Pass non existent settings path to force DEFAULT_SETTINGS loading
-            gauth = GoogleAuth(settings_file="")
-            gauth.CommandLineAuth()
-
-            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
-                os.remove(self.gdrive_user_credentials_path)
-
-            self._gdrive = GoogleDrive(gauth)
-
-            self.root_id = self.get_remote_id(self.path_info, create=True)
-            self._cached_dirs, self._cached_ids = self.cache_root_dirs()
-
-        return self._gdrive
+        return item["id"]

    @gdrive_retry
-    def create_remote_dir(self, parent_id, title):
+    def _create_remote_dir(self, parent_id, title):
+        parent = {"id": parent_id}
        item = self.drive.CreateFile(
-            {
-                "title": title,
-                "parents": [{"id": parent_id}],
-                "mimeType": FOLDER_MIME_TYPE,
-            }
+            {"title": title, "parents": [parent], "mimeType": FOLDER_MIME_TYPE}
        )
        item.Upload()
        return item

    @gdrive_retry
-    def get_remote_item(self, name, parents_ids):
+    def _delete_remote_file(self, remote_id):
+        from pydrive2.files import ApiRequestError
+
+        param = {"id": remote_id}
+        # it does not create a file on the remote
+        item = self.drive.CreateFile(param)
+
+        try:
+            item.Trash() if self._trash_only else item.Delete()
+        except ApiRequestError as exc:
+            http_error_code = exc.error.get("code", 0)
+            if (
+                http_error_code == 403
+                and self.list_params["corpora"] == "drive"
+                and _location(exc) == "file.permissions"
+            ):
+                raise DvcException(
+                    "Insufficient permissions to {}. You should have {} "
+                    "access level for the used shared drive. More details "
+                    "at {}.".format(
+                        "move the file into Trash"
+                        if self._trash_only
+                        else "permanently delete the file",
+                        "Manager or Content Manager"
+                        if self._trash_only
+                        else "Manager",
+                        "https://support.google.com/a/answer/7337554",
+                    )
+                ) from exc
+            raise
+
+    @gdrive_retry
+    def _get_remote_item(self, name, parents_ids):
        if not parents_ids:
            return None
-        query = " or ".join(
-            "'{}' in parents".format(parent_id) for parent_id in parents_ids
+        query = "({})".format(
+            " or ".join(
+                "'{}' in parents".format(parent_id)
+                for parent_id in parents_ids
+            )
        )

        query += " and trashed=false and title='{}'".format(name)

+        # Remote might contain items with duplicated path (titles).
+        # We thus limit number of items.
+        param = {"q": query, "maxResults": 1}
+        param.update(self.list_params)
+
        # Limit found remote items count to 1 in response
-        item_list = self.drive.ListFile(
-            {"q": query, "maxResults": 1}
-        ).GetList()
+        item_list = self.drive.ListFile(param).GetList()
        return next(iter(item_list), None)

-    def resolve_remote_item_from_path(self, path_parts, create):
-        parents_ids = ["root"]
-        current_path = ""
-        for path_part in path_parts:
-            current_path = posixpath.join(current_path, path_part)
-            remote_ids = self.get_remote_id_from_cache(current_path)
-            if remote_ids:
-                parents_ids = remote_ids
-                continue
-            item = self.get_remote_item(path_part, parents_ids)
-            if not item and create:
-                item = self.create_remote_dir(parents_ids[0], path_part)
-            elif not item:
-                return None
-            parents_ids = [item["id"]]
-        return item
+    @gdrive_retry
+    def _get_remote_drive_id(self, remote_id):
+        param = {"id": remote_id}
+        # it does not create a file on the remote
+        item = self.drive.CreateFile(param)
+        item.FetchMetadata("driveId")
+        return item.get("driveId", None)

-    def get_remote_id_from_cache(self, remote_path):
-        if hasattr(self, "_cached_dirs"):
-            return self.cached_dirs.get(remote_path, [])
+    def _get_cached_remote_ids(self, path):
+        if not path:
+            return [self._bucket]
+        if "cache" in self.__dict__:
+            return self.cache["dirs"].get(path, [])
        return []

-    def get_remote_id(self, path_info, create=False):
-        remote_ids = self.get_remote_id_from_cache(path_info.path)
-
+    def _path_to_remote_ids(self, path, create):
+        remote_ids = self._get_cached_remote_ids(path)
        if remote_ids:
-            return remote_ids[0]
+            return remote_ids

-        file1 = self.resolve_remote_item_from_path(
-            path_info.path.split("/"), create
-        )
-        return file1["id"] if file1 else ""
+        parent_path, part = posixpath.split(path)
+        parent_ids = self._path_to_remote_ids(parent_path, create)
+        item = self._get_remote_item(part, parent_ids)
+
+        if not item:
+            return (
+                [self.gdrive_create_dir(parent_ids[0], part, path)]
+                if create
+                else []
+            )
+
+        return [item["id"]]
+
+    def _get_remote_id(self, path_info, create=False):
+        assert path_info.bucket == self._bucket
+
+        remote_ids = self._path_to_remote_ids(path_info.path, create)
+        if not remote_ids:
+            raise GDrivePathNotFound(path_info)
+
+        return remote_ids[0]

    def exists(self, path_info):
-        return self.get_remote_id(path_info) != ""
+        try:
+            self._get_remote_id(path_info)
+        except GDrivePathNotFound:
+            return False
+        else:
+            return True

    def _upload(self, from_file, to_info, name, no_progress_bar):
        dirname = to_info.parent
-        if dirname:
-            parent_id = self.get_remote_id(dirname, True)
-        else:
-            parent_id = to_info.bucket
+        assert dirname
+        parent_id = self._get_remote_id(dirname, True)

        self.gdrive_upload_file(
-            {"title": to_info.name, "parent_id": parent_id},
-            no_progress_bar,
-            from_file,
-            name,
+            parent_id, to_info.name, no_progress_bar, from_file, name
        )

    def _download(self, from_info, to_file, name, no_progress_bar):
-        file_id = self.get_remote_id(from_info)
+        file_id = self._get_remote_id(from_info)
        self.gdrive_download_file(file_id, to_file, name, no_progress_bar)

-    def all(self):
-        if not self.cached_ids:
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        if not self.cache["ids"]:
            return

-        query = " or ".join(
-            "'{}' in parents".format(dir_id) for dir_id in self.cached_ids
+        if prefix:
+            dir_ids = self.cache["dirs"].get(prefix[:2])
+            if not dir_ids:
+                return
+        else:
+            dir_ids = self.cache["ids"]
+        parents_query = " or ".join(
+            "'{}' in parents".format(dir_id) for dir_id in dir_ids
        )
+        query = "({}) and trashed=false".format(parents_query)

-        query += " and trashed=false"
-        for file1 in self.gdrive_list_item(query):
-            parent_id = file1["parents"][0]["id"]
-            path = posixpath.join(self.cached_ids[parent_id], file1["title"])
-            try:
-                yield self.path_to_checksum(path)
-            except ValueError:
-                # We ignore all the non-cache looking files
-                logger.debug('Ignoring path as "non-cache looking"')
+        for item in self.gdrive_list_item(query):
+            if progress_callback:
+                progress_callback()
+            parent_id = item["parents"][0]["id"]
+            yield posixpath.join(self.cache["ids"][parent_id], item["title"])
+
+    def remove(self, path_info):
+        remote_id = self._get_remote_id(path_info)
+        self._delete_remote_file(remote_id)
@@ -1,21 +1,18 @@
-from __future__ import unicode_literals, division
-
 import logging
 from datetime import timedelta
 from functools import wraps
 import io
 import os.path
+import posixpath
 import threading

 from funcy import cached_property, wrap_prop

-from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
-from dvc.utils.compat import FileNotFoundError  # skipcq: PYL-W0622

 logger = logging.getLogger(__name__)

@@ -55,7 +52,7 @@ def _upload_to_bucket(
    to_info,
    chunk_size=None,
    name=None,
-    no_progress_bar=True,
+    no_progress_bar=False,
 ):
    blob = bucket.blob(to_info.path, chunk_size=chunk_size)
    with io.open(from_file, mode="rb") as fobj:
@@ -76,13 +73,13 @@ class RemoteGS(RemoteBASE):
    PARAM_CHECKSUM = "md5"

    def __init__(self, repo, config):
-        super(RemoteGS, self).__init__(repo, config)
+        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL, "gs:///")
+        url = config.get("url", "gs:///")
        self.path_info = self.path_cls(url)

-        self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
-        self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
+        self.projectname = config.get("projectname", None)
+        self.credentialpath = config.get("credentialpath")

    @wrap_prop(threading.Lock())
    @cached_property
@@ -130,14 +127,24 @@ class RemoteGS(RemoteBASE):

        blob.delete()

-    def _list_paths(self, path_info, max_items=None):
+    def _list_paths(
+        self, path_info, max_items=None, prefix=None, progress_callback=None
+    ):
+        if prefix:
+            prefix = posixpath.join(path_info.path, prefix[:2], prefix[2:])
+        else:
+            prefix = path_info.path
        for blob in self.gs.bucket(path_info.bucket).list_blobs(
            prefix=path_info.path, max_results=max_items
        ):
+            if progress_callback:
+                progress_callback()
            yield blob.name

-    def list_cache_paths(self):
-        return self._list_paths(self.path_info)
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        return self._list_paths(
+            self.path_info, prefix=prefix, progress_callback=progress_callback
+        )

    def walk_files(self, path_info):
        for fname in self._list_paths(path_info / ""):
@@ -147,6 +154,9 @@ class RemoteGS(RemoteBASE):
            yield path_info.replace(fname)

    def makedirs(self, path_info):
+        if not path_info.path:
+            return
+
        self.gs.bucket(path_info.bucket).blob(
            (path_info / "").path
        ).upload_from_string("")
@@ -170,7 +180,7 @@ class RemoteGS(RemoteBASE):
        """
        return self.isfile(path_info) or self.isdir(path_info)

-    def _upload(self, from_file, to_info, name=None, no_progress_bar=True):
+    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
        bucket = self.gs.bucket(to_info.bucket)
        _upload_to_bucket(
            bucket,
@@ -180,7 +190,7 @@ class RemoteGS(RemoteBASE):
            no_progress_bar=no_progress_bar,
        )

-    def _download(self, from_info, to_file, name=None, no_progress_bar=True):
+    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
        bucket = self.gs.bucket(from_info.bucket)
        blob = bucket.get_blob(from_info.path)
        with io.open(to_file, mode="wb") as fobj:
@@ -1,18 +1,15 @@
-from __future__ import unicode_literals
-
-from dvc.utils.compat import FileNotFoundError, urlparse
 import io
 import logging
 import os
 import posixpath
 import re
+import subprocess
 from collections import deque
 from contextlib import closing, contextmanager
-import subprocess
+from urllib.parse import urlparse

 from .base import RemoteBASE, RemoteCmdError
 from .pool import get_connection
-from dvc.config import Config
 from dvc.scheme import Schemes
 from dvc.utils import fix_env, tmp_fname

@@ -24,17 +21,17 @@ class RemoteHDFS(RemoteBASE):
    REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
    PARAM_CHECKSUM = "checksum"
    REQUIRES = {"pyarrow": "pyarrow"}
+    TRAVERSE_PREFIX_LEN = 2

    def __init__(self, repo, config):
-        super(RemoteHDFS, self).__init__(repo, config)
+        super().__init__(repo, config)
        self.path_info = None
-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
        if not url:
            return

        parsed = urlparse(url)
-
-        user = parsed.username or config.get(Config.SECTION_REMOTE_USER)
+        user = parsed.username or config.get("user")

        self.path_info = self.path_cls.from_parts(
            scheme=self.scheme,
@@ -156,16 +153,29 @@ class RemoteHDFS(RemoteBASE):
                raise FileNotFoundError(*e.args)
            raise

-    def list_cache_paths(self):
+    def list_cache_paths(self, prefix=None, progress_callback=None):
        if not self.exists(self.path_info):
            return

-        dirs = deque([self.path_info.path])
+        if prefix:
+            root = posixpath.join(self.path_info.path, prefix[:2])
+        else:
+            root = self.path_info.path
+        dirs = deque([root])

        with self.hdfs(self.path_info) as hdfs:
            while dirs:
-                for entry in hdfs.ls(dirs.pop(), detail=True):
-                    if entry["kind"] == "directory":
-                        dirs.append(urlparse(entry["name"]).path)
-                    elif entry["kind"] == "file":
-                        yield urlparse(entry["name"]).path
+                try:
+                    entries = hdfs.ls(dirs.pop(), detail=True)
+                    for entry in entries:
+                        if entry["kind"] == "directory":
+                            dirs.append(urlparse(entry["name"]).path)
+                        elif entry["kind"] == "file":
+                            if progress_callback:
+                                progress_callback()
+                            yield urlparse(entry["name"]).path
+                except IOError as e:
+                    # When searching for a specific prefix pyarrow raises an
+                    # exception if the specified cache dir does not exist
+                    if not prefix:
+                        raise e
@@ -1,56 +1,96 @@
-from __future__ import unicode_literals
-
 import logging
+import os.path
 import threading

-from funcy import cached_property, wrap_prop
+from funcy import cached_property, memoize, wrap_prop, wrap_with

-from dvc.config import Config
-from dvc.config import ConfigError
+from dvc.path_info import HTTPURLInfo
+import dvc.prompt as prompt
 from dvc.exceptions import DvcException, HTTPError
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
-from dvc.utils.compat import open

 logger = logging.getLogger(__name__)


+@wrap_with(threading.Lock())
+@memoize
+def ask_password(host, user):
+    return prompt.password(
+        "Enter a password for "
+        "host '{host}' user '{user}'".format(host=host, user=user)
+    )
+
+
 class RemoteHTTP(RemoteBASE):
    scheme = Schemes.HTTP
+    path_cls = HTTPURLInfo
    SESSION_RETRIES = 5
    SESSION_BACKOFF_FACTOR = 0.1
    REQUEST_TIMEOUT = 10
    CHUNK_SIZE = 2 ** 16
    PARAM_CHECKSUM = "etag"
+    CAN_TRAVERSE = False

    def __init__(self, repo, config):
-        super(RemoteHTTP, self).__init__(repo, config)
+        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL)
-        self.path_info = self.path_cls(url) if url else None
+        url = config.get("url")
+        if url:
+            self.path_info = self.path_cls(url)
+            user = config.get("user", None)
+            if user:
+                self.path_info.user = user
+        else:
+            self.path_info = None

-        if not self.no_traverse:
-            raise ConfigError(
-                "HTTP doesn't support traversing the remote to list existing "
-                "files. Use: `dvc remote modify <name> no_traverse true`"
-            )
+        self.auth = config.get("auth", None)
+        self.custom_auth_header = config.get("custom_auth_header", None)
+        self.password = config.get("password", None)
+        self.ask_password = config.get("ask_password", False)
+        self.headers = {}

    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
        response = self._request("GET", from_info.url, stream=True)
        if response.status_code != 200:
            raise HTTPError(response.status_code, response.reason)
-        with Tqdm(
-            total=None if no_progress_bar else self._content_length(response),
-            leave=False,
-            bytes=True,
-            desc=from_info.url if name is None else name,
-            disable=no_progress_bar,
-        ) as pbar:
-            with open(to_file, "wb") as fd:
+        with open(to_file, "wb") as fd:
+            with Tqdm.wrapattr(
+                fd,
+                "write",
+                total=None
+                if no_progress_bar
+                else self._content_length(response),
+                leave=False,
+                desc=from_info.url if name is None else name,
+                disable=no_progress_bar,
+            ) as fd_wrapped:
                for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE):
-                    fd.write(chunk)
-                    pbar.update(len(chunk))
+                    fd_wrapped.write(chunk)
+
+    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
+        def chunks():
+            with open(from_file, "rb") as fd:
+                with Tqdm.wrapattr(
+                    fd,
+                    "read",
+                    total=None
+                    if no_progress_bar
+                    else os.path.getsize(from_file),
+                    leave=False,
+                    desc=to_info.url if name is None else name,
+                    disable=no_progress_bar,
+                ) as fd_wrapped:
+                    while True:
+                        chunk = fd_wrapped.read(self.CHUNK_SIZE)
+                        if not chunk:
+                            break
+                        yield chunk
+
+        response = self._request("POST", to_info.url, data=chunks())
+        if response.status_code not in (200, 201):
+            raise HTTPError(response.status_code, response.reason)

    def exists(self, path_info):
        return bool(self._request("HEAD", path_info.url))
@@ -78,6 +118,24 @@ class RemoteHTTP(RemoteBASE):

        return etag

+    def auth_method(self, path_info=None):
+        from requests.auth import HTTPBasicAuth, HTTPDigestAuth
+
+        if path_info is None:
+            path_info = self.path_info
+
+        if self.auth:
+            if self.ask_password and self.password is None:
+                host, user = path_info.host, path_info.user
+                self.password = ask_password(host, user)
+            if self.auth == "basic":
+                return HTTPBasicAuth(path_info.user, self.password)
+            if self.auth == "digest":
+                return HTTPDigestAuth(path_info.user, self.password)
+            if self.auth == "custom" and self.custom_auth_header:
+                self.headers.update({self.custom_auth_header: self.password})
+        return None
+
    @wrap_prop(threading.Lock())
    @cached_property
    def _session(self):
@@ -104,7 +162,28 @@ class RemoteHTTP(RemoteBASE):
        kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)

        try:
-            return self._session.request(method, url, **kwargs)
+            res = self._session.request(
+                method,
+                url,
+                auth=self.auth_method(),
+                headers=self.headers,
+                **kwargs,
+            )
+
+            redirect_no_location = (
+                kwargs["allow_redirects"]
+                and res.status_code in (301, 302)
+                and "location" not in res.headers
+            )
+
+            if redirect_no_location:
+                # AWS s3 doesn't like to add a location header to its redirects
+                # from https://s3.amazonaws.com/<bucket name>/* type URLs.
+                # This should be treated as an error
+                raise requests.exceptions.RequestException
+
+            return res
+
        except requests.exceptions.RequestException:
            raise DvcException("could not perform a {} request".format(method))

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from .http import RemoteHTTP
 from dvc.scheme import Schemes

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import errno
 import logging
 import os
@@ -9,30 +7,17 @@ from functools import partial

 from shortuuid import uuid

-from dvc.config import Config
-from dvc.exceptions import DownloadError
-from dvc.exceptions import DvcException
-from dvc.exceptions import UploadError
+from dvc.compat import fspath_py35
+from dvc.exceptions import DvcException, DownloadError, UploadError
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
-from dvc.remote.base import STATUS_DELETED
-from dvc.remote.base import STATUS_MAP
-from dvc.remote.base import STATUS_MISSING
-from dvc.remote.base import STATUS_NEW
+from dvc.remote.base import RemoteBASE, STATUS_MAP
+from dvc.remote.base import STATUS_DELETED, STATUS_MISSING, STATUS_NEW
 from dvc.scheme import Schemes
+from dvc.scm.tree import is_working_tree
 from dvc.system import System
-from dvc.utils import copyfile
-from dvc.utils import file_md5
-from dvc.utils import makedirs
-from dvc.utils import relpath
-from dvc.utils import tmp_fname
-from dvc.utils import walk_files
-from dvc.utils.compat import fspath_py35
-from dvc.utils.compat import open
-from dvc.utils.compat import str
-from dvc.utils.fs import move
-from dvc.utils.fs import remove
+from dvc.utils import file_md5, relpath, tmp_fname
+from dvc.utils.fs import copyfile, move, makedirs, remove, walk_files

 logger = logging.getLogger(__name__)

@@ -47,26 +32,12 @@ class RemoteLOCAL(RemoteBASE):

    DEFAULT_CACHE_TYPES = ["reflink", "copy"]

+    CACHE_MODE = 0o444
    SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}

    def __init__(self, repo, config):
-        super(RemoteLOCAL, self).__init__(repo, config)
-        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
-
-        shared = config.get(Config.SECTION_CACHE_SHARED)
-        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
-
-        if self.protected:
-            # cache files are set to be read-only for everyone
-            self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
-
-        cache_dir = config.get(Config.SECTION_REMOTE_URL)
-
-        if cache_dir is not None and not os.path.isabs(cache_dir):
-            cwd = config[Config.PRIVATE_CWD]
-            cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
-
-        self.path_info = PathInfo(cache_dir) if cache_dir else None
+        super().__init__(repo, config)
+        self.cache_dir = config.get("url")
        self._dir_info = {}

    @property
@@ -77,13 +48,23 @@ class RemoteLOCAL(RemoteBASE):
    def cache_dir(self):
        return self.path_info.fspath if self.path_info else None

+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.path_info = PathInfo(value) if value else None
+
    @classmethod
    def supported(cls, config):
        return True

-    def list_cache_paths(self):
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        assert prefix is None
        assert self.path_info is not None
-        return walk_files(self.path_info, None)
+        if progress_callback:
+            for path in walk_files(self.path_info):
+                progress_callback()
+                yield path
+        else:
+            yield from walk_files(self.path_info)

    def get(self, md5):
        if not md5:
@@ -109,6 +90,12 @@ class RemoteLOCAL(RemoteBASE):

        return not self.changed_cache(current_md5)

+    def _verify_link(self, path_info, link_type):
+        if link_type == "hardlink" and self.getsize(path_info) == 0:
+            return
+
+        super()._verify_link(path_info, link_type)
+
    def is_empty(self, path_info):
        path = path_info.fspath

@@ -138,7 +125,9 @@ class RemoteLOCAL(RemoteBASE):
        return os.path.getsize(fspath_py35(path_info))

    def walk_files(self, path_info):
-        for fname in walk_files(path_info, self.repo.dvcignore):
+        assert is_working_tree(self.repo.tree)
+
+        for fname in self.repo.tree.walk_files(path_info):
            yield PathInfo(fname)

    def get_file_checksum(self, path_info):
@@ -151,16 +140,17 @@ class RemoteLOCAL(RemoteBASE):
        if self.exists(path_info):
            remove(path_info.fspath)

-    def move(self, from_info, to_info):
+    def move(self, from_info, to_info, mode=None):
        if from_info.scheme != "local" or to_info.scheme != "local":
            raise NotImplementedError

        self.makedirs(to_info.parent)

-        if self.isfile(from_info):
-            mode = self._file_mode
-        else:
-            mode = self._dir_mode
+        if mode is None:
+            if self.isfile(from_info):
+                mode = self._file_mode
+            else:
+                mode = self._dir_mode

        move(from_info, to_info, mode=mode)

@@ -168,6 +158,7 @@ class RemoteLOCAL(RemoteBASE):
        tmp_info = to_info.parent / tmp_fname(to_info.name)
        try:
            System.copy(from_info, tmp_info)
+            os.chmod(fspath_py35(tmp_info), self._file_mode)
            os.rename(fspath_py35(tmp_info), fspath_py35(to_info))
        except Exception:
            self.remove(tmp_info)
@@ -177,6 +168,10 @@ class RemoteLOCAL(RemoteBASE):
    def symlink(from_info, to_info):
        System.symlink(from_info, to_info)

+    @staticmethod
+    def is_symlink(path_info):
+        return System.is_symlink(path_info)
+
    def hardlink(self, from_info, to_info):
        # If there are a lot of empty files (which happens a lot in datasets),
        # and the cache type is `hardlink`, we might reach link limits and
@@ -204,8 +199,16 @@ class RemoteLOCAL(RemoteBASE):
        System.hardlink(from_info, to_info)

    @staticmethod
-    def reflink(from_info, to_info):
-        System.reflink(from_info, to_info)
+    def is_hardlink(path_info):
+        return System.is_hardlink(path_info)
+
+    def reflink(self, from_info, to_info):
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        System.reflink(from_info, tmp_info)
+        # NOTE: reflink has its own separate inode, so you can set permissions
+        # that are different from the source.
+        os.chmod(fspath_py35(tmp_info), self._file_mode)
+        os.rename(fspath_py35(tmp_info), fspath_py35(to_info))

    def cache_exists(self, checksums, jobs=None, name=None):
        return [
@@ -403,11 +406,10 @@ class RemoteLOCAL(RemoteBASE):
            )
            logger.warning(msg)

-    @staticmethod
-    def _unprotect_file(path):
+    def _unprotect_file(self, path):
        if System.is_symlink(path) or System.is_hardlink(path):
            logger.debug("Unprotecting '{}'".format(path))
-            tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
+            tmp = os.path.join(os.path.dirname(path), "." + uuid())

            # The operations order is important here - if some application
            # would access the file during the process of copyfile then it
@@ -424,11 +426,13 @@ class RemoteLOCAL(RemoteBASE):
                "a symlink or a hardlink.".format(path)
            )

-        os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
+        os.chmod(path, self._file_mode)

    def _unprotect_dir(self, path):
-        for fname in walk_files(path, self.repo.dvcignore):
-            RemoteLOCAL._unprotect_file(fname)
+        assert is_working_tree(self.repo.tree)
+
+        for fname in self.repo.tree.walk_files(path):
+            self._unprotect_file(fname)

    def unprotect(self, path_info):
        path = path_info.fspath
@@ -440,29 +444,36 @@ class RemoteLOCAL(RemoteBASE):
        if os.path.isdir(path):
            self._unprotect_dir(path)
        else:
-            RemoteLOCAL._unprotect_file(path)
+            self._unprotect_file(path)

-    @staticmethod
-    def protect(path_info):
+    def protect(self, path_info):
        path = fspath_py35(path_info)
-        mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
+        mode = self.CACHE_MODE

        try:
            os.chmod(path, mode)
        except OSError as exc:
-            # In share cache scenario, we might not own the cache file, so we
+            # There is nothing we need to do in case of a read-only file system
+            if exc.errno == errno.EROFS:
+                return
+
+            # In shared cache scenario, we might not own the cache file, so we
            # need to check if cache file is already protected.
            if exc.errno not in [errno.EPERM, errno.EACCES]:
                raise

-            actual = os.stat(path).st_mode
-            if actual & mode != mode:
+            actual = stat.S_IMODE(os.stat(path).st_mode)
+            if actual != mode:
                raise

    def _get_unpacked_dir_path_info(self, checksum):
        info = self.checksum_to_path_info(checksum)
        return info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)

+    def _remove_unpacked_dir(self, checksum):
+        path_info = self._get_unpacked_dir_path_info(checksum)
+        self.remove(path_info)
+
    def _path_info_changed(self, path_info):
        if self.exists(path_info) and self.state.get(path_info):
            return False
@@ -514,3 +525,11 @@ class RemoteLOCAL(RemoteBASE):
            if self.is_dir_checksum(c):
                unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
        return unpacked
+
+    def is_protected(self, path_info):
+        if not self.exists(path_info):
+            return False
+
+        mode = os.stat(fspath_py35(path_info)).st_mode
+
+        return stat.S_IMODE(mode) == self.CACHE_MODE
@@ -1,13 +1,10 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 import logging
 import os
+import posixpath
 import threading

 from funcy import cached_property, wrap_prop

-from dvc.config import Config
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
@@ -41,25 +38,24 @@ class RemoteOSS(RemoteBASE):
    REQUIRES = {"oss2": "oss2"}
    PARAM_CHECKSUM = "etag"
    COPY_POLL_SECONDS = 5
+    LIST_OBJECT_PAGE_SIZE = 100

    def __init__(self, repo, config):
-        super(RemoteOSS, self).__init__(repo, config)
+        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
        self.path_info = self.path_cls(url) if url else None

-        self.endpoint = config.get(Config.SECTION_OSS_ENDPOINT) or os.getenv(
-            "OSS_ENDPOINT"
-        )
+        self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")

        self.key_id = (
-            config.get(Config.SECTION_OSS_ACCESS_KEY_ID)
+            config.get("oss_key_id")
            or os.getenv("OSS_ACCESS_KEY_ID")
            or "defaultId"
        )

        self.key_secret = (
-            config.get(Config.SECTION_OSS_ACCESS_KEY_SECRET)
+            config.get("oss_key_secret")
            or os.getenv("OSS_ACCESS_KEY_SECRET")
            or "defaultSecret"
        )
@@ -69,9 +65,9 @@ class RemoteOSS(RemoteBASE):
    def oss_service(self):
        import oss2

-        logger.debug("URL {}".format(self.path_info))
-        logger.debug("key id {}".format(self.key_id))
-        logger.debug("key secret {}".format(self.key_secret))
+        logger.debug("URL: {}".format(self.path_info))
+        logger.debug("key id: {}".format(self.key_id))
+        logger.debug("key secret: {}".format(self.key_secret))

        auth = oss2.Auth(self.key_id, self.key_secret)
        bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)
@@ -95,14 +91,22 @@ class RemoteOSS(RemoteBASE):
        logger.debug("Removing oss://{}".format(path_info))
        self.oss_service.delete_object(path_info.path)

-    def _list_paths(self, prefix):
+    def _list_paths(self, prefix, progress_callback=None):
        import oss2

        for blob in oss2.ObjectIterator(self.oss_service, prefix=prefix):
+            if progress_callback:
+                progress_callback()
            yield blob.key

-    def list_cache_paths(self):
-        return self._list_paths(self.path_info.path)
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            prefix = posixpath.join(
+                self.path_info.path, prefix[:2], prefix[2:]
+            )
+        else:
+            prefix = self.path_info.path
+        return self._list_paths(prefix, progress_callback)

    def _upload(
        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals

 import logging
 import os
+import posixpath
 import threading

 from funcy import cached_property, wrap_prop

-from dvc.config import Config, ConfigError
+from dvc.config import ConfigError
 from dvc.exceptions import DvcException
 from dvc.exceptions import ETagMismatchError
 from dvc.path_info import CloudURLInfo
@@ -25,37 +25,35 @@ class RemoteS3(RemoteBASE):
    PARAM_CHECKSUM = "etag"

    def __init__(self, repo, config):
-        super(RemoteS3, self).__init__(repo, config)
+        super().__init__(repo, config)

-        url = config.get(Config.SECTION_REMOTE_URL, "s3://")
+        url = config.get("url", "s3://")
        self.path_info = self.path_cls(url)

-        self.region = config.get(Config.SECTION_AWS_REGION)
+        self.region = config.get("region")
+        self.profile = config.get("profile")
+        self.endpoint_url = config.get("endpointurl")

-        self.profile = config.get(Config.SECTION_AWS_PROFILE)
-
-        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
-
-        if config.get(Config.SECTION_AWS_LIST_OBJECTS):
+        if config.get("listobjects"):
            self.list_objects_api = "list_objects"
        else:
            self.list_objects_api = "list_objects_v2"

-        self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
+        self.use_ssl = config.get("use_ssl", True)

        self.extra_args = {}

-        self.sse = config.get(Config.SECTION_AWS_SSE, "")
+        self.sse = config.get("sse")
        if self.sse:
            self.extra_args["ServerSideEncryption"] = self.sse

-        self.acl = config.get(Config.SECTION_AWS_ACL, "")
+        self.acl = config.get("acl")
        if self.acl:
            self.extra_args["ACL"] = self.acl

        self._append_aws_grants_to_extra_args(config)

-        shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
+        shared_creds = config.get("credentialpath")
        if shared_creds:
            os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)

@@ -88,8 +86,8 @@ class RemoteS3(RemoteBASE):
            obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
        except Exception as exc:
            raise DvcException(
-                "s3://{}/{} does not exist".format(bucket, path), exc
-            )
+                "s3://{}/{} does not exist".format(bucket, path)
+            ) from exc
        return obj

    @classmethod
@@ -157,6 +155,7 @@ class RemoteS3(RemoteBASE):
        # the same throughout the transfer, so it means that in order to
        # preserve etag, we need to transfer each part separately, so the
        # object is transfered in the same chunks as it was originally.
+        from boto3.s3.transfer import TransferConfig

        obj = cls.get_head_object(s3, from_info.bucket, from_info.path)
        etag = obj["ETag"].strip('"')
@@ -170,7 +169,13 @@ class RemoteS3(RemoteBASE):
            )
        else:
            source = {"Bucket": from_info.bucket, "Key": from_info.path}
-            s3.copy(source, to_info.bucket, to_info.path, ExtraArgs=extra_args)
+            s3.copy(
+                source,
+                to_info.bucket,
+                to_info.path,
+                ExtraArgs=extra_args,
+                Config=TransferConfig(multipart_threshold=size + 1),
+            )

        cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
        if etag != cached_etag:
@@ -186,28 +191,41 @@ class RemoteS3(RemoteBASE):
        logger.debug("Removing {}".format(path_info))
        self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)

-    def _list_objects(self, path_info, max_items=None):
+    def _list_objects(
+        self, path_info, max_items=None, prefix=None, progress_callback=None
+    ):
        """ Read config for list object api, paginate through list objects."""
        kwargs = {
            "Bucket": path_info.bucket,
            "Prefix": path_info.path,
            "PaginationConfig": {"MaxItems": max_items},
        }
+        if prefix:
+            kwargs["Prefix"] = posixpath.join(path_info.path, prefix[:2])
        paginator = self.s3.get_paginator(self.list_objects_api)
        for page in paginator.paginate(**kwargs):
-            contents = page.get("Contents", None)
-            if not contents:
-                continue
-            for item in contents:
-                yield item
+            contents = page.get("Contents", ())
+            if progress_callback:
+                for item in contents:
+                    progress_callback()
+                    yield item
+            else:
+                yield from contents

-    def _list_paths(self, path_info, max_items=None):
+    def _list_paths(
+        self, path_info, max_items=None, prefix=None, progress_callback=None
+    ):
        return (
-            item["Key"] for item in self._list_objects(path_info, max_items)
+            item["Key"]
+            for item in self._list_objects(
+                path_info, max_items, prefix, progress_callback
+            )
        )

-    def list_cache_paths(self):
-        return self._list_paths(self.path_info)
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        return self._list_paths(
+            self.path_info, prefix=prefix, progress_callback=progress_callback
+        )

    def isfile(self, path_info):
        from botocore.exceptions import ClientError
@@ -238,6 +256,9 @@ class RemoteS3(RemoteBASE):
        #
        # We are not creating directory objects for every parent prefix,
        # as it is not required.
+        if not path_info.path:
+            return
+
        dir_path = path_info / ""
        self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")

@@ -320,14 +341,14 @@ class RemoteS3(RemoteBASE):
        """

        grants = {
-            Config.SECTION_AWS_GRANT_FULL_CONTROL: "GrantFullControl",
-            Config.SECTION_AWS_GRANT_READ: "GrantRead",
-            Config.SECTION_AWS_GRANT_READ_ACP: "GrantReadACP",
-            Config.SECTION_AWS_GRANT_WRITE_ACP: "GrantWriteACP",
+            "grant_full_control": "GrantFullControl",
+            "grant_read": "GrantRead",
+            "grant_read_acp": "GrantReadACP",
+            "grant_write_acp": "GrantWriteACP",
        }

        for grant_option, extra_args_key in grants.items():
-            if config.get(grant_option, ""):
+            if config.get(grant_option):
                if self.acl:
                    raise ConfigError(
                        "`acl` and `grant_*` AWS S3 config options "
@@ -1,12 +1,10 @@
 import logging
 import sys
 import time
+
 from functools import wraps
-
 import colorama

-from dvc.config import Config
-

 logger = logging.getLogger(__name__)
 this = sys.modules[__name__]
@@ -31,11 +29,9 @@ def slow_link_guard(f):
        if this.already_displayed:
            return f(remote, *args, **kwargs)

-        config = remote.repo.config.config.get(Config.SECTION_CACHE, {})
-        cache_type = config.get(Config.SECTION_CACHE_TYPE)
-        should_warn = config.get(Config.SECTION_CACHE_SLOW_LINK_WARNING, True)
-
-        if not should_warn or cache_type:
+        cache_conf = remote.repo.config["cache"]
+        slow_link_warning = cache_conf.get("slow_link_warning", True)
+        if not slow_link_warning or cache_conf.get("type"):
            return f(remote, *args, **kwargs)

        start = time.time()
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import errno
 import getpass
 import io
@@ -9,18 +7,16 @@ import os
 import threading
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import closing, contextmanager
+from urllib.parse import urlparse

-from funcy import memoize, wrap_with
+from funcy import memoize, wrap_with, silent, first

 import dvc.prompt as prompt
-from dvc.config import Config
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.remote.pool import get_connection
 from dvc.scheme import Schemes
 from dvc.utils import to_chunks
-from dvc.utils.compat import StringIO
-from dvc.utils.compat import urlparse

 logger = logging.getLogger(__name__)

@@ -52,21 +48,21 @@ class RemoteSSH(RemoteBASE):
    DEFAULT_CACHE_TYPES = ["copy"]

    def __init__(self, repo, config):
-        super(RemoteSSH, self).__init__(repo, config)
-        url = config.get(Config.SECTION_REMOTE_URL)
+        super().__init__(repo, config)
+        url = config.get("url")
        if url:
            parsed = urlparse(url)
            user_ssh_config = self._load_user_ssh_config(parsed.hostname)

            host = user_ssh_config.get("hostname", parsed.hostname)
            user = (
-                config.get(Config.SECTION_REMOTE_USER)
+                config.get("user")
                or parsed.username
                or user_ssh_config.get("user")
                or getpass.getuser()
            )
            port = (
-                config.get(Config.SECTION_REMOTE_PORT)
+                config.get("port")
                or parsed.port
                or self._try_get_ssh_config_port(user_ssh_config)
                or self.DEFAULT_PORT
@@ -83,14 +79,12 @@ class RemoteSSH(RemoteBASE):
            user_ssh_config = {}

        self.keyfile = config.get(
-            Config.SECTION_REMOTE_KEY_FILE
+            "keyfile"
        ) or self._try_get_ssh_config_keyfile(user_ssh_config)
-        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
-        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
-        self.ask_password = config.get(
-            Config.SECTION_REMOTE_ASK_PASSWORD, False
-        )
-        self.gss_auth = config.get(Config.SECTION_REMOTE_GSS_AUTH, False)
+        self.timeout = config.get("timeout", self.TIMEOUT)
+        self.password = config.get("password", None)
+        self.ask_password = config.get("ask_password", False)
+        self.gss_auth = config.get("gss_auth", False)

    @staticmethod
    def ssh_config_filename():
@@ -106,24 +100,18 @@ class RemoteSSH(RemoteBASE):
            ssh_config = paramiko.SSHConfig()
            with open(user_config_file) as f:
                # For whatever reason parsing directly from f is unreliable
-                f_copy = StringIO(f.read())
+                f_copy = io.StringIO(f.read())
                ssh_config.parse(f_copy)
            user_ssh_config = ssh_config.lookup(hostname)
        return user_ssh_config

    @staticmethod
    def _try_get_ssh_config_port(user_ssh_config):
-        try:
-            return int(user_ssh_config.get("port"))
-        except (ValueError, TypeError):
-            return None
+        return silent(int)(user_ssh_config.get("port"))

    @staticmethod
    def _try_get_ssh_config_keyfile(user_ssh_config):
-        identity_file = user_ssh_config.get("identityfile")
-        if identity_file and len(identity_file) > 0:
-            return identity_file[0]
-        return None
+        return first(user_ssh_config.get("identityfile") or ())

    def ensure_credentials(self, path_info=None):
        if path_info is None:
@@ -221,7 +209,8 @@ class RemoteSSH(RemoteBASE):
        with self.ssh(path_info) as ssh:
            ssh.remove(path_info.path)

-    def move(self, from_info, to_info):
+    def move(self, from_info, to_info, mode=None):
+        assert mode is None
        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
            raise NotImplementedError

@@ -260,11 +249,16 @@ class RemoteSSH(RemoteBASE):
            else:
                yield io.TextIOWrapper(fd, encoding=encoding)

-    def list_cache_paths(self):
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        assert prefix is None
        with self.ssh(self.path_info) as ssh:
            # If we simply return an iterator then with above closes instantly
-            for path in ssh.walk_files(self.path_info.path):
-                yield path
+            if progress_callback:
+                for path in ssh.walk_files(self.path_info.path):
+                    progress_callback()
+                    yield path
+            else:
+                yield from ssh.walk_files(self.path_info.path)

    def walk_files(self, path_info):
        with self.ssh(path_info) as ssh:
@@ -309,7 +303,7 @@ class RemoteSSH(RemoteBASE):
        faster than current approach (relying on exists(path_info)) applied in
        remote/base.
        """
-        if not self.no_traverse:
+        if not self.CAN_TRAVERSE:
            return list(set(checksums) & set(self.all()))

        # possibly prompt for credentials before "Querying" progress output
@@ -3,6 +3,7 @@ import logging
 import os
 import posixpath
 import stat
+from contextlib import suppress

 from funcy import cached_property

@@ -12,7 +13,6 @@ except ImportError:
    paramiko = None

 from dvc.utils import tmp_fname
-from dvc.utils.compat import ignore_file_not_found
 from dvc.progress import Tqdm
 from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteCmdError
@@ -60,13 +60,13 @@ class SSHConnection:
        self._ssh.close()

    def st_mode(self, path):
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
            return self.sftp.lstat(path).st_mode

        return 0

    def getsize(self, path):
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
            return self.sftp.lstat(path).st_size

        return 0
@@ -103,14 +103,13 @@ class SSHConnection:
        if tail:
            try:
                self.sftp.mkdir(path)
-            except IOError as e:
+            except IOError as exc:
                # Since paramiko errors are very vague we need to recheck
                # whether it's because path already exists or something else
-                if e.errno == errno.EACCES or not self.exists(path):
+                if exc.errno == errno.EACCES or not self.exists(path):
                    raise DvcException(
-                        "unable to create remote directory '{}'".format(path),
-                        cause=e,
-                    )
+                        "unable to create remote directory '{}'".format(path)
+                    ) from exc

    def walk(self, directory, topdown=True):
        # NOTE: original os.walk() implementation [1] with default options was
@@ -123,9 +122,8 @@ class SSHConnection:
            raise DvcException(
                "couldn't get the '{}' remote directory files list".format(
                    directory
-                ),
-                cause=exc,
-            )
+                )
+            ) from exc

        dirs = []
        nondirs = []
@@ -141,8 +139,7 @@ class SSHConnection:

        for dname in dirs:
            newpath = posixpath.join(directory, dname)
-            for entry in self.walk(newpath, topdown=topdown):
-                yield entry
+            yield from self.walk(newpath, topdown=topdown)

        if not topdown:
            yield directory, dirs, nondirs
@@ -153,22 +150,20 @@ class SSHConnection:
                yield posixpath.join(root, fname)

    def _remove_file(self, path):
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
            self.sftp.remove(path)

    def _remove_dir(self, path):
        for root, dirs, files in self.walk(path, topdown=False):
            for fname in files:
-                path = posixpath.join(root, fname)
-                with ignore_file_not_found():
-                    self._remove_file(path)
+                with suppress(FileNotFoundError):
+                    self._remove_file(posixpath.join(root, fname))

            for dname in dirs:
-                path = posixpath.join(root, dname)
-                with ignore_file_not_found():
-                    self.sftp.rmdir(dname)
+                with suppress(FileNotFoundError):
+                    self.sftp.rmdir(posixpath.join(root, dname))

-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
            self.sftp.rmdir(path)

    def remove(self, path):
@@ -1,38 +1,30 @@
-from __future__ import unicode_literals
-
-import logging
 import os
 from contextlib import contextmanager
 from functools import wraps
-from itertools import chain

-from funcy import cached_property
+from dvc.ignore import CleanTree
+from dvc.compat import fspath_py35
+
+from funcy import cached_property, cat, first

-from .graph import check_acyclic
-from .graph import get_pipeline
-from .graph import get_pipelines
-from .graph import get_stages
 from dvc.config import Config
-from dvc.exceptions import FileMissingError
-from dvc.exceptions import NotDvcRepoError
-from dvc.exceptions import OutputNotFoundError
-from dvc.ignore import DvcIgnoreFilter
+from dvc.exceptions import (
+    FileMissingError,
+    IsADirectoryError,
+    NotDvcRepoError,
+    OutputNotFoundError,
+)
 from dvc.path_info import PathInfo
 from dvc.remote.base import RemoteActionNotImplemented
-from dvc.utils import relpath
 from dvc.utils.fs import path_isin
-from dvc.utils.compat import FileNotFoundError
-from dvc.utils.compat import fspath_py35
-from dvc.utils.compat import open as _open
-
-
-logger = logging.getLogger(__name__)
+from .graph import check_acyclic, get_pipeline, get_pipelines


 def locked(f):
    @wraps(f)
    def wrapper(repo, *args, **kwargs):
        with repo.lock, repo.state:
+            repo._reset()
            ret = f(repo, *args, **kwargs)
            # Our graph cache is no longer valid after we release the repo.lock
            repo._reset()
@@ -48,6 +40,7 @@ class Repo(object):
    from dvc.repo.install import install
    from dvc.repo.add import add
    from dvc.repo.remove import remove
+    from dvc.repo.ls import ls
    from dvc.repo.lock import lock as lock_stage
    from dvc.repo.move import move
    from dvc.repo.run import run
@@ -76,7 +69,7 @@ class Repo(object):
        from dvc.repo.metrics import Metrics
        from dvc.scm.tree import WorkingTree
        from dvc.repo.tag import Tag
-        from dvc.utils import makedirs
+        from dvc.utils.fs import makedirs

        root_dir = self.find_root(root_dir)

@@ -85,14 +78,15 @@ class Repo(object):

        self.config = Config(self.dvc_dir)

-        self.scm = SCM(self.root_dir)
+        no_scm = self.config["core"].get("no_scm", False)
+        self.scm = SCM(self.root_dir, no_scm=no_scm)

        self.tree = WorkingTree(self.root_dir)

        self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
        makedirs(self.tmp_dir, exist_ok=True)

-        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        hardlink_lock = self.config["core"].get("hardlink_lock", False)
        self.lock = make_lock(
            os.path.join(self.dvc_dir, "lock"),
            tmp_dir=os.path.join(self.dvc_dir, "tmp"),
@@ -102,13 +96,7 @@ class Repo(object):

        # NOTE: storing state and link_state in the repository itself to avoid
        # any possible state corruption in 'shared cache dir' scenario.
-        self.state = State(self, self.config.config)
-
-        core = self.config.config[Config.SECTION_CORE]
-
-        level = core.get(Config.SECTION_CORE_LOGLEVEL)
-        if level:
-            logger.setLevel(level.upper())
+        self.state = State(self)

        self.cache = Cache(self)
        self.cloud = DataCloud(self)
@@ -124,29 +112,34 @@ class Repo(object):

    @tree.setter
    def tree(self, tree):
-        self._tree = tree
+        self._tree = tree if isinstance(tree, CleanTree) else CleanTree(tree)
        # Our graph cache is no longer valid, as it was based on the previous
        # tree.
        self._reset()

    def __repr__(self):
-        return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
+        return "{}: '{}'".format(self.__class__.__name__, self.root_dir)

    @classmethod
    def find_root(cls, root=None):
-        if root is None:
-            root = os.getcwd()
-        else:
-            root = os.path.abspath(os.path.realpath(root))
+        root_dir = os.path.realpath(root or os.curdir)
+
+        if not os.path.isdir(root_dir):
+            raise NotDvcRepoError("directory '{}' does not exist".format(root))

        while True:
-            dvc_dir = os.path.join(root, cls.DVC_DIR)
+            dvc_dir = os.path.join(root_dir, cls.DVC_DIR)
            if os.path.isdir(dvc_dir):
-                return root
-            if os.path.ismount(root):
+                return root_dir
+            if os.path.ismount(root_dir):
                break
-            root = os.path.dirname(root)
-        raise NotDvcRepoError(root)
+            root_dir = os.path.dirname(root_dir)
+
+        message = (
+            "you are not inside of a DVC repository "
+            "(checked up to mount point '{}')"
+        ).format(root_dir)
+        raise NotDvcRepoError(message)

    @classmethod
    def find_dvc_dir(cls, root=None):
@@ -154,10 +147,10 @@ class Repo(object):
        return os.path.join(root_dir, cls.DVC_DIR)

    @staticmethod
-    def init(root_dir=os.curdir, no_scm=False, force=False):
+    def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
        from dvc.repo.init import init

-        init(root_dir=root_dir, no_scm=no_scm, force=force)
+        init(root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir)
        return Repo(root_dir)

    def unprotect(self, target):
@@ -169,7 +162,7 @@ class Repo(object):
        updater = Updater(self.dvc_dir)

        flist = (
-            [self.config.config_local_file, updater.updater_file]
+            [self.config.files["local"], updater.updater_file]
            + [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir]
            + self.state.files
        )
@@ -181,41 +174,60 @@ class Repo(object):

    def check_modified_graph(self, new_stages):
        """Generate graph including the new stage to check for errors"""
-        self._collect_graph(self.stages + new_stages)
+        # Building graph might be costly for the ones with many DVC-files,
+        # so we provide this undocumented hack to skip it. See [1] for
+        # more details. The hack can be used as:
+        #
+        #     repo = Repo(...)
+        #     repo._skip_graph_checks = True
+        #     repo.add(...)
+        #
+        # A user should care about not duplicating outs and not adding cycles,
+        # otherwise DVC might have an undefined behaviour.
+        #
+        # [1] https://github.com/iterative/dvc/issues/2671
+        if not getattr(self, "_skip_graph_checks", False):
+            self._collect_graph(self.stages + new_stages)

    def collect(self, target, with_deps=False, recursive=False, graph=None):
        import networkx as nx
        from dvc.stage import Stage

-        G = graph or self.graph
-
        if not target:
-            return get_stages(G)
+            return list(graph) if graph else self.stages

        target = os.path.abspath(target)

        if recursive and os.path.isdir(target):
-            attrs = nx.get_node_attributes(G, "stage")
-            nodes = [node for node in nx.dfs_postorder_nodes(G)]
-
-            ret = []
-            for node in nodes:
-                stage = attrs[node]
-                if path_isin(stage.path, target):
-                    ret.append(stage)
-            return ret
+            stages = nx.dfs_postorder_nodes(graph or self.graph)
+            return [stage for stage in stages if path_isin(stage.path, target)]

        stage = Stage.load(self, target)
+
+        # Optimization: do not collect the graph for a specific target
        if not with_deps:
            return [stage]

-        node = relpath(stage.path, self.root_dir)
-        pipeline = get_pipeline(get_pipelines(G), node)
+        pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
+        return list(nx.dfs_postorder_nodes(pipeline, stage))

-        return [
-            pipeline.node[n]["stage"]
-            for n in nx.dfs_postorder_nodes(pipeline, node)
-        ]
+    def collect_granular(self, target, *args, **kwargs):
+        from dvc.stage import Stage
+
+        if not target:
+            return [(stage, None) for stage in self.stages]
+
+        # Optimization: do not collect the graph for a specific .dvc target
+        if Stage.is_valid_filename(target) and not kwargs.get("with_deps"):
+            return [(Stage.load(self, target), None)]
+
+        try:
+            (out,) = self.find_outs_by_path(target, strict=False)
+            filter_info = PathInfo(os.path.abspath(target))
+            return [(out.stage, filter_info)]
+        except OutputNotFoundError:
+            stages = self.collect(target, *args, **kwargs)
+            return [(stage, None) for stage in stages]

    def used_cache(
        self,
@@ -236,7 +248,7 @@ class Repo(object):
        (namely, a file described as an output on a stage).

        The scope is, by default, the working directory, but you can use
-        `all_branches` or `all_tags` to expand scope.
+        `all_branches`/`all_tags`/`all_commits` to expand the scope.

        Returns:
            A dictionary with Schemes (representing output's location) as keys,
@@ -251,28 +263,24 @@ class Repo(object):
            all_tags=all_tags,
            all_commits=all_commits,
        ):
-            if targets:
-                stages = []
-                for target in targets:
-                    collected = self.collect(
-                        target, recursive=recursive, with_deps=with_deps
-                    )
-                    stages.extend(collected)
-            else:
-                stages = self.stages
+            targets = targets or [None]

-            for stage in stages:
-                if stage.is_repo_import:
-                    dep, = stage.deps
-                    cache.external[dep.repo_pair].add(dep.def_path)
-                    continue
+            pairs = cat(
+                self.collect_granular(
+                    target, recursive=recursive, with_deps=with_deps
+                )
+                for target in targets
+            )

-                for out in stage.outs:
-                    used_cache = out.get_used_cache(
-                        remote=remote, force=force, jobs=jobs
-                    )
-                    suffix = "({})".format(branch) if branch else ""
-                    cache.update(used_cache, suffix=suffix)
+            suffix = "({})".format(branch) if branch else ""
+            for stage, filter_info in pairs:
+                used_cache = stage.get_used_cache(
+                    remote=remote,
+                    force=force,
+                    jobs=jobs,
+                    filter_info=filter_info,
+                )
+                cache.update(used_cache, suffix=suffix)

        return cache

@@ -316,6 +324,7 @@ class Repo(object):
            CyclicGraphError: resulting graph has cycles
        """
        import networkx as nx
+        from pygtrie import Trie
        from dvc.exceptions import (
            OutputDuplicationError,
            StagePathAsOutputError,
@@ -323,44 +332,59 @@ class Repo(object):
        )

        G = nx.DiGraph()
-        stages = stages or self.collect_stages()
+        stages = stages or self.stages
        stages = [stage for stage in stages if stage]
-        outs = {}
+        outs = Trie()  # Use trie to efficiently find overlapping outs and deps

        for stage in stages:
            for out in stage.outs:
-                if out.path_info in outs:
-                    stages = [stage.relpath, outs[out.path_info].stage.relpath]
-                    raise OutputDuplicationError(str(out), stages)
-                outs[out.path_info] = out
+                out_key = out.path_info.parts
+
+                # Check for dup outs
+                if out_key in outs:
+                    dup_stages = [stage, outs[out_key].stage]
+                    raise OutputDuplicationError(str(out), dup_stages)
+
+                # Check for overlapping outs
+                if outs.has_subtrie(out_key):
+                    parent = out
+                    overlapping = first(outs.values(prefix=out_key))
+                else:
+                    parent = outs.shortest_prefix(out_key).value
+                    overlapping = out
+                if parent and overlapping:
+                    msg = (
+                        "Paths for outs:\n'{}'('{}')\n'{}'('{}')\n"
+                        "overlap. To avoid unpredictable behaviour, "
+                        "rerun command with non overlapping outs paths."
+                    ).format(
+                        str(parent),
+                        parent.stage.relpath,
+                        str(overlapping),
+                        overlapping.stage.relpath,
+                    )
+                    raise OverlappingOutputPathsError(parent, overlapping, msg)
+
+                outs[out_key] = out

        for stage in stages:
-            for out in stage.outs:
-                for p in out.path_info.parents:
-                    if p in outs:
-                        raise OverlappingOutputPathsError(outs[p], out)
+            out = outs.shortest_prefix(PathInfo(stage.path).parts).value
+            if out:
+                raise StagePathAsOutputError(stage, str(out))

+        # Building graph
+        G.add_nodes_from(stages)
        for stage in stages:
-            stage_path_info = PathInfo(stage.path)
-            for p in chain([stage_path_info], stage_path_info.parents):
-                if p in outs:
-                    raise StagePathAsOutputError(stage.wdir, stage.relpath)
-
-        for stage in stages:
-            node = relpath(stage.path, self.root_dir)
-
-            G.add_node(node, stage=stage)
-
            for dep in stage.deps:
                if dep.path_info is None:
                    continue

-                for out in outs:
-                    if out.overlaps(dep.path_info):
-                        dep_stage = outs[out].stage
-                        dep_node = relpath(dep_stage.path, self.root_dir)
-                        G.add_node(dep_node, stage=dep_stage)
-                        G.add_edge(node, dep_node)
+                dep_key = dep.path_info.parts
+                overlapping = list(n.value for n in outs.prefixes(dep_key))
+                if outs.has_subtrie(dep_key):
+                    overlapping.extend(outs.values(prefix=dep_key))
+
+                G.add_edges_from((stage, out.stage) for out in overlapping)

        check_acyclic(G)

@@ -374,18 +398,8 @@ class Repo(object):
    def pipelines(self):
        return get_pipelines(self.graph)

-    @staticmethod
-    def _filter_out_dirs(dirs, outs, root_dir):
-        def filter_dirs(dname):
-            path = os.path.join(root_dir, dname)
-            for out in outs:
-                if path == os.path.normpath(out):
-                    return False
-            return True
-
-        return list(filter(filter_dirs, dirs))
-
-    def collect_stages(self):
+    @cached_property
+    def stages(self):
        """
        Walks down the root directory looking for Dvcfiles,
        skipping the directories that are related with
@@ -398,41 +412,37 @@ class Repo(object):
        from dvc.stage import Stage

        stages = []
-        outs = []
+        outs = set()

-        for root, dirs, files in self.tree.walk(
-            self.root_dir, dvcignore=self.dvcignore
-        ):
+        for root, dirs, files in self.tree.walk(self.root_dir):
            for fname in files:
                path = os.path.join(root, fname)
                if not Stage.is_valid_filename(path):
                    continue
                stage = Stage.load(self, path)
-                for out in stage.outs:
-                    if out.scheme == "local":
-                        outs.append(out.fspath + out.sep)
                stages.append(stage)

-            dirs[:] = self._filter_out_dirs(dirs, outs, root)
+                for out in stage.outs:
+                    if out.scheme == "local":
+                        outs.add(out.fspath)
+
+            dirs[:] = [d for d in dirs if os.path.join(root, d) not in outs]

        return stages

-    @cached_property
-    def stages(self):
-        return get_stages(self.graph)
-
-    def find_outs_by_path(self, path, outs=None, recursive=False):
+    def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
        if not outs:
            outs = [out for stage in self.stages for out in stage.outs]

        abs_path = os.path.abspath(path)
-        is_dir = self.tree.isdir(abs_path)
+        path_info = PathInfo(abs_path)
+        match = path_info.__eq__ if strict else path_info.isin_or_eq

        def func(out):
-            if out.scheme == "local" and out.fspath == abs_path:
+            if out.scheme == "local" and match(out.path_info):
                return True

-            if is_dir and recursive and out.path_info.isin(abs_path):
+            if recursive and out.path_info.isin(path_info):
                return True

            return False
@@ -445,7 +455,7 @@ class Repo(object):

    def find_out_by_relpath(self, relpath):
        path = os.path.join(self.root_dir, relpath)
-        out, = self.find_outs_by_path(path)
+        (out,) = self.find_outs_by_path(path)
        return out

    def is_dvc_internal(self, path):
@@ -453,39 +463,40 @@ class Repo(object):
        return self.DVC_DIR in path_parts

    @contextmanager
-    def open(self, path, remote=None, mode="r", encoding=None):
+    def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
        """Opens a specified resource as a file descriptor"""
        cause = None
        try:
-            out, = self.find_outs_by_path(path)
-        except OutputNotFoundError as e:
+            out = self.find_out_by_relpath(path)
+        except OutputNotFoundError as exc:
            out = None
-            cause = e
+            cause = exc

        if out and out.use_cache:
            try:
                with self._open_cached(out, remote, mode, encoding) as fd:
                    yield fd
                return
-            except FileNotFoundError as e:
-                raise FileMissingError(relpath(path, self.root_dir), cause=e)
+            except FileNotFoundError as exc:
+                raise FileMissingError(path) from exc

-        if self.tree.exists(path):
-            with self.tree.open(path, mode, encoding) as fd:
+        abs_path = os.path.join(self.root_dir, path)
+        if os.path.exists(abs_path):
+            with open(abs_path, mode=mode, encoding=encoding) as fd:
                yield fd
            return

-        raise FileMissingError(relpath(path, self.root_dir), cause=cause)
+        raise FileMissingError(path) from cause

    def _open_cached(self, out, remote=None, mode="r", encoding=None):
        if out.isdir():
-            raise ValueError("Can't open a dir")
+            raise IsADirectoryError("Can't open a dir")

        cache_file = self.cache.local.checksum_to_path_info(out.checksum)
        cache_file = fspath_py35(cache_file)

        if os.path.exists(cache_file):
-            return _open(cache_file, mode=mode, encoding=encoding)
+            return open(cache_file, mode=mode, encoding=encoding)

        try:
            remote_obj = self.cloud.get_remote(remote)
@@ -496,11 +507,7 @@ class Repo(object):
                cache_info = out.get_used_cache(remote=remote)
                self.cloud.pull(cache_info, remote=remote)

-            return _open(cache_file, mode=mode, encoding=encoding)
-
-    @cached_property
-    def dvcignore(self):
-        return DvcIgnoreFilter(self.root_dir, self.tree)
+            return open(cache_file, mode=mode, encoding=encoding)

    def close(self):
        self.scm.close()
@@ -1,18 +1,18 @@
-from __future__ import unicode_literals
-
 import logging
 import os

 import colorama
-from six import string_types

 from . import locked
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
-from dvc.progress import Tqdm
-from dvc.repo.scm_context import scm_context
-from dvc.stage import Stage
-from dvc.utils import LARGE_DIR_SIZE
-from dvc.utils import walk_files
+from ..exceptions import (
+    RecursiveAddingWhileUsingFilename,
+    OverlappingOutputPathsError,
+)
+from ..output.base import OutputDoesNotExistError
+from ..progress import Tqdm
+from ..repo.scm_context import scm_context
+from ..stage import Stage
+from ..utils import LARGE_DIR_SIZE

 logger = logging.getLogger(__name__)

@@ -23,11 +23,16 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
    if recursive and fname:
        raise RecursiveAddingWhileUsingFilename()

-    if isinstance(targets, string_types):
+    if isinstance(targets, str):
        targets = [targets]

    stages_list = []
-    with Tqdm(total=len(targets), desc="Add", unit="file", leave=True) as pbar:
+    num_targets = len(targets)
+    with Tqdm(total=num_targets, desc="Add", unit="file", leave=True) as pbar:
+        if num_targets == 1:
+            # clear unneeded top-level progress bar for single target
+            pbar.bar_format = "Adding..."
+            pbar.refresh()
        for target in targets:
            sub_targets = _find_all_targets(repo, target, recursive)
            pbar.total += len(sub_targets) - 1
@@ -37,7 +42,7 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
                    "You are adding a large directory '{target}' recursively,"
                    " consider tracking it as a whole instead.\n"
                    "{purple}HINT:{nc} Remove the generated DVC-file and then"
-                    " run {cyan}dvc add {target}{nc}".format(
+                    " run `{cyan}dvc add {target}{nc}`".format(
                        purple=colorama.Fore.MAGENTA,
                        cyan=colorama.Fore.CYAN,
                        nc=colorama.Style.RESET_ALL,
@@ -47,19 +52,46 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):

            stages = _create_stages(repo, sub_targets, fname, pbar=pbar)

-            repo.check_modified_graph(stages)
+            try:
+                repo.check_modified_graph(stages)
+            except OverlappingOutputPathsError as exc:
+                msg = (
+                    "Cannot add '{out}', because it is overlapping with other "
+                    "DVC tracked output: '{parent}'.\n"
+                    "To include '{out}' in '{parent}', run "
+                    "'dvc commit {parent_stage}'"
+                ).format(
+                    out=exc.overlapping_out.path_info,
+                    parent=exc.parent.path_info,
+                    parent_stage=exc.parent.stage.relpath,
+                )
+                raise OverlappingOutputPathsError(
+                    exc.parent, exc.overlapping_out, msg
+                )

-            for stage in stages:
-                stage.save()
+            with Tqdm(
+                total=len(stages),
+                desc="Processing",
+                unit="file",
+                disable=len(stages) == 1,
+            ) as pbar_stages:
+                for stage in stages:
+                    try:
+                        stage.save()
+                    except OutputDoesNotExistError:
+                        pbar.n -= 1
+                        raise

-                if not no_commit:
-                    stage.commit()
+                    if not no_commit:
+                        stage.commit()

-                stage.dump()
+                    stage.dump()
+                    pbar_stages.update()

            stages_list += stages
-        # remove filled bar bit of progress, leaving stats
-        pbar.bar_format = pbar.BAR_FMT_DEFAULT.replace("|{bar:10}|", " ")
+
+        if num_targets == 1:  # restore bar format for stats
+            pbar.bar_format = pbar.BAR_FMT_DEFAULT

    return stages_list

@@ -68,7 +100,12 @@ def _find_all_targets(repo, target, recursive):
    if os.path.isdir(target) and recursive:
        return [
            fname
-            for fname in walk_files(target, repo.dvcignore)
+            for fname in Tqdm(
+                repo.tree.walk_files(target),
+                desc="Searching " + target,
+                bar_format=Tqdm.BAR_FMT_NOTOTAL,
+                unit="file",
+            )
            if not repo.is_dvc_internal(fname)
            if not Stage.is_stage_file(fname)
            if not repo.scm.belongs_to_scm(fname)
@@ -80,8 +117,16 @@ def _find_all_targets(repo, target, recursive):
 def _create_stages(repo, targets, fname, pbar=None):
    stages = []

-    for out in targets:
-        stage = Stage.create(repo, outs=[out], add=True, fname=fname)
+    for out in Tqdm(
+        targets,
+        desc="Creating DVC-files",
+        disable=len(targets) < LARGE_DIR_SIZE,
+        unit="file",
+    ):
+        stage = Stage.create(
+            repo, outs=[out], accompany_outs=True, fname=fname
+        )
+        repo._reset()

        if not stage:
            if pbar is not None:
--- a/Show More
+++ b/Show More