dvc: bump to 0.82.3

push: improve 'no remote specified' hint

.travis.yml

@@ -2,7 +2,6 @@ cache:
   directories:
     - $HOME/.cache/pip
     - $HOME/.pyenv
-sudo: true
 dist: xenial
 env:
   global:
@@ -14,19 +13,19 @@ stages:
   - check
   - test
   - build
-  - name: deploy
-    if: tag is present
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
 # Note that we specify language as ruby-2.4.3 to help us satisfy
 # fpm dependencies, as travis runs el capitan with an old ruby
 # version, that causes fpm to crash.
+#
+# Each job will use the previous job's `stage` tag if not specified.
 jobs:
   include:
   # patch formatting checks
   - name: "Check patch formatting"
-    stage: "check"
+    stage: check
     os: linux
     language: python
     python: 3.7
@@ -34,44 +33,22 @@ jobs:
     install:
     script: ./scripts/ci/check_patch.sh
   # test jobs
-  - name: "2.7 on Windows"
-    stage: test
-    os: windows           # Windows 10.0.17134 N/A Build 17134
-    language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-    - PY2=1
   - name: "3.7 on Windows"
     stage: test
     os: windows           # Windows 10.0.17134 N/A Build 17134
     language: shell       # 'language: python' is an error on Travis CI Windows
-  - name: "2.7 on Linux"
-    stage: test
-    language: python
-    python: 2.7
-    env:
-    - PY2=1
   - name: "3.5 on Linux"
-    stage: test
     language: python
     python: 3.5
   - name: "3.6 on Linux"
-    stage: test
     language: python
     python: 3.6
   - name: "3.7 on Linux"
-    stage: test
     language: python
     python: 3.7
-  - name: "2.7 on Mac"
-    stage: test
+  - name: "3.7 on Mac"
     os: osx
-    osx_image: xcode9.4
-    env:
-    - PY2=1
-  - name: "3.6 on Mac"
-    stage: test
-    os: osx
-    osx_image: xcode9.4
+    osx_image: xcode11.3
   # build jobs
   - name: "Mac pkg"
     stage: build
@@ -82,44 +59,40 @@ jobs:
     install:
     script: ./scripts/build_posix.sh
   - name: "Windows pkg"
-    stage: build
     os: windows
     language: shell
     install:
     script: ./scripts/build_windows.cmd
   - name: "Linux pkgs"
-    stage: build
     language: python
     python: 3.7
     before_install:
     install:
     script: ./scripts/build_posix.sh
-  - name: "Pypi pkgs"
-    stage: build
+  - name: "PyPI pkgs"
     language: python
     python: 3.7
     script: ./scripts/build_package.sh
-  # deploy jobs
-  - name: "Pypi pkgs"
-    stage: deploy
+  - name: Snapcraft snap
     language: python
     python: 3.7
-    script: ./scripts/build_package.sh
-    deploy:
-    - provider: pypi
-      user:
-        secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
-      password:
-        secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
-      skip_existing: true
-      file_glob: true
-      file:
-        - dist/dvc*.whl
-        - dist/dvc*.tar.gz
-      skip_cleanup: true
-      on:
-        tags: true
-        repo: iterative/dvc
+    addons:
+     snaps:
+     - name: snapcraft
+       channel: stable
+       confinement: classic
+     - name: lxd
+       channel: stable
+    env:
+    - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}'
+    install:
+    - sudo /snap/bin/lxd.migrate -yes
+    - sudo /snap/bin/lxd waitready
+    - sudo /snap/bin/lxd init --auto
+    script:
+    - ./scripts/build_snap.sh
+    after_failure:
+    - sudo journalctl -u snapd
 
 before_install:
   - bash ./scripts/ci/before_install.sh
@@ -131,7 +104,6 @@ script:
   - bash ./scripts/ci/script.sh
 deploy:
   - provider: releases
-    condition: "[[ $(./scripts/ci/deploy_condition.sh) == 'true' ]]"
     api_key:
       secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=
     file_glob: true
@@ -142,6 +114,32 @@ deploy:
       - dvc*.exe
     skip_cleanup: true
     on:
+      condition: "$(./scripts/ci/deploy_condition.sh dvc*.rpm dvc*.deb dvc*.pkg dvc*.exe)"
       tags: true
       repo: iterative/dvc
       stage: build
+  - provider: pypi
+    user:
+      secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
+    password:
+      secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
+    skip_existing: true
+    file_glob: true
+    file:
+      - dist/dvc*.whl
+      - dist/dvc*.tar.gz
+    skip_cleanup: true
+    on:
+      condition: "$(./scripts/ci/deploy_condition.sh dist/dvc*.whl dist/dvc*.tar.gz)"
+      tags: true
+      repo: iterative/dvc
+      stage: build
+  - provider: snap
+    snap: dvc_*.snap
+    channel: $SNAP_CHANNEL
+    skip_cleanup: true
+    on:
+      all_branches: true
+      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG)"
+      repo: iterative/dvc
+      stage: build

README.rst

@@ -10,9 +10,13 @@
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
-.. image:: https://travis-ci.com/iterative/dvc.svg?branch=master
+.. image:: https://img.shields.io/badge/release-ok-brightgreen
   :target: https://travis-ci.com/iterative/dvc
-  :alt: Travis
+  :alt: Release
+
+.. image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev
+  :target: https://travis-ci.com/iterative/dvc
+  :alt: Travis dev branch
 
 .. image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
   :target: https://codeclimate.com/github/iterative/dvc
@@ -30,25 +34,30 @@
   :target: https://anaconda.org/conda-forge/dvc
   :alt: Conda-forge
 
+.. image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
+  :target: https://snapcraft.io/dvc
+  :alt: Snapcraft
+
 |
 
 **Data Version Control** or **DVC** is an **open-source** tool for data science and machine
 learning projects. Key features:
 
-#. simple **command line** Git-like experience. Does not require installing and maintaining
-   any databases. Does not depend on any proprietary online services;
+#. Simple **command line** Git-like experience. Does not require installing and maintaining
+   any databases. Does not depend on any proprietary online services.
 
-#. it manages and versions **datasets** and **machine learning models**. Data is saved in
-   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS or even local HDD RAID;
+#. Management and versioning of **datasets** and **machine learning models**. Data is saved in
+   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
 
-#. it makes projects **reproducible** and **shareable**, it helps answering question how
-   the model was build;
+#. Makes projects **reproducible** and **shareable**; helping to answer questions about how
+   a model was built.
 
-#. it helps manage experiments with Git tags or branches and **metrics** tracking;
+#. Helps manage experiments with Git tags/branches and **metrics** tracking.
 
-**DVC** aims to replace tools like Excel and Google Docs that are being commonly used as a knowledge repo and
-a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
-data file suffixes and prefixes.
+**DVC** aims to replace spreadsheet and document sharing tools (such as Excel or Google Docs)
+which are being used frequently as both knowledge repositories and team ledgers.
+DVC also replaces both ad-hoc scripts to track, move, and deploy different model versions;
+as well as ad-hoc data file suffixes and prefixes.
 
 .. contents:: **Contents**
   :backlinks: none
@@ -56,34 +65,33 @@ data file suffixes and prefixes.
 How DVC works
 =============
 
-We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ to better understand what DVC
-is and how does it fit your scenarios.
+We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ guide to better understand what DVC
+is and how it can fit your scenarios.
 
-The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-lfs to be precise) + ``makefiles``
+The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-LFS to be precise) & Makefiles
 made right and tailored specifically for ML and Data Science scenarios.
 
-#. ``Git/Git-lfs`` part - DVC helps you storing and sharing data artifacts, models. It connects them with your
-   Git repository.
-#. ``Makefiles`` part - DVC describes how one data or model artifact was build from another data.
+#. ``Git/Git-LFS`` part - DVC helps store and share data artifacts and models, connecting them with a Git repository.
+#. ``Makefile``\ s part - DVC describes how one data or model artifact was built from other data and code.
 
-DVC usually runs along with Git. Git is used as usual to store and version code and DVC meta-files. DVC helps
-to store data and model files seamlessly out of Git while preserving almost the same user experience as if they
-were stored in Git itself. To store and share data files cache DVC supports remotes - any cloud (S3, Azure,
+DVC usually runs along with Git. Git is used as usual to store and version code (including DVC meta-files). DVC helps
+to store data and model files seamlessly out of Git, while preserving almost the same user experience as if they
+were stored in Git itself. To store and share the data cache, DVC supports multiple remotes - any cloud (S3, Azure,
 Google Cloud, etc) or any on-premise network storage (via SSH, for example).
 
 .. image:: https://dvc.org/static/img/flow.gif
    :target: https://dvc.org/static/img/flow.gif
    :alt: how_dvc_works
 
-DVC pipelines (aka computational graph) feature connects code and data together. In a very explicit way you can
-specify, run, and save information that a certain command with certain dependencies needs to be run to produce
-a model. See the quick start section below or check `Get Started <https://dvc.org/doc/get-started>`_ tutorial to
-learn more.
+The DVC pipelines (computational graph) feature connects code and data together. It is possible to explicitly
+specify all steps required to produce a model: input dependencies including data, commands to run,
+and output information to be saved. See the quick start section below or
+the `Get Started <https://dvc.org/doc/get-started>`_ tutorial to learn more.
 
 Quick start
 ===========
 
-Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full version. Common workflow commands include:
+Please read `Get Started <https://dvc.org/doc/get-started>`_ guide for a full version. Common workflow commands include:
 
 +-----------------------------------+-------------------------------------------------------------------+
 | Step                              | Command                                                           |
@@ -108,8 +116,8 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 Installation
 ============
 
-Read this `instruction <https://dvc.org/doc/get-started/install>`_ to get more details. There are four
-options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package:
+There are four options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package.
+Full instructions are `available here <https://dvc.org/doc/get-started/install>`_.
 
 pip (PyPI)
 ----------
@@ -120,8 +128,8 @@ pip (PyPI)
 
 Depending on the remote storage type you plan to use to keep and share your data, you might need to specify
 one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all`` to include them all.
-The command should look like this: ``pip install dvc[s3]`` - it installs the ``boto3`` library along with
-DVC to support the AWS S3 storage.
+The command should look like this: ``pip install dvc[s3]`` (in this case AWS S3 dependencies such as ``boto3``
+will be installed automatically).
 
 To install the development version, run:
 
@@ -144,13 +152,23 @@ Conda (Anaconda)
 
    conda install -c conda-forge dvc
 
-Currently, it supports only python version 2.7, 3.6 and 3.7.
+Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
+
+Snap (Snapcraft)
+----------------
+
+.. code-block:: bash
+
+   snap install dvc --classic
+
+This corresponds to the latest tagged release.
+Add ``--edge`` for the latest ``master`` version.
 
 Package
 -------
 
-Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
-GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+Self-contained packages for Linux, Windows, and Mac are available. The latest version of the packages
+can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
 
 Ubuntu / Debian (deb)
 ^^^^^^^^^^^^^^^^^^^^^
@@ -168,23 +186,23 @@ Fedora / CentOS (rpm)
    sudo yum update
    sudo yum install dvc
 
-Related technologies
-====================
+Comparison to related technologies
+==================================
 
-#. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (that you
-   don't want to see in your Git repository) in a local key-value store and uses file hardlinks/symlinks instead of the
-   copying actual files.
+#. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (which should
+   not be in a Git repository) in a local key-value store, and uses file hardlinks/symlinks instead of
+   copying/duplicating files.
 
 #. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with any remote storage (S3, Google Cloud, Azure, SSH,
-   etc). DVC utilizes reflinks or hardlinks to avoid copy operation on checkouts which makes much more efficient for
-   large data files.
+   etc). DVC also uses reflinks or hardlinks to avoid copy operations on checkouts; thus handling large data files
+   much more efficiently.
 
-#. *Makefile* (and its analogues). DVC tracks dependencies (DAG).
+#. *Makefile* (and analogues including ad-hoc scripts) - DVC tracks dependencies (in a directed acyclic graph).
 
-#. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_. DVC is a workflow
+#. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_ - DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-#. `DAGsHub <https://dagshub.com/>`_ Is a Github equivalent for DVC - pushing your Git+DVC based repo to DAGsHub will give you a high level dashboard of your project, including DVC pipeline and metrics visualizations, as well as links to DVC managed files if they are in cloud storage.
+#. `DAGsHub <https://dagshub.com/>`_ - This is a Github equivalent for DVC. Pushing Git+DVC based repositories to DAGsHub will produce in a high level project dashboard; including DVC pipelines and metrics visualizations, as well as links to any DVC-managed files present in cloud storage.
 
 Contributing
 ============
@@ -233,5 +251,5 @@ Copyright
 
 This project is distributed under the Apache license version 2.0 (see the LICENSE file in the project root).
 
-By submitting a pull request for this project, you agree to license your contribution under the Apache license version
+By submitting a pull request to this project, you agree to license your contribution under the Apache license version
 2.0 to this project.

debian/changelog

@@ -0,0 +1,5 @@
+dvc (0.75.b1) experimental; urgency=low
+
+  * First native Debian package version.
+
+ -- Dmitry Petrov <[email protected]>  Sat, 07 Dec 2019 09:49:28 +0300

debian/compat

@@ -0,0 +1 @@
+10

debian/control

@@ -0,0 +1,33 @@
+Source: dvc
+Maintainer: Dmitry Petrov <[email protected]>
+Section: python
+Priority: optional
+Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
+Standards-Version: 4.0.0
+Homepage: http://dvc.org
+Vcs-Git: https://github.com/iterative/dvc.git
+Vcs-browser: https://github.com/iterative/dvc
+
+Package: dvc
+Architecture: all
+Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
+Recommends: python3-arrow (=0.14.0),
+            python3-paramiko (>=2.5.0),
+            python3-gssapi (>= 2.5.0),
+            python3-boto3 (>=1.9.86),
+            python3-pydrive (=1.3.1),
+            python3-azure-storage-blob (=12.1.0),
+            python3-oss2 (=2.6.1),
+            python3-google-cloud-storage (=1.19.0)
+Description: Git for data scientists - manage your code and data together
+ Data Version Control or DVC is an open-source tool for data science and machine learning projects.
+ .
+ Key features:
+ .
+ 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
+ .
+ 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
+ .
+ 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
+ .
+ 4. Helps manage experiments with Git tags/branches and metrics tracking.

debian/copyright

@@ -0,0 +1,11 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: DVC
+Upstream-Contact: Dmitry Petrov <[email protected]>
+Source: https://github.com/iterative/dvc
+
+Files: *
+Copyright: Dmitry Petrov <[email protected]>
+License: Apache 2.0
+ On Debian systems, the full text of the Apache 2.0
+ License can be found in the file
+ `/usr/share/common-licenses/Apache-2.0`.

debian/rules

@@ -0,0 +1,6 @@
+#!/usr/bin/make -f
+
+export PYBUILD_NAME=dvc
+%:
+	dh $@ --with python3 --buildsystem=pybuild
+

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

debian/watch

@@ -0,0 +1,4 @@
+# please also check http://pypi.debian.net/dvc/watch
+version=3
+opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
+http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

dvc/__init__.py

@@ -3,8 +3,6 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-from __future__ import unicode_literals
-
 import warnings
 
 import dvc.logger

dvc/__main__.py

@@ -1,6 +1,4 @@
-"""Main entry point for dvc command line tool."""
-from __future__ import unicode_literals
-
+"""Main entry point for DVC command line tool."""
 import sys
 
 from dvc.main import main

dvc/analytics.py

@@ -1,256 +1,169 @@
-"""Collect and send usage analytics"""
-from __future__ import unicode_literals
-
-import errno
 import json
 import logging
 import os
+import platform
+import requests
+import sys
+import tempfile
+import uuid
+
+import distro
 
 from dvc import __version__
-from dvc.utils import env2bool
-from dvc.utils.compat import str
+from dvc.config import Config, to_bool
+from dvc.daemon import daemon
+from dvc.exceptions import NotDvcRepoError
+from dvc.lock import Lock, LockError
+from dvc.repo import Repo
+from dvc.scm import SCM
+from dvc.utils import env2bool, is_binary
+from dvc.utils.fs import makedirs
 
 
 logger = logging.getLogger(__name__)
 
 
-class Analytics(object):
-    """Class for collecting and sending usage analytics.
-
-    Args:
-        info (dict): optional existing analytics report.
+def collect_and_send_report(args=None, return_code=None):
     """
+    Collect information from the runtime/environment and the command
+    being executed into a report and send it over the network.
 
-    URL = "https://analytics.dvc.org"
-    TIMEOUT_POST = 5
+    To prevent analytics from blocking the execution of the main thread,
+    sending the report is done in a separate process.
 
-    USER_ID_FILE = "user_id"
+    The inter-process communication happens through a file containing the
+    report as a JSON, where the _collector_ generates it and the _sender_
+    removes it after sending it.
+    """
+    report = _runtime_info()
 
-    PARAM_DVC_VERSION = "dvc_version"
-    PARAM_USER_ID = "user_id"
-    PARAM_SYSTEM_INFO = "system_info"
+    # Include command execution information on the report only when available.
+    if args and hasattr(args, "func"):
+        report.update({"cmd_class": args.func.__name__})
 
-    PARAM_OS = "os"
+    if return_code is not None:
+        report.update({"cmd_return_code": return_code})
 
-    PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major"
-    PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor"
-    PARAM_WINDOWS_VERSION_BUILD = "windows_version_build"
-    PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack"
+    with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
+        json.dump(report, fobj)
+        daemon(["analytics", fobj.name])
 
-    PARAM_MAC_VERSION = "mac_version"
 
-    PARAM_LINUX_DISTRO = "linux_distro"
-    PARAM_LINUX_DISTRO_VERSION = "linux_distro_version"
-    PARAM_LINUX_DISTRO_LIKE = "linux_distro_like"
+def is_enabled():
+    if env2bool("DVC_TEST"):
+        return False
 
-    PARAM_SCM_CLASS = "scm_class"
-    PARAM_IS_BINARY = "is_binary"
-    PARAM_CMD_CLASS = "cmd_class"
-    PARAM_CMD_RETURN_CODE = "cmd_return_code"
+    enabled = to_bool(
+        Config(validate=False)
+        .config.get(Config.SECTION_CORE, {})
+        .get(Config.SECTION_CORE_ANALYTICS, "true")
+    )
 
-    def __init__(self, info=None):
-        from dvc.config import Config
-        from dvc.lock import Lock
+    logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
 
-        if info is None:
-            info = {}
+    return enabled
 
-        self.info = info
 
-        cdir = Config.get_global_config_dir()
-        try:
-            os.makedirs(cdir)
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                raise
+def send(report):
+    """
+    Side effect: Removes the report after sending it.
 
-        self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
-        self.user_id_file_lock = Lock(self.user_id_file + ".lock")
+    The report is generated and stored in a temporary file, see:
+    `collect_and_send_report`. Sending happens on another process,
+    thus, the need of removing such file afterwards.
+    """
+    url = "https://analytics.dvc.org"
+    headers = {"content-type": "application/json"}
 
-    @staticmethod
-    def load(path):
-        """Loads analytics report from json file specified by path.
+    with open(report, "rb") as fobj:
+        requests.post(url, data=fobj, headers=headers, timeout=5)
 
-        Args:
-            path (str): path to json file with analytics report.
-        """
-        with open(path, "r") as fobj:
-            analytics = Analytics(info=json.load(fobj))
-        os.unlink(path)
-        return analytics
+    os.remove(report)
 
-    def _write_user_id(self):
-        import uuid
 
-        with open(self.user_id_file, "w+") as fobj:
-            user_id = str(uuid.uuid4())
-            info = {self.PARAM_USER_ID: user_id}
-            json.dump(info, fobj)
+def _scm_in_use():
+    try:
+        scm = SCM(root_dir=Repo.find_root())
+        return type(scm).__name__
+    except NotDvcRepoError:
+        pass
+
+
+def _runtime_info():
+    """
+    Gather information from the environment where DVC runs to fill a report.
+    """
+    return {
+        "dvc_version": __version__,
+        "is_binary": is_binary(),
+        "scm_class": _scm_in_use(),
+        "system_info": _system_info(),
+        "user_id": _find_or_create_user_id(),
+    }
+
+
+def _system_info():
+    system = platform.system()
+
+    if system == "Windows":
+        version = sys.getwindowsversion()
+
+        return {
+            "os": "windows",
+            "windows_version_build": version.build,
+            "windows_version_major": version.major,
+            "windows_version_minor": version.minor,
+            "windows_version_service_pack": version.service_pack,
+        }
+
+    if system == "Darwin":
+        return {"os": "mac", "mac_version": platform.mac_ver()[0]}
+
+    if system == "Linux":
+        return {
+            "os": "linux",
+            "linux_distro": distro.id(),
+            "linux_distro_like": distro.like(),
+            "linux_distro_version": distro.version(),
+        }
+
+    # We don't collect data for any other system.
+    raise NotImplementedError
+
+
+def _find_or_create_user_id():
+    """
+    The user's ID is stored on a file under the global config directory.
+
+    The file should contain a JSON with a "user_id" key:
+
+        {"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"}
+
+    IDs are generated randomly with UUID.
+    """
+    config_dir = Config.get_global_config_dir()
+    fname = os.path.join(config_dir, "user_id")
+    lockfile = os.path.join(config_dir, "user_id.lock")
+
+    # Since the `fname` and `lockfile` are under the global config,
+    # we need to make sure such directory exist already.
+    makedirs(config_dir, exist_ok=True)
+
+    try:
+        with Lock(lockfile):
+            try:
+                with open(fname, "r") as fobj:
+                    user_id = json.load(fobj)["user_id"]
+
+            except (FileNotFoundError, ValueError, KeyError):
+                user_id = str(uuid.uuid4())
+
+                with open(fname, "w") as fobj:
+                    json.dump({"user_id": user_id}, fobj)
+
             return user_id
 
-    def _read_user_id(self):
-        if not os.path.exists(self.user_id_file):
-            return None
-
-        with open(self.user_id_file, "r") as fobj:
-            try:
-                info = json.load(fobj)
-            except ValueError as exc:
-                logger.debug("Failed to load user_id: {}".format(exc))
-                return None
-
-            return info[self.PARAM_USER_ID]
-
-    def _get_user_id(self):
-        from dvc.lock import LockError
-
-        try:
-            with self.user_id_file_lock:
-                user_id = self._read_user_id()
-                if user_id is None:
-                    user_id = self._write_user_id()
-                return user_id
-        except LockError:
-            msg = "Failed to acquire '{}'"
-            logger.debug(msg.format(self.user_id_file_lock.lockfile))
-
-    def _collect_windows(self):
-        import sys
-
-        version = sys.getwindowsversion()  # pylint: disable=no-member
-        info = {}
-        info[self.PARAM_OS] = "windows"
-        info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
-        info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
-        info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
-        info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack
-        return info
-
-    def _collect_darwin(self):
-        import platform
-
-        info = {}
-        info[self.PARAM_OS] = "mac"
-        info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
-        return info
-
-    def _collect_linux(self):
-        import distro
-
-        info = {}
-        info[self.PARAM_OS] = "linux"
-        info[self.PARAM_LINUX_DISTRO] = distro.id()
-        info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
-        info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
-        return info
-
-    def _collect_system_info(self):
-        import platform
-
-        system = platform.system()
-
-        if system == "Windows":
-            return self._collect_windows()
-
-        if system == "Darwin":
-            return self._collect_darwin()
-
-        if system == "Linux":
-            return self._collect_linux()
-
-        raise NotImplementedError
-
-    def collect(self):
-        """Collect analytics report."""
-        from dvc.scm import SCM
-        from dvc.utils import is_binary
-        from dvc.repo import Repo
-        from dvc.exceptions import NotDvcRepoError
-
-        self.info[self.PARAM_DVC_VERSION] = __version__
-        self.info[self.PARAM_IS_BINARY] = is_binary()
-        self.info[self.PARAM_USER_ID] = self._get_user_id()
-
-        self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
-
-        try:
-            scm = SCM(root_dir=Repo.find_root())
-            self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
-        except NotDvcRepoError:
-            pass
-
-    def collect_cmd(self, args, ret):
-        """Collect analytics info from a CLI command."""
-        from dvc.command.daemon import CmdDaemonAnalytics
-
-        assert isinstance(ret, int) or ret is None
-
-        if ret is not None:
-            self.info[self.PARAM_CMD_RETURN_CODE] = ret
-
-        if args is not None and hasattr(args, "func"):
-            assert args.func != CmdDaemonAnalytics
-            self.info[self.PARAM_CMD_CLASS] = args.func.__name__
-
-    def dump(self):
-        """Save analytics report to a temporary file.
-
-        Returns:
-            str: path to the temporary file that contains the analytics report.
-        """
-        import tempfile
-
-        with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
-            json.dump(self.info, fobj)
-            return fobj.name
-
-    @staticmethod
-    def is_enabled(cmd=None):
-        from dvc.config import Config, to_bool
-        from dvc.command.daemon import CmdDaemonBase
-
-        if env2bool("DVC_TEST"):
-            return False
-
-        if isinstance(cmd, CmdDaemonBase):
-            return False
-
-        core = Config(validate=False).config.get(Config.SECTION_CORE, {})
-        enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true"))
+    except LockError:
         logger.debug(
-            "Analytics is {}.".format("enabled" if enabled else "disabled")
+            "Failed to acquire '{lockfile}'".format(lockfile=lockfile)
         )
-        return enabled
-
-    @staticmethod
-    def send_cmd(cmd, args, ret):
-        """Collect and send analytics for CLI command.
-
-        Args:
-            args (list): parsed args for the CLI command.
-            ret (int): return value of the CLI command.
-        """
-        from dvc.daemon import daemon
-
-        if not Analytics.is_enabled(cmd):
-            return
-
-        analytics = Analytics()
-        analytics.collect_cmd(args, ret)
-        daemon(["analytics", analytics.dump()])
-
-    def send(self):
-        """Collect and send analytics."""
-        import requests
-
-        if not self.is_enabled():
-            return
-
-        self.collect()
-
-        logger.debug("Sending analytics: {}".format(self.info))
-
-        try:
-            requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
-        except requests.exceptions.RequestException as exc:
-            logger.debug("Failed to send analytics: {}".format(str(exc)))

dvc/api.py

@@ -1,27 +1,74 @@
+from builtins import open as builtin_open
+import importlib
 import os
-from contextlib import contextmanager
+import sys
+from contextlib import contextmanager, _GeneratorContextManager as GCM
+import threading
 
-try:
-    from contextlib import _GeneratorContextManager as GCM
-except ImportError:
-    from contextlib import GeneratorContextManager as GCM
+from funcy import wrap_with
+import ruamel.yaml
+from voluptuous import Schema, Required, Invalid
 
-from dvc.utils.compat import urlparse
 from dvc.repo import Repo
+from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.external_repo import external_repo
 
 
+SUMMON_FILE_SCHEMA = Schema(
+    {
+        Required("objects"): [
+            {
+                Required("name"): str,
+                "meta": dict,
+                Required("summon"): {
+                    Required("type"): str,
+                    "deps": [str],
+                    str: object,
+                },
+            }
+        ]
+    }
+)
+SUMMON_PYTHON_SCHEMA = Schema(
+    {
+        Required("type"): "python",
+        Required("call"): str,
+        "args": dict,
+        "deps": [str],
+    }
+)
+
+
+class SummonError(DvcException):
+    pass
+
+
+class UrlNotDvcRepoError(DvcException):
+    """Thrown if given url is not a DVC repository.
+
+    Args:
+        url (str): URL to the repository
+    """
+
+    def __init__(self, url):
+        super().__init__("'{}' is not a DVC repository.".format(url))
+
+
 def get_url(path, repo=None, rev=None, remote=None):
-    """Returns an url of a resource specified by path in repo"""
+    """
+    Returns the full URL to the data artifact specified by its `path` in a
+    `repo`.
+    NOTE: There is no guarantee that the file actually exists in that location.
+    """
     with _make_repo(repo, rev=rev) as _repo:
-        abspath = os.path.join(_repo.root_dir, path)
-        out, = _repo.find_outs_by_path(abspath)
+        _require_dvc(_repo)
+        out = _repo.find_out_by_relpath(path)
         remote_obj = _repo.cloud.get_remote(remote)
         return str(remote_obj.checksum_to_path_info(out.checksum))
 
 
 def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Opens a specified resource as a file descriptor"""
+    """Context manager to open a file artifact as a file object."""
     args = (path,)
     kwargs = {
         "repo": repo,
@@ -40,21 +87,20 @@ class _OpenContextManager(GCM):
 
     def __getattr__(self, name):
         raise AttributeError(
-            "dvc.api.open() should be used in a with statement"
+            "dvc.api.open() should be used in a with statement."
         )
 
 
 def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
     with _make_repo(repo, rev=rev) as _repo:
-        abspath = os.path.join(_repo.root_dir, path)
-        with _repo.open(
-            abspath, remote=remote, mode=mode, encoding=encoding
+        with _repo.open_by_relpath(
+            path, remote=remote, mode=mode, encoding=encoding
         ) as fd:
             yield fd
 
 
 def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Read a specified resource into string"""
+    """Returns the contents of a file artifact."""
     with open(
         path, repo=repo, rev=rev, remote=remote, mode=mode, encoding=encoding
     ) as fd:
@@ -62,10 +108,140 @@ def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
 
 
 @contextmanager
-def _make_repo(repo_url, rev=None):
-    if not repo_url or urlparse(repo_url).scheme == "":
-        assert rev is None, "Custom revision is not supported for local repo"
-        yield Repo(repo_url)
+def _make_repo(repo_url=None, rev=None):
+    repo_url = repo_url or os.getcwd()
+    if rev is None and os.path.exists(repo_url):
+        try:
+            yield Repo(repo_url)
+            return
+        except NotDvcRepoError:
+            pass  # fallthrough to external_repo
+    with external_repo(url=repo_url, rev=rev) as repo:
+        yield repo
+
+
+def summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml", args=None):
+    """Instantiate an object described in the `summon_file`."""
+    with prepare_summon(
+        name, repo=repo, rev=rev, summon_file=summon_file
+    ) as desc:
+        try:
+            summon_dict = SUMMON_PYTHON_SCHEMA(desc.obj["summon"])
+        except Invalid as exc:
+            raise SummonError(str(exc)) from exc
+
+        _args = {**summon_dict.get("args", {}), **(args or {})}
+        return _invoke_method(summon_dict["call"], _args, desc.repo.root_dir)
+
+
+@contextmanager
+def prepare_summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml"):
+    """Does a couple of things every summon needs as a prerequisite:
+    clones the repo, parses the summon file and pulls the deps.
+
+    Calling code is expected to complete the summon logic following
+    instructions stated in "summon" dict of the object spec.
+
+    Returns a SummonDesc instance, which contains references to a Repo object,
+    named object specification and resolved paths to deps.
+    """
+    with _make_repo(repo, rev=rev) as _repo:
+        _require_dvc(_repo)
+        try:
+            path = os.path.join(_repo.root_dir, summon_file)
+            obj = _get_object_spec(name, path)
+            yield SummonDesc(_repo, obj)
+        except SummonError as exc:
+            raise SummonError(
+                str(exc) + " at '{}' in '{}'".format(summon_file, repo)
+            ) from exc.__cause__
+
+
+class SummonDesc:
+    def __init__(self, repo, obj):
+        self.repo = repo
+        self.obj = obj
+        self._pull_deps()
+
+    @property
+    def deps(self):
+        return [os.path.join(self.repo.root_dir, d) for d in self._deps]
+
+    @property
+    def _deps(self):
+        return self.obj["summon"].get("deps", [])
+
+    def _pull_deps(self):
+        if not self._deps:
+            return
+
+        outs = [self.repo.find_out_by_relpath(d) for d in self._deps]
+
+        with self.repo.state:
+            for out in outs:
+                self.repo.cloud.pull(out.get_used_cache())
+                out.checkout()
+
+
+def _get_object_spec(name, path):
+    """
+    Given a summonable object's name, search for it on the given file
+    and return its description.
+    """
+    try:
+        with builtin_open(path, "r") as fobj:
+            content = SUMMON_FILE_SCHEMA(ruamel.yaml.safe_load(fobj.read()))
+            objects = [x for x in content["objects"] if x["name"] == name]
+
+        if not objects:
+            raise SummonError("No object with name '{}'".format(name))
+        elif len(objects) >= 2:
+            raise SummonError(
+                "More than one object with name '{}'".format(name)
+            )
+
+        return objects[0]
+
+    except FileNotFoundError as exc:
+        raise SummonError("Summon file not found") from exc
+    except ruamel.yaml.YAMLError as exc:
+        raise SummonError("Failed to parse summon file") from exc
+    except Invalid as exc:
+        raise SummonError(str(exc)) from exc
+
+
+@wrap_with(threading.Lock())
+def _invoke_method(call, args, path):
+    # XXX: Some issues with this approach:
+    #   * Import will pollute sys.modules
+    #   * sys.path manipulation is "theoretically" not needed,
+    #     but tests are failing for an unknown reason.
+    cwd = os.getcwd()
+
+    try:
+        os.chdir(path)
+        sys.path.insert(0, path)
+        method = _import_string(call)
+        return method(**args)
+    finally:
+        os.chdir(cwd)
+        sys.path.pop(0)
+
+
+def _import_string(import_name):
+    """Imports an object based on a string.
+    Useful to delay import to not load everything on startup.
+    Use dotted notaion in `import_name`, e.g. 'dvc.remote.gs.RemoteGS'.
+
+    :return: imported object
+    """
+    if "." in import_name:
+        module, obj = import_name.rsplit(".", 1)
     else:
-        with external_repo(url=repo_url, rev=rev) as repo:
-            yield repo
+        return importlib.import_module(import_name)
+    return getattr(importlib.import_module(module), obj)
+
+
+def _require_dvc(repo):
+    if not isinstance(repo, Repo):
+        raise UrlNotDvcRepoError(repo.url)

dvc/cache.py

@@ -1,13 +1,10 @@
-"""Manages cache of a dvc repo."""
-from __future__ import unicode_literals
-
+"""Manages cache of a DVC repo."""
 import os
 from collections import defaultdict
 
 from funcy import cached_property
 
 from dvc.config import Config
-from dvc.utils.compat import builtin_str
 
 
 class CacheConfig(object):
@@ -56,12 +53,12 @@ def _make_remote_property(name):
 
         return Remote(self.repo, name=remote)
 
-    getter.__name__ = builtin_str(name)
+    getter.__name__ = name
     return cached_property(getter)
 
 
 class Cache(object):
-    """Class that manages cache locations of a dvc repo.
+    """Class that manages cache locations of a DVC repo.
 
     Args:
         repo (dvc.repo.Repo): repo instance that this cache belongs to.

dvc/cli.py

@@ -1,7 +1,4 @@
-"""Command line interface for dvc."""
-from __future__ import print_function
-from __future__ import unicode_literals
-
+"""DVC command line interface"""
 import argparse
 import logging
 import sys
@@ -173,7 +170,7 @@ def parse_args(argv=None):
         title="Available Commands",
         metavar="COMMAND",
         dest="cmd",
-        help="Use dvc COMMAND --help for command-specific help.",
+        help="Use `dvc COMMAND --help` for command-specific help.",
     )
 
     fix_subparsers(subparsers)

dvc/command/add.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/base.py

@@ -1,9 +1,5 @@
-from __future__ import unicode_literals
-
 import logging
 
-import colorama
-
 
 logger = logging.getLogger(__name__)
 
@@ -16,23 +12,18 @@ def fix_subparsers(subparsers):
         Args:
             subparsers: subparsers to fix.
     """
-    from dvc.utils.compat import is_py3
-
-    if is_py3:  # pragma: no cover
-        subparsers.required = True
-        subparsers.dest = "cmd"
+    subparsers.required = True
+    subparsers.dest = "cmd"
 
 
 def append_doc_link(help_message, path):
+    from dvc.utils import format_link
+
     if not path:
         return help_message
     doc_base = "https://man.dvc.org/"
-    return "{message}\nDocumentation: <{blue}{base}{path}{nc}>".format(
-        message=help_message,
-        base=doc_base,
-        path=path,
-        blue=colorama.Fore.CYAN,
-        nc=colorama.Fore.RESET,
+    return "{message}\nDocumentation: {link}".format(
+        message=help_message, link=format_link(doc_base + path)
     )
 
 
@@ -44,7 +35,8 @@ class CmdBase(object):
         self.repo = Repo()
         self.config = self.repo.config
         self.args = args
-        updater = Updater(self.repo.dvc_dir)
+        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
         updater.check()
 
     @property

dvc/command/cache.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 
 from dvc.cache import CacheConfig
@@ -10,7 +8,7 @@ from dvc.command.config import CmdConfig
 
 class CmdCacheDir(CmdConfig):
     def __init__(self, args):
-        super(CmdCacheDir, self).__init__(args)
+        super().__init__(args)
         self.cache_config = CacheConfig(self.config)
 
     def run(self):
@@ -32,7 +30,8 @@ def add_parser(subparsers, parent_parser):
     )
 
     cache_subparsers = cache_parser.add_subparsers(
-        dest="cmd", help="Use dvc cache CMD --help for command-specific help."
+        dest="cmd",
+        help="Use `dvc cache CMD --help` for command-specific " "help.",
     )
 
     fix_subparsers(cache_subparsers)

dvc/command/checkout.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 
 from dvc.command.base import append_doc_link
@@ -12,6 +10,7 @@ class CmdCheckout(CmdBase):
             targets=self.args.targets,
             with_deps=self.args.with_deps,
             force=self.args.force,
+            relink=self.args.relink,
             recursive=self.args.recursive,
         )
         return 0
@@ -27,13 +26,6 @@ def add_parser(subparsers, parent_parser):
         help=CHECKOUT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    checkout_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Do not prompt when removing working directory files.",
-    )
     checkout_parser.add_argument(
         "-d",
         "--with-deps",
@@ -48,6 +40,19 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Checkout all subdirectories of the specified directory.",
     )
+    checkout_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Do not prompt when removing working directory files.",
+    )
+    checkout_parser.add_argument(
+        "--relink",
+        action="store_true",
+        default=False,
+        help="Recreate links or copies from cache to workspace.",
+    )
     checkout_parser.add_argument(
         "targets",
         nargs="*",

dvc/command/commit.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/config.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -13,7 +11,7 @@ logger = logging.getLogger(__name__)
 
 class CmdConfig(CmdBaseNoRepo):
     def __init__(self, args):
-        super(CmdConfig, self).__init__(args)
+        super().__init__(args)
 
         self.config = Config(validate=False)
 
@@ -29,6 +27,14 @@ class CmdConfig(CmdBaseNoRepo):
                 section, opt, self.args.value, level=self.args.level
             )
 
+        is_write = self.args.unset or self.args.value is not None
+        if is_write and self.args.name == "cache.type":
+            logger.warning(
+                "You have changed the 'cache.type' option. This doesn't update"
+                " any existing workspace file links, but it can be done with:"
+                "\n             dvc checkout --relink"
+            )
+
         return 0
 
 

dvc/command/daemon.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.command.base import CmdBaseNoRepo
 from dvc.command.base import fix_subparsers
 
@@ -12,11 +10,16 @@ class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
         from dvc.repo import Repo
+        from dvc.config import Config
         from dvc.updater import Updater
 
         root_dir = Repo.find_root()
         dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
-        updater = Updater(dvc_dir)
+        config = Config(dvc_dir, validate=False)
+        hardlink_lock = config.config.get("core", {}).get(
+            "hardlink_lock", False
+        )
+        updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
         updater.fetch(detach=False)
 
         return 0
@@ -24,10 +27,9 @@ class CmdDaemonUpdater(CmdDaemonBase):
 
 class CmdDaemonAnalytics(CmdDaemonBase):
     def run(self):
-        from dvc.analytics import Analytics
+        from dvc import analytics
 
-        analytics = Analytics.load(self.args.target)
-        analytics.send()
+        analytics.send(self.args.target)
 
         return 0
 
@@ -42,7 +44,8 @@ def add_parser(subparsers, parent_parser):
     )
 
     daemon_subparsers = daemon_parser.add_subparsers(
-        dest="cmd", help="Use dvc daemon CMD --help for command-specific help."
+        dest="cmd",
+        help="Use `dvc daemon CMD --help` for command-specific " "help.",
     )
 
     fix_subparsers(daemon_subparsers)

dvc/command/data_sync.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/destroy.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/diff.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/gc.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 import os

dvc/command/get.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -12,7 +10,27 @@ logger = logging.getLogger(__name__)
 
 
 class CmdGet(CmdBaseNoRepo):
+    def _show_url(self):
+        from dvc.api import get_url
+
+        try:
+            url = get_url(
+                self.args.path, repo=self.args.url, rev=self.args.rev
+            )
+            logger.info(url)
+        except DvcException:
+            logger.exception("failed to show URL")
+            return 1
+
+        return 0
+
     def run(self):
+        if self.args.show_url:
+            return self._show_url()
+
+        return self._get_file_from_repo()
+
+    def _get_file_from_repo(self):
         from dvc.repo import Repo
 
         try:
@@ -33,7 +51,9 @@ class CmdGet(CmdBaseNoRepo):
 
 
 def add_parser(subparsers, parent_parser):
-    GET_HELP = "Download data from DVC repository."
+    GET_HELP = (
+        "Download file or directory from any DVC project or Git repository."
+    )
     get_parser = subparsers.add_parser(
         "get",
         parents=[parent_parser],
@@ -42,13 +62,23 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     get_parser.add_argument(
-        "url", help="URL of Git repository with DVC project to download from."
-    )
-    get_parser.add_argument("path", help="Path to data within DVC repository.")
-    get_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to put data to."
+        "url",
+        help="Location of DVC project or Git repository to download from",
     )
     get_parser.add_argument(
-        "--rev", nargs="?", help="DVC repository git revision."
+        "path",
+        help="Path to a file or directory within the project or repository",
+    )
+    get_parser.add_argument(
+        "-o", "--out", nargs="?", help="Destination path to download files to"
+    )
+    get_parser.add_argument(
+        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
+    )
+    get_parser.add_argument(
+        "--show-url",
+        action="store_true",
+        help="Print the storage location (URL) the target data would be "
+        "downloaded from, and exit.",
     )
     get_parser.set_defaults(func=CmdGet)

dvc/command/get_url.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/imp.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -32,7 +30,8 @@ class CmdImport(CmdBase):
 
 def add_parser(subparsers, parent_parser):
     IMPORT_HELP = (
-        "Download data from DVC repository and take it under DVC control."
+        "Download file or directory from any DVC project or Git repository "
+        "into the workspace, and track it."
     )
 
     import_parser = subparsers.add_parser(
@@ -43,15 +42,17 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawTextHelpFormatter,
     )
     import_parser.add_argument(
-        "url", help="URL of Git repository with DVC project to download from."
+        "url",
+        help="Location of DVC project or Git repository to download from",
     )
     import_parser.add_argument(
-        "path", help="Path to data within DVC repository."
+        "path",
+        help="Path to a file or directory within the project or repository",
     )
     import_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to put data to."
+        "-o", "--out", nargs="?", help="Destination path to download files to"
     )
     import_parser.add_argument(
-        "--rev", nargs="?", help="DVC repository git revision."
+        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
     )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -44,12 +42,17 @@ def add_parser(subparsers, parent_parser):
         "url",
         help="Supported urls:\n"
         "/path/to/file\n"
+        "/path/to/directory\n"
         "C:\\\\path\\to\\file\n"
+        "C:\\\\path\\to\\directory\n"
         "https://example.com/path/to/file\n"
         "s3://bucket/path/to/file\n"
+        "s3://bucket/path/to/directory\n"
         "gs://bucket/path/to/file\n"
+        "gs://bucket/path/to/directory\n"
         "hdfs://example.com/path/to/file\n"
         "ssh://example.com:/path/to/file\n"
+        "ssh://example.com:/path/to/directory\n"
         "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(

dvc/command/init.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -21,7 +19,7 @@ class CmdInit(CmdBaseNoRepo):
             )
             self.config = self.repo.config
         except InitError:
-            logger.exception("failed to initiate dvc")
+            logger.exception("failed to initiate DVC")
             return 1
         return 0
 
@@ -45,7 +43,7 @@ def add_parser(subparsers, parent_parser):
         "--no-scm",
         action="store_true",
         default=False,
-        help="Initiate dvc in directory that is "
+        help="Initiate DVC in directory that is "
         "not tracked by any scm tool (e.g. git).",
     )
     init_parser.add_argument(

dvc/command/install.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -15,7 +13,7 @@ class CmdInstall(CmdBase):
         try:
             self.repo.install()
         except Exception:
-            logger.exception("failed to install dvc hooks")
+            logger.exception("failed to install DVC Git hooks")
             return 1
         return 0
 

dvc/command/lock.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/metrics.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -28,7 +26,12 @@ def show_metrics(metrics, all_branches=False, all_tags=False):
             logger.info("{branch}:".format(branch=branch))
 
         for fname, metric in val.items():
-            lines = metric if type(metric) is list else metric.splitlines()
+            if isinstance(metric, dict):
+                lines = list(metric.values())
+            elif isinstance(metric, list):
+                lines = metric
+            else:
+                lines = metric.splitlines()
 
             if len(lines) > 1:
                 logger.info("\t{fname}:".format(fname=fname))
@@ -102,6 +105,59 @@ class CmdMetricsRemove(CmdBase):
         return 0
 
 
+def _show_diff(diff):
+    from texttable import Texttable
+
+    if not diff:
+        return "No changes."
+
+    table = Texttable()
+
+    # remove borders to make it easier for users to copy stuff
+    table.set_chars(("", "", "", ""))
+    table.set_deco(0)
+
+    rows = [["Path", "Metric", "Value", "Change"]]
+    for fname, mdiff in diff.items():
+        for metric, change in mdiff.items():
+            rows.append(
+                [
+                    fname,
+                    metric,
+                    change["new"],
+                    change.get("diff", "diff not supported"),
+                ]
+            )
+    table.add_rows(rows)
+    return table.draw()
+
+
+class CmdMetricsDiff(CmdBase):
+    def run(self):
+        try:
+            diff = self.repo.metrics.diff(
+                a_ref=self.args.a_ref,
+                b_ref=self.args.b_ref,
+                targets=self.args.targets,
+                typ=self.args.type,
+                xpath=self.args.xpath,
+                recursive=self.args.recursive,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                logger.info(_show_diff(diff))
+
+        except DvcException:
+            logger.exception("failed to show metrics diff")
+            return 1
+
+        return 0
+
+
 def add_parser(subparsers, parent_parser):
     METRICS_HELP = "Commands to add, manage, collect and display metrics."
 
@@ -115,7 +171,7 @@ def add_parser(subparsers, parent_parser):
 
     metrics_subparsers = metrics_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc metrics CMD --help to display command-specific help.",
+        help="Use `dvc metrics CMD --help` to display command-specific help.",
     )
 
     fix_subparsers(metrics_subparsers)
@@ -216,3 +272,65 @@ def add_parser(subparsers, parent_parser):
     )
     metrics_remove_parser.add_argument("path", help="Path to a metric file.")
     metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
+
+    METRICS_DIFF_HELP = "Output metric values."
+    metrics_diff_parser = metrics_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_DIFF_HELP, "metrics/diff"),
+        help=METRICS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_diff_parser.add_argument(
+        "a_ref",
+        nargs="?",
+        help=(
+            "Git reference from which diff is calculated. "
+            "If omitted `HEAD`(latest commit) is used."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "b_ref",
+        nargs="?",
+        help=(
+            "Git reference to which diff is calculated. "
+            "If omitted current working tree is used."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help=(
+            "Metric files or directories (see -R) to show diff for. "
+            "Shows diff for all metric files by default."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "-t",
+        "--type",
+        help=(
+            "Type of metrics (json/tsv/htsv/csv/hcsv). "
+            "It can be detected by the file extension automatically. "
+            "Unsupported types will be treated as raw."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
+    )
+    metrics_diff_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help=(
+            "If any target is a directory, recursively search and process "
+            "metric files."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    metrics_diff_parser.set_defaults(func=CmdMetricsDiff)

dvc/command/move.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/pipeline.py

@@ -1,14 +1,8 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.command.base import fix_subparsers
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.utils import relpath
-from dvc.utils.compat import str
 
 
 logger = logging.getLogger(__name__)
@@ -21,23 +15,21 @@ class CmdPipelineShow(CmdBase):
 
         stage = Stage.load(self.repo, target)
         G = self.repo.graph
-        stages = networkx.get_node_attributes(G, "stage")
-        node = relpath(stage.path, self.repo.root_dir)
-        nodes = networkx.dfs_postorder_nodes(G, node)
+        stages = networkx.dfs_postorder_nodes(G, stage)
 
         if locked:
-            nodes = [n for n in nodes if stages[n].locked]
+            stages = [s for s in stages if s.locked]
 
-        for n in nodes:
+        for stage in stages:
             if commands:
-                if stages[n].cmd is None:
+                if stage.cmd is None:
                     continue
-                logger.info(stages[n].cmd)
+                logger.info(stage.cmd)
             elif outs:
-                for out in stages[n].outs:
+                for out in stage.outs:
                     logger.info(str(out))
             else:
-                logger.info(n)
+                logger.info(stage.path_in_repo)
 
     def __build_graph(self, target, commands, outs):
         import networkx
@@ -45,14 +37,10 @@ class CmdPipelineShow(CmdBase):
         from dvc.repo.graph import get_pipeline
 
         stage = Stage.load(self.repo, target)
-        node = relpath(stage.path, self.repo.root_dir)
-
-        G = get_pipeline(self.repo.pipelines, node)
-        stages = networkx.get_node_attributes(G, "stage")
+        G = get_pipeline(self.repo.pipelines, stage)
 
         nodes = []
-        for n in G:
-            stage = stages[n]
+        for stage in G:
             if commands:
                 if stage.cmd is None:
                     continue
@@ -64,9 +52,7 @@ class CmdPipelineShow(CmdBase):
                 nodes.append(stage.relpath)
 
         edges = []
-        for e in G.edges():
-            from_stage = stages[e[0]]
-            to_stage = stages[e[1]]
+        for from_stage, to_stage in G.edges():
             if commands:
                 if to_stage.cmd is None:
                     continue
@@ -115,7 +101,7 @@ class CmdPipelineShow(CmdBase):
         tree.show()
 
     def __write_dot(self, target, commands, outs):
-        from dvc.utils.compat import StringIO
+        import io
         import networkx
         from networkx.drawing.nx_pydot import write_dot
 
@@ -125,7 +111,7 @@ class CmdPipelineShow(CmdBase):
         simple_g = networkx.DiGraph()
         simple_g.add_edges_from(edges)
 
-        dot_file = StringIO()
+        dot_file = io.StringIO()
         write_dot(simple_g, dot_file)
         logger.info(dot_file.getvalue())
 
@@ -163,14 +149,11 @@ class CmdPipelineShow(CmdBase):
 
 class CmdPipelineList(CmdBase):
     def run(self):
-        import networkx
-
         pipelines = self.repo.pipelines
-        for p in pipelines:
-            stages = networkx.get_node_attributes(p, "stage")
-            for stage in stages:
-                logger.info(stage)
-            if len(stages) != 0:
+        for pipeline in pipelines:
+            for stage in pipeline:
+                logger.info(stage.relpath)
+            if len(pipeline) != 0:
                 logger.info("=" * 80)
         logger.info("{} pipelines total".format(len(pipelines)))
 
@@ -189,7 +172,7 @@ def add_parser(subparsers, parent_parser):
 
     pipeline_subparsers = pipeline_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc pipeline CMD --help for command-specific help.",
+        help="Use `dvc pipeline CMD --help` for command-specific help.",
     )
 
     fix_subparsers(pipeline_subparsers)

dvc/command/remote.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -7,18 +5,23 @@ from dvc.command.base import append_doc_link
 from dvc.command.base import fix_subparsers
 from dvc.command.config import CmdConfig
 from dvc.remote.config import RemoteConfig
+from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
 
 class CmdRemoteConfig(CmdConfig):
     def __init__(self, args):
-        super(CmdRemoteConfig, self).__init__(args)
+        super().__init__(args)
         self.remote_config = RemoteConfig(self.config)
 
 
 class CmdRemoteAdd(CmdRemoteConfig):
     def run(self):
+        if self.args.default:
+            logger.info(
+                "Setting '{}' as a default remote.".format(self.args.name)
+            )
         self.remote_config.add(
             self.args.name,
             self.args.url,
@@ -81,7 +84,7 @@ def add_parser(subparsers, parent_parser):
 
     remote_subparsers = remote_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc remote CMD --help for " "command-specific help.",
+        help="Use `dvc remote CMD --help` for " "command-specific help.",
     )
 
     fix_subparsers(remote_subparsers)
@@ -97,7 +100,9 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser.add_argument("name", help="Name of the remote")
     remote_add_parser.add_argument(
         "url",
-        help="URL. See full list of supported urls at " "man.dvc.org/remote",
+        help="URL. See full list of supported urls at {}".format(
+            format_link("https://man.dvc.org/remote")
+        ),
     )
     remote_add_parser.add_argument(
         "-d",

dvc/command/remove.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -20,7 +18,7 @@ class CmdRemove(CmdBase):
         if self.args.force:
             return False
 
-        msg = "Are you sure you want to remove {} with its outputs?".format(
+        msg = "Are you sure you want to remove '{}' with its outputs?".format(
             target
         )
 
@@ -38,7 +36,7 @@ class CmdRemove(CmdBase):
                 outs_only = self._is_outs_only(target)
                 self.repo.remove(target, outs_only=outs_only)
             except DvcException:
-                logger.exception("failed to remove {}".format(target))
+                logger.exception("failed to remove '{}'".format(target))
                 return 1
         return 0
 

dvc/command/repro.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 import os

dvc/command/root.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/run.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/status.py

@@ -1,9 +1,6 @@
-from __future__ import unicode_literals
-
 import logging
 
 from dvc.command.data_sync import CmdDataBase
-from dvc.utils.compat import str
 
 
 logger = logging.getLogger(__name__)

dvc/command/unprotect.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/update.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/version.py

@@ -1,11 +1,10 @@
-from __future__ import unicode_literals
-
 import argparse
 import itertools
 import logging
 import os
 import platform
 import uuid
+import pathlib
 
 try:
     import psutil
@@ -13,7 +12,6 @@ except ImportError:
     psutil = None
 
 from dvc.utils import is_binary, relpath
-from dvc.utils.compat import pathlib
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
 from dvc.exceptions import DvcException, NotDvcRepoError
@@ -27,23 +25,13 @@ class CmdVersion(CmdBaseNoRepo):
     def run(self):
         from dvc.repo import Repo
 
-        dvc_version = __version__
-        python_version = platform.python_version()
-        platform_type = platform.platform()
-        binary = is_binary()
-        info = (
-            "DVC version: {dvc_version}\n"
-            "Python version: {python_version}\n"
-            "Platform: {platform_type}\n"
-            "Binary: {binary}\n"
-            "Package: {package}\n"
-        ).format(
-            dvc_version=dvc_version,
-            python_version=python_version,
-            platform_type=platform_type,
-            binary=binary,
-            package=PKG,
-        )
+        info = [
+            "DVC version: {}".format(__version__),
+            "Python version: {}".format(platform.python_version()),
+            "Platform: {}".format(platform.platform()),
+            "Binary: {}".format(is_binary()),
+            "Package: {}".format(PKG),
+        ]
 
         try:
             repo = Repo()
@@ -54,9 +42,14 @@ class CmdVersion(CmdBaseNoRepo):
             # later decides to enable shared cache mode with
             # `dvc config cache.shared group`.
             if os.path.exists(repo.cache.local.cache_dir):
-                info += "Cache: {cache}\n".format(
-                    cache=self.get_linktype_support_info(repo)
+                info.append(
+                    "Cache: {}".format(self.get_linktype_support_info(repo))
                 )
+                if psutil:
+                    fs_type = self.get_fs_type(repo.cache.local.cache_dir)
+                    info.append(
+                        "Filesystem type (cache directory): {}".format(fs_type)
+                    )
             else:
                 logger.warning(
                     "Unable to detect supported link types, as cache "
@@ -66,19 +59,14 @@ class CmdVersion(CmdBaseNoRepo):
                     "check.".format(relpath(repo.cache.local.cache_dir))
                 )
 
-            if psutil:
-                info += (
-                    "Filesystem type (cache directory): {fs_cache}\n"
-                ).format(fs_cache=self.get_fs_type(repo.cache.local.cache_dir))
         except NotDvcRepoError:
             root_directory = os.getcwd()
 
         if psutil:
-            info += ("Filesystem type (workspace): {fs_root}").format(
-                fs_root=self.get_fs_type(os.path.abspath(root_directory))
-            )
+            fs_root = self.get_fs_type(os.path.abspath(root_directory))
+            info.append("Filesystem type (workspace): {}".format(fs_root))
 
-        logger.info(info)
+        logger.info("\n".join(info))
         return 0
 
     @staticmethod
@@ -98,9 +86,9 @@ class CmdVersion(CmdBaseNoRepo):
     @staticmethod
     def get_linktype_support_info(repo):
         links = {
-            "reflink": System.reflink,
-            "hardlink": System.hardlink,
-            "symlink": System.symlink,
+            "reflink": (System.reflink, None),
+            "hardlink": (System.hardlink, System.is_hardlink),
+            "symlink": (System.symlink, System.is_symlink),
         }
 
         fname = "." + str(uuid.uuid4())
@@ -110,18 +98,16 @@ class CmdVersion(CmdBaseNoRepo):
 
         cache = []
 
-        for name, link in links.items():
+        for name, (link, is_link) in links.items():
             try:
                 link(src, dst)
+                status = "supported"
+                if is_link and not is_link(dst):
+                    status = "broken"
                 os.unlink(dst)
-                supported = True
             except DvcException:
-                supported = False
-            cache.append(
-                "{name} - {supported}".format(
-                    name=name, supported=True if supported else False
-                )
-            )
+                status = "not supported"
+            cache.append("{name} - {status}".format(name=name, status=status))
         os.remove(src)
 
         return ", ".join(cache)

dvc/compat.py

@@ -0,0 +1,42 @@
+"""Python version compatibility code"""
+
+# Backport os.fspath() from Python 3.6
+try:
+    from os import fspath  # noqa: F821
+
+    fspath_py35 = lambda s: s  # noqa: E731
+except ImportError:
+
+    def fspath(path):
+        """Return the path representation of a path-like object.
+
+        If str or bytes is passed in, it is returned unchanged. Otherwise the
+        os.PathLike interface is used to get the path representation. If the
+        path representation is not str or bytes, TypeError is raised. If the
+        provided path is not str, bytes, or os.PathLike, TypeError is raised.
+        """
+        if isinstance(path, (str, bytes)):
+            return path
+
+        # Work from the object's type to match method resolution of other magic
+        # methods.
+        path_type = type(path)
+        try:
+            path_repr = path_type.__fspath__(path)
+        except AttributeError:
+            if hasattr(path_type, "__fspath__"):
+                raise
+            else:
+                raise TypeError(
+                    "expected str, bytes or os.PathLike object, "
+                    "not " + path_type.__name__
+                )
+        if isinstance(path_repr, (str, bytes)):
+            return path_repr
+        else:
+            raise TypeError(
+                "expected {}.__fspath__() to return str or bytes, "
+                "not {}".format(path_type.__name__, type(path_repr).__name__)
+            )
+
+    fspath_py35 = fspath

dvc/config.py

@@ -1,5 +1,4 @@
 """DVC config objects."""
-from __future__ import unicode_literals
 
 import copy
 import errno
@@ -13,36 +12,19 @@ from voluptuous import All, Any, Lower, Range, Coerce, Match
 
 from dvc.exceptions import DvcException
 from dvc.exceptions import NotDvcRepoError
-from dvc.utils.compat import open
-from dvc.utils.compat import str
 
 logger = logging.getLogger(__name__)
 
 
 class ConfigError(DvcException):
-    """DVC config exception.
+    """DVC config exception."""
 
-    Args:
-        msg (str): error message.
-        cause (Exception): optional exception that has caused this error.
-    """
-
-    def __init__(self, msg, cause=None):
-        super(ConfigError, self).__init__(
-            "config file error: {}".format(msg), cause=cause
-        )
+    def __init__(self, msg):
+        super().__init__("config file error: {}".format(msg))
 
 
 class NoRemoteError(ConfigError):
-    def __init__(self, command, cause=None):
-        msg = (
-            "no remote specified. Setup default remote with\n"
-            "    dvc config core.remote <name>\n"
-            "or use:\n"
-            "    dvc {} -r <name>\n".format(command)
-        )
-
-        super(NoRemoteError, self).__init__(msg, cause=cause)
+    pass
 
 
 def supported_cache_type(types):
@@ -82,11 +64,11 @@ def Choices(*choices):
         *choices: pass allowed values as arguments, or pass a list or
             tuple as a single argument
     """
-    return Any(*choices, msg="expected one of {}".format(",".join(choices)))
+    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
 
 
 class Config(object):  # pylint: disable=too-many-instance-attributes
-    """Class that manages configuration files for a dvc repo.
+    """Class that manages configuration files for a DVC repo.
 
     Args:
         dvc_dir (str): optional path to `.dvc` directory, that is used to
@@ -110,6 +92,8 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
+    CREDENTIALPATH = "credentialpath"
+
     LEVEL_LOCAL = 0
     LEVEL_REPO = 1
     LEVEL_GLOBAL = 2
@@ -124,6 +108,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_CORE_INTERACTIVE = "interactive"
     SECTION_CORE_ANALYTICS = "analytics"
     SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
+    SECTION_CORE_HARDLINK_LOCK = "hardlink_lock"
 
     SECTION_CACHE = "cache"
     SECTION_CACHE_DIR = "dir"
@@ -159,10 +144,11 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
         Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
         SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
+        Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool,
     }
 
     # aws specific options
-    SECTION_AWS_CREDENTIALPATH = "credentialpath"
+    SECTION_AWS_CREDENTIALPATH = CREDENTIALPATH
     SECTION_AWS_ENDPOINT_URL = "endpointurl"
     SECTION_AWS_LIST_OBJECTS = "listobjects"
     SECTION_AWS_REGION = "region"
@@ -170,9 +156,13 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_AWS_USE_SSL = "use_ssl"
     SECTION_AWS_SSE = "sse"
     SECTION_AWS_ACL = "acl"
+    SECTION_AWS_GRANT_READ = "grant_read"
+    SECTION_AWS_GRANT_READ_ACP = "grant_read_acp"
+    SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp"
+    SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control"
 
     # gcp specific options
-    SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH
+    SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH
     SECTION_GCP_PROJECTNAME = "projectname"
 
     # azure specific option
@@ -183,6 +173,12 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
     SECTION_OSS_ENDPOINT = "oss_endpoint"
 
+    # GDrive options
+    SECTION_GDRIVE_CLIENT_ID = "gdrive_client_id"
+    SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret"
+    SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file"
+
+    SECTION_REMOTE_CHECKSUM_JOBS = "checksum_jobs"
     SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
     SECTION_REMOTE_FMT = 'remote "{}"'
     SECTION_REMOTE_URL = "url"
@@ -194,6 +190,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_REMOTE_ASK_PASSWORD = "ask_password"
     SECTION_REMOTE_GSS_AUTH = "gss_auth"
     SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
+    SECTION_REMOTE_VERIFY = "verify"
     SECTION_REMOTE_SCHEMA = {
         Required(SECTION_REMOTE_URL): str,
         SECTION_AWS_REGION: str,
@@ -204,9 +201,14 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_USE_SSL, default=True): Bool,
         SECTION_AWS_SSE: str,
         SECTION_AWS_ACL: str,
+        SECTION_AWS_GRANT_READ: str,
+        SECTION_AWS_GRANT_READ_ACP: str,
+        SECTION_AWS_GRANT_WRITE_ACP: str,
+        SECTION_AWS_GRANT_FULL_CONTROL: str,
         SECTION_GCP_PROJECTNAME: str,
         SECTION_CACHE_TYPE: supported_cache_type,
         Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
+        SECTION_REMOTE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
         SECTION_REMOTE_USER: str,
         SECTION_REMOTE_PORT: Coerce(int),
         SECTION_REMOTE_KEY_FILE: str,
@@ -218,8 +220,12 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         SECTION_OSS_ACCESS_KEY_ID: str,
         SECTION_OSS_ACCESS_KEY_SECRET: str,
         SECTION_OSS_ENDPOINT: str,
+        SECTION_GDRIVE_CLIENT_ID: str,
+        SECTION_GDRIVE_CLIENT_SECRET: str,
+        SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
         PRIVATE_CWD: str,
-        Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): Bool,
+        SECTION_REMOTE_NO_TRAVERSE: Bool,
+        SECTION_REMOTE_VERIFY: Bool,
     }
 
     SECTION_STATE = "state"
@@ -377,7 +383,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         try:
             d = self.COMPILED_SCHEMA(d)
         except Invalid as exc:
-            raise ConfigError(str(exc), cause=exc)
+            raise ConfigError(str(exc)) from exc
         self.config = configobj.ConfigObj(d, write_empty_values=True)
 
     def save(self, config=None):

dvc/daemon.py

@@ -1,5 +1,4 @@
 """Launch `dvc daemon` command in a separate detached process."""
-from __future__ import unicode_literals
 
 import inspect
 import logging
@@ -10,7 +9,6 @@ from subprocess import Popen
 from dvc.env import DVC_DAEMON
 from dvc.utils import fix_env
 from dvc.utils import is_binary
-from dvc.utils.compat import cast_bytes_py2
 
 
 logger = logging.getLogger(__name__)
@@ -99,9 +97,7 @@ def daemon(args):
 
     env = fix_env()
     file_path = os.path.abspath(inspect.stack()[0][1])
-    env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2(
-        os.path.dirname(os.path.dirname(file_path))
-    )
-    env[cast_bytes_py2(DVC_DAEMON)] = cast_bytes_py2("1")
+    env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
+    env[DVC_DAEMON] = "1"
 
     _spawn(cmd, env)

dvc/dagascii.py

@@ -1,8 +1,9 @@
 """Draws DAG in ASCII."""
-from __future__ import print_function
-from __future__ import unicode_literals
 
+import logging
 import math
+import os
+import pydoc
 import sys
 
 from grandalf.graphs import Edge
@@ -12,6 +13,44 @@ from grandalf.layouts import SugiyamaLayout
 from grandalf.routing import EdgeViewer
 from grandalf.routing import route_with_lines
 
+from dvc.env import DVC_PAGER
+from dvc.utils import format_link
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_PAGER = "less"
+DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
+    DEFAULT_PAGER
+)
+
+
+def make_pager(cmd):
+    def pager(text):
+        return pydoc.tempfilepager(pydoc.plain(text), cmd)
+
+    return pager
+
+
+def find_pager():
+    if not sys.stdout.isatty():
+        return pydoc.plainpager
+
+    env_pager = os.getenv(DVC_PAGER)
+    if env_pager:
+        return make_pager(env_pager)
+
+    if os.system("({}) 2>{}".format(DEFAULT_PAGER, os.devnull)) == 0:
+        return make_pager(DEFAULT_PAGER_FORMATTED)
+
+    logger.warning(
+        "Unable to find `less` in the PATH. Check out "
+        "{} for more info.".format(
+            format_link("https://man.dvc.org/pipeline/show")
+        )
+    )
+    return pydoc.plainpager
+
 
 class VertexViewer(object):
     """Class to define vertex box boundaries that will be accounted for during
@@ -60,99 +99,10 @@ class AsciiCanvas(object):
 
     def draw(self):
         """Draws ASCII canvas on the screen."""
-        if sys.stdout.isatty():  # pragma: no cover
-            from asciimatics.screen import Screen
-
-            Screen.wrapper(self._do_draw)
-        else:
-            for line in self.canvas:
-                print("".join(line))
-
-    def _do_draw(self, screen):  # pragma: no cover
-        # pylint: disable=too-many-locals
-        # pylint: disable=too-many-branches, too-many-statements
-        from dvc.system import System
-        from asciimatics.event import KeyboardEvent
-
-        offset_x = 0
-        offset_y = 0
-        smaxrow, smaxcol = screen.dimensions
-        assert smaxrow > 1
-        assert smaxcol > 1
-        smaxrow -= 1
-        smaxcol -= 1
-
-        if self.lines + 1 > smaxrow:
-            max_y = self.lines + 1 - smaxrow
-        else:
-            max_y = 0
-
-        if self.cols + 1 > smaxcol:
-            max_x = self.cols + 1 - smaxcol
-        else:
-            max_x = 0
-
-        while True:
-            for y in range(smaxrow + 1):
-                y_index = offset_y + y
-                line = []
-                for x in range(smaxcol + 1):
-                    x_index = offset_x + x
-                    if (
-                        len(self.canvas) > y_index
-                        and len(self.canvas[y_index]) > x_index
-                    ):
-                        line.append(self.canvas[y_index][x_index])
-                    else:
-                        line.append(" ")
-                assert len(line) == (smaxcol + 1)
-                screen.print_at("".join(line), 0, y)
-
-            screen.refresh()
-
-            # NOTE: get_event() doesn't block by itself,
-            # so we have to do the blocking ourselves.
-            #
-            # NOTE: using this workaround while waiting for PR [1]
-            # to get merged and released. After that need to adjust
-            # asciimatics version requirements.
-            #
-            # [1] https://github.com/peterbrittain/asciimatics/pull/188
-            System.wait_for_input(self.TIMEOUT)
-
-            event = screen.get_event()
-            if not isinstance(event, KeyboardEvent):
-                continue
-
-            k = event.key_code
-            if k == screen.KEY_DOWN or k == ord("s"):
-                offset_y += 1
-            elif k == screen.KEY_PAGE_DOWN or k == ord("S"):
-                offset_y += smaxrow
-            elif k == screen.KEY_UP or k == ord("w"):
-                offset_y -= 1
-            elif k == screen.KEY_PAGE_UP or k == ord("W"):
-                offset_y -= smaxrow
-            elif k == screen.KEY_RIGHT or k == ord("d"):
-                offset_x += 1
-            elif k == ord("D"):
-                offset_x += smaxcol
-            elif k == screen.KEY_LEFT or k == ord("a"):
-                offset_x -= 1
-            elif k == ord("A"):
-                offset_x -= smaxcol
-            elif k == ord("q") or k == ord("Q"):
-                break
-
-            if offset_y > max_y:
-                offset_y = max_y
-            elif offset_y < 0:
-                offset_y = 0
-
-            if offset_x > max_x:
-                offset_x = max_x
-            elif offset_x < 0:
-                offset_x = 0
+        pager = find_pager()
+        lines = map("".join, self.canvas)
+        joined_lines = os.linesep.join(lines)
+        pager(joined_lines)
 
     def point(self, x, y, char):
         """Create a point on ASCII canvas.

dvc/data_cloud.py

@@ -1,5 +1,4 @@
 """Manages dvc remotes that user can use with push/pull/status commands."""
-from __future__ import unicode_literals
 
 import logging
 
@@ -40,7 +39,26 @@ class DataCloud(object):
         if remote:
             return self._init_remote(remote)
 
-        raise NoRemoteError(command)
+        has_non_default_remote = bool(
+            self.repo.config.list_options(
+                Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL
+            )
+        )
+
+        if has_non_default_remote:
+            error_msg = (
+                "no remote specified. Setup default remote with\n"
+                "    dvc remote default <remote name>\n"
+                "or use:\n"
+                "    dvc {} -r <remote name>".format(command)
+            )
+        else:
+            error_msg = (
+                "no remote specified. Create a default remote with\n"
+                "    dvc remote add -d <remote name> <remote url>"
+            )
+
+        raise NoRemoteError(error_msg)
 
     def _init_remote(self, remote):
         return Remote(self.repo, name=remote)
@@ -74,13 +92,25 @@ class DataCloud(object):
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
-        return self.repo.cache.local.pull(
-            cache,
-            jobs=jobs,
-            remote=self.get_remote(remote, "pull"),
-            show_checksums=show_checksums,
+        remote = self.get_remote(remote, "pull")
+        downloaded_items_num = self.repo.cache.local.pull(
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )
 
+        if not remote.verify:
+            self._save_pulled_checksums(cache)
+
+        return downloaded_items_num
+
+    def _save_pulled_checksums(self, cache):
+        for checksum in cache["local"].keys():
+            cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
+            if self.repo.cache.local.exists(cache_file):
+                # We can safely save here, as existing corrupted files will be
+                # removed upon status, while files corrupted during download
+                # will not be moved from tmp_file (see `RemoteBASE.download()`)
+                self.repo.state.save(cache_file, checksum)
+
     def status(self, cache, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
 

dvc/dependency/__init__.py

@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+from urllib.parse import urlparse
 
 import dvc.output as output
 from dvc.dependency.gs import DependencyGS
@@ -46,8 +46,6 @@ SCHEMA[DependencyREPO.PARAM_REPO] = DependencyREPO.REPO_SCHEMA
 
 
 def _get(stage, p, info):
-    from dvc.utils.compat import urlparse
-
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":

dvc/dependency/base.py

@@ -1,23 +1,21 @@
-from __future__ import unicode_literals
-
 from dvc.exceptions import DvcException
 
 
 class DependencyDoesNotExistError(DvcException):
     def __init__(self, path):
         msg = "dependency '{}' does not exist".format(path)
-        super(DependencyDoesNotExistError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class DependencyIsNotFileOrDirError(DvcException):
     def __init__(self, path):
         msg = "dependency '{}' is not a file or directory".format(path)
-        super(DependencyIsNotFileOrDirError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class DependencyIsStageFileError(DvcException):
     def __init__(self, path):
-        super(DependencyIsStageFileError, self).__init__(
+        super().__init__(
             "Stage file '{}' cannot be a dependency.".format(path)
         )
 

dvc/dependency/gs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.gs import OutputGS
 

dvc/dependency/hdfs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.hdfs import OutputHDFS
 

dvc/dependency/http.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP

dvc/dependency/https.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from .http import DependencyHTTP
 from dvc.remote.https import RemoteHTTPS
 

dvc/dependency/local.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.local import OutputLOCAL
 

dvc/dependency/repo.py

@@ -1,13 +1,10 @@
-from __future__ import unicode_literals
+import os
 
-import copy
-from contextlib import contextmanager
-
-from funcy import merge
+from voluptuous import Required
 
 from .local import DependencyLOCAL
-from dvc.external_repo import external_repo
-from dvc.utils.compat import str
+from dvc.exceptions import OutputNotFoundError
+from dvc.path_info import PathInfo
 
 
 class DependencyREPO(DependencyLOCAL):
@@ -16,11 +13,15 @@ class DependencyREPO(DependencyLOCAL):
     PARAM_REV = "rev"
     PARAM_REV_LOCK = "rev_lock"
 
-    REPO_SCHEMA = {PARAM_URL: str, PARAM_REV: str, PARAM_REV_LOCK: str}
+    REPO_SCHEMA = {
+        Required(PARAM_URL): str,
+        PARAM_REV: str,
+        PARAM_REV_LOCK: str,
+    }
 
     def __init__(self, def_repo, stage, *args, **kwargs):
         self.def_repo = def_repo
-        super(DependencyREPO, self).__init__(stage, *args, **kwargs)
+        super().__init__(stage, *args, **kwargs)
 
     def _parse_path(self, remote, path):
         return None
@@ -32,24 +33,33 @@ class DependencyREPO(DependencyLOCAL):
     @property
     def repo_pair(self):
         d = self.def_repo
-        return d[self.PARAM_URL], d[self.PARAM_REV_LOCK] or d[self.PARAM_REV]
+        rev = d.get(self.PARAM_REV_LOCK) or d.get(self.PARAM_REV)
+        return d[self.PARAM_URL], rev
 
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 
-    @contextmanager
-    def _make_repo(self, **overrides):
-        with external_repo(**merge(self.def_repo, overrides)) as repo:
-            yield repo
+    def _make_repo(self, *, locked=True):
+        from dvc.external_repo import external_repo
+
+        d = self.def_repo
+        rev = (d.get("rev_lock") if locked else None) or d.get("rev")
+        return external_repo(d["url"], rev=rev)
+
+    def _get_checksum(self, locked=True):
+        with self._make_repo(locked=locked) as repo:
+            try:
+                return repo.find_out_by_relpath(self.def_path).info["md5"]
+            except OutputNotFoundError:
+                path = PathInfo(os.path.join(repo.root_dir, self.def_path))
+                # We are polluting our repo cache with some dir listing here
+                return self.repo.cache.local.get_checksum(path)
 
     def status(self):
-        with self._make_repo() as repo:
-            current = repo.find_out_by_relpath(self.def_path).info
+        current_checksum = self._get_checksum(locked=True)
+        updated_checksum = self._get_checksum(locked=False)
 
-        with self._make_repo(rev_lock=None) as repo:
-            updated = repo.find_out_by_relpath(self.def_path).info
-
-        if current != updated:
+        if current_checksum != updated_checksum:
             return {str(self): "update available"}
 
         return {}
@@ -60,23 +70,16 @@ class DependencyREPO(DependencyLOCAL):
     def dumpd(self):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
-    def fetch(self):
-        with self._make_repo(
-            cache_dir=self.repo.cache.local.cache_dir
-        ) as repo:
-            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
-
-            out = repo.find_out_by_relpath(self.def_path)
-            with repo.state:
-                repo.cloud.pull(out.get_used_cache())
-
-        return out
-
     def download(self, to):
-        out = self.fetch()
-        to.info = copy.copy(out.info)
-        to.checkout()
+        with self._make_repo() as repo:
+            if self.def_repo.get(self.PARAM_REV_LOCK) is None:
+                self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+
+            if hasattr(repo, "cache"):
+                repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
+
+            repo.pull_to(self.def_path, to.path_info)
 
     def update(self):
-        with self._make_repo(rev_lock=None) as repo:
+        with self._make_repo(locked=False) as repo:
             self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()

dvc/dependency/s3.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.s3 import OutputS3
 

dvc/dependency/ssh.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.ssh import OutputSSH
 

dvc/env.py

@@ -1 +1,2 @@
 DVC_DAEMON = "DVC_DAEMON"
+DVC_PAGER = "DVC_PAGER"

dvc/exceptions.py

@@ -1,32 +1,10 @@
 """Exceptions raised by the dvc."""
-from __future__ import unicode_literals
 
-import traceback
-
-from dvc.utils import relpath
-from dvc.utils.compat import builtin_str
-from dvc.utils.compat import str
+from dvc.utils import relpath, format_link
 
 
 class DvcException(Exception):
-    """Base class for all dvc exceptions.
-
-    Args:
-        msg (unicode): message for this exception.
-        cause (Exception): optional cause exception.
-    """
-
-    def __init__(self, msg, cause=None):
-        # NOTE: unlike python 3, python 2 doesn't have built-in support
-        # for chained exceptions, so we are using our own implementation.
-        self.cause = cause
-        self.cause_tb = None
-        if cause:
-            try:
-                self.cause_tb = traceback.format_exc()
-            except AttributeError:  # pragma: no cover
-                pass
-        super(DvcException, self).__init__(msg)
+    """Base class for all dvc exceptions."""
 
 
 class OutputDuplicationError(DvcException):
@@ -39,17 +17,14 @@ class OutputDuplicationError(DvcException):
     """
 
     def __init__(self, output, stages):
-        assert isinstance(output, str) or isinstance(output, builtin_str)
-        assert isinstance(stages, list)
-        assert all(
-            isinstance(stage, str) or isinstance(stage, builtin_str)
-            for stage in stages
-        )
+        assert isinstance(output, str)
+        assert all(hasattr(stage, "relpath") for stage in stages)
         msg = (
             "file/directory '{}' is specified as an output in more than one "
-            "stage: {}"
-        ).format(output, "\n    ".join(stages))
-        super(OutputDuplicationError, self).__init__(msg)
+            "stage: {}\n"
+            "This is not allowed. Consider using a different output name."
+        ).format(output, "\n    ".join(s.relpath for s in stages))
+        super().__init__(msg)
 
 
 class OutputNotFoundError(DvcException):
@@ -62,8 +37,8 @@ class OutputNotFoundError(DvcException):
     def __init__(self, output, repo=None):
         self.output = output
         self.repo = repo
-        super(OutputNotFoundError, self).__init__(
-            "unable to find DVC-file with output '{path}'".format(
+        super().__init__(
+            "Unable to find DVC-file with output '{path}'".format(
                 path=relpath(self.output)
             )
         )
@@ -74,21 +49,17 @@ class StagePathAsOutputError(DvcException):
     an output of another stage.
 
     Args:
-        cwd (str): path to the directory.
-        fname (str): path to the DVC-file that has cwd specified as an
-            output.
+        stage (Stage): a stage that is in some other stages output
+        output (str): an output covering the stage above
     """
 
-    def __init__(self, wdir, fname):
-        assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
-        assert isinstance(fname, str) or isinstance(fname, builtin_str)
-        msg = (
-            "current working directory '{cwd}' is specified as an output in "
-            "'{fname}'. Use another CWD to prevent any data removal.".format(
-                cwd=wdir, fname=fname
+    def __init__(self, stage, output):
+        assert isinstance(output, str)
+        super().__init__(
+            "'{stage}' is within an output '{output}' of another stage".format(
+                stage=stage.relpath, output=output
             )
         )
-        super(StagePathAsOutputError, self).__init__(msg)
 
 
 class CircularDependencyError(DvcException):
@@ -100,13 +71,13 @@ class CircularDependencyError(DvcException):
     """
 
     def __init__(self, dependency):
-        assert isinstance(dependency, (str, builtin_str))
+        assert isinstance(dependency, str)
 
         msg = (
             "file/directory '{}' is specified as an output and as a "
             "dependency."
         )
-        super(CircularDependencyError, self).__init__(msg.format(dependency))
+        super().__init__(msg.format(dependency))
 
 
 class ArgumentDuplicationError(DvcException):
@@ -118,9 +89,8 @@ class ArgumentDuplicationError(DvcException):
     """
 
     def __init__(self, path):
-        assert isinstance(path, (str, builtin_str))
-        msg = "file '{}' is specified more than once."
-        super(ArgumentDuplicationError, self).__init__(msg.format(path))
+        assert isinstance(path, str)
+        super().__init__("file '{}' is specified more than once.".format(path))
 
 
 class MoveNotDataSourceError(DvcException):
@@ -138,11 +108,11 @@ class MoveNotDataSourceError(DvcException):
             "it by hand, or remove '{path}' and create a new one at the "
             "desired location."
         )
-        super(MoveNotDataSourceError, self).__init__(msg.format(path=path))
+        super().__init__(msg.format(path=path))
 
 
 class NotDvcRepoError(DvcException):
-    """Thrown if a directory is not a dvc repo.
+    """Thrown if a directory is not a DVC repo.
 
     Args:
         root (str): path to the directory.
@@ -150,56 +120,54 @@ class NotDvcRepoError(DvcException):
 
     def __init__(self, root):
         msg = (
-            "you are not inside of a dvc repository "
+            "you are not inside of a DVC repository "
             "(checked up to mount point '{}')"
         )
-        super(NotDvcRepoError, self).__init__(msg.format(root))
+        super().__init__(msg.format(root))
 
 
 class DvcParserError(DvcException):
     """Base class for CLI parser errors."""
 
     def __init__(self):
-        super(DvcParserError, self).__init__("parser error")
+        super().__init__("parser error")
 
 
 class CyclicGraphError(DvcException):
     def __init__(self, stages):
         assert isinstance(stages, list)
-        stages = "\n".join("\t- {}".format(stage) for stage in stages)
+        stages = "\n".join("\t- {}".format(stage.relpath) for stage in stages)
         msg = (
             "you've introduced a cycle in your pipeline that involves "
             "the following stages:"
             "\n"
             "{stages}".format(stages=stages)
         )
-        super(CyclicGraphError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class ConfirmRemoveError(DvcException):
     def __init__(self, path):
-        super(ConfirmRemoveError, self).__init__(
+        super().__init__(
             "unable to remove '{}' without a confirmation from the user. Use "
             "`-f` to force.".format(path)
         )
 
 
 class InitError(DvcException):
-    def __init__(self, msg):
-        super(InitError, self).__init__(msg)
+    pass
 
 
 class ReproductionError(DvcException):
-    def __init__(self, dvc_file_name, ex):
+    def __init__(self, dvc_file_name):
         self.path = dvc_file_name
-        msg = "failed to reproduce '{}'".format(dvc_file_name)
-        super(ReproductionError, self).__init__(msg, cause=ex)
+        super().__init__("failed to reproduce '{}'".format(dvc_file_name))
 
 
 class BadMetricError(DvcException):
     def __init__(self, paths):
-        super(BadMetricError, self).__init__(
-            "the following metrics do not exists, "
+        super().__init__(
+            "the following metrics do not exist, "
             "are not metric files or are malformed: {paths}".format(
                 paths=", ".join("'{}'".format(path) for path in paths)
             )
@@ -208,32 +176,31 @@ class BadMetricError(DvcException):
 
 class NoMetricsError(DvcException):
     def __init__(self):
-        super(NoMetricsError, self).__init__(
+        super().__init__(
             "no metric files in this repository. "
             "Use `dvc metrics add` to add a metric file to track."
         )
 
 
 class StageFileCorruptedError(DvcException):
-    def __init__(self, path, cause=None):
+    def __init__(self, path):
         path = relpath(path)
-        super(StageFileCorruptedError, self).__init__(
+        super().__init__(
             "unable to read DVC-file: {} "
-            "YAML file structure is corrupted".format(path),
-            cause=cause,
+            "YAML file structure is corrupted".format(path)
         )
 
 
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
-        super(RecursiveAddingWhileUsingFilename, self).__init__(
+        super().__init__(
             "cannot use `fname` with multiple targets or `-R|--recursive`"
         )
 
 
 class OverlappingOutputPathsError(DvcException):
     def __init__(self, out_1, out_2):
-        super(OverlappingOutputPathsError, self).__init__(
+        super().__init__(
             "Paths for outs:\n'{}'('{}')\n'{}'('{}')\noverlap. To avoid "
             "unpredictable behaviour, rerun command with non overlapping outs "
             "paths.".format(
@@ -246,57 +213,41 @@ class OverlappingOutputPathsError(DvcException):
 
 
 class CheckoutErrorSuggestGit(DvcException):
-    def __init__(self, target, cause):
-        super(CheckoutErrorSuggestGit, self).__init__(
-            "Did you mean `git checkout {}`?".format(target), cause=cause
-        )
+    def __init__(self, target):
+        super().__init__("Did you mean `git checkout {}`?".format(target))
 
 
 class ETagMismatchError(DvcException):
     def __init__(self, etag, cached_etag):
-        super(ETagMismatchError, self).__init__(
+        super().__init__(
             "ETag mismatch detected when copying file to cache! "
             "(expected: '{}', actual: '{}')".format(etag, cached_etag)
         )
 
 
 class FileMissingError(DvcException):
-    def __init__(self, path, cause=None):
-        super(FileMissingError, self).__init__(
-            "Can't find '{}' neither locally nor on remote".format(path),
-            cause=cause,
+    def __init__(self, path):
+        self.path = path
+        super().__init__(
+            "Can't find '{}' neither locally nor on remote".format(path)
         )
 
 
 class DvcIgnoreInCollectedDirError(DvcException):
     def __init__(self, ignore_dirname):
-        super(DvcIgnoreInCollectedDirError, self).__init__(
+        super().__init__(
             ".dvcignore file should not be in collected dir path: "
             "'{}'".format(ignore_dirname)
         )
 
 
-class UrlNotDvcRepoError(DvcException):
-    def __init__(self, url):
-        super(UrlNotDvcRepoError, self).__init__(
-            "URL '{}' is not a dvc repository.".format(url)
-        )
-
-
-class GetDVCFileError(DvcException):
-    def __init__(self):
-        super(GetDVCFileError, self).__init__(
-            "the given path is a DVC-file, you must specify a data file "
-            "or a directory"
-        )
-
-
 class GitHookAlreadyExistsError(DvcException):
     def __init__(self, hook_name):
-        super(GitHookAlreadyExistsError, self).__init__(
-            "Hook '{}' already exists. Please refer to "
-            "https://man.dvc.org/install "
-            "for more info.".format(hook_name)
+        super().__init__(
+            "Hook '{}' already exists. Please refer to {} for more "
+            "info.".format(
+                hook_name, format_link("https://man.dvc.org/install")
+            )
         )
 
 
@@ -304,7 +255,7 @@ class DownloadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super(DownloadError, self).__init__(
+        super().__init__(
             "{amount} files failed to download".format(amount=amount)
         )
 
@@ -313,7 +264,7 @@ class UploadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super(UploadError, self).__init__(
+        super().__init__(
             "{amount} files failed to upload".format(amount=amount)
         )
 
@@ -325,27 +276,53 @@ class CheckoutError(DvcException):
             "Checkout failed for following targets:\n {}\nDid you "
             "forget to fetch?".format("\n".join(targets))
         )
-        super(CheckoutError, self).__init__(m)
+        super().__init__(m)
 
 
 class CollectCacheError(DvcException):
     pass
 
 
-class RemoteNotSpecifiedInExternalRepoError(DvcException):
-    def __init__(self, url, cause=None):
-        super(RemoteNotSpecifiedInExternalRepoError, self).__init__(
-            "No DVC remote is specified in the target repository '{}'".format(
-                url
-            ),
-            cause=cause,
+class NoRemoteInExternalRepoError(DvcException):
+    def __init__(self, url):
+        super().__init__(
+            "No DVC remote is specified in target repository '{}'.".format(url)
         )
 
 
 class NoOutputInExternalRepoError(DvcException):
     def __init__(self, path, external_repo_path, external_repo_url):
-        super(NoOutputInExternalRepoError, self).__init__(
+        super().__init__(
             "Output '{}' not found in target repository '{}'".format(
                 relpath(path, external_repo_path), external_repo_url
             )
         )
+
+
+class HTTPError(DvcException):
+    def __init__(self, code, reason):
+        super().__init__("'{} {}'".format(code, reason))
+
+
+class PathMissingError(DvcException):
+    def __init__(self, path, repo):
+        msg = (
+            "The path '{}' does not exist in the target repository '{}'"
+            " neither as an output nor a git-handled file."
+        )
+        super().__init__(msg.format(path, repo))
+
+
+class RemoteCacheRequiredError(DvcException):
+    def __init__(self, path_info):
+        super().__init__(
+            (
+                "Current operation was unsuccessful because '{}' requires "
+                "existing cache on '{}' remote. See {} for information on how "
+                "to set up remote cache."
+            ).format(
+                path_info,
+                path_info.scheme,
+                format_link("https://man.dvc.org/config#cache"),
+            )
+        )

dvc/external_repo.py

@@ -1,56 +1,169 @@
-from __future__ import unicode_literals
-
+import logging
 import os
 import tempfile
 from contextlib import contextmanager
 from distutils.dir_util import copy_tree
+import threading
 
-from funcy import retry
+from funcy import retry, suppress, wrap_with, cached_property
 
-from dvc.config import NoRemoteError
-from dvc.exceptions import RemoteNotSpecifiedInExternalRepoError
-from dvc.exceptions import NoOutputInExternalRepoError
-from dvc.exceptions import OutputNotFoundError
-from dvc.utils.fs import remove
+from dvc.compat import fspath
+from dvc.repo import Repo
+from dvc.config import Config, NoRemoteError, NotDvcRepoError
+from dvc.exceptions import NoRemoteInExternalRepoError
+from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
+from dvc.exceptions import FileMissingError, PathMissingError
+from dvc.remote import RemoteConfig
+from dvc.utils.fs import remove, fs_copy
+from dvc.scm import SCM
 
 
-REPO_CACHE = {}
+logger = logging.getLogger(__name__)
 
 
 @contextmanager
-def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
-    from dvc.repo import Repo
+def external_repo(url, rev=None):
+    path = _cached_clone(url, rev)
+    try:
+        repo = ExternalRepo(path, url)
+    except NotDvcRepoError:
+        repo = ExternalGitRepo(path, url)
 
-    path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
-    repo = Repo(path)
     try:
         yield repo
-    except NoRemoteError as exc:
-        raise RemoteNotSpecifiedInExternalRepoError(url, cause=exc)
+    except NoRemoteError:
+        raise NoRemoteInExternalRepoError(url)
     except OutputNotFoundError as exc:
         if exc.repo is repo:
             raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url)
         raise
-    repo.close()
+    except FileMissingError as exc:
+        raise PathMissingError(exc.path, url)
+    finally:
+        repo.close()
 
 
-def _external_repo(url=None, rev=None, cache_dir=None):
-    from dvc.config import Config
-    from dvc.cache import CacheConfig
-    from dvc.repo import Repo
+def clean_repos():
+    # Outside code should not see cache while we are removing
+    repo_paths = list(REPOS_CACHE.values())
+    REPOS_CACHE.clear()
 
-    key = (url, rev, cache_dir)
-    if key in REPO_CACHE:
-        return REPO_CACHE[key]
+    for path in repo_paths:
+        _remove(path)
+
+
+class ExternalRepo(Repo):
+    def __init__(self, root_dir, url):
+        super().__init__(root_dir)
+        self.url = url
+        self._set_upstream()
+
+    def pull_to(self, path, to_info):
+        try:
+            out = None
+            with suppress(OutputNotFoundError):
+                out = self.find_out_by_relpath(path)
+
+            if out and out.use_cache:
+                self._pull_cached(out, to_info)
+                return
+
+            # Git handled files can't have absolute path
+            if os.path.isabs(path):
+                raise FileNotFoundError
+
+            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+    def _pull_cached(self, out, to_info):
+        with self.state:
+            # Only pull unless all needed cache is present
+            if out.changed_cache():
+                self.cloud.pull(out.get_used_cache())
+
+            out.path_info = to_info
+            failed = out.checkout()
+            # This might happen when pull haven't really pulled all the files
+            if failed:
+                raise FileNotFoundError
+
+    def _set_upstream(self):
+        # check if the URL is local and no default remote is present
+        # add default remote pointing to the original repo's cache location
+        if os.path.isdir(self.url):
+            rconfig = RemoteConfig(self.config)
+            if not rconfig.has_default():
+                src_repo = Repo(self.url)
+                try:
+                    rconfig.add(
+                        "auto-generated-upstream",
+                        src_repo.cache.local.cache_dir,
+                        default=True,
+                        level=Config.LEVEL_LOCAL,
+                    )
+                finally:
+                    src_repo.close()
+
+
+class ExternalGitRepo:
+    def __init__(self, root_dir, url):
+        self.root_dir = root_dir
+        self.url = url
+
+    @cached_property
+    def scm(self):
+        return SCM(self.root_dir)
+
+    def close(self):
+        if "scm" in self.__dict__:
+            self.scm.close()
+
+    def find_out_by_relpath(self, path):
+        raise OutputNotFoundError(path, self)
+
+    def pull_to(self, path, to_info):
+        try:
+            # Git handled files can't have absolute path
+            if os.path.isabs(path):
+                raise FileNotFoundError
+
+            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+    @contextmanager
+    def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
+        try:
+            abs_path = os.path.join(self.root_dir, path)
+            with open(abs_path, mode, encoding=encoding) as fd:
+                yield fd
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+
+REPOS_CACHE = {}
+
+
+@wrap_with(threading.Lock())
+def _cached_clone(url, rev):
+    """Clone an external git repo to a temporary directory.
+
+    Returns the path to a local temporary directory with the specified
+    revision checked out.
+    """
+    if (url, rev) in REPOS_CACHE:
+        path = REPOS_CACHE[url, rev]
+        _git_pull(path, rev)
+        return path
 
     new_path = tempfile.mkdtemp("dvc-erepo")
 
-    # Copy and adjust existing clone
-    if (url, None, None) in REPO_CACHE:
-        old_path = REPO_CACHE[url, None, None]
-
+    if url in REPOS_CACHE:
+        # Copy and an existing clean clone
         # This one unlike shutil.copytree() works with an existing dir
-        copy_tree(old_path, new_path)
+        copy_tree(REPOS_CACHE[url], new_path)
+        _git_pull(new_path, None)
     else:
         # Create a new clone
         _clone_repo(url, new_path)
@@ -58,32 +171,39 @@ def _external_repo(url=None, rev=None, cache_dir=None):
         # Save clean clone dir so that we will have access to a default branch
         clean_clone_path = tempfile.mkdtemp("dvc-erepo")
         copy_tree(new_path, clean_clone_path)
-        REPO_CACHE[url, None, None] = clean_clone_path
+        REPOS_CACHE[url] = clean_clone_path
 
-    # Adjust new clone/copy to fit rev and cache_dir
-    repo = Repo(new_path)
-    try:
-        if rev is not None:
-            repo.scm.checkout(rev)
+    # Check out the specified revision
+    if rev is not None:
+        _git_checkout(new_path, rev)
 
-        if cache_dir is not None:
-            cache_config = CacheConfig(repo.config)
-            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
-    finally:
-        # Need to close/reopen repo to force config reread
-        repo.close()
-
-    REPO_CACHE[key] = new_path
+    REPOS_CACHE[url, rev] = new_path
     return new_path
 
 
-def clean_repos():
-    # Outside code should not see cache while we are removing
-    repo_paths = list(REPO_CACHE.values())
-    REPO_CACHE.clear()
+def _git_checkout(repo_path, rev):
+    from dvc.scm import Git
 
-    for path in repo_paths:
-        _remove(path)
+    git = Git(repo_path)
+    try:
+        git.checkout(rev)
+    finally:
+        git.close()
+
+
+def _git_pull(repo_path, rev):
+    import git
+
+    # Do not try to pull in a detached mode
+    if rev and git.Repo.re_hexsha_only.search(rev):
+        return
+
+    git = git.Repo(repo_path)
+    try:
+        msg = git.git.pull()
+        logger.debug("external repo: git pull: {}", msg)
+    finally:
+        git.close()
 
 
 def _remove(path):

dvc/ignore.py

@@ -1,11 +1,11 @@
-from __future__ import unicode_literals
-
 import logging
 import os
 
+from funcy import cached_property
 from pathspec import PathSpec
 from pathspec.patterns import GitWildMatchPattern
 
+from dvc.scm.tree import BaseTree
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
@@ -72,13 +72,12 @@ class DvcIgnoreDirs(DvcIgnore):
 
 
 class DvcIgnoreFilter(object):
-    def __init__(self, root_dir, tree):
+    def __init__(self, tree):
         self.tree = tree
         self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
-        self._update(root_dir)
-        for root, dirs, _ in self.tree.walk(root_dir, dvcignore=self):
-            for d in dirs:
-                self._update(os.path.join(root, d))
+        for root, dirs, files in self.tree.walk(self.tree.tree_root):
+            self._update(root)
+            dirs[:], files[:] = self(root, dirs, files)
 
     def _update(self, dirname):
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
@@ -90,3 +89,39 @@ class DvcIgnoreFilter(object):
             dirs, files = ignore(root, dirs, files)
 
         return dirs, files
+
+
+class CleanTree(BaseTree):
+    def __init__(self, tree):
+        self.tree = tree
+
+    @cached_property
+    def dvcignore(self):
+        return DvcIgnoreFilter(self.tree)
+
+    @property
+    def tree_root(self):
+        return self.tree.tree_root
+
+    def open(self, path, mode="r", encoding="utf-8"):
+        return self.tree.open(path, mode, encoding)
+
+    def exists(self, path):
+        return self.tree.exists(path)
+
+    def isdir(self, path):
+        return self.tree.isdir(path)
+
+    def isfile(self, path):
+        return self.tree.isfile(path)
+
+    def walk(self, top, topdown=True):
+        for root, dirs, files in self.tree.walk(top, topdown):
+            dirs[:], files[:] = self.dvcignore(root, dirs, files)
+
+            yield root, dirs, files
+
+    def walk_files(self, top):
+        for root, _, files in self.walk(top):
+            for file in files:
+                yield os.path.join(root, file)

dvc/istextfile.py

@@ -1,24 +1,10 @@
 """Use heuristics to guess if it is a text file or a binary file."""
-from __future__ import unicode_literals
-
-from dvc.utils.compat import is_py3
-from dvc.utils.compat import open
 
 # Based on https://eli.thegreenplace.net/2011/10/19/
 # perls-guess-if-file-is-text-or-binary-implemented-in-python
 
 
-# A function that takes an integer in the 8-bit range and returns
-# a single-character byte object in py3 / a single-character string
-# in py2.
-#
-def _int2byte(i):
-    if is_py3:
-        return bytes((i,))
-    return chr(i)
-
-
-TEXT_CHARS = b"".join(_int2byte(i) for i in range(32, 127)) + b"\n\r\t\f\b"
+TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
 
 
 def istextfile(fname, blocksize=512):

dvc/lock.py

@@ -1,17 +1,16 @@
 """Manages dvc lock file."""
-from __future__ import unicode_literals
 
 import hashlib
 import os
 import time
 from datetime import timedelta
 
-from funcy.py3 import lkeep
+import zc.lockfile
+import flufl.lock
 
 from dvc.exceptions import DvcException
-from dvc.utils import makedirs
-from dvc.utils.compat import is_py3
-
+from dvc.utils import format_link
+from dvc.progress import Tqdm
 
 DEFAULT_TIMEOUT = 5
 
@@ -23,122 +22,127 @@ FAILED_TO_LOCK_MESSAGE = (
 
 
 class LockError(DvcException):
-    """Thrown when unable to acquire the lock for dvc repo."""
+    """Thrown when unable to acquire the lock for DVC repo."""
 
 
-if is_py3:
-    import flufl.lock
+class Lock(object):
+    """Class for DVC repo lock.
 
-    class Lock(flufl.lock.Lock):
-        """Class for dvc repo lock.
+    Uses zc.lockfile as backend.
+    """
 
-        Args:
-            lockfile (str): the lock filename
-                in.
-            tmp_dir (str): a directory to store claim files.
-        """
+    def __init__(self, lockfile, friendly=False, **kwargs):
+        self._friendly = friendly
+        self.lockfile = lockfile
+        self._lock = None
 
-        def __init__(self, lockfile, tmp_dir=None):
-            import socket
+    @property
+    def files(self):
+        return [self.lockfile]
 
-            self._tmp_dir = tmp_dir
-            if self._tmp_dir is not None:
-                makedirs(self._tmp_dir, exist_ok=True)
-
-            # NOTE: this is basically Lock.__init__ copy-paste, except that
-            # instead of using `socket.getfqdn()` we use `socket.gethostname()`
-            # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
-            # anything, which is way too slow. `gethostname()` is actually a
-            # fallback for `getfqdn()` when it is not able to resolve a
-            # canonical hostname through network. The claimfile that uses
-            # `self._hostname` is still usable, as it uses `pid` and random
-            # number to generate the resulting lock file name, which is unique
-            # enough for our application.
-            #
-            # [1] https://github.com/iterative/dvc/issues/2582
-            self._hostname = socket.gethostname()
-
-            self._lockfile = lockfile
-            self._lifetime = timedelta(days=365)  # Lock for good by default
-            self._separator = flufl.lock.SEP
-            self._set_claimfile()
-            self._owned = True
-            self._retry_errnos = []
-
-        @property
-        def lockfile(self):
-            return self._lockfile
-
-        @property
-        def files(self):
-            return lkeep([self._lockfile, self._tmp_dir])
-
-        def lock(self):
-            try:
-                super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
-            except flufl.lock.TimeOutError:
-                raise LockError(FAILED_TO_LOCK_MESSAGE)
-
-        def _set_claimfile(self, pid=None):
-            super(Lock, self)._set_claimfile(pid)
-
-            if self._tmp_dir is not None:
-                # Under Windows file path length is limited so we hash it
-                filename = hashlib.md5(self._claimfile.encode()).hexdigest()
-                self._claimfile = os.path.join(
-                    self._tmp_dir, filename + ".lock"
-                )
-
-        # Fix for __del__ bug in flufl.lock [1] which is causing errors on
-        # Python shutdown [2].
-        # [1] https://gitlab.com/warsaw/flufl.lock/issues/7
-        # [2] https://github.com/iterative/dvc/issues/2573
-        def __del__(self):
-            try:
-                if self._owned:
-                    self.finalize()
-            except ImportError:
-                pass
-
-
-else:
-    import zc.lockfile
-
-    class Lock(object):
-        """Class for dvc repo lock.
-
-        Uses zc.lockfile as backend.
-        """
-
-        def __init__(self, lockfile, tmp_dir=None):
-            self.lockfile = lockfile
-            self._lock = None
-
-        @property
-        def files(self):
-            return [self.lockfile]
-
-        def _do_lock(self):
-            try:
+    def _do_lock(self):
+        try:
+            with Tqdm(
+                bar_format="{desc}",
+                disable=not self._friendly,
+                desc=(
+                    "If DVC froze, see `hardlink_lock` in {}".format(
+                        format_link("man.dvc.org/config#core")
+                    )
+                ),
+            ):
                 self._lock = zc.lockfile.LockFile(self.lockfile)
-            except zc.lockfile.LockError:
-                raise LockError(FAILED_TO_LOCK_MESSAGE)
-
-        def lock(self):
-            try:
-                self._do_lock()
-                return
-            except LockError:
-                time.sleep(DEFAULT_TIMEOUT)
+        except zc.lockfile.LockError:
+            raise LockError(FAILED_TO_LOCK_MESSAGE)
 
+    def lock(self):
+        try:
             self._do_lock()
+            return
+        except LockError:
+            time.sleep(DEFAULT_TIMEOUT)
 
-        def unlock(self):
-            self._lock.close()
-            self._lock = None
+        self._do_lock()
 
-        def __enter__(self):
-            self.lock()
+    def unlock(self):
+        self._lock.close()
+        self._lock = None
 
-        def __exit__(self, typ, value, tbck):
-            self.unlock()
+    @property
+    def is_locked(self):
+        return bool(self._lock)
+
+    def __enter__(self):
+        self.lock()
+
+    def __exit__(self, typ, value, tbck):
+        self.unlock()
+
+
+class HardlinkLock(flufl.lock.Lock):
+    """Class for DVC repo lock.
+
+    Args:
+        lockfile (str): the lock filename
+            in.
+        tmp_dir (str): a directory to store claim files.
+    """
+
+    def __init__(self, lockfile, tmp_dir=None, **kwargs):
+        import socket
+
+        self._tmp_dir = tmp_dir
+
+        # NOTE: this is basically Lock.__init__ copy-paste, except that
+        # instead of using `socket.getfqdn()` we use `socket.gethostname()`
+        # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
+        # anything, which is way too slow. `gethostname()` is actually a
+        # fallback for `getfqdn()` when it is not able to resolve a
+        # canonical hostname through network. The claimfile that uses
+        # `self._hostname` is still usable, as it uses `pid` and random
+        # number to generate the resulting lock file name, which is unique
+        # enough for our application.
+        #
+        # [1] https://github.com/iterative/dvc/issues/2582
+        self._hostname = socket.gethostname()
+
+        self._lockfile = lockfile
+        self._lifetime = timedelta(days=365)  # Lock for good by default
+        self._separator = flufl.lock.SEP
+        self._set_claimfile()
+        self._owned = True
+        self._retry_errnos = []
+
+    @property
+    def lockfile(self):
+        return self._lockfile
+
+    def lock(self):
+        try:
+            super().lock(timedelta(seconds=DEFAULT_TIMEOUT))
+        except flufl.lock.TimeOutError:
+            raise LockError(FAILED_TO_LOCK_MESSAGE)
+
+    def _set_claimfile(self, pid=None):
+        super()._set_claimfile(pid)
+
+        if self._tmp_dir is not None:
+            # Under Windows file path length is limited so we hash it
+            filename = hashlib.md5(self._claimfile.encode()).hexdigest()
+            self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
+
+    # Fix for __del__ bug in flufl.lock [1] which is causing errors on
+    # Python shutdown [2].
+    # [1] https://gitlab.com/warsaw/flufl.lock/issues/7
+    # [2] https://github.com/iterative/dvc/issues/2573
+    def __del__(self):
+        try:
+            if self._owned:
+                self.finalize()
+        except ImportError:
+            pass
+
+
+def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False):
+    cls = HardlinkLock if hardlink_lock else Lock
+    return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly)

dvc/logger.py

@@ -1,15 +1,12 @@
-"""Manages logging configuration for dvc repo."""
-from __future__ import unicode_literals
+"""Manages logging configuration for DVC repo."""
 
+import traceback
 import logging.config
 import logging.handlers
 
 import colorama
 
 from dvc.progress import Tqdm
-from dvc.utils.compat import RecursionError
-from dvc.utils.compat import str
-from dvc.utils.compat import StringIO
 
 
 FOOTER = (
@@ -26,7 +23,7 @@ FOOTER = (
 class LoggingException(Exception):
     def __init__(self, record):
         msg = "failed to log {}".format(str(record))
-        super(LoggingException, self).__init__(msg)
+        super().__init__(msg)
 
 
 class ExcludeErrorsFilter(logging.Filter):
@@ -61,11 +58,13 @@ class ColorFormatter(logging.Formatter):
     }
 
     def format(self, record):
+        msg = record.msg.format(*record.args) if record.args else record.msg
+
         if record.levelname == "INFO":
-            return record.msg
+            return msg
 
         if record.levelname == "ERROR" or record.levelname == "CRITICAL":
-            exception, stack_trace = self._parse_exc(record.exc_info)
+            exception, stack_trace = self._parse_exc(record)
 
             return (
                 "{color}{levelname}{nc}: {description}" "{stack_trace}\n"
@@ -73,8 +72,8 @@ class ColorFormatter(logging.Formatter):
                 color=self.color_code.get(record.levelname, ""),
                 nc=colorama.Fore.RESET,
                 levelname=record.levelname,
-                description=self._description(record.msg, exception),
-                msg=record.msg,
+                description=self._description(msg, exception),
+                msg=msg,
                 stack_trace=stack_trace,
             )
 
@@ -82,7 +81,7 @@ class ColorFormatter(logging.Formatter):
             color=self.color_code.get(record.levelname, ""),
             nc=colorama.Fore.RESET,
             levelname=record.levelname,
-            msg=record.msg,
+            msg=msg,
         )
 
     def _current_level(self):
@@ -104,34 +103,26 @@ class ColorFormatter(logging.Formatter):
         return description.format(message=message, exception=exception)
 
     def _walk_exc(self, exc_info):
-        import traceback
-
-        buffer = StringIO()
-
-        traceback.print_exception(*exc_info, file=buffer)
-
         exc = exc_info[1]
-        tb = buffer.getvalue()
 
         exc_list = [str(exc)]
-        tb_list = [tb]
 
-        # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info
-        while hasattr(exc, "cause") and exc.cause:
-            exc_list.append(str(exc.cause))
-            if hasattr(exc, "cause_tb") and exc.cause_tb:
-                tb_list.insert(0, str(exc.cause_tb))
-            exc = exc.cause
+        while hasattr(exc, "__cause__") and exc.__cause__:
+            exc_list.append(str(exc.__cause__))
+            exc = exc.__cause__
 
-        return exc_list, tb_list
+        return exc_list
 
-    def _parse_exc(self, exc_info):
-        if not exc_info:
+    def _parse_exc(self, record):
+        tb_only = getattr(record, "tb_only", False)
+
+        if not record.exc_info:
             return (None, "")
 
-        exc_list, tb_list = self._walk_exc(exc_info)
+        exc_list = self._walk_exc(record.exc_info)
+        tb = traceback.format_exception(*record.exc_info)
 
-        exception = ": ".join(exc_list)
+        exception = None if tb_only else ": ".join(exc_list)
 
         if self._current_level() == logging.DEBUG:
             stack_trace = (
@@ -140,7 +131,7 @@ class ColorFormatter(logging.Formatter):
                 red=colorama.Fore.RED,
                 nc=colorama.Fore.RESET,
                 line="-" * 60,
-                stack_trace="\n".join(tb_list),
+                stack_trace="".join(tb),
             )
         else:
             stack_trace = ""
@@ -150,7 +141,7 @@ class ColorFormatter(logging.Formatter):
 
 class LoggerHandler(logging.StreamHandler):
     def handleError(self, record):
-        super(LoggerHandler, self).handleError(record)
+        super().handleError(record)
         raise LoggingException(record)
 
     def emit(self, record):

dvc/main.py

@@ -1,22 +1,21 @@
 """Main entry point for dvc CLI."""
-from __future__ import unicode_literals
 
+import errno
 import logging
 
-from dvc.analytics import Analytics
+from dvc import analytics
 from dvc.cli import parse_args
 from dvc.config import ConfigError
 from dvc.exceptions import DvcParserError
 from dvc.exceptions import NotDvcRepoError
 from dvc.external_repo import clean_repos
 from dvc.logger import FOOTER
-from dvc.remote.pool import close_pools
-from dvc.utils.compat import is_py2
-
+from dvc.utils import format_link
 
 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
 # [2] https://bugs.python.org/issue29288
+
 "".encode("idna")
 
 
@@ -59,10 +58,14 @@ def main(argv=None):
     except DvcParserError:
         ret = 254
     except Exception as exc:  # pylint: disable=broad-except
-        if isinstance(exc, UnicodeError) and is_py2:
+        if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
             logger.exception(
-                "unicode is not supported in DVC for Python 2 "
-                "(end-of-life January 1, 2020), please upgrade to Python 3"
+                "too many open files, please visit "
+                "{} to see how to handle this "
+                "problem".format(
+                    format_link("https://error.dvc.org/many-files")
+                ),
+                extra={"tb_only": True},
             )
         else:
             logger.exception("unexpected error")
@@ -70,10 +73,6 @@ def main(argv=None):
     finally:
         logger.setLevel(outerLogLevel)
 
-        # Python 2 fails to close these clean occasionally and users see
-        # weird error messages, so we do it manually
-        close_pools()
-
         # Remove cached repos in the end of the call, these are anonymous
         # so won't be reused by any other subsequent run anyway.
         clean_repos()
@@ -81,6 +80,7 @@ def main(argv=None):
     if ret != 0:
         logger.info(FOOTER)
 
-    Analytics().send_cmd(cmd, args, ret)
+    if analytics.is_enabled():
+        analytics.collect_and_send_report(args, ret)
 
     return ret

dvc/output/__init__.py

@@ -1,5 +1,4 @@
-from __future__ import unicode_literals
-
+from urllib.parse import urlparse
 from voluptuous import Any, Required
 
 from dvc.output.base import OutputBase
@@ -13,8 +12,6 @@ from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.local import RemoteLOCAL
 from dvc.remote.s3 import RemoteS3
 from dvc.scheme import Schemes
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
 
 OUTS = [
     OutputHDFS,

dvc/output/base.py

@@ -1,17 +1,14 @@
-from __future__ import unicode_literals
-
 import logging
+from urllib.parse import urlparse
 from copy import copy
 
 from voluptuous import Any
 
 import dvc.prompt as prompt
 from dvc.cache import NamedCache
-from dvc.exceptions import CollectCacheError
+from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError
 from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteBASE
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -20,26 +17,24 @@ logger = logging.getLogger(__name__)
 class OutputDoesNotExistError(DvcException):
     def __init__(self, path):
         msg = "output '{}' does not exist".format(path)
-        super(OutputDoesNotExistError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class OutputIsNotFileOrDirError(DvcException):
     def __init__(self, path):
         msg = "output '{}' is not a file or directory".format(path)
-        super(OutputIsNotFileOrDirError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class OutputAlreadyTrackedError(DvcException):
     def __init__(self, path):
         msg = "output '{}' is already tracked by scm (e.g. git)".format(path)
-        super(OutputAlreadyTrackedError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class OutputIsStageFileError(DvcException):
     def __init__(self, path):
-        super(OutputIsStageFileError, self).__init__(
-            "Stage file '{}' cannot be an output.".format(path)
-        )
+        super().__init__("Stage file '{}' cannot be an output.".format(path))
 
 
 class OutputBase(object):
@@ -103,14 +98,9 @@ class OutputBase(object):
         self.persist = persist
         self.tags = None if self.IS_DEPENDENCY else (tags or {})
 
-        if self.use_cache and self.cache is None:
-            raise DvcException(
-                "no cache location setup for '{}' outputs.".format(
-                    self.REMOTE.scheme
-                )
-            )
-
         self.path_info = self._parse_path(remote, path)
+        if self.use_cache and self.cache is None:
+            raise RemoteCacheRequiredError(self.path_info)
 
     def _parse_path(self, remote, path):
         if remote:
@@ -170,12 +160,7 @@ class OutputBase(object):
         return self.remote.exists(self.path_info)
 
     def changed_checksum(self):
-        return (
-            self.checksum
-            != self.remote.save_info(self.path_info)[
-                self.remote.PARAM_CHECKSUM
-            ]
-        )
+        return self.checksum != self.remote.get_checksum(self.path_info)
 
     def changed_cache(self):
         if not self.use_cache or not self.checksum:
@@ -287,10 +272,19 @@ class OutputBase(object):
     def download(self, to):
         self.remote.download(self.path_info, to.path_info)
 
-    def checkout(self, force=False, progress_callback=None, tag=None):
+    def checkout(
+        self,
+        force=False,
+        progress_callback=None,
+        tag=None,
+        relink=False,
+        filter_info=None,
+    ):
         if not self.use_cache:
             if progress_callback:
-                progress_callback(str(self.path_info), self.get_files_number())
+                progress_callback(
+                    str(self.path_info), self.get_files_number(filter_info)
+                )
             return None
 
         if tag:
@@ -303,6 +297,8 @@ class OutputBase(object):
             info,
             force=force,
             progress_callback=progress_callback,
+            relink=relink,
+            filter_info=filter_info,
         )
 
     def remove(self, ignore_remove=False):
@@ -326,17 +322,21 @@ class OutputBase(object):
         if self.scheme == "local" and self.use_scm_ignore:
             self.repo.scm.ignore(self.fspath)
 
-    def get_files_number(self):
+    def get_files_number(self, filter_info=None):
         if not self.use_cache:
             return 0
 
-        return self.cache.get_files_number(self.checksum)
+        return self.cache.get_files_number(
+            self.path_info, self.checksum, filter_info
+        )
 
     def unprotect(self):
         if self.exists:
             self.remote.unprotect(self.path_info)
 
-    def _collect_used_dir_cache(self, remote=None, force=False, jobs=None):
+    def _collect_used_dir_cache(
+        self, remote=None, force=False, jobs=None, filter_info=None
+    ):
         """Get a list of `info`s related to the given directory.
 
         - Pull the directory entry from the remote cache if it was changed.
@@ -385,8 +385,9 @@ class OutputBase(object):
 
         for entry in self.dir_cache:
             checksum = entry[self.remote.PARAM_CHECKSUM]
-            path_info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            cache.add(self.scheme, checksum, str(path_info))
+            info = self.path_info / entry[self.remote.PARAM_RELPATH]
+            if not filter_info or info.isin_or_eq(filter_info):
+                cache.add(self.scheme, checksum, str(info))
 
         return cache
 
@@ -402,6 +403,12 @@ class OutputBase(object):
         if not self.use_cache:
             return NamedCache()
 
+        if self.stage.is_repo_import:
+            cache = NamedCache()
+            dep, = self.stage.deps
+            cache.external[dep.repo_pair].add(dep.def_path)
+            return cache
+
         if not self.info:
             logger.warning(
                 "Output '{}'({}) is missing version info. Cache for it will "

dvc/output/gs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.s3 import OutputS3
 from dvc.remote.gs import RemoteGS
 

dvc/output/hdfs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.base import OutputBase
 from dvc.remote.hdfs import RemoteHDFS
 

dvc/output/local.py

@@ -1,16 +1,13 @@
-from __future__ import unicode_literals
-
 import logging
 import os
+from urllib.parse import urlparse
 
 from dvc.exceptions import DvcException
 from dvc.istextfile import istextfile
 from dvc.output.base import OutputBase
 from dvc.remote.local import RemoteLOCAL
 from dvc.utils import relpath
-from dvc.utils.compat import fspath_py35
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
+from dvc.compat import fspath_py35
 from dvc.utils.fs import path_isin
 
 
@@ -59,7 +56,7 @@ class OutputLOCAL(OutputBase):
         return def_scheme != "remote" and not os.path.isabs(self.def_path)
 
     def dumpd(self):
-        ret = super(OutputLOCAL, self).dumpd()
+        ret = super().dumpd()
         if self.is_in_repo:
             path = self.path_info.relpath(self.stage.wdir).as_posix()
         else:

dvc/output/s3.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.base import OutputBase
 from dvc.remote.s3 import RemoteS3
 

dvc/output/ssh.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.base import OutputBase
 from dvc.remote.ssh import RemoteSSH
 

dvc/path_info.py

@@ -1,29 +1,26 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 import os
 import posixpath
-import sys
+import pathlib
+from urllib.parse import urlparse
 
 from funcy import cached_property
 
 from dvc.utils import relpath
-from dvc.utils.compat import basestring
-from dvc.utils.compat import builtin_str
-from dvc.utils.compat import is_py2
-from dvc.utils.compat import pathlib
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
 
 
-# On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
-# which is lossy, thus we can't use that,
-# see https://github.com/mcmtroffaes/pathlib2/issues/56.
-if is_py2:
-    fs_encoding = "utf-8"
+class _BasePath(object):
+    def overlaps(self, other):
+        if isinstance(other, (str, bytes)):
+            other = self.__class__(other)
+        elif self.__class__ != other.__class__:
+            return False
+        return self.isin_or_eq(other) or other.isin(self)
+
+    def isin_or_eq(self, other):
+        return self == other or self.isin(other)
 
 
-class PathInfo(pathlib.PurePath):
+class PathInfo(pathlib.PurePath, _BasePath):
     # Use __slots__ in PathInfo objects following PurePath implementation.
     # This makes objects smaller and speeds up attribute access.
     # We don't add any fields so it's empty.
@@ -54,7 +51,7 @@ class PathInfo(pathlib.PurePath):
         return relpath(path)
 
     def __repr__(self):
-        return builtin_str("{}: '{}'").format(type(self).__name__, self)
+        return "{}: '{}'".format(type(self).__name__, self)
 
     # This permits passing it to file utils directly in Python 3.6+
     # With Python 2.7, Python 3.5+ we are stuck with path_info.fspath for now
@@ -71,7 +68,7 @@ class PathInfo(pathlib.PurePath):
         return self.__class__(relpath(self, other))
 
     def isin(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         elif self.__class__ != other.__class__:
             return False
@@ -79,34 +76,6 @@ class PathInfo(pathlib.PurePath):
         n = len(other._cparts)
         return len(self._cparts) > n and self._cparts[:n] == other._cparts
 
-    # pathlib2 uses bytes internally in Python 2, and we use unicode everywhere
-    # for paths in both pythons, thus we need this glue.
-    if is_py2:
-        __unicode__ = __str__
-
-        def __str__(self):
-            return self.__unicode__().encode(sys.getfilesystemencoding())
-
-        @classmethod
-        def _parse_args(cls, args):
-            args = [
-                a.encode(fs_encoding)
-                if isinstance(a, unicode)  # noqa: F821
-                else a
-                for a in args
-            ]
-            return super(PathInfo, cls)._parse_args(args)
-
-        @property
-        def name(self):
-            return super(PathInfo, self).name.decode(fs_encoding)
-
-        def __fspath__(self):  # noqa: F811
-            return pathlib.PurePath.__str__(self).decode(fs_encoding)
-
-        def with_name(self, name):
-            return pathlib.PurePath.with_name(self, name.encode(fs_encoding))
-
 
 class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
     pass
@@ -138,7 +107,7 @@ class _URLPathParents(object):
         return "<{}.parents>".format(self.src)
 
 
-class URLInfo(object):
+class URLInfo(_BasePath):
     DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22, "hdfs": 0}
 
     def __init__(self, url):
@@ -163,13 +132,13 @@ class URLInfo(object):
 
     def fill_parts(self, scheme, host, user, port, path):
         assert scheme != "remote"
-        assert isinstance(path, (basestring, _URLPathInfo))
+        assert isinstance(path, (str, bytes, _URLPathInfo))
 
         self.scheme, self.host, self.user = scheme, host, user
         self.port = int(port) if port else self.DEFAULT_PORTS.get(self.scheme)
 
         if isinstance(path, _URLPathInfo):
-            self._spath = builtin_str(path)
+            self._spath = str(path)
             self._path = path
         else:
             if path and path[0] != "/":
@@ -198,7 +167,7 @@ class URLInfo(object):
         return "{}: '{}'".format(type(self).__name__, self)
 
     def __eq__(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         return (
             self.__class__ == other.__class__
@@ -248,7 +217,7 @@ class URLInfo(object):
         return _URLPathParents(self)
 
     def relative_to(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         if self.__class__ != other.__class__:
             msg = "'{}' has incompatible class with '{}'".format(self, other)
@@ -259,7 +228,7 @@ class URLInfo(object):
         return self._path.relative_to(other._path)
 
     def isin(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         elif self.__class__ != other.__class__:
             return False

dvc/progress.py

@@ -1,5 +1,4 @@
-"""Manages progress bars for dvc repo."""
-from __future__ import print_function
+"""Manages progress bars for DVC repo."""
 
 import logging
 import sys
@@ -20,8 +19,14 @@ class Tqdm(tqdm):
     """
 
     BAR_FMT_DEFAULT = (
-        "{percentage:3.0f}%|{bar:10}|"
-        "{desc:{ncols_desc}.{ncols_desc}}{n_fmt}/{total_fmt}"
+        "{percentage:3.0f}% {desc}|{bar}|"
+        "{n_fmt}/{total_fmt}"
+        " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
+    )
+    # nested bars should have fixed bar widths to align nicely
+    BAR_FMT_DEFAULT_NESTED = (
+        "{percentage:3.0f}%|{bar:10}|{desc:{ncols_desc}.{ncols_desc}}"
+        "{n_fmt}/{total_fmt}"
         " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
     )
     BAR_FMT_NOTOTAL = (
@@ -72,7 +77,7 @@ class Tqdm(tqdm):
             and hasattr(file, "isatty")
         ):
             disable = not file.isatty()
-        super(Tqdm, self).__init__(
+        super().__init__(
             iterable=iterable,
             disable=disable,
             leave=leave,
@@ -83,7 +88,11 @@ class Tqdm(tqdm):
         )
         if bar_format is None:
             if self.__len__():
-                self.bar_format = self.BAR_FMT_DEFAULT
+                self.bar_format = (
+                    self.BAR_FMT_DEFAULT_NESTED
+                    if self.pos
+                    else self.BAR_FMT_DEFAULT
+                )
             else:
                 self.bar_format = self.BAR_FMT_NOTOTAL
         else:
@@ -105,12 +114,12 @@ class Tqdm(tqdm):
     def close(self):
         if self.desc_persist is not None:
             self.set_description_str(self.desc_persist, refresh=False)
-        super(Tqdm, self).close()
+        super().close()
 
     @property
     def format_dict(self):
         """inject `ncols_desc` to fill the display width (`ncols`)"""
-        d = super(Tqdm, self).format_dict
+        d = super().format_dict
         ncols = d["ncols"] or 80
         ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1
         ncols_desc = max(ncols_desc, 0)

dvc/prompt.py

@@ -1,14 +1,9 @@
 """Manages user prompts."""
-from __future__ import print_function
-from __future__ import unicode_literals
 
 import logging
 import sys
 from getpass import getpass
 
-from dvc.utils.compat import input
-
-
 logger = logging.getLogger(__name__)
 
 

dvc/remote/__init__.py

@@ -1,7 +1,6 @@
-from __future__ import unicode_literals
-
 from .config import RemoteConfig
 from dvc.remote.azure import RemoteAZURE
+from dvc.remote.gdrive import RemoteGDrive
 from dvc.remote.gs import RemoteGS
 from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.http import RemoteHTTP
@@ -14,6 +13,7 @@ from dvc.remote.ssh import RemoteSSH
 
 REMOTES = [
     RemoteAZURE,
+    RemoteGDrive,
     RemoteGS,
     RemoteHDFS,
     RemoteHTTP,

dvc/remote/azure.py

@@ -1,20 +1,17 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 import logging
 import os
 import re
-from datetime import datetime
-from datetime import timedelta
+from datetime import datetime, timedelta
+from urllib.parse import urlparse
+import threading
 
-from funcy import cached_property
+from funcy import cached_property, wrap_prop
 
 from dvc.config import Config
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
-from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -34,7 +31,7 @@ class RemoteAZURE(RemoteBASE):
     COPY_POLL_SECONDS = 5
 
     def __init__(self, repo, config):
-        super(RemoteAZURE, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL, "azure://")
 
@@ -64,6 +61,7 @@ class RemoteAZURE(RemoteBASE):
             else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
         )
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def blob_service(self):
         from azure.storage.blob import BlockBlobService

dvc/remote/base.py

@@ -1,34 +1,32 @@
-from __future__ import unicode_literals
-
+import errno
 import itertools
 import json
 import logging
 import tempfile
+from urllib.parse import urlparse
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
 from functools import partial
 from multiprocessing import cpu_count
 from operator import itemgetter
 
+from shortuuid import uuid
+
 import dvc.prompt as prompt
 from dvc.config import Config
-from dvc.exceptions import ConfirmRemoveError
-from dvc.exceptions import DvcException
-from dvc.exceptions import DvcIgnoreInCollectedDirError
+from dvc.exceptions import (
+    DvcException,
+    ConfirmRemoveError,
+    DvcIgnoreInCollectedDirError,
+    RemoteCacheRequiredError,
+)
 from dvc.ignore import DvcIgnore
-from dvc.path_info import PathInfo
-from dvc.path_info import URLInfo
+from dvc.path_info import PathInfo, URLInfo
 from dvc.progress import Tqdm
 from dvc.remote.slow_link_detection import slow_link_guard
 from dvc.state import StateNoop
-from dvc.utils import makedirs
-from dvc.utils import relpath
 from dvc.utils import tmp_fname
-from dvc.utils.compat import basestring
-from dvc.utils.compat import FileNotFoundError
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
-from dvc.utils.fs import move
+from dvc.utils.fs import move, makedirs
 from dvc.utils.http import open_url
 
 logger = logging.getLogger(__name__)
@@ -47,16 +45,9 @@ STATUS_MAP = {
 }
 
 
-class DataCloudError(DvcException):
-    """ Data Cloud exception """
-
-    def __init__(self, msg):
-        super(DataCloudError, self).__init__("Data sync error: {}".format(msg))
-
-
 class RemoteCmdError(DvcException):
     def __init__(self, remote, cmd, ret, err):
-        super(RemoteCmdError, self).__init__(
+        super().__init__(
             "{remote} command '{cmd}' finished with non-zero return code"
             " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
         )
@@ -64,8 +55,8 @@ class RemoteCmdError(DvcException):
 
 class RemoteActionNotImplemented(DvcException):
     def __init__(self, action, scheme):
-        m = "{} is not supported by {} remote".format(action, scheme)
-        super(RemoteActionNotImplemented, self).__init__(m)
+        m = "{} is not supported for {} remotes".format(action, scheme)
+        super().__init__(m)
 
 
 class RemoteMissingDepsError(DvcException):
@@ -73,10 +64,9 @@ class RemoteMissingDepsError(DvcException):
 
 
 class DirCacheError(DvcException):
-    def __init__(self, checksum, cause=None):
-        super(DirCacheError, self).__init__(
-            "Failed to load dir cache for checksum: '{}'.".format(checksum),
-            cause=cause,
+    def __init__(self, checksum):
+        super().__init__(
+            "Failed to load dir cache for checksum: '{}'.".format(checksum)
         )
 
 
@@ -90,6 +80,8 @@ class RemoteBASE(object):
     CHECKSUM_DIR_SUFFIX = ".dir"
     CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
     DEFAULT_CACHE_TYPES = ["copy"]
+    DEFAULT_NO_TRAVERSE = True
+    DEFAULT_VERIFY = False
 
     state = StateNoop()
 
@@ -97,14 +89,15 @@ class RemoteBASE(object):
         self.repo = repo
 
         self._check_requires(config)
-
-        core = config.get(Config.SECTION_CORE, {})
-        self.checksum_jobs = core.get(
-            Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
-        )
+        self.checksum_jobs = self._get_checksum_jobs(config)
 
         self.protected = False
-        self.no_traverse = config.get(Config.SECTION_REMOTE_NO_TRAVERSE, True)
+        self.no_traverse = config.get(
+            Config.SECTION_REMOTE_NO_TRAVERSE, self.DEFAULT_NO_TRAVERSE
+        )
+        self.verify = config.get(
+            Config.SECTION_REMOTE_VERIFY, self.DEFAULT_VERIFY
+        )
         self._dir_info = {}
 
         types = config.get(Config.SECTION_CACHE_TYPE, None)
@@ -153,6 +146,19 @@ class RemoteBASE(object):
         ).format(url, missing, " ".join(missing), self.scheme)
         raise RemoteMissingDepsError(msg)
 
+    def _get_checksum_jobs(self, config):
+        checksum_jobs = config.get(Config.SECTION_REMOTE_CHECKSUM_JOBS)
+        if checksum_jobs:
+            return checksum_jobs
+
+        if self.repo:
+            core = self.repo.config.config.get(Config.SECTION_CORE, {})
+            return core.get(
+                Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
+            )
+
+        return self.CHECKSUM_JOBS
+
     def __repr__(self):
         return "{class_name}: '{path_info}'".format(
             class_name=type(self).__name__,
@@ -161,7 +167,7 @@ class RemoteBASE(object):
 
     @classmethod
     def supported(cls, config):
-        if isinstance(config, basestring):
+        if isinstance(config, (str, bytes)):
             url = config
         else:
             url = config[Config.SECTION_REMOTE_URL]
@@ -228,6 +234,9 @@ class RemoteBASE(object):
         return sorted(result, key=itemgetter(self.PARAM_RELPATH))
 
     def get_dir_checksum(self, path_info):
+        if not self.cache:
+            raise RemoteCacheRequiredError(path_info)
+
         dir_info = self._collect_dir(path_info)
         checksum, tmp_info = self._get_dir_info_checksum(dir_info)
         new_info = self.cache.checksum_to_path_info(checksum)
@@ -274,11 +283,13 @@ class RemoteBASE(object):
             with self.cache.open(path_info, "r") as fobj:
                 d = json.load(fobj)
         except (ValueError, FileNotFoundError) as exc:
-            raise DirCacheError(checksum, cause=exc)
+            raise DirCacheError(checksum) from exc
 
         if not isinstance(d, list):
-            msg = "dir cache file format error '{}' [skipping the file]"
-            logger.error(msg.format(relpath(path_info)))
+            logger.error(
+                "dir cache file format error '{}' [skipping the file]",
+                path_info,
+            )
             return []
 
         for info in d:
@@ -293,6 +304,8 @@ class RemoteBASE(object):
         return checksum.endswith(cls.CHECKSUM_DIR_SUFFIX)
 
     def get_checksum(self, path_info):
+        assert path_info.scheme == self.scheme
+
         if not self.exists(path_info):
             return None
 
@@ -322,7 +335,6 @@ class RemoteBASE(object):
         return checksum
 
     def save_info(self, path_info):
-        assert path_info.scheme == self.scheme
         return {self.PARAM_CHECKSUM: self.get_checksum(path_info)}
 
     def changed(self, path_info, checksum_info):
@@ -343,36 +355,35 @@ class RemoteBASE(object):
         """
 
         logger.debug(
-            "checking if '{}'('{}') has changed.".format(
-                path_info, checksum_info
-            )
+            "checking if '{}'('{}') has changed.", path_info, checksum_info
         )
 
         if not self.exists(path_info):
-            logger.debug("'{}' doesn't exist.".format(path_info))
+            logger.debug("'{}' doesn't exist.", path_info)
             return True
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
         if checksum is None:
-            logger.debug("checksum for '{}' is missing.".format(path_info))
+            logger.debug("checksum for '{}' is missing.", path_info)
             return True
 
         if self.changed_cache(checksum):
             logger.debug(
-                "cache for '{}'('{}') has changed.".format(path_info, checksum)
+                "cache for '{}'('{}') has changed.", path_info, checksum
             )
             return True
 
-        actual = self.save_info(path_info)[self.PARAM_CHECKSUM]
+        actual = self.get_checksum(path_info)
         if checksum != actual:
             logger.debug(
-                "checksum '{}'(actual '{}') for '{}' has changed.".format(
-                    checksum, actual, path_info
-                )
+                "checksum '{}'(actual '{}') for '{}' has changed.",
+                checksum,
+                actual,
+                path_info,
             )
             return True
 
-        logger.debug("'{}' hasn't changed.".format(path_info))
+        logger.debug("'{}' hasn't changed.", path_info)
         return False
 
     def link(self, from_info, to_info):
@@ -385,18 +396,30 @@ class RemoteBASE(object):
 
         self._try_links(from_info, to_info, link_types)
 
+    def _verify_link(self, path_info, link_type):
+        if self.cache_type_confirmed:
+            return
+
+        is_link = getattr(self, "is_{}".format(link_type), None)
+        if is_link and not is_link(path_info):
+            self.remove(path_info)
+            raise DvcException("failed to verify {}".format(link_type))
+
+        self.cache_type_confirmed = True
+
     @slow_link_guard
     def _try_links(self, from_info, to_info, link_types):
         while link_types:
             link_method = getattr(self, link_types[0])
             try:
                 self._do_link(from_info, to_info, link_method)
-                self.cache_type_confirmed = True
+                self._verify_link(to_info, link_types[0])
                 return
 
             except DvcException as exc:
-                msg = "Cache type '{}' is not supported: {}"
-                logger.debug(msg.format(link_types[0], str(exc)))
+                logger.debug(
+                    "Cache type '{}' is not supported: {}", link_types[0], exc
+                )
                 del link_types[0]
 
         raise DvcException("no possible cache types left to try out.")
@@ -410,13 +433,13 @@ class RemoteBASE(object):
         if self.protected:
             self.protect(to_info)
 
-        msg = "Created {}'{}': {} -> {}".format(
+        logger.debug(
+            "Created {}'{}': {} -> {}",
             "protected " if self.protected else "",
             self.cache_types[0],
             from_info,
             to_info,
         )
-        logger.debug(msg)
 
     def _save_file(self, path_info, checksum, save_link=True):
         assert checksum
@@ -424,10 +447,16 @@ class RemoteBASE(object):
         cache_info = self.checksum_to_path_info(checksum)
         if self.changed_cache(checksum):
             self.move(path_info, cache_info)
+            self.link(cache_info, path_info)
+        elif self.iscopy(path_info) and self._cache_is_copy(path_info):
+            # Default relink procedure involves unneeded copy
+            if self.protected:
+                self.protect(path_info)
+            else:
+                self.unprotect(path_info)
         else:
             self.remove(path_info)
-
-        self.link(cache_info, path_info)
+            self.link(cache_info, path_info)
 
         if save_link:
             self.state.save_link(path_info)
@@ -438,6 +467,28 @@ class RemoteBASE(object):
         self.state.save(path_info, checksum)
         self.state.save(cache_info, checksum)
 
+    def _cache_is_copy(self, path_info):
+        """Checks whether cache uses copies."""
+        if self.cache_type_confirmed:
+            return self.cache_types[0] == "copy"
+
+        if set(self.cache_types) <= {"copy"}:
+            return True
+
+        workspace_file = path_info.with_name("." + uuid())
+        test_cache_file = self.path_info / ".cache_type_test_file"
+        if not self.exists(test_cache_file):
+            with self.open(test_cache_file, "wb") as fobj:
+                fobj.write(bytes(1))
+        try:
+            self.link(test_cache_file, workspace_file)
+        finally:
+            self.remove(workspace_file)
+            self.remove(test_cache_file)
+
+        self.cache_type_confirmed = True
+        return self.cache_types[0] == "copy"
+
     def _save_dir(self, path_info, checksum):
         cache_info = self.checksum_to_path_info(checksum)
         dir_info = self.get_dir_cache(checksum)
@@ -466,6 +517,10 @@ class RemoteBASE(object):
         """
         return False
 
+    def iscopy(self, path_info):
+        """Check if this file is an independent copy."""
+        return False  # We can't be sure by default
+
     def walk_files(self, path_info):
         """Return a generator with `PathInfo`s to all the files"""
         raise NotImplementedError
@@ -482,20 +537,27 @@ class RemoteBASE(object):
             )
 
         checksum = checksum_info[self.PARAM_CHECKSUM]
-        if not self.changed_cache(checksum):
-            self._checkout(path_info, checksum)
-            return
-
         self._save(path_info, checksum)
 
     def _save(self, path_info, checksum):
         to_info = self.checksum_to_path_info(checksum)
-        logger.debug("Saving '{}' to '{}'.".format(path_info, to_info))
+        logger.debug("Saving '{}' to '{}'.", path_info, to_info)
         if self.isdir(path_info):
             self._save_dir(path_info, checksum)
             return
         self._save_file(path_info, checksum)
 
+    def _handle_transfer_exception(
+        self, from_info, to_info, exception, operation
+    ):
+        if isinstance(exception, OSError) and exception.errno == errno.EMFILE:
+            raise exception
+
+        logger.exception(
+            "failed to {} '{}' to '{}'", operation, from_info, to_info
+        )
+        return 1
+
     def upload(self, from_info, to_info, name=None, no_progress_bar=False):
         if not hasattr(self, "_upload"):
             raise RemoteActionNotImplemented("upload", self.scheme)
@@ -506,7 +568,7 @@ class RemoteBASE(object):
         if from_info.scheme != "local":
             raise NotImplementedError
 
-        logger.debug("Uploading '{}' to '{}'".format(from_info, to_info))
+        logger.debug("Uploading '{}' to '{}'", from_info, to_info)
 
         name = name or from_info.name
 
@@ -517,10 +579,10 @@ class RemoteBASE(object):
                 name=name,
                 no_progress_bar=no_progress_bar,
             )
-        except Exception:
-            msg = "failed to upload '{}' to '{}'"
-            logger.exception(msg.format(from_info, to_info))
-            return 1  # 1 fail
+        except Exception as e:
+            return self._handle_transfer_exception(
+                from_info, to_info, e, "upload"
+            )
 
         return 0
 
@@ -546,21 +608,58 @@ class RemoteBASE(object):
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
+        if self.isdir(from_info):
+            return self._download_dir(
+                from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+            )
+        return self._download_file(
+            from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+        )
 
+    def _download_dir(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        from_infos = list(self.walk_files(from_info))
+        to_infos = (
+            to_info / info.relative_to(from_info) for info in from_infos
+        )
+
+        with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
+            download_files = partial(
+                self._download_file,
+                name=name,
+                no_progress_bar=True,
+                file_mode=file_mode,
+                dir_mode=dir_mode,
+            )
+            futures = executor.map(download_files, from_infos, to_infos)
+            with Tqdm(
+                futures,
+                total=len(from_infos),
+                desc="Downloading directory",
+                unit="Files",
+                disable=no_progress_bar,
+            ) as futures:
+                return sum(futures)
+
+    def _download_file(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
+
+        logger.debug("Downloading '{}' to '{}'", from_info, to_info)
         name = name or to_info.name
 
-        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
         tmp_file = tmp_fname(to_info)
 
         try:
             self._download(
                 from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
             )
-        except Exception:
-            msg = "failed to download '{}' to '{}'"
-            logger.exception(msg.format(from_info, to_info))
-            return 1  # 1 fail
+        except Exception as e:
+            return self._handle_transfer_exception(
+                from_info, to_info, e, "download"
+            )
 
         move(tmp_file, to_info, mode=file_mode)
 
@@ -632,6 +731,8 @@ class RemoteBASE(object):
             if checksum in used:
                 continue
             path_info = self.checksum_to_path_info(checksum)
+            if self.is_dir_checksum(checksum):
+                self._remove_unpacked_dir(checksum)
             self.remove(path_info)
             removed = True
         return removed
@@ -652,9 +753,10 @@ class RemoteBASE(object):
         actual = self.get_checksum(cache_info)
 
         logger.debug(
-            "cache '{}' expected '{}' actual '{}'".format(
-                str(cache_info), checksum, actual
-            )
+            "cache '{}' expected '{}' actual '{}'",
+            cache_info,
+            checksum,
+            actual,
         )
 
         if not checksum or not actual:
@@ -664,29 +766,41 @@ class RemoteBASE(object):
             return False
 
         if self.exists(cache_info):
-            logger.warning("corrupted cache file '{}'.".format(cache_info))
+            logger.warning("corrupted cache file '{}'.", cache_info)
             self.remove(cache_info)
 
         return True
 
-    def _changed_dir_cache(self, checksum):
+    def _changed_dir_cache(self, checksum, path_info=None, filter_info=None):
         if self.changed_cache_file(checksum):
             return True
 
-        if not self._changed_unpacked_dir(checksum):
+        if not (path_info and filter_info) and not self._changed_unpacked_dir(
+            checksum
+        ):
             return False
 
         for entry in self.get_dir_cache(checksum):
             entry_checksum = entry[self.PARAM_CHECKSUM]
+
+            if path_info and filter_info:
+                entry_info = path_info / entry[self.PARAM_RELPATH]
+                if not entry_info.isin_or_eq(filter_info):
+                    continue
+
             if self.changed_cache_file(entry_checksum):
                 return True
 
-        self._update_unpacked_dir(checksum)
+        if not (path_info and filter_info):
+            self._update_unpacked_dir(checksum)
+
         return False
 
-    def changed_cache(self, checksum):
+    def changed_cache(self, checksum, path_info=None, filter_info=None):
         if self.is_dir_checksum(checksum):
-            return self._changed_dir_cache(checksum)
+            return self._changed_dir_cache(
+                checksum, path_info=path_info, filter_info=filter_info
+            )
         return self.changed_cache_file(checksum)
 
     def cache_exists(self, checksums, jobs=None, name=None):
@@ -757,45 +871,22 @@ class RemoteBASE(object):
         self.remove(path_info)
 
     def _checkout_file(
-        self,
-        path_info,
-        checksum,
-        force,
-        progress_callback=None,
-        save_link=True,
+        self, path_info, checksum, force, progress_callback=None
     ):
-        # NOTE: In case if path_info is already cached and path_info's
-        # link type matches cache link type, we would like to avoid
-        # relinking.
-        if self.changed(
-            path_info, {self.PARAM_CHECKSUM: checksum}
-        ) or not self._link_matches(path_info):
+        """The file is changed we need to checkout a new copy"""
+        cache_info = self.checksum_to_path_info(checksum)
+        if self.exists(path_info):
+            logger.warning(
+                "data '{}' exists. Removing before checkout.", path_info
+            )
             self.safe_remove(path_info, force=force)
 
-            cache_info = self.checksum_to_path_info(checksum)
-            self.link(cache_info, path_info)
-
-            if save_link:
-                self.state.save_link(path_info)
-
-            self.state.save(path_info, checksum)
-        else:
-            # NOTE: performing (un)protection costs us +/- the same as checking
-            # if path_info is protected. Instead of implementing logic,
-            # just (un)protect according to self.protected.
-            if self.protected:
-                self.protect(path_info)
-            else:
-                # NOTE dont allow copy, because we checked before that link
-                # type matches cache, and we don't want data duplication
-                self.unprotect(path_info, allow_copy=False)
-
+        self.link(cache_info, path_info)
+        self.state.save_link(path_info)
+        self.state.save(path_info, checksum)
         if progress_callback:
             progress_callback(str(path_info))
 
-    def _link_matches(self, path_info):
-        return True
-
     def makedirs(self, path_info):
         """Optional: Implement only if the remote needs to create
         directories before copying/linking/moving data
@@ -803,7 +894,13 @@ class RemoteBASE(object):
         pass
 
     def _checkout_dir(
-        self, path_info, checksum, force, progress_callback=None
+        self,
+        path_info,
+        checksum,
+        force,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
     ):
         # Create dir separately so that dir is created
         # even if there are no files in it
@@ -812,19 +909,24 @@ class RemoteBASE(object):
 
         dir_info = self.get_dir_cache(checksum)
 
-        logger.debug("Linking directory '{}'.".format(path_info))
+        logger.debug("Linking directory '{}'.", path_info)
 
         for entry in dir_info:
             relative_path = entry[self.PARAM_RELPATH]
             entry_checksum = entry[self.PARAM_CHECKSUM]
+            entry_cache_info = self.checksum_to_path_info(entry_checksum)
             entry_info = path_info / relative_path
-            self._checkout_file(
-                entry_info,
-                entry_checksum,
-                force,
-                progress_callback,
-                save_link=False,
-            )
+
+            if filter_info and not entry_info.isin_or_eq(filter_info):
+                continue
+
+            entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
+            if relink or self.changed(entry_info, entry_checksum_info):
+                self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+                self.state.save(entry_info, entry_checksum)
+            if progress_callback:
+                progress_callback(str(entry_info))
 
         self._remove_redundant_files(path_info, dir_info, force)
 
@@ -842,62 +944,95 @@ class RemoteBASE(object):
             self.safe_remove(path, force)
 
     def checkout(
-        self, path_info, checksum_info, force=False, progress_callback=None
+        self,
+        path_info,
+        checksum_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
     ):
         if path_info.scheme not in ["local", self.scheme]:
             raise NotImplementedError
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
         failed = None
+        skip = False
         if not checksum:
             logger.warning(
-                "No checksum info found for '{}'. "
-                "It won't be created.".format(str(path_info))
+                "No checksum info found for '{}'. " "It won't be created.",
+                path_info,
             )
             self.safe_remove(path_info, force=force)
             failed = path_info
 
-        elif self.changed_cache(checksum):
-            msg = "Cache '{}' not found. File '{}' won't be created."
-            logger.warning(msg.format(checksum, str(path_info)))
+        elif not relink and not self.changed(path_info, checksum_info):
+            logger.debug("Data '{}' didn't change.", path_info)
+            skip = True
+
+        elif self.changed_cache(
+            checksum, path_info=path_info, filter_info=filter_info
+        ):
+            logger.warning(
+                "Cache '{}' not found. File '{}' won't be created.",
+                checksum,
+                path_info,
+            )
             self.safe_remove(path_info, force=force)
             failed = path_info
 
-        if failed:
+        if failed or skip:
             if progress_callback:
                 progress_callback(
-                    str(path_info), self.get_files_number(checksum)
+                    str(path_info),
+                    self.get_files_number(
+                        self.path_info, checksum, filter_info
+                    ),
                 )
             return failed
 
-        msg = "Checking out '{}' with cache '{}'."
-        logger.debug(msg.format(str(path_info), checksum))
+        logger.debug("Checking out '{}' with cache '{}'.", path_info, checksum)
 
-        self._checkout(path_info, checksum, force, progress_callback)
-        return None
+        self._checkout(
+            path_info, checksum, force, progress_callback, relink, filter_info
+        )
 
     def _checkout(
-        self, path_info, checksum, force=False, progress_callback=None
+        self,
+        path_info,
+        checksum,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
     ):
         if not self.is_dir_checksum(checksum):
             return self._checkout_file(
                 path_info, checksum, force, progress_callback=progress_callback
             )
         return self._checkout_dir(
-            path_info, checksum, force, progress_callback=progress_callback
+            path_info, checksum, force, progress_callback, relink, filter_info
         )
 
-    def get_files_number(self, checksum):
+    def get_files_number(self, path_info, checksum, filter_info):
+        from funcy.py3 import ilen
+
         if not checksum:
             return 0
 
-        if self.is_dir_checksum(checksum):
+        if not self.is_dir_checksum(checksum):
+            return 1
+
+        if not filter_info:
             return len(self.get_dir_cache(checksum))
 
-        return 1
+        return ilen(
+            filter_info.isin_or_eq(path_info / entry[self.PARAM_CHECKSUM])
+            for entry in self.get_dir_cache(checksum)
+        )
 
     @staticmethod
-    def unprotect(path_info, allow_copy=True):
+    def unprotect(path_info):
         pass
 
     def _get_unpacked_dir_names(self, checksums):
@@ -913,3 +1048,6 @@ class RemoteBASE(object):
 
     def _update_unpacked_dir(self, checksum):
         pass
+
+    def _remove_unpacked_dir(self, checksum):
+        pass

dvc/remote/config.py

@@ -1,11 +1,10 @@
 import logging
 import os
 import posixpath
+from urllib.parse import urlparse
 
-from dvc.config import Config
-from dvc.config import ConfigError
+from dvc.config import Config, ConfigError
 from dvc.utils import relpath
-from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -34,7 +33,7 @@ class RemoteConfig(object):
 
         if settings is None:
             raise ConfigError(
-                "unable to find remote section '{}'".format(name)
+                "Unable to find remote section '{}'".format(name)
             )
 
         parsed = urlparse(settings["url"])
@@ -105,7 +104,6 @@ class RemoteConfig(object):
             level=level,
         )
         if default:
-            logger.info("Setting '{}' as a default remote.".format(name))
             self.config.set(
                 Config.SECTION_CORE,
                 Config.SECTION_CORE_REMOTE,
@@ -135,6 +133,9 @@ class RemoteConfig(object):
                 break
 
     def modify(self, name, option, value, level=None):
+        # try to get remote settings to verify that section exists
+        self.get_settings(name)
+
         self.config.set(
             Config.SECTION_REMOTE_FMT.format(name), option, value, level=level
         )
@@ -156,3 +157,7 @@ class RemoteConfig(object):
         return self.config.get(
             Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level
         )
+
+    def has_default(self):
+        core = self.config.config[Config.SECTION_CORE]
+        return bool(core.get(Config.SECTION_CORE_REMOTE))

dvc/remote/gdrive.py

@@ -0,0 +1,353 @@
+import os
+import posixpath
+import logging
+import threading
+import re
+
+from funcy import retry, compose, decorator, wrap_with
+from funcy.py3 import cat
+
+from dvc.progress import Tqdm
+from dvc.scheme import Schemes
+from dvc.path_info import CloudURLInfo
+from dvc.remote.base import RemoteBASE
+from dvc.config import Config
+from dvc.exceptions import DvcException
+from dvc.utils import tmp_fname, format_link
+
+logger = logging.getLogger(__name__)
+FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
+
+
+class GDriveRetriableError(DvcException):
+    pass
+
+
+class GDriveAccessTokenRefreshError(DvcException):
+    pass
+
+
+class GDriveMissedCredentialKeyError(DvcException):
+    pass
+
+
+@decorator
+def _wrap_pydrive_retriable(call):
+    from pydrive2.files import ApiRequestError
+
+    try:
+        result = call()
+    except ApiRequestError as exception:
+        retry_codes = ["403", "500", "502", "503", "504"]
+        if any(
+            "HttpError {}".format(code) in str(exception)
+            for code in retry_codes
+        ):
+            raise GDriveRetriableError("Google API request failed")
+        raise
+    return result
+
+
+gdrive_retry = compose(
+    # 8 tries, start at 0.5s, multiply by golden ratio, cap at 10s
+    retry(
+        8, GDriveRetriableError, timeout=lambda a: min(0.5 * 1.618 ** a, 10)
+    ),
+    _wrap_pydrive_retriable,
+)
+
+
+class RemoteGDrive(RemoteBASE):
+    scheme = Schemes.GDRIVE
+    path_cls = CloudURLInfo
+    REQUIRES = {"pydrive2": "pydrive2"}
+    DEFAULT_NO_TRAVERSE = False
+    DEFAULT_VERIFY = True
+
+    GDRIVE_USER_CREDENTIALS_DATA = "GDRIVE_USER_CREDENTIALS_DATA"
+    DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"
+
+    def __init__(self, repo, config):
+        super().__init__(repo, config)
+        self.path_info = self.path_cls(config[Config.SECTION_REMOTE_URL])
+
+        bucket = re.search(
+            "{}://(.*)".format(self.scheme),
+            config[Config.SECTION_REMOTE_URL],
+            re.IGNORECASE,
+        )
+        self.bucket = (
+            bucket.group(1).split("/")[0] if bucket else self.path_info.bucket
+        )
+
+        self.config = config
+        self.init_drive()
+
+    def init_drive(self):
+        self.client_id = self.config.get(Config.SECTION_GDRIVE_CLIENT_ID, None)
+        self.client_secret = self.config.get(
+            Config.SECTION_GDRIVE_CLIENT_SECRET, None
+        )
+        if not self.client_id or not self.client_secret:
+            raise DvcException(
+                "Please specify Google Drive's client id and "
+                "secret in DVC's config. Learn more at "
+                "{}.".format(format_link("https://man.dvc.org/remote/add"))
+            )
+        self.gdrive_user_credentials_path = (
+            tmp_fname(os.path.join(self.repo.tmp_dir, ""))
+            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
+            else self.config.get(
+                Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE,
+                os.path.join(
+                    self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE
+                ),
+            )
+        )
+
+    @gdrive_retry
+    def gdrive_upload_file(
+        self, args, no_progress_bar=True, from_file="", progress_name=""
+    ):
+        item = self.drive.CreateFile(
+            {"title": args["title"], "parents": [{"id": args["parent_id"]}]}
+        )
+
+        with open(from_file, "rb") as fobj:
+            total = os.path.getsize(from_file)
+            with Tqdm.wrapattr(
+                fobj,
+                "read",
+                desc=progress_name,
+                total=total,
+                disable=no_progress_bar,
+            ) as wrapped:
+                # PyDrive doesn't like content property setting for empty files
+                # https://github.com/gsuitedevs/PyDrive/issues/121
+                if total:
+                    item.content = wrapped
+                item.Upload()
+        return item
+
+    @gdrive_retry
+    def gdrive_download_file(
+        self, file_id, to_file, progress_name, no_progress_bar
+    ):
+        gdrive_file = self.drive.CreateFile({"id": file_id})
+        bar_format = (
+            "Donwloading {desc:{ncols_desc}.{ncols_desc}}... "
+            + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
+        )
+        with Tqdm(
+            bar_format=bar_format, desc=progress_name, disable=no_progress_bar
+        ):
+            gdrive_file.GetContentFile(to_file)
+
+    def gdrive_list_item(self, query):
+        file_list = self.drive.ListFile({"q": query, "maxResults": 1000})
+
+        # Isolate and decorate fetching of remote drive items in pages
+        get_list = gdrive_retry(lambda: next(file_list, None))
+
+        # Fetch pages until None is received, lazily flatten the thing
+        return cat(iter(get_list, None))
+
+    def cache_root_dirs(self):
+        cached_dirs = {}
+        cached_ids = {}
+        for dir1 in self.gdrive_list_item(
+            "'{}' in parents and trashed=false".format(self.remote_root_id)
+        ):
+            remote_path = posixpath.join(self.path_info.path, dir1["title"])
+            cached_dirs.setdefault(remote_path, []).append(dir1["id"])
+            cached_ids[dir1["id"]] = dir1["title"]
+        return cached_dirs, cached_ids
+
+    @property
+    def cached_dirs(self):
+        if not hasattr(self, "_cached_dirs"):
+            self.drive
+        return self._cached_dirs
+
+    @property
+    def cached_ids(self):
+        if not hasattr(self, "_cached_ids"):
+            self.drive
+        return self._cached_ids
+
+    @property
+    @wrap_with(threading.RLock())
+    def drive(self):
+        from pydrive2.auth import RefreshError
+
+        if not hasattr(self, "_gdrive"):
+            from pydrive2.auth import GoogleAuth
+            from pydrive2.drive import GoogleDrive
+
+            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                with open(
+                    self.gdrive_user_credentials_path, "w"
+                ) as credentials_file:
+                    credentials_file.write(
+                        os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
+                    )
+
+            GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
+            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                "token_uri": "https://oauth2.googleapis.com/token",
+                "revoke_uri": "https://oauth2.googleapis.com/revoke",
+                "redirect_uri": "",
+            }
+            GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
+            GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
+            GoogleAuth.DEFAULT_SETTINGS[
+                "save_credentials_file"
+            ] = self.gdrive_user_credentials_path
+            GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
+            GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
+                "https://www.googleapis.com/auth/drive",
+                # drive.appdata grants access to appDataFolder GDrive directory
+                "https://www.googleapis.com/auth/drive.appdata",
+            ]
+
+            # Pass non existent settings path to force DEFAULT_SETTINGS loading
+            gauth = GoogleAuth(settings_file="")
+
+            try:
+                gauth.CommandLineAuth()
+            except RefreshError as exc:
+                raise GDriveAccessTokenRefreshError(
+                    "Google Drive's access token refreshment is failed"
+                ) from exc
+            except KeyError as exc:
+                raise GDriveMissedCredentialKeyError(
+                    "Google Drive's user credentials file '{}' "
+                    "misses value for key '{}'".format(
+                        self.gdrive_user_credentials_path, str(exc)
+                    )
+                )
+            # Handle pydrive2.auth.AuthenticationError and others auth failures
+            except Exception as exc:
+                raise DvcException(
+                    "Google Drive authentication failed"
+                ) from exc
+            finally:
+                if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                    os.remove(self.gdrive_user_credentials_path)
+
+            self._gdrive = GoogleDrive(gauth)
+
+            self.remote_root_id = self.get_remote_id(
+                self.path_info, create=True
+            )
+            self._cached_dirs, self._cached_ids = self.cache_root_dirs()
+
+        return self._gdrive
+
+    @gdrive_retry
+    def create_remote_dir(self, parent_id, title):
+        item = self.drive.CreateFile(
+            {
+                "title": title,
+                "parents": [{"id": parent_id}],
+                "mimeType": FOLDER_MIME_TYPE,
+            }
+        )
+        item.Upload()
+        return item
+
+    @gdrive_retry
+    def get_remote_item(self, name, parents_ids):
+        if not parents_ids:
+            return None
+        query = "({})".format(
+            " or ".join(
+                "'{}' in parents".format(parent_id)
+                for parent_id in parents_ids
+            )
+        )
+
+        query += " and trashed=false and title='{}'".format(name)
+
+        # Limit found remote items count to 1 in response
+        item_list = self.drive.ListFile(
+            {"q": query, "maxResults": 1}
+        ).GetList()
+        return next(iter(item_list), None)
+
+    def resolve_remote_item_from_path(self, path_parts, create):
+        parents_ids = [self.bucket]
+        current_path = ""
+        for path_part in path_parts:
+            current_path = posixpath.join(current_path, path_part)
+            remote_ids = self.get_remote_id_from_cache(current_path)
+            if remote_ids:
+                parents_ids = remote_ids
+                continue
+            item = self.get_remote_item(path_part, parents_ids)
+            if not item and create:
+                item = self.create_remote_dir(parents_ids[0], path_part)
+            elif not item:
+                return None
+            parents_ids = [item["id"]]
+        return item
+
+    def get_remote_id_from_cache(self, remote_path):
+        if hasattr(self, "_cached_dirs"):
+            return self.cached_dirs.get(remote_path, [])
+        return []
+
+    def get_remote_id(self, path_info, create=False):
+        remote_ids = self.get_remote_id_from_cache(path_info.path)
+
+        if remote_ids:
+            return remote_ids[0]
+
+        file1 = self.resolve_remote_item_from_path(
+            path_info.path.split("/"), create
+        )
+        return file1["id"] if file1 else ""
+
+    def exists(self, path_info):
+        return self.get_remote_id(path_info) != ""
+
+    def _upload(self, from_file, to_info, name, no_progress_bar):
+        dirname = to_info.parent
+        if dirname:
+            parent_id = self.get_remote_id(dirname, True)
+        else:
+            parent_id = to_info.bucket
+
+        self.gdrive_upload_file(
+            {"title": to_info.name, "parent_id": parent_id},
+            no_progress_bar,
+            from_file,
+            name,
+        )
+
+    def _download(self, from_info, to_file, name, no_progress_bar):
+        file_id = self.get_remote_id(from_info)
+        self.gdrive_download_file(file_id, to_file, name, no_progress_bar)
+
+    def all(self):
+        if not self.cached_ids:
+            return
+
+        query = "({})".format(
+            " or ".join(
+                "'{}' in parents".format(dir_id) for dir_id in self.cached_ids
+            )
+        )
+
+        query += " and trashed=false"
+        for file1 in self.gdrive_list_item(query):
+            parent_id = file1["parents"][0]["id"]
+            path = posixpath.join(self.cached_ids[parent_id], file1["title"])
+            try:
+                yield self.path_to_checksum(path)
+            except ValueError:
+                # We ignore all the non-cache looking files
+                logger.debug('Ignoring path as "non-cache looking"')

dvc/remote/gs.py

@@ -1,12 +1,11 @@
-from __future__ import unicode_literals, division
-
 import logging
 from datetime import timedelta
 from functools import wraps
 import io
 import os.path
+import threading
 
-from funcy import cached_property
+from funcy import cached_property, wrap_prop
 
 from dvc.config import Config
 from dvc.exceptions import DvcException
@@ -14,7 +13,6 @@ from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
-from dvc.utils.compat import FileNotFoundError  # skipcq: PYL-W0622
 
 logger = logging.getLogger(__name__)
 
@@ -57,23 +55,15 @@ def _upload_to_bucket(
     no_progress_bar=True,
 ):
     blob = bucket.blob(to_info.path, chunk_size=chunk_size)
-    with Tqdm(
-        desc=name or to_info.path,
-        total=os.path.getsize(from_file),
-        bytes=True,
-        disable=no_progress_bar,
-    ) as pbar:
-        with io.open(from_file, mode="rb") as fobj:
-            raw_read = fobj.read
-
-            def read(size=chunk_size):
-                res = raw_read(size)
-                if res:
-                    pbar.update(len(res))
-                return res
-
-            fobj.read = read
-            blob.upload_from_file(fobj)
+    with io.open(from_file, mode="rb") as fobj:
+        with Tqdm.wrapattr(
+            fobj,
+            "read",
+            desc=name or to_info.path,
+            total=os.path.getsize(from_file),
+            disable=no_progress_bar,
+        ) as wrapped:
+            blob.upload_from_file(wrapped)
 
 
 class RemoteGS(RemoteBASE):
@@ -83,7 +73,7 @@ class RemoteGS(RemoteBASE):
     PARAM_CHECKSUM = "md5"
 
     def __init__(self, repo, config):
-        super(RemoteGS, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL, "gs:///")
         self.path_info = self.path_cls(url)
@@ -91,6 +81,7 @@ class RemoteGS(RemoteBASE):
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
         self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def gs(self):
         from google.cloud.storage import Client
@@ -136,16 +127,45 @@ class RemoteGS(RemoteBASE):
 
         blob.delete()
 
-    def _list_paths(self, bucket, prefix):
-        for blob in self.gs.bucket(bucket).list_blobs(prefix=prefix):
+    def _list_paths(self, path_info, max_items=None):
+        for blob in self.gs.bucket(path_info.bucket).list_blobs(
+            prefix=path_info.path, max_results=max_items
+        ):
             yield blob.name
 
     def list_cache_paths(self):
-        return self._list_paths(self.path_info.bucket, self.path_info.path)
+        return self._list_paths(self.path_info)
+
+    def walk_files(self, path_info):
+        for fname in self._list_paths(path_info / ""):
+            # skip nested empty directories
+            if fname.endswith("/"):
+                continue
+            yield path_info.replace(fname)
+
+    def makedirs(self, path_info):
+        self.gs.bucket(path_info.bucket).blob(
+            (path_info / "").path
+        ).upload_from_string("")
+
+    def isdir(self, path_info):
+        dir_path = path_info / ""
+        return bool(list(self._list_paths(dir_path, max_items=1)))
+
+    def isfile(self, path_info):
+        if path_info.path.endswith("/"):
+            return False
+
+        blob = self.gs.bucket(path_info.bucket).blob(path_info.path)
+        return blob.exists()
 
     def exists(self, path_info):
-        paths = set(self._list_paths(path_info.bucket, path_info.path))
-        return any(path_info.path == path for path in paths)
+        """Check if the blob exists. If it does not exist,
+        it could be a part of a directory path.
+
+        eg: if `data/file.txt` exists, check for `data` should return True
+        """
+        return self.isfile(path_info) or self.isdir(path_info)
 
     def _upload(self, from_file, to_info, name=None, no_progress_bar=True):
         bucket = self.gs.bucket(to_info.bucket)
@@ -160,21 +180,15 @@ class RemoteGS(RemoteBASE):
     def _download(self, from_info, to_file, name=None, no_progress_bar=True):
         bucket = self.gs.bucket(from_info.bucket)
         blob = bucket.get_blob(from_info.path)
-        with Tqdm(
-            desc=name or from_info.path,
-            total=blob.size,
-            bytes=True,
-            disable=no_progress_bar,
-        ) as pbar:
-            with io.open(to_file, mode="wb") as fobj:
-                raw_write = fobj.write
-
-                def write(byte_string):
-                    raw_write(byte_string)
-                    pbar.update(len(byte_string))
-
-                fobj.write = write
-                blob.download_to_file(fobj)
+        with io.open(to_file, mode="wb") as fobj:
+            with Tqdm.wrapattr(
+                fobj,
+                "write",
+                desc=name or from_info.path,
+                total=blob.size,
+                disable=no_progress_bar,
+            ) as wrapped:
+                blob.download_to_file(wrapped)
 
     def _generate_download_url(self, path_info, expires=3600):
         expiration = timedelta(seconds=int(expires))

dvc/remote/hdfs.py

@@ -1,24 +1,18 @@
-from __future__ import unicode_literals
-
 import io
 import logging
 import os
 import posixpath
 import re
-from contextlib import closing
-from contextlib import contextmanager
-from subprocess import PIPE
-from subprocess import Popen
+import subprocess
+from collections import deque
+from contextlib import closing, contextmanager
+from urllib.parse import urlparse
 
-from .base import RemoteBASE
-from .base import RemoteCmdError
+from .base import RemoteBASE, RemoteCmdError
 from .pool import get_connection
 from dvc.config import Config
 from dvc.scheme import Schemes
-from dvc.utils import fix_env
-from dvc.utils import tmp_fname
-from dvc.utils.compat import FileNotFoundError
-from dvc.utils.compat import urlparse
+from dvc.utils import fix_env, tmp_fname
 
 logger = logging.getLogger(__name__)
 
@@ -30,7 +24,7 @@ class RemoteHDFS(RemoteBASE):
     REQUIRES = {"pyarrow": "pyarrow"}
 
     def __init__(self, repo, config):
-        super(RemoteHDFS, self).__init__(repo, config)
+        super().__init__(repo, config)
         self.path_info = None
         url = config.get(Config.SECTION_REMOTE_URL)
         if not url:
@@ -68,15 +62,15 @@ class RemoteHDFS(RemoteBASE):
         close_fds = os.name != "nt"
 
         executable = os.getenv("SHELL") if os.name != "nt" else None
-        p = Popen(
+        p = subprocess.Popen(
             cmd,
             shell=True,
             close_fds=close_fds,
             executable=executable,
             env=fix_env(os.environ),
-            stdin=PIPE,
-            stdout=PIPE,
-            stderr=PIPE,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
         )
         out, err = p.communicate()
         if p.returncode != 0:
@@ -104,8 +98,14 @@ class RemoteHDFS(RemoteBASE):
             # NOTE: this is how `hadoop fs -cp` works too: it copies through
             # your local machine.
             with hdfs.open(from_info.path, "rb") as from_fobj:
-                with hdfs.open(to_info.path, "wb") as to_fobj:
-                    to_fobj.upload(from_fobj)
+                tmp_info = to_info.parent / tmp_fname(to_info.name)
+                try:
+                    with hdfs.open(tmp_info.path, "wb") as tmp_fobj:
+                        tmp_fobj.upload(from_fobj)
+                    hdfs.rename(tmp_info.path, to_info.path)
+                except Exception:
+                    self.remove(tmp_info)
+                    raise
 
     def remove(self, path_info):
         if path_info.scheme != "hdfs":
@@ -156,10 +156,9 @@ class RemoteHDFS(RemoteBASE):
 
     def list_cache_paths(self):
         if not self.exists(self.path_info):
-            return []
+            return
 
-        files = []
-        dirs = [self.path_info.path]
+        dirs = deque([self.path_info.path])
 
         with self.hdfs(self.path_info) as hdfs:
             while dirs:
@@ -167,6 +166,4 @@ class RemoteHDFS(RemoteBASE):
                     if entry["kind"] == "directory":
                         dirs.append(urlparse(entry["name"]).path)
                     elif entry["kind"] == "file":
-                        files.append(urlparse(entry["name"]).path)
-
-        return files
+                        yield urlparse(entry["name"]).path

dvc/remote/http.py

@@ -1,16 +1,14 @@
-from __future__ import unicode_literals
-
 import logging
+import threading
 
-from funcy import cached_property
+from funcy import cached_property, wrap_prop
 
 from dvc.config import Config
 from dvc.config import ConfigError
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, HTTPError
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
-from dvc.utils.compat import open
 
 logger = logging.getLogger(__name__)
 
@@ -24,7 +22,7 @@ class RemoteHTTP(RemoteBASE):
     PARAM_CHECKSUM = "etag"
 
     def __init__(self, repo, config):
-        super(RemoteHTTP, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL)
         self.path_info = self.path_cls(url) if url else None
@@ -36,30 +34,26 @@ class RemoteHTTP(RemoteBASE):
             )
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        request = self._request("GET", from_info.url, stream=True)
+        response = self._request("GET", from_info.url, stream=True)
+        if response.status_code != 200:
+            raise HTTPError(response.status_code, response.reason)
         with Tqdm(
-            total=None if no_progress_bar else self._content_length(from_info),
+            total=None if no_progress_bar else self._content_length(response),
             leave=False,
             bytes=True,
             desc=from_info.url if name is None else name,
             disable=no_progress_bar,
         ) as pbar:
             with open(to_file, "wb") as fd:
-                for chunk in request.iter_content(chunk_size=self.CHUNK_SIZE):
+                for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE):
                     fd.write(chunk)
-                    fd.flush()
                     pbar.update(len(chunk))
 
     def exists(self, path_info):
         return bool(self._request("HEAD", path_info.url))
 
-    def _content_length(self, url_or_request):
-        headers = getattr(
-            url_or_request,
-            "headers",
-            self._request("HEAD", url_or_request).headers,
-        )
-        res = headers.get("Content-Length")
+    def _content_length(self, response):
+        res = response.headers.get("Content-Length")
         return int(res) if res else None
 
     def get_file_checksum(self, path_info):
@@ -81,6 +75,7 @@ class RemoteHTTP(RemoteBASE):
 
         return etag
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def _session(self):
         import requests
@@ -106,7 +101,22 @@ class RemoteHTTP(RemoteBASE):
         kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
 
         try:
-            return self._session.request(method, url, **kwargs)
+            res = self._session.request(method, url, **kwargs)
+
+            redirect_no_location = (
+                kwargs["allow_redirects"]
+                and res.status_code in (301, 302)
+                and "location" not in res.headers
+            )
+
+            if redirect_no_location:
+                # AWS s3 doesn't like to add a location header to its redirects
+                # from https://s3.amazonaws.com/<bucket name>/* type URLs.
+                # This should be treated as an error
+                raise requests.exceptions.RequestException
+
+            return res
+
         except requests.exceptions.RequestException:
             raise DvcException("could not perform a {} request".format(method))
 

dvc/remote/https.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from .http import RemoteHTTP
 from dvc.scheme import Schemes
 

dvc/remote/local.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import errno
 import logging
 import os
@@ -21,18 +19,15 @@ from dvc.remote.base import STATUS_MAP
 from dvc.remote.base import STATUS_MISSING
 from dvc.remote.base import STATUS_NEW
 from dvc.scheme import Schemes
+from dvc.scm.tree import is_working_tree
 from dvc.system import System
-from dvc.utils import copyfile
+from dvc.utils.fs import copyfile
 from dvc.utils import file_md5
-from dvc.utils import makedirs
 from dvc.utils import relpath
 from dvc.utils import tmp_fname
-from dvc.utils import walk_files
-from dvc.utils.compat import fspath_py35
-from dvc.utils.compat import open
-from dvc.utils.compat import str
-from dvc.utils.fs import move
-from dvc.utils.fs import remove
+from dvc.compat import fspath_py35
+from dvc.utils.fs import move, makedirs, remove
+from dvc.utils.fs import walk_files
 
 logger = logging.getLogger(__name__)
 
@@ -50,7 +45,7 @@ class RemoteLOCAL(RemoteBASE):
     SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
 
     def __init__(self, repo, config):
-        super(RemoteLOCAL, self).__init__(repo, config)
+        super().__init__(repo, config)
         self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
 
         shared = config.get(Config.SECTION_CACHE_SHARED)
@@ -66,7 +61,7 @@ class RemoteLOCAL(RemoteBASE):
             cwd = config[Config.PRIVATE_CWD]
             cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
 
-        self.path_info = PathInfo(cache_dir) if cache_dir else None
+        self.cache_dir = cache_dir
         self._dir_info = {}
 
     @property
@@ -77,23 +72,17 @@ class RemoteLOCAL(RemoteBASE):
     def cache_dir(self):
         return self.path_info.fspath if self.path_info else None
 
+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.path_info = PathInfo(value) if value else None
+
     @classmethod
     def supported(cls, config):
         return True
 
     def list_cache_paths(self):
         assert self.path_info is not None
-
-        clist = []
-        for entry in os.listdir(fspath_py35(self.path_info)):
-            subdir = self.path_info / entry
-            if not os.path.isdir(fspath_py35(subdir)):
-                continue
-            clist.extend(
-                subdir / cache for cache in os.listdir(fspath_py35(subdir))
-            )
-
-        return clist
+        return walk_files(self.path_info)
 
     def get(self, md5):
         if not md5:
@@ -138,12 +127,19 @@ class RemoteLOCAL(RemoteBASE):
     def isdir(path_info):
         return os.path.isdir(fspath_py35(path_info))
 
+    def iscopy(self, path_info):
+        return not (
+            System.is_symlink(path_info) or System.is_hardlink(path_info)
+        )
+
     @staticmethod
     def getsize(path_info):
         return os.path.getsize(fspath_py35(path_info))
 
     def walk_files(self, path_info):
-        for fname in walk_files(path_info, self.repo.dvcignore):
+        assert is_working_tree(self.repo.tree)
+
+        for fname in self.repo.tree.walk_files(path_info):
             yield PathInfo(fname)
 
     def get_file_checksum(self, path_info):
@@ -169,14 +165,23 @@ class RemoteLOCAL(RemoteBASE):
 
         move(from_info, to_info, mode=mode)
 
-    @staticmethod
-    def copy(from_info, to_info):
-        System.copy(from_info, to_info)
+    def copy(self, from_info, to_info):
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        try:
+            System.copy(from_info, tmp_info)
+            os.rename(fspath_py35(tmp_info), fspath_py35(to_info))
+        except Exception:
+            self.remove(tmp_info)
+            raise
 
     @staticmethod
     def symlink(from_info, to_info):
         System.symlink(from_info, to_info)
 
+    @staticmethod
+    def is_symlink(path_info):
+        return System.is_symlink(path_info)
+
     def hardlink(self, from_info, to_info):
         # If there are a lot of empty files (which happens a lot in datasets),
         # and the cache type is `hardlink`, we might reach link limits and
@@ -203,6 +208,10 @@ class RemoteLOCAL(RemoteBASE):
 
         System.hardlink(from_info, to_info)
 
+    @staticmethod
+    def is_hardlink(path_info):
+        return System.is_hardlink(path_info)
+
     @staticmethod
     def reflink(from_info, to_info):
         System.reflink(from_info, to_info)
@@ -404,22 +413,19 @@ class RemoteLOCAL(RemoteBASE):
             logger.warning(msg)
 
     @staticmethod
-    def _unprotect_file(path, allow_copy=True):
+    def _unprotect_file(path):
         if System.is_symlink(path) or System.is_hardlink(path):
-            if allow_copy:
-                logger.debug("Unprotecting '{}'".format(path))
-                tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
+            logger.debug("Unprotecting '{}'".format(path))
+            tmp = os.path.join(os.path.dirname(path), "." + uuid())
 
-                # The operations order is important here - if some application
-                # would access the file during the process of copyfile then it
-                # would get only the part of file. So, at first, the file
-                # should be copied with the temporary name, and then
-                # original file should be replaced by new.
-                copyfile(
-                    path, tmp, name="Unprotecting '{}'".format(relpath(path))
-                )
-                remove(path)
-                os.rename(tmp, path)
+            # The operations order is important here - if some application
+            # would access the file during the process of copyfile then it
+            # would get only the part of file. So, at first, the file should be
+            # copied with the temporary name, and then original file should be
+            # replaced by new.
+            copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
+            remove(path)
+            os.rename(tmp, path)
 
         else:
             logger.debug(
@@ -429,11 +435,13 @@ class RemoteLOCAL(RemoteBASE):
 
         os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
 
-    def _unprotect_dir(self, path, allow_copy=True):
-        for fname in walk_files(path, self.repo.dvcignore):
-            self._unprotect_file(fname, allow_copy)
+    def _unprotect_dir(self, path):
+        assert is_working_tree(self.repo.tree)
 
-    def unprotect(self, path_info, allow_copy=True):
+        for fname in self.repo.tree.walk_files(path):
+            RemoteLOCAL._unprotect_file(fname)
+
+    def unprotect(self, path_info):
         path = path_info.fspath
         if not os.path.exists(path):
             raise DvcException(
@@ -441,9 +449,9 @@ class RemoteLOCAL(RemoteBASE):
             )
 
         if os.path.isdir(path):
-            self._unprotect_dir(path, allow_copy)
+            self._unprotect_dir(path)
         else:
-            self._unprotect_file(path, allow_copy)
+            RemoteLOCAL._unprotect_file(path)
 
     @staticmethod
     def protect(path_info):
@@ -466,6 +474,10 @@ class RemoteLOCAL(RemoteBASE):
         info = self.checksum_to_path_info(checksum)
         return info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
 
+    def _remove_unpacked_dir(self, checksum):
+        path_info = self._get_unpacked_dir_path_info(checksum)
+        self.remove(path_info)
+
     def _path_info_changed(self, path_info):
         if self.exists(path_info) and self.state.get(path_info):
             return False
@@ -517,36 +529,3 @@ class RemoteLOCAL(RemoteBASE):
             if self.is_dir_checksum(c):
                 unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
         return unpacked
-
-    def _get_cache_type(self, path_info):
-        if self.cache_type_confirmed:
-            return self.cache_types[0]
-
-        workspace_file = path_info.with_name("." + uuid())
-        test_cache_file = self.path_info / ".cache_type_test_file"
-        if not self.exists(test_cache_file):
-            with open(fspath_py35(test_cache_file), "wb") as fobj:
-                fobj.write(bytes(1))
-        try:
-            self.link(test_cache_file, workspace_file)
-        finally:
-            self.remove(workspace_file)
-            self.remove(test_cache_file)
-
-        self.cache_type_confirmed = True
-        return self.cache_types[0]
-
-    def _link_matches(self, path_info):
-        is_hardlink = System.is_hardlink(path_info)
-        is_symlink = System.is_symlink(path_info)
-        is_copy_or_reflink = not is_hardlink and not is_symlink
-
-        cache_type = self._get_cache_type(path_info)
-
-        if cache_type == "symlink":
-            return is_symlink
-
-        if cache_type == "hardlink":
-            return is_hardlink
-
-        return is_copy_or_reflink

dvc/remote/oss.py

@@ -1,8 +1,8 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 import logging
 import os
+import threading
+
+from funcy import cached_property, wrap_prop
 
 from dvc.config import Config
 from dvc.path_info import CloudURLInfo
@@ -40,7 +40,7 @@ class RemoteOSS(RemoteBASE):
     COPY_POLL_SECONDS = 5
 
     def __init__(self, repo, config):
-        super(RemoteOSS, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL)
         self.path_info = self.path_cls(url) if url else None
@@ -61,30 +61,29 @@ class RemoteOSS(RemoteBASE):
             or "defaultSecret"
         )
 
-        self._bucket = None
-
-    @property
+    @wrap_prop(threading.Lock())
+    @cached_property
     def oss_service(self):
         import oss2
 
-        if self._bucket is None:
-            logger.debug("URL {}".format(self.path_info))
-            logger.debug("key id {}".format(self.key_id))
-            logger.debug("key secret {}".format(self.key_secret))
-            auth = oss2.Auth(self.key_id, self.key_secret)
-            self._bucket = oss2.Bucket(
-                auth, self.endpoint, self.path_info.bucket
+        logger.debug("URL: {}".format(self.path_info))
+        logger.debug("key id: {}".format(self.key_id))
+        logger.debug("key secret: {}".format(self.key_secret))
+
+        auth = oss2.Auth(self.key_id, self.key_secret)
+        bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)
+
+        # Ensure bucket exists
+        try:
+            bucket.get_bucket_info()
+        except oss2.exceptions.NoSuchBucket:
+            bucket.create_bucket(
+                oss2.BUCKET_ACL_PUBLIC_READ,
+                oss2.models.BucketCreateConfig(
+                    oss2.BUCKET_STORAGE_CLASS_STANDARD
+                ),
             )
-            try:  # verify that bucket exists
-                self._bucket.get_bucket_info()
-            except oss2.exceptions.NoSuchBucket:
-                self._bucket.create_bucket(
-                    oss2.BUCKET_ACL_PUBLIC_READ,
-                    oss2.models.BucketCreateConfig(
-                        oss2.BUCKET_STORAGE_CLASS_STANDARD
-                    ),
-                )
-        return self._bucket
+        return bucket
 
     def remove(self, path_info):
         if path_info.scheme != self.scheme:

dvc/remote/pool.py

@@ -1,7 +1,8 @@
 from collections import deque
 from contextlib import contextmanager
+import threading
 
-from funcy import memoize
+from funcy import memoize, wrap_with
 
 
 @contextmanager
@@ -17,6 +18,7 @@ def get_connection(conn_func, *args, **kwargs):
         pool.release(conn)
 
 
+@wrap_with(threading.Lock())
 @memoize
 def get_pool(conn_func, *args, **kwargs):
     return Pool(conn_func, *args, **kwargs)

dvc/remote/s3.py

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
 
 import logging
 import os
+import threading
 
-from funcy import cached_property
+from funcy import cached_property, wrap_prop
 
-from dvc.config import Config
+from dvc.config import Config, ConfigError
 from dvc.exceptions import DvcException
 from dvc.exceptions import ETagMismatchError
 from dvc.path_info import CloudURLInfo
@@ -24,7 +24,7 @@ class RemoteS3(RemoteBASE):
     PARAM_CHECKSUM = "etag"
 
     def __init__(self, repo, config):
-        super(RemoteS3, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL, "s3://")
         self.path_info = self.path_cls(url)
@@ -52,10 +52,13 @@ class RemoteS3(RemoteBASE):
         if self.acl:
             self.extra_args["ACL"] = self.acl
 
+        self._append_aws_grants_to_extra_args(config)
+
         shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
         if shared_creds:
             os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def s3(self):
         import boto3
@@ -84,8 +87,8 @@ class RemoteS3(RemoteBASE):
             obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
         except Exception as exc:
             raise DvcException(
-                "s3://{}/{} does not exist".format(bucket, path), exc
-            )
+                "s3://{}/{} does not exist".format(bucket, path)
+            ) from exc
         return obj
 
     @classmethod
@@ -153,6 +156,7 @@ class RemoteS3(RemoteBASE):
         # the same throughout the transfer, so it means that in order to
         # preserve etag, we need to transfer each part separately, so the
         # object is transfered in the same chunks as it was originally.
+        from boto3.s3.transfer import TransferConfig
 
         obj = cls.get_head_object(s3, from_info.bucket, from_info.path)
         etag = obj["ETag"].strip('"')
@@ -166,7 +170,13 @@ class RemoteS3(RemoteBASE):
             )
         else:
             source = {"Bucket": from_info.bucket, "Key": from_info.path}
-            s3.copy(source, to_info.bucket, to_info.path, ExtraArgs=extra_args)
+            s3.copy(
+                source,
+                to_info.bucket,
+                to_info.path,
+                ExtraArgs=extra_args,
+                Config=TransferConfig(multipart_threshold=size + 1),
+            )
 
         cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
         if etag != cached_etag:
@@ -191,11 +201,7 @@ class RemoteS3(RemoteBASE):
         }
         paginator = self.s3.get_paginator(self.list_objects_api)
         for page in paginator.paginate(**kwargs):
-            contents = page.get("Contents", None)
-            if not contents:
-                continue
-            for item in contents:
-                yield item
+            yield from page.get("Contents", ())
 
     def _list_paths(self, path_info, max_items=None):
         return (
@@ -205,10 +211,37 @@ class RemoteS3(RemoteBASE):
     def list_cache_paths(self):
         return self._list_paths(self.path_info)
 
+    def isfile(self, path_info):
+        from botocore.exceptions import ClientError
+
+        if path_info.path.endswith("/"):
+            return False
+
+        try:
+            self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
+        except ClientError as exc:
+            if exc.response["Error"]["Code"] != "404":
+                raise
+            return False
+
+        return True
+
     def exists(self, path_info):
+        """Check if the blob exists. If it does not exist,
+        it could be a part of a directory path.
+
+        eg: if `data/file.txt` exists, check for `data` should return True
+        """
+        return self.isfile(path_info) or self.isdir(path_info)
+
+    def makedirs(self, path_info):
+        # We need to support creating empty directories, which means
+        # creating an object with an empty body and a trailing slash `/`.
+        #
+        # We are not creating directory objects for every parent prefix,
+        # as it is not required.
         dir_path = path_info / ""
-        fname = next(self._list_paths(path_info, max_items=1), "")
-        return path_info.path == fname or fname.startswith(dir_path.path)
+        self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
 
     def isdir(self, path_info):
         # S3 doesn't have a concept for directories.
@@ -268,5 +301,39 @@ class RemoteS3(RemoteBASE):
         )
 
     def walk_files(self, path_info, max_items=None):
-        for fname in self._list_paths(path_info, max_items):
+        for fname in self._list_paths(path_info / "", max_items):
+            if fname.endswith("/"):
+                continue
+
             yield path_info.replace(path=fname)
+
+    def _append_aws_grants_to_extra_args(self, config):
+        # Keys for extra_args can be one of the following list:
+        # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS
+        """
+          ALLOWED_UPLOAD_ARGS = [
+            'ACL', 'CacheControl', 'ContentDisposition', 'ContentEncoding',
+            'ContentLanguage', 'ContentType', 'Expires', 'GrantFullControl',
+            'GrantRead', 'GrantReadACP', 'GrantWriteACP', 'Metadata',
+            'RequestPayer', 'ServerSideEncryption', 'StorageClass',
+            'SSECustomerAlgorithm', 'SSECustomerKey', 'SSECustomerKeyMD5',
+            'SSEKMSKeyId', 'WebsiteRedirectLocation'
+          ]
+        """
+
+        grants = {
+            Config.SECTION_AWS_GRANT_FULL_CONTROL: "GrantFullControl",
+            Config.SECTION_AWS_GRANT_READ: "GrantRead",
+            Config.SECTION_AWS_GRANT_READ_ACP: "GrantReadACP",
+            Config.SECTION_AWS_GRANT_WRITE_ACP: "GrantWriteACP",
+        }
+
+        for grant_option, extra_args_key in grants.items():
+            if config.get(grant_option, ""):
+                if self.acl:
+                    raise ConfigError(
+                        "`acl` and `grant_*` AWS S3 config options "
+                        "are mutually exclusive"
+                    )
+
+                self.extra_args[extra_args_key] = config.get(grant_option)

dvc/remote/ssh/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import errno
 import getpass
 import io
@@ -8,8 +6,10 @@ import logging
 import os
 import threading
 from concurrent.futures import ThreadPoolExecutor
-from contextlib import closing
-from contextlib import contextmanager
+from contextlib import closing, contextmanager
+from urllib.parse import urlparse
+
+from funcy import memoize, wrap_with
 
 import dvc.prompt as prompt
 from dvc.config import Config
@@ -18,14 +18,19 @@ from dvc.remote.base import RemoteBASE
 from dvc.remote.pool import get_connection
 from dvc.scheme import Schemes
 from dvc.utils import to_chunks
-from dvc.utils.compat import StringIO
-from dvc.utils.compat import urlparse
 
 logger = logging.getLogger(__name__)
 
 
-saved_passwords = {}
-saved_passwords_lock = threading.Lock()
+@wrap_with(threading.Lock())
+@memoize
+def ask_password(host, user, port):
+    return prompt.password(
+        "Enter a private key passphrase or a password for "
+        "host '{host}' port '{port}' user '{user}'".format(
+            host=host, port=port, user=user
+        )
+    )
 
 
 class RemoteSSH(RemoteBASE):
@@ -44,7 +49,7 @@ class RemoteSSH(RemoteBASE):
     DEFAULT_CACHE_TYPES = ["copy"]
 
     def __init__(self, repo, config):
-        super(RemoteSSH, self).__init__(repo, config)
+        super().__init__(repo, config)
         url = config.get(Config.SECTION_REMOTE_URL)
         if url:
             parsed = urlparse(url)
@@ -98,7 +103,7 @@ class RemoteSSH(RemoteBASE):
             ssh_config = paramiko.SSHConfig()
             with open(user_config_file) as f:
                 # For whatever reason parsing directly from f is unreliable
-                f_copy = StringIO(f.read())
+                f_copy = io.StringIO(f.read())
                 ssh_config.parse(f_copy)
             user_ssh_config = ssh_config.lookup(hostname)
         return user_ssh_config
@@ -120,21 +125,11 @@ class RemoteSSH(RemoteBASE):
     def ensure_credentials(self, path_info=None):
         if path_info is None:
             path_info = self.path_info
-        host, user, port = path_info.host, path_info.user, path_info.port
+
         # NOTE: we use the same password regardless of the server :(
         if self.ask_password and self.password is None:
-            with saved_passwords_lock:
-                server_key = (host, user, port)
-                password = saved_passwords.get(server_key)
-
-                if password is None:
-                    saved_passwords[server_key] = password = prompt.password(
-                        "Enter a private key passphrase or a password for "
-                        "host '{host}' port '{port}' user '{user}'".format(
-                            host=host, port=port, user=user
-                        )
-                    )
-                self.password = password
+            host, user, port = path_info.host, path_info.user, path_info.port
+            self.password = ask_password(host, user, port)
 
     def ssh(self, path_info):
         self.ensure_credentials(path_info)
@@ -180,7 +175,7 @@ class RemoteSSH(RemoteBASE):
             raise NotImplementedError
 
         with self.ssh(from_info) as ssh:
-            ssh.copy(from_info.path, to_info.path)
+            ssh.atomic_copy(from_info.path, to_info.path)
 
     def symlink(self, from_info, to_info):
         if not from_info.scheme == to_info.scheme == self.scheme:
@@ -264,7 +259,8 @@ class RemoteSSH(RemoteBASE):
 
     def list_cache_paths(self):
         with self.ssh(self.path_info) as ssh:
-            return list(ssh.walk_files(self.path_info.path))
+            # If we simply return an iterator then with above closes instantly
+            yield from ssh.walk_files(self.path_info.path)
 
     def walk_files(self, path_info):
         with self.ssh(path_info) as ssh:

dvc/remote/ssh/connection.py

@@ -3,6 +3,7 @@ import logging
 import os
 import posixpath
 import stat
+from contextlib import suppress
 
 from funcy import cached_property
 
@@ -12,7 +13,6 @@ except ImportError:
     paramiko = None
 
 from dvc.utils import tmp_fname
-from dvc.utils.compat import ignore_file_not_found
 from dvc.progress import Tqdm
 from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteCmdError
@@ -60,13 +60,13 @@ class SSHConnection:
         self._ssh.close()
 
     def st_mode(self, path):
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
             return self.sftp.lstat(path).st_mode
 
         return 0
 
     def getsize(self, path):
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
             return self.sftp.lstat(path).st_size
 
         return 0
@@ -103,14 +103,13 @@ class SSHConnection:
         if tail:
             try:
                 self.sftp.mkdir(path)
-            except IOError as e:
+            except IOError as exc:
                 # Since paramiko errors are very vague we need to recheck
                 # whether it's because path already exists or something else
-                if e.errno == errno.EACCES or not self.exists(path):
+                if exc.errno == errno.EACCES or not self.exists(path):
                     raise DvcException(
-                        "unable to create remote directory '{}'".format(path),
-                        cause=e,
-                    )
+                        "unable to create remote directory '{}'".format(path)
+                    ) from exc
 
     def walk(self, directory, topdown=True):
         # NOTE: original os.walk() implementation [1] with default options was
@@ -123,9 +122,8 @@ class SSHConnection:
             raise DvcException(
                 "couldn't get the '{}' remote directory files list".format(
                     directory
-                ),
-                cause=exc,
-            )
+                )
+            ) from exc
 
         dirs = []
         nondirs = []
@@ -141,8 +139,7 @@ class SSHConnection:
 
         for dname in dirs:
             newpath = posixpath.join(directory, dname)
-            for entry in self.walk(newpath, topdown=topdown):
-                yield entry
+            yield from self.walk(newpath, topdown=topdown)
 
         if not topdown:
             yield directory, dirs, nondirs
@@ -153,22 +150,20 @@ class SSHConnection:
                 yield posixpath.join(root, fname)
 
     def _remove_file(self, path):
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
             self.sftp.remove(path)
 
     def _remove_dir(self, path):
         for root, dirs, files in self.walk(path, topdown=False):
             for fname in files:
-                path = posixpath.join(root, fname)
-                with ignore_file_not_found():
-                    self._remove_file(path)
+                with suppress(FileNotFoundError):
+                    self._remove_file(posixpath.join(root, fname))
 
             for dname in dirs:
-                path = posixpath.join(root, dname)
-                with ignore_file_not_found():
-                    self.sftp.rmdir(dname)
+                with suppress(FileNotFoundError):
+                    self.sftp.rmdir(posixpath.join(root, dname))
 
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
             self.sftp.rmdir(path)
 
     def remove(self, path):
@@ -195,10 +190,10 @@ class SSHConnection:
         try:
             self.sftp.rename(src, dst)
         except OSError:
-            self._atomic_copy(src, dst)
+            self.atomic_copy(src, dst)
             self.remove(src)
 
-    def _atomic_copy(self, src, dst):
+    def atomic_copy(self, src, dst):
         tmp = tmp_fname(dst)
 
         try:

dvc/repo/__init__.py

@@ -1,29 +1,24 @@
-from __future__ import unicode_literals
-
 import logging
 import os
 from contextlib import contextmanager
 from functools import wraps
 from itertools import chain
 
-from funcy import cached_property
+from dvc.ignore import CleanTree
+from dvc.compat import fspath_py35
+
+from funcy import cached_property, cat
 
-from .graph import check_acyclic
-from .graph import get_pipeline
-from .graph import get_pipelines
-from .graph import get_stages
 from dvc.config import Config
-from dvc.exceptions import FileMissingError
-from dvc.exceptions import NotDvcRepoError
-from dvc.exceptions import OutputNotFoundError
-from dvc.ignore import DvcIgnoreFilter
+from dvc.exceptions import (
+    FileMissingError,
+    NotDvcRepoError,
+    OutputNotFoundError,
+)
 from dvc.path_info import PathInfo
 from dvc.remote.base import RemoteActionNotImplemented
-from dvc.utils import relpath
 from dvc.utils.fs import path_isin
-from dvc.utils.compat import FileNotFoundError
-from dvc.utils.compat import fspath_py35
-from dvc.utils.compat import open as _open
+from .graph import check_acyclic, get_pipeline, get_pipelines
 
 
 logger = logging.getLogger(__name__)
@@ -69,13 +64,14 @@ class Repo(object):
 
     def __init__(self, root_dir=None):
         from dvc.state import State
-        from dvc.lock import Lock
+        from dvc.lock import make_lock
         from dvc.scm import SCM
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
         from dvc.repo.metrics import Metrics
         from dvc.scm.tree import WorkingTree
         from dvc.repo.tag import Tag
+        from dvc.utils.fs import makedirs
 
         root_dir = self.find_root(root_dir)
 
@@ -88,10 +84,17 @@ class Repo(object):
 
         self.tree = WorkingTree(self.root_dir)
 
-        self.lock = Lock(
+        self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
+        makedirs(self.tmp_dir, exist_ok=True)
+
+        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        self.lock = make_lock(
             os.path.join(self.dvc_dir, "lock"),
             tmp_dir=os.path.join(self.dvc_dir, "tmp"),
+            hardlink_lock=hardlink_lock,
+            friendly=True,
         )
+
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
         self.state = State(self, self.config.config)
@@ -116,7 +119,7 @@ class Repo(object):
 
     @tree.setter
     def tree(self, tree):
-        self._tree = tree
+        self._tree = tree if isinstance(tree, CleanTree) else CleanTree(tree)
         # Our graph cache is no longer valid, as it was based on the previous
         # tree.
         self._reset()
@@ -162,9 +165,8 @@ class Repo(object):
 
         flist = (
             [self.config.config_local_file, updater.updater_file]
+            + [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir]
             + self.state.files
-            + self.lock.files
-            + updater.lock.files
         )
 
         if path_isin(self.cache.local.cache_dir, self.root_dir):
@@ -183,32 +185,32 @@ class Repo(object):
         G = graph or self.graph
 
         if not target:
-            return get_stages(G)
+            return list(G)
 
         target = os.path.abspath(target)
 
         if recursive and os.path.isdir(target):
-            attrs = nx.get_node_attributes(G, "stage")
-            nodes = [node for node in nx.dfs_postorder_nodes(G)]
-
-            ret = []
-            for node in nodes:
-                stage = attrs[node]
-                if path_isin(stage.path, target):
-                    ret.append(stage)
-            return ret
+            stages = nx.dfs_postorder_nodes(G)
+            return [stage for stage in stages if path_isin(stage.path, target)]
 
         stage = Stage.load(self, target)
         if not with_deps:
             return [stage]
 
-        node = relpath(stage.path, self.root_dir)
-        pipeline = get_pipeline(get_pipelines(G), node)
+        pipeline = get_pipeline(get_pipelines(G), stage)
+        return list(nx.dfs_postorder_nodes(pipeline, stage))
 
-        return [
-            pipeline.node[n]["stage"]
-            for n in nx.dfs_postorder_nodes(pipeline, node)
-        ]
+    def collect_granular(self, target, *args, **kwargs):
+        if not target:
+            return [(stage, None) for stage in self.stages]
+
+        try:
+            out, = self.find_outs_by_path(target, strict=False)
+            filter_info = PathInfo(os.path.abspath(target))
+            return [(out.stage, filter_info)]
+        except OutputNotFoundError:
+            stages = self.collect(target, *args, **kwargs)
+            return [(stage, None) for stage in stages]
 
     def used_cache(
         self,
@@ -244,28 +246,24 @@ class Repo(object):
             all_tags=all_tags,
             all_commits=all_commits,
         ):
-            if targets:
-                stages = []
-                for target in targets:
-                    collected = self.collect(
-                        target, recursive=recursive, with_deps=with_deps
-                    )
-                    stages.extend(collected)
-            else:
-                stages = self.stages
+            targets = targets or [None]
 
-            for stage in stages:
-                if stage.is_repo_import:
-                    dep, = stage.deps
-                    cache.external[dep.repo_pair].add(dep.def_path)
-                    continue
+            pairs = cat(
+                self.collect_granular(
+                    target, recursive=recursive, with_deps=with_deps
+                )
+                for target in targets
+            )
 
-                for out in stage.outs:
-                    used_cache = out.get_used_cache(
-                        remote=remote, force=force, jobs=jobs
-                    )
-                    suffix = "({})".format(branch) if branch else ""
-                    cache.update(used_cache, suffix=suffix)
+            suffix = "({})".format(branch) if branch else ""
+            for stage, filter_info in pairs:
+                used_cache = stage.get_used_cache(
+                    remote=remote,
+                    force=force,
+                    jobs=jobs,
+                    filter_info=filter_info,
+                )
+                cache.update(used_cache, suffix=suffix)
 
         return cache
 
@@ -316,15 +314,15 @@ class Repo(object):
         )
 
         G = nx.DiGraph()
-        stages = stages or self.collect_stages()
+        stages = stages or self.stages
         stages = [stage for stage in stages if stage]
         outs = {}
 
         for stage in stages:
             for out in stage.outs:
                 if out.path_info in outs:
-                    stages = [stage.relpath, outs[out.path_info].stage.relpath]
-                    raise OutputDuplicationError(str(out), stages)
+                    dup_stages = [stage, outs[out.path_info].stage]
+                    raise OutputDuplicationError(str(out), dup_stages)
                 outs[out.path_info] = out
 
         for stage in stages:
@@ -337,27 +335,19 @@ class Repo(object):
             stage_path_info = PathInfo(stage.path)
             for p in chain([stage_path_info], stage_path_info.parents):
                 if p in outs:
-                    raise StagePathAsOutputError(stage.wdir, stage.relpath)
+                    raise StagePathAsOutputError(stage, str(outs[p]))
 
         for stage in stages:
-            node = relpath(stage.path, self.root_dir)
-
-            G.add_node(node, stage=stage)
+            G.add_node(stage)
 
             for dep in stage.deps:
                 if dep.path_info is None:
                     continue
 
-                for out in outs:
-                    if (
-                        out == dep.path_info
-                        or dep.path_info.isin(out)
-                        or out.isin(dep.path_info)
-                    ):
-                        dep_stage = outs[out].stage
-                        dep_node = relpath(dep_stage.path, self.root_dir)
-                        G.add_node(dep_node, stage=dep_stage)
-                        G.add_edge(node, dep_node)
+                for out_path_info, out in outs.items():
+                    if out_path_info.overlaps(dep.path_info):
+                        G.add_node(out.stage)
+                        G.add_edge(stage, out.stage)
 
         check_acyclic(G)
 
@@ -371,18 +361,8 @@ class Repo(object):
     def pipelines(self):
         return get_pipelines(self.graph)
 
-    @staticmethod
-    def _filter_out_dirs(dirs, outs, root_dir):
-        def filter_dirs(dname):
-            path = os.path.join(root_dir, dname)
-            for out in outs:
-                if path == os.path.normpath(out):
-                    return False
-            return True
-
-        return list(filter(filter_dirs, dirs))
-
-    def collect_stages(self):
+    @cached_property
+    def stages(self):
         """
         Walks down the root directory looking for Dvcfiles,
         skipping the directories that are related with
@@ -395,41 +375,38 @@ class Repo(object):
         from dvc.stage import Stage
 
         stages = []
-        outs = []
+        outs = set()
 
-        for root, dirs, files in self.tree.walk(
-            self.root_dir, dvcignore=self.dvcignore
-        ):
+        for root, dirs, files in self.tree.walk(self.root_dir):
             for fname in files:
                 path = os.path.join(root, fname)
                 if not Stage.is_valid_filename(path):
                     continue
                 stage = Stage.load(self, path)
-                for out in stage.outs:
-                    if out.scheme == "local":
-                        outs.append(out.fspath + out.sep)
                 stages.append(stage)
 
-            dirs[:] = self._filter_out_dirs(dirs, outs, root)
+                for out in stage.outs:
+                    if out.scheme == "local":
+                        outs.add(out.fspath)
+
+            dirs[:] = [d for d in dirs if os.path.join(root, d) not in outs]
 
         return stages
 
-    @cached_property
-    def stages(self):
-        return get_stages(self.graph)
-
-    def find_outs_by_path(self, path, outs=None, recursive=False):
+    def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
         if not outs:
             outs = [out for stage in self.stages for out in stage.outs]
 
         abs_path = os.path.abspath(path)
+        path_info = PathInfo(abs_path)
         is_dir = self.tree.isdir(abs_path)
+        match = path_info.__eq__ if strict else path_info.isin_or_eq
 
         def func(out):
-            if out.scheme == "local" and out.fspath == abs_path:
+            if out.scheme == "local" and match(out.path_info):
                 return True
 
-            if is_dir and recursive and out.path_info.isin(abs_path):
+            if is_dir and recursive and out.path_info.isin(path_info):
                 return True
 
             return False
@@ -450,29 +427,30 @@ class Repo(object):
         return self.DVC_DIR in path_parts
 
     @contextmanager
-    def open(self, path, remote=None, mode="r", encoding=None):
+    def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
         cause = None
         try:
-            out, = self.find_outs_by_path(path)
-        except OutputNotFoundError as e:
+            out = self.find_out_by_relpath(path)
+        except OutputNotFoundError as exc:
             out = None
-            cause = e
+            cause = exc
 
         if out and out.use_cache:
             try:
                 with self._open_cached(out, remote, mode, encoding) as fd:
                     yield fd
                 return
-            except FileNotFoundError as e:
-                raise FileMissingError(relpath(path, self.root_dir), cause=e)
+            except FileNotFoundError as exc:
+                raise FileMissingError(path) from exc
 
-        if self.tree.exists(path):
-            with self.tree.open(path, mode, encoding) as fd:
+        abs_path = os.path.join(self.root_dir, path)
+        if os.path.exists(abs_path):
+            with open(abs_path, mode=mode, encoding=encoding) as fd:
                 yield fd
             return
 
-        raise FileMissingError(relpath(path, self.root_dir), cause=cause)
+        raise FileMissingError(path) from cause
 
     def _open_cached(self, out, remote=None, mode="r", encoding=None):
         if out.isdir():
@@ -482,7 +460,7 @@ class Repo(object):
         cache_file = fspath_py35(cache_file)
 
         if os.path.exists(cache_file):
-            return _open(cache_file, mode=mode, encoding=encoding)
+            return open(cache_file, mode=mode, encoding=encoding)
 
         try:
             remote_obj = self.cloud.get_remote(remote)
@@ -493,11 +471,7 @@ class Repo(object):
                 cache_info = out.get_used_cache(remote=remote)
                 self.cloud.pull(cache_info, remote=remote)
 
-            return _open(cache_file, mode=mode, encoding=encoding)
-
-    @cached_property
-    def dvcignore(self):
-        return DvcIgnoreFilter(self.root_dir, self.tree)
+            return open(cache_file, mode=mode, encoding=encoding)
 
     def close(self):
         self.scm.close()

dvc/repo/add.py

@@ -1,10 +1,7 @@
-from __future__ import unicode_literals
-
 import logging
 import os
 
 import colorama
-from six import string_types
 
 from . import locked
 from dvc.exceptions import RecursiveAddingWhileUsingFilename
@@ -12,7 +9,6 @@ from dvc.progress import Tqdm
 from dvc.repo.scm_context import scm_context
 from dvc.stage import Stage
 from dvc.utils import LARGE_DIR_SIZE
-from dvc.utils import walk_files
 
 logger = logging.getLogger(__name__)
 
@@ -23,7 +19,7 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
     if recursive and fname:
         raise RecursiveAddingWhileUsingFilename()
 
-    if isinstance(targets, string_types):
+    if isinstance(targets, str):
         targets = [targets]
 
     stages_list = []
@@ -37,7 +33,7 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
                     "You are adding a large directory '{target}' recursively,"
                     " consider tracking it as a whole instead.\n"
                     "{purple}HINT:{nc} Remove the generated DVC-file and then"
-                    " run {cyan}dvc add {target}{nc}".format(
+                    " run `{cyan}dvc add {target}{nc}`".format(
                         purple=colorama.Fore.MAGENTA,
                         cyan=colorama.Fore.CYAN,
                         nc=colorama.Style.RESET_ALL,
@@ -68,7 +64,7 @@ def _find_all_targets(repo, target, recursive):
     if os.path.isdir(target) and recursive:
         return [
             fname
-            for fname in walk_files(target, repo.dvcignore)
+            for fname in repo.tree.walk_files(target)
             if not repo.is_dvc_internal(fname)
             if not Stage.is_stage_file(fname)
             if not repo.scm.belongs_to_scm(fname)

dvc/repo/brancher.py

@@ -4,12 +4,7 @@ from dvc.scm.tree import WorkingTree
 
 
 def brancher(  # noqa: E302
-    self,
-    branches=None,
-    all_branches=False,
-    tags=None,
-    all_tags=False,
-    all_commits=False,
+    self, revs=None, all_branches=False, all_tags=False, all_commits=False
 ):
     """Generator that iterates over specified revisions.
 
@@ -25,12 +20,12 @@ def brancher(  # noqa: E302
             - empty string it there is no branches to iterate over
             - "Working Tree" if there are uncommitted changes in the SCM repo
     """
-    if not any([branches, all_branches, tags, all_tags, all_commits]):
+    if not any([revs, all_branches, all_tags, all_commits]):
         yield ""
         return
 
     saved_tree = self.tree
-    revs = []
+    revs = revs or []
 
     scm = self.scm
 
@@ -41,24 +36,15 @@ def brancher(  # noqa: E302
         revs = scm.list_all_commits()
     else:
         if all_branches:
-            branches = scm.list_branches()
+            revs.extend(scm.list_branches())
 
         if all_tags:
-            tags = scm.list_tags()
+            revs.extend(scm.list_tags())
 
-        if branches is not None:
-            revs.extend(branches)
-
-        if tags is not None:
-            revs.extend(tags)
-
-    # NOTE: it might be a good idea to wrap this loop in try/finally block
-    # to don't leave the tree on some unexpected branch after the
-    # `brancher()`, but this could cause problems on exception handling
-    # code which might expect the tree on which exception was raised to
-    # stay in place. This behavior is a subject to change.
-    for sha, names in group_by(scm.resolve_rev, revs).items():
-        self.tree = scm.get_tree(sha)
-        yield ", ".join(names)
-
-    self.tree = saved_tree
+    try:
+        if revs:
+            for sha, names in group_by(scm.resolve_rev, revs).items():
+                self.tree = scm.get_tree(sha)
+                yield ", ".join(names)
+    finally:
+        self.tree = saved_tree

dvc/repo/checkout.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import logging
 
 from dvc.exceptions import CheckoutError
@@ -20,42 +18,54 @@ def _cleanup_unused_links(repo):
     repo.state.remove_unused_links(used)
 
 
-def get_all_files_numbers(stages):
-    return sum(stage.get_all_files_number() for stage in stages)
+def get_all_files_numbers(pairs):
+    return sum(
+        stage.get_all_files_number(filter_info) for stage, filter_info in pairs
+    )
 
 
 def _checkout(
-    self, targets=None, with_deps=False, force=False, recursive=False
+    self,
+    targets=None,
+    with_deps=False,
+    force=False,
+    relink=False,
+    recursive=False,
 ):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    stages = set()
-
     if not targets:
         targets = [None]
         _cleanup_unused_links(self)
 
+    pairs = set()
     for target in targets:
         try:
-            new = self.collect(
-                target, with_deps=with_deps, recursive=recursive
+            pairs.update(
+                self.collect_granular(
+                    target, with_deps=with_deps, recursive=recursive
+                )
             )
-            stages.update(new)
         except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
             if not target:
                 raise
-            raise CheckoutErrorSuggestGit(target, exc)
+            raise CheckoutErrorSuggestGit(target) from exc
 
-    total = get_all_files_numbers(stages)
+    total = get_all_files_numbers(pairs)
     if total == 0:
         logger.info("Nothing to do")
     failed = []
     with Tqdm(
         total=total, unit="file", desc="Checkout", disable=total == 0
     ) as pbar:
-        for stage in stages:
+        for stage, filter_info in pairs:
             failed.extend(
-                stage.checkout(force=force, progress_callback=pbar.update_desc)
+                stage.checkout(
+                    force=force,
+                    progress_callback=pbar.update_desc,
+                    relink=relink,
+                    filter_info=filter_info,
+                )
             )
     if failed:
         raise CheckoutError(failed)

dvc/repo/destroy.py

@@ -1,8 +1,15 @@
 from dvc.utils.fs import remove
+from . import locked
 
 
-def destroy(self):
-    for stage in self.stages:
+@locked
+def _destroy_stages(repo):
+    for stage in repo.stages:
         stage.remove(remove_outs=False)
 
-    remove(self.dvc_dir)
+
+# NOTE: not locking `destroy`, as `remove` will need to delete `.dvc` dir,
+# which will cause issues on Windows, as `.dvc/lock` will be busy.
+def destroy(repo):
+    _destroy_stages(repo)
+    remove(repo.dvc_dir)

dvc/repo/diff.py

@@ -1,16 +1,11 @@
-from __future__ import unicode_literals
-
 import os
 from errno import ENOENT
 
-import dvc.logger as logger
-from . import locked
+from dvc import logger
 from dvc.scm.base import FileNotInCommitError
-from dvc.scm.git import DIFF_A_REF
-from dvc.scm.git import DIFF_A_TREE
-from dvc.scm.git import DIFF_B_REF
-from dvc.scm.git import DIFF_B_TREE
+from dvc.scm.git import DIFF_A_REF, DIFF_B_REF, DIFF_A_TREE, DIFF_B_TREE
 from dvc.scm.git import DIFF_EQUAL
+from . import locked
 
 
 DIFF_TARGET = "target"

dvc/repo/fetch.py

@@ -1,12 +1,9 @@
-from __future__ import unicode_literals
-
 import logging
 
 from dvc.cache import NamedCache
 from dvc.config import NoRemoteError
 from dvc.exceptions import DownloadError
 from dvc.exceptions import OutputNotFoundError
-from dvc.external_repo import external_repo
 from dvc.scm.base import CloneError
 
 
@@ -72,11 +69,13 @@ def _fetch(
 
 
 def _fetch_external(self, repo_url, repo_rev, files):
-    failed = 0
+    from dvc.external_repo import external_repo
 
-    cache_dir = self.cache.local.cache_dir
+    failed = 0
     try:
-        with external_repo(repo_url, repo_rev, cache_dir=cache_dir) as repo:
+        with external_repo(repo_url, repo_rev) as repo:
+            repo.cache.local.cache_dir = self.cache.local.cache_dir
+
             with repo.state:
                 cache = NamedCache()
                 for name in files: