dvc: bump to 0.82.9

pipeline: show: outs: eliminate extra edges in DAG

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,4 +5,3 @@
 * [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect everything to be addressed. Please review them carefully and fix those that actually improve code or fix bugs.
 
 Thank you for the contribution - we'll try to review it as soon as possible. 🙏
-

.pre-commit-config.yaml

@@ -9,3 +9,9 @@ repos:
     hooks:
     - id: flake8
       language_version: python3
+-   repo: https://github.com/lovesegfault/beautysh
+    rev: master
+    hooks:
+    - id: beautysh
+      language_version: python3
+      args: [-i, '2']  # 2-space indentaion

.travis.yml

@@ -13,8 +13,6 @@ stages:
   - check
   - test
   - build
-  - name: deploy
-    if: (branch = master OR tag is present) AND repo = iterative/dvc
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
@@ -35,20 +33,10 @@ jobs:
     install:
     script: ./scripts/ci/check_patch.sh
   # test jobs
-  - name: "2.7 on Windows"
+  - name: "3.7 on Windows"
     stage: test
     os: windows           # Windows 10.0.17134 N/A Build 17134
     language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-    - PY2=1
-  - name: "3.7 on Windows"
-    os: windows           # Windows 10.0.17134 N/A Build 17134
-    language: shell       # 'language: python' is an error on Travis CI Windows
-  - name: "2.7 on Linux"
-    language: python
-    python: 2.7
-    env:
-    - PY2=1
   - name: "3.5 on Linux"
     language: python
     python: 3.5
@@ -58,14 +46,9 @@ jobs:
   - name: "3.7 on Linux"
     language: python
     python: 3.7
-  - name: "2.7 on Mac"
+  - name: "3.7 on Mac"
     os: osx
-    osx_image: xcode9.4
-    env:
-    - PY2=1
-  - name: "3.6 on Mac"
-    os: osx
-    osx_image: xcode9.4
+    osx_image: xcode11.3
   # build jobs
   - name: "Mac pkg"
     stage: build
@@ -86,11 +69,13 @@ jobs:
     before_install:
     install:
     script: ./scripts/build_posix.sh
-  - name: "Pypi pkgs"
+  - name: "PyPI pkgs"
     language: python
     python: 3.7
     script: ./scripts/build_package.sh
   - name: Snapcraft snap
+    language: python
+    python: 3.7
     addons:
      snaps:
      - name: snapcraft
@@ -100,37 +85,14 @@ jobs:
        channel: stable
     env:
     - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}'
-    before_install:
+    install:
     - sudo /snap/bin/lxd.migrate -yes
     - sudo /snap/bin/lxd waitready
     - sudo /snap/bin/lxd init --auto
-    install:
     script:
-    - sudo snapcraft --use-lxd
+    - ./scripts/build_snap.sh
     after_failure:
     - sudo journalctl -u snapd
-  # deploy jobs
-  - name: "Pypi pkgs"
-    if: tag is present
-    stage: deploy
-    language: python
-    python: 3.7
-    script: ./scripts/build_package.sh
-    deploy:
-    - provider: pypi
-      user:
-        secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
-      password:
-        secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
-      skip_existing: true
-      file_glob: true
-      file:
-        - dist/dvc*.whl
-        - dist/dvc*.tar.gz
-      skip_cleanup: true
-      on:
-        tags: true
-        repo: iterative/dvc
 
 before_install:
   - bash ./scripts/ci/before_install.sh
@@ -142,7 +104,6 @@ script:
   - bash ./scripts/ci/script.sh
 deploy:
   - provider: releases
-    condition: "[[ $(./scripts/ci/deploy_condition.sh) == 'true' ]]"
     api_key:
       secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=
     file_glob: true
@@ -151,9 +112,34 @@ deploy:
       - dvc*.deb
       - dvc*.pkg
       - dvc*.exe
-      - dvc*.snap
     skip_cleanup: true
     on:
+      condition: "$(./scripts/ci/deploy_condition.sh dvc*.rpm dvc*.deb dvc*.pkg dvc*.exe)"
       tags: true
       repo: iterative/dvc
       stage: build
+  - provider: pypi
+    user:
+      secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
+    password:
+      secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
+    skip_existing: true
+    file_glob: true
+    file:
+      - dist/dvc*.whl
+      - dist/dvc*.tar.gz
+    skip_cleanup: true
+    on:
+      condition: "$(./scripts/ci/deploy_condition.sh dist/dvc*.whl dist/dvc*.tar.gz)"
+      tags: true
+      repo: iterative/dvc
+      stage: build
+  - provider: snap
+    snap: dvc_*.snap
+    channel: $SNAP_CHANNEL
+    skip_cleanup: true
+    on:
+      all_branches: true
+      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG)"
+      repo: iterative/dvc
+      stage: build

README.rst

@@ -10,9 +10,13 @@
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
-.. image:: https://travis-ci.com/iterative/dvc.svg?branch=master
+.. image:: https://img.shields.io/badge/release-ok-brightgreen
   :target: https://travis-ci.com/iterative/dvc
-  :alt: Travis
+  :alt: Release
+
+.. image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev
+  :target: https://travis-ci.com/iterative/dvc
+  :alt: Travis dev branch
 
 .. image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
   :target: https://codeclimate.com/github/iterative/dvc
@@ -153,17 +157,12 @@ Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
 Snap (Snapcraft)
 ----------------
 
-Download the latest ``dvc_*.snap`` from the
-GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
-
 .. code-block:: bash
 
-   snap install dvc_*.snap --dangerous --classic
+   snap install dvc --classic
 
-Once ``dvc`` is released on the snap store
-(`pending request <https://forum.snapcraft.io/t/classic-confinement-request-for-dvc/14124>`_)
-there will be no need to download ``dvc_*.snap`` or use ``--dangerous``
-(``snap install dvc --classic`` would suffice).
+This corresponds to the latest tagged release.
+Add ``--edge`` for the latest ``master`` version.
 
 Package
 -------

debian/changelog

@@ -0,0 +1,5 @@
+dvc (0.75.b1) experimental; urgency=low
+
+  * First native Debian package version.
+
+ -- Dmitry Petrov <[email protected]>  Sat, 07 Dec 2019 09:49:28 +0300

debian/compat

@@ -0,0 +1 @@
+10

debian/control

@@ -0,0 +1,33 @@
+Source: dvc
+Maintainer: Dmitry Petrov <[email protected]>
+Section: python
+Priority: optional
+Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
+Standards-Version: 4.0.0
+Homepage: http://dvc.org
+Vcs-Git: https://github.com/iterative/dvc.git
+Vcs-browser: https://github.com/iterative/dvc
+
+Package: dvc
+Architecture: all
+Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
+Recommends: python3-arrow (=0.14.0),
+            python3-paramiko (>=2.5.0),
+            python3-gssapi (>= 2.5.0),
+            python3-boto3 (>=1.9.86),
+            python3-pydrive (=1.3.1),
+            python3-azure-storage-blob (=12.1.0),
+            python3-oss2 (=2.6.1),
+            python3-google-cloud-storage (=1.19.0)
+Description: Git for data scientists - manage your code and data together
+ Data Version Control or DVC is an open-source tool for data science and machine learning projects.
+ .
+ Key features:
+ .
+ 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
+ .
+ 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
+ .
+ 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
+ .
+ 4. Helps manage experiments with Git tags/branches and metrics tracking.

debian/copyright

@@ -0,0 +1,11 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: DVC
+Upstream-Contact: Dmitry Petrov <[email protected]>
+Source: https://github.com/iterative/dvc
+
+Files: *
+Copyright: Dmitry Petrov <[email protected]>
+License: Apache 2.0
+ On Debian systems, the full text of the Apache 2.0
+ License can be found in the file
+ `/usr/share/common-licenses/Apache-2.0`.

debian/rules

@@ -0,0 +1,6 @@
+#!/usr/bin/make -f
+
+export PYBUILD_NAME=dvc
+%:
+	dh $@ --with python3 --buildsystem=pybuild
+

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

debian/watch

@@ -0,0 +1,4 @@
+# please also check http://pypi.debian.net/dvc/watch
+version=3
+opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
+http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

dvc/__init__.py

@@ -3,8 +3,6 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-from __future__ import unicode_literals
-
 import warnings
 
 import dvc.logger

dvc/__main__.py

@@ -1,6 +1,4 @@
-"""Main entry point for dvc command line tool."""
-from __future__ import unicode_literals
-
+"""Main entry point for DVC command line tool."""
 import sys
 
 from dvc.main import main

dvc/analytics.py

@@ -1,256 +1,169 @@
-"""Collect and send usage analytics"""
-from __future__ import unicode_literals
-
-import errno
 import json
 import logging
 import os
+import platform
+import requests
+import sys
+import tempfile
+import uuid
+
+import distro
 
 from dvc import __version__
-from dvc.utils import env2bool
-from dvc.utils.compat import str
+from dvc.config import Config, to_bool
+from dvc.daemon import daemon
+from dvc.exceptions import NotDvcRepoError
+from dvc.lock import Lock, LockError
+from dvc.repo import Repo
+from dvc.scm import SCM
+from dvc.utils import env2bool, is_binary
+from dvc.utils.fs import makedirs
 
 
 logger = logging.getLogger(__name__)
 
 
-class Analytics(object):
-    """Class for collecting and sending usage analytics.
-
-    Args:
-        info (dict): optional existing analytics report.
+def collect_and_send_report(args=None, return_code=None):
     """
+    Collect information from the runtime/environment and the command
+    being executed into a report and send it over the network.
 
-    URL = "https://analytics.dvc.org"
-    TIMEOUT_POST = 5
+    To prevent analytics from blocking the execution of the main thread,
+    sending the report is done in a separate process.
 
-    USER_ID_FILE = "user_id"
+    The inter-process communication happens through a file containing the
+    report as a JSON, where the _collector_ generates it and the _sender_
+    removes it after sending it.
+    """
+    report = _runtime_info()
 
-    PARAM_DVC_VERSION = "dvc_version"
-    PARAM_USER_ID = "user_id"
-    PARAM_SYSTEM_INFO = "system_info"
+    # Include command execution information on the report only when available.
+    if args and hasattr(args, "func"):
+        report.update({"cmd_class": args.func.__name__})
 
-    PARAM_OS = "os"
+    if return_code is not None:
+        report.update({"cmd_return_code": return_code})
 
-    PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major"
-    PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor"
-    PARAM_WINDOWS_VERSION_BUILD = "windows_version_build"
-    PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack"
+    with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
+        json.dump(report, fobj)
+        daemon(["analytics", fobj.name])
 
-    PARAM_MAC_VERSION = "mac_version"
 
-    PARAM_LINUX_DISTRO = "linux_distro"
-    PARAM_LINUX_DISTRO_VERSION = "linux_distro_version"
-    PARAM_LINUX_DISTRO_LIKE = "linux_distro_like"
+def is_enabled():
+    if env2bool("DVC_TEST"):
+        return False
 
-    PARAM_SCM_CLASS = "scm_class"
-    PARAM_IS_BINARY = "is_binary"
-    PARAM_CMD_CLASS = "cmd_class"
-    PARAM_CMD_RETURN_CODE = "cmd_return_code"
+    enabled = to_bool(
+        Config(validate=False)
+        .config.get(Config.SECTION_CORE, {})
+        .get(Config.SECTION_CORE_ANALYTICS, "true")
+    )
 
-    def __init__(self, info=None):
-        from dvc.config import Config
-        from dvc.lock import Lock
+    logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
 
-        if info is None:
-            info = {}
+    return enabled
 
-        self.info = info
 
-        cdir = Config.get_global_config_dir()
-        try:
-            os.makedirs(cdir)
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                raise
+def send(report):
+    """
+    Side effect: Removes the report after sending it.
 
-        self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
-        self.user_id_file_lock = Lock(self.user_id_file + ".lock")
+    The report is generated and stored in a temporary file, see:
+    `collect_and_send_report`. Sending happens on another process,
+    thus, the need of removing such file afterwards.
+    """
+    url = "https://analytics.dvc.org"
+    headers = {"content-type": "application/json"}
 
-    @staticmethod
-    def load(path):
-        """Loads analytics report from json file specified by path.
+    with open(report, "rb") as fobj:
+        requests.post(url, data=fobj, headers=headers, timeout=5)
 
-        Args:
-            path (str): path to json file with analytics report.
-        """
-        with open(path, "r") as fobj:
-            analytics = Analytics(info=json.load(fobj))
-        os.unlink(path)
-        return analytics
+    os.remove(report)
 
-    def _write_user_id(self):
-        import uuid
 
-        with open(self.user_id_file, "w+") as fobj:
-            user_id = str(uuid.uuid4())
-            info = {self.PARAM_USER_ID: user_id}
-            json.dump(info, fobj)
+def _scm_in_use():
+    try:
+        scm = SCM(root_dir=Repo.find_root())
+        return type(scm).__name__
+    except NotDvcRepoError:
+        pass
+
+
+def _runtime_info():
+    """
+    Gather information from the environment where DVC runs to fill a report.
+    """
+    return {
+        "dvc_version": __version__,
+        "is_binary": is_binary(),
+        "scm_class": _scm_in_use(),
+        "system_info": _system_info(),
+        "user_id": _find_or_create_user_id(),
+    }
+
+
+def _system_info():
+    system = platform.system()
+
+    if system == "Windows":
+        version = sys.getwindowsversion()
+
+        return {
+            "os": "windows",
+            "windows_version_build": version.build,
+            "windows_version_major": version.major,
+            "windows_version_minor": version.minor,
+            "windows_version_service_pack": version.service_pack,
+        }
+
+    if system == "Darwin":
+        return {"os": "mac", "mac_version": platform.mac_ver()[0]}
+
+    if system == "Linux":
+        return {
+            "os": "linux",
+            "linux_distro": distro.id(),
+            "linux_distro_like": distro.like(),
+            "linux_distro_version": distro.version(),
+        }
+
+    # We don't collect data for any other system.
+    raise NotImplementedError
+
+
+def _find_or_create_user_id():
+    """
+    The user's ID is stored on a file under the global config directory.
+
+    The file should contain a JSON with a "user_id" key:
+
+        {"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"}
+
+    IDs are generated randomly with UUID.
+    """
+    config_dir = Config.get_global_config_dir()
+    fname = os.path.join(config_dir, "user_id")
+    lockfile = os.path.join(config_dir, "user_id.lock")
+
+    # Since the `fname` and `lockfile` are under the global config,
+    # we need to make sure such directory exist already.
+    makedirs(config_dir, exist_ok=True)
+
+    try:
+        with Lock(lockfile):
+            try:
+                with open(fname, "r") as fobj:
+                    user_id = json.load(fobj)["user_id"]
+
+            except (FileNotFoundError, ValueError, KeyError):
+                user_id = str(uuid.uuid4())
+
+                with open(fname, "w") as fobj:
+                    json.dump({"user_id": user_id}, fobj)
+
             return user_id
 
-    def _read_user_id(self):
-        if not os.path.exists(self.user_id_file):
-            return None
-
-        with open(self.user_id_file, "r") as fobj:
-            try:
-                info = json.load(fobj)
-            except ValueError as exc:
-                logger.debug("Failed to load user_id: {}".format(exc))
-                return None
-
-            return info[self.PARAM_USER_ID]
-
-    def _get_user_id(self):
-        from dvc.lock import LockError
-
-        try:
-            with self.user_id_file_lock:
-                user_id = self._read_user_id()
-                if user_id is None:
-                    user_id = self._write_user_id()
-                return user_id
-        except LockError:
-            msg = "Failed to acquire '{}'"
-            logger.debug(msg.format(self.user_id_file_lock.lockfile))
-
-    def _collect_windows(self):
-        import sys
-
-        version = sys.getwindowsversion()  # pylint: disable=no-member
-        info = {}
-        info[self.PARAM_OS] = "windows"
-        info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
-        info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
-        info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
-        info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack
-        return info
-
-    def _collect_darwin(self):
-        import platform
-
-        info = {}
-        info[self.PARAM_OS] = "mac"
-        info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
-        return info
-
-    def _collect_linux(self):
-        import distro
-
-        info = {}
-        info[self.PARAM_OS] = "linux"
-        info[self.PARAM_LINUX_DISTRO] = distro.id()
-        info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
-        info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
-        return info
-
-    def _collect_system_info(self):
-        import platform
-
-        system = platform.system()
-
-        if system == "Windows":
-            return self._collect_windows()
-
-        if system == "Darwin":
-            return self._collect_darwin()
-
-        if system == "Linux":
-            return self._collect_linux()
-
-        raise NotImplementedError
-
-    def collect(self):
-        """Collect analytics report."""
-        from dvc.scm import SCM
-        from dvc.utils import is_binary
-        from dvc.repo import Repo
-        from dvc.exceptions import NotDvcRepoError
-
-        self.info[self.PARAM_DVC_VERSION] = __version__
-        self.info[self.PARAM_IS_BINARY] = is_binary()
-        self.info[self.PARAM_USER_ID] = self._get_user_id()
-
-        self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
-
-        try:
-            scm = SCM(root_dir=Repo.find_root())
-            self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
-        except NotDvcRepoError:
-            pass
-
-    def collect_cmd(self, args, ret):
-        """Collect analytics info from a CLI command."""
-        from dvc.command.daemon import CmdDaemonAnalytics
-
-        assert isinstance(ret, int) or ret is None
-
-        if ret is not None:
-            self.info[self.PARAM_CMD_RETURN_CODE] = ret
-
-        if args is not None and hasattr(args, "func"):
-            assert args.func != CmdDaemonAnalytics
-            self.info[self.PARAM_CMD_CLASS] = args.func.__name__
-
-    def dump(self):
-        """Save analytics report to a temporary file.
-
-        Returns:
-            str: path to the temporary file that contains the analytics report.
-        """
-        import tempfile
-
-        with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
-            json.dump(self.info, fobj)
-            return fobj.name
-
-    @staticmethod
-    def is_enabled(cmd=None):
-        from dvc.config import Config, to_bool
-        from dvc.command.daemon import CmdDaemonBase
-
-        if env2bool("DVC_TEST"):
-            return False
-
-        if isinstance(cmd, CmdDaemonBase):
-            return False
-
-        core = Config(validate=False).config.get(Config.SECTION_CORE, {})
-        enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true"))
+    except LockError:
         logger.debug(
-            "Analytics is {}.".format("enabled" if enabled else "disabled")
+            "Failed to acquire '{lockfile}'".format(lockfile=lockfile)
         )
-        return enabled
-
-    @staticmethod
-    def send_cmd(cmd, args, ret):
-        """Collect and send analytics for CLI command.
-
-        Args:
-            args (list): parsed args for the CLI command.
-            ret (int): return value of the CLI command.
-        """
-        from dvc.daemon import daemon
-
-        if not Analytics.is_enabled(cmd):
-            return
-
-        analytics = Analytics()
-        analytics.collect_cmd(args, ret)
-        daemon(["analytics", analytics.dump()])
-
-    def send(self):
-        """Collect and send analytics."""
-        import requests
-
-        if not self.is_enabled():
-            return
-
-        self.collect()
-
-        logger.debug("Sending analytics: {}".format(self.info))
-
-        try:
-            requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
-        except requests.exceptions.RequestException as exc:
-            logger.debug("Failed to send analytics: {}".format(str(exc)))

dvc/api.py

@@ -1,27 +1,37 @@
 import os
-from contextlib import contextmanager
+from contextlib import contextmanager, _GeneratorContextManager as GCM
 
-try:
-    from contextlib import _GeneratorContextManager as GCM
-except ImportError:
-    from contextlib import GeneratorContextManager as GCM
-
-from dvc.utils.compat import urlparse
 from dvc.repo import Repo
+from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.external_repo import external_repo
 
 
+class UrlNotDvcRepoError(DvcException):
+    """Thrown if given url is not a DVC repository.
+
+    Args:
+        url (str): URL to the repository
+    """
+
+    def __init__(self, url):
+        super().__init__("'{}' is not a DVC repository.".format(url))
+
+
 def get_url(path, repo=None, rev=None, remote=None):
-    """Returns an url of a resource specified by path in repo"""
+    """
+    Returns the full URL to the data artifact specified by its `path` in a
+    `repo`.
+    NOTE: There is no guarantee that the file actually exists in that location.
+    """
     with _make_repo(repo, rev=rev) as _repo:
-        abspath = os.path.join(_repo.root_dir, path)
-        out, = _repo.find_outs_by_path(abspath)
+        _require_dvc(_repo)
+        out = _repo.find_out_by_relpath(path)
         remote_obj = _repo.cloud.get_remote(remote)
         return str(remote_obj.checksum_to_path_info(out.checksum))
 
 
 def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Opens a specified resource as a file descriptor"""
+    """Context manager to open a file artifact as a file object."""
     args = (path,)
     kwargs = {
         "repo": repo,
@@ -40,21 +50,20 @@ class _OpenContextManager(GCM):
 
     def __getattr__(self, name):
         raise AttributeError(
-            "dvc.api.open() should be used in a with statement"
+            "dvc.api.open() should be used in a with statement."
         )
 
 
 def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
     with _make_repo(repo, rev=rev) as _repo:
-        abspath = os.path.join(_repo.root_dir, path)
-        with _repo.open(
-            abspath, remote=remote, mode=mode, encoding=encoding
+        with _repo.open_by_relpath(
+            path, remote=remote, mode=mode, encoding=encoding
         ) as fd:
             yield fd
 
 
 def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Read a specified resource into string"""
+    """Returns the contents of a file artifact."""
     with open(
         path, repo=repo, rev=rev, remote=remote, mode=mode, encoding=encoding
     ) as fd:
@@ -62,10 +71,18 @@ def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
 
 
 @contextmanager
-def _make_repo(repo_url, rev=None):
-    if not repo_url or urlparse(repo_url).scheme == "":
-        assert rev is None, "Custom revision is not supported for local repo"
-        yield Repo(repo_url)
-    else:
-        with external_repo(url=repo_url, rev=rev) as repo:
-            yield repo
+def _make_repo(repo_url=None, rev=None):
+    repo_url = repo_url or os.getcwd()
+    if rev is None and os.path.exists(repo_url):
+        try:
+            yield Repo(repo_url)
+            return
+        except NotDvcRepoError:
+            pass  # fallthrough to external_repo
+    with external_repo(url=repo_url, rev=rev) as repo:
+        yield repo
+
+
+def _require_dvc(repo):
+    if not isinstance(repo, Repo):
+        raise UrlNotDvcRepoError(repo.url)

dvc/cache.py

@@ -1,13 +1,10 @@
-"""Manages cache of a dvc repo."""
-from __future__ import unicode_literals
-
+"""Manages cache of a DVC repo."""
 import os
 from collections import defaultdict
 
 from funcy import cached_property
 
 from dvc.config import Config
-from dvc.utils.compat import builtin_str
 
 
 class CacheConfig(object):
@@ -56,12 +53,12 @@ def _make_remote_property(name):
 
         return Remote(self.repo, name=remote)
 
-    getter.__name__ = builtin_str(name)
+    getter.__name__ = name
     return cached_property(getter)
 
 
 class Cache(object):
-    """Class that manages cache locations of a dvc repo.
+    """Class that manages cache locations of a DVC repo.
 
     Args:
         repo (dvc.repo.Repo): repo instance that this cache belongs to.

dvc/cli.py

@@ -1,42 +1,41 @@
-"""Command line interface for dvc."""
-from __future__ import print_function
-from __future__ import unicode_literals
-
+"""DVC command line interface"""
 import argparse
 import logging
 import sys
 
-import dvc.command.add as add
-import dvc.command.cache as cache
-import dvc.command.checkout as checkout
-import dvc.command.commit as commit
-import dvc.command.config as config
-import dvc.command.daemon as daemon
-import dvc.command.data_sync as data_sync
-import dvc.command.destroy as destroy
-import dvc.command.diff as diff
-import dvc.command.gc as gc
-import dvc.command.get as get
-import dvc.command.get_url as get_url
-import dvc.command.imp as imp
-import dvc.command.imp_url as imp_url
-import dvc.command.init as init
-import dvc.command.install as install
-import dvc.command.lock as lock
-import dvc.command.metrics as metrics
-import dvc.command.move as move
-import dvc.command.pipeline as pipeline
-import dvc.command.remote as remote
-import dvc.command.remove as remove
-import dvc.command.repro as repro
-import dvc.command.root as root
-import dvc.command.run as run
-import dvc.command.tag as tag
-import dvc.command.unprotect as unprotect
-import dvc.command.update as update
-import dvc.command.version as version
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import DvcParserError
+from .command import (
+    add,
+    cache,
+    checkout,
+    commit,
+    config,
+    daemon,
+    data_sync,
+    destroy,
+    diff,
+    gc,
+    get,
+    get_url,
+    imp,
+    imp_url,
+    init,
+    install,
+    lock,
+    metrics,
+    move,
+    pipeline,
+    remote,
+    remove,
+    repro,
+    root,
+    run,
+    tag,
+    unprotect,
+    update,
+    version,
+)
+from .command.base import fix_subparsers
+from .exceptions import DvcParserError
 
 
 logger = logging.getLogger(__name__)
@@ -92,7 +91,7 @@ class DvcParser(argparse.ArgumentParser):
 
 class VersionAction(argparse.Action):  # pragma: no cover
     # pylint: disable=too-few-public-methods
-    """Shows dvc version and exits."""
+    """Shows DVC version and exits."""
 
     def __call__(self, parser, namespace, values, option_string=None):
         from dvc import __version__
@@ -173,7 +172,7 @@ def parse_args(argv=None):
         title="Available Commands",
         metavar="COMMAND",
         dest="cmd",
-        help="Use dvc COMMAND --help for command-specific help.",
+        help="Use `dvc COMMAND --help` for command-specific help.",
     )
 
     fix_subparsers(subparsers)

dvc/command/add.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/base.py

@@ -1,9 +1,5 @@
-from __future__ import unicode_literals
-
 import logging
 
-import colorama
-
 
 logger = logging.getLogger(__name__)
 
@@ -16,23 +12,18 @@ def fix_subparsers(subparsers):
         Args:
             subparsers: subparsers to fix.
     """
-    from dvc.utils.compat import is_py3
-
-    if is_py3:  # pragma: no cover
-        subparsers.required = True
-        subparsers.dest = "cmd"
+    subparsers.required = True
+    subparsers.dest = "cmd"
 
 
 def append_doc_link(help_message, path):
+    from dvc.utils import format_link
+
     if not path:
         return help_message
     doc_base = "https://man.dvc.org/"
-    return "{message}\nDocumentation: <{blue}{base}{path}{nc}>".format(
-        message=help_message,
-        base=doc_base,
-        path=path,
-        blue=colorama.Fore.CYAN,
-        nc=colorama.Fore.RESET,
+    return "{message}\nDocumentation: {link}".format(
+        message=help_message, link=format_link(doc_base + path)
     )
 
 
@@ -44,7 +35,8 @@ class CmdBase(object):
         self.repo = Repo()
         self.config = self.repo.config
         self.args = args
-        updater = Updater(self.repo.dvc_dir)
+        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
         updater.check()
 
     @property

dvc/command/cache.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 
 from dvc.cache import CacheConfig
@@ -10,7 +8,7 @@ from dvc.command.config import CmdConfig
 
 class CmdCacheDir(CmdConfig):
     def __init__(self, args):
-        super(CmdCacheDir, self).__init__(args)
+        super().__init__(args)
         self.cache_config = CacheConfig(self.config)
 
     def run(self):
@@ -32,7 +30,8 @@ def add_parser(subparsers, parent_parser):
     )
 
     cache_subparsers = cache_parser.add_subparsers(
-        dest="cmd", help="Use dvc cache CMD --help for command-specific help."
+        dest="cmd",
+        help="Use `dvc cache CMD --help` for command-specific " "help.",
     )
 
     fix_subparsers(cache_subparsers)

dvc/command/checkout.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 
 from dvc.command.base import append_doc_link
@@ -12,6 +10,7 @@ class CmdCheckout(CmdBase):
             targets=self.args.targets,
             with_deps=self.args.with_deps,
             force=self.args.force,
+            relink=self.args.relink,
             recursive=self.args.recursive,
         )
         return 0
@@ -27,13 +26,6 @@ def add_parser(subparsers, parent_parser):
         help=CHECKOUT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    checkout_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Do not prompt when removing working directory files.",
-    )
     checkout_parser.add_argument(
         "-d",
         "--with-deps",
@@ -48,6 +40,19 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Checkout all subdirectories of the specified directory.",
     )
+    checkout_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Do not prompt when removing working directory files.",
+    )
+    checkout_parser.add_argument(
+        "--relink",
+        action="store_true",
+        default=False,
+        help="Recreate links or copies from cache to workspace.",
+    )
     checkout_parser.add_argument(
         "targets",
         nargs="*",

dvc/command/commit.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -49,7 +47,7 @@ def add_parser(subparsers, parent_parser):
         "--force",
         action="store_true",
         default=False,
-        help="Commit even if checksums for dependencies/outputs changed.",
+        help="Commit even if hash value for dependencies/outputs changed.",
     )
     commit_parser.add_argument(
         "-d",

dvc/command/config.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -13,7 +11,7 @@ logger = logging.getLogger(__name__)
 
 class CmdConfig(CmdBaseNoRepo):
     def __init__(self, args):
-        super(CmdConfig, self).__init__(args)
+        super().__init__(args)
 
         self.config = Config(validate=False)
 
@@ -29,6 +27,14 @@ class CmdConfig(CmdBaseNoRepo):
                 section, opt, self.args.value, level=self.args.level
             )
 
+        is_write = self.args.unset or self.args.value is not None
+        if is_write and self.args.name == "cache.type":
+            logger.warning(
+                "You have changed the 'cache.type' option. This doesn't update"
+                " any existing workspace file links, but it can be done with:"
+                "\n             dvc checkout --relink"
+            )
+
         return 0
 
 

dvc/command/daemon.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.command.base import CmdBaseNoRepo
 from dvc.command.base import fix_subparsers
 
@@ -12,11 +10,16 @@ class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
         from dvc.repo import Repo
+        from dvc.config import Config
         from dvc.updater import Updater
 
         root_dir = Repo.find_root()
         dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
-        updater = Updater(dvc_dir)
+        config = Config(dvc_dir, validate=False)
+        hardlink_lock = config.config.get("core", {}).get(
+            "hardlink_lock", False
+        )
+        updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
         updater.fetch(detach=False)
 
         return 0
@@ -24,10 +27,9 @@ class CmdDaemonUpdater(CmdDaemonBase):
 
 class CmdDaemonAnalytics(CmdDaemonBase):
     def run(self):
-        from dvc.analytics import Analytics
+        from dvc import analytics
 
-        analytics = Analytics.load(self.args.target)
-        analytics.send()
+        analytics.send(self.args.target)
 
         return 0
 
@@ -42,7 +44,8 @@ def add_parser(subparsers, parent_parser):
     )
 
     daemon_subparsers = daemon_parser.add_subparsers(
-        dest="cmd", help="Use dvc daemon CMD --help for command-specific help."
+        dest="cmd",
+        help="Use `dvc daemon CMD --help` for command-specific " "help.",
     )
 
     fix_subparsers(daemon_subparsers)

dvc/command/data_sync.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/destroy.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/diff.py

@@ -1,14 +1,11 @@
-from __future__ import unicode_literals
-
 import argparse
+import json
 import logging
+import os
 
-import humanize
-import inflect
-from funcy import compact
+import colorama
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 
@@ -17,156 +14,149 @@ logger = logging.getLogger(__name__)
 
 class CmdDiff(CmdBase):
     @staticmethod
-    def _print_size(size):
-        if size < 0:
-            change = "decreased by {}"
-        elif size > 0:
-            change = "increased by {}"
-        else:
-            change = "not changed"
-        natur_size = humanize.naturalsize(abs(size))
-        return change.format(natur_size)
+    def _format(diff):
+        """
+        Given a diff structure, generate a string of paths separated
+        by new lines and grouped together by their state.
 
-    @staticmethod
-    def _get_md5_string(sign, file_name, checksum):
-        sample_msg = ""
-        if file_name:
-            sample_msg = "{}{} with md5 {}\n"
-            sample_msg = sample_msg.format(sign, file_name, checksum)
-        return sample_msg
+        A group's header is colored and its entries are sorted to enhance
+        readability, for example:
 
-    @classmethod
-    def _get_dir_changes(cls, dct):
-        import dvc.repo.diff as diff
+            Added:
+                another_file.txt
+                backup.tar
+                dir/
+                dir/1
 
-        engine = inflect.engine()
-        changes_msg = (
-            "{} {} untouched, {} {} modified, {} {} added, "
-            "{} {} deleted, size was {}"
+        An example of a diff formatted when entries contain checksums:
+
+            Added:
+                d3b07384 foo
+
+            Modified:
+                c157a790..f98bf6f1 bar
+
+        If a group has no entries, it won't be included in the result.
+
+        At the bottom, include a summary with the number of files per state.
+        """
+
+        def _digest(checksum):
+            if type(checksum) is str:
+                return checksum[0:8]
+            return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
+
+        colors = {
+            "added": colorama.Fore.GREEN,
+            "modified": colorama.Fore.YELLOW,
+            "deleted": colorama.Fore.RED,
+        }
+
+        summary = {}
+        groups = []
+
+        for state in ["added", "deleted", "modified"]:
+            summary[state] = 0
+            entries = diff[state]
+
+            if not entries:
+                continue
+
+            content = []
+
+            for entry in entries:
+                path = entry["path"]
+                checksum = entry.get("checksum")
+                summary[state] += 1 if not path.endswith(os.sep) else 0
+                content.append(
+                    "{space}{checksum}{separator}{path}".format(
+                        space="    ",
+                        checksum=_digest(checksum) if checksum else "",
+                        separator="  " if checksum else "",
+                        path=entry["path"],
+                    )
+                )
+
+            groups.append(
+                "{color}{header}{nc}:\n{content}".format(
+                    color=colors[state],
+                    header=state.capitalize(),
+                    nc=colorama.Fore.RESET,
+                    content="\n".join(content),
+                )
+            )
+
+        groups.append(
+            "files summary: {added} added, {deleted} deleted,"
+            " {modified} modified".format_map(summary)
         )
-        changes_msg = changes_msg.format(
-            dct[diff.DIFF_IDENT],
-            engine.plural("file", dct[diff.DIFF_IDENT]),
-            dct[diff.DIFF_CHANGE],
-            engine.plural("file", dct[diff.DIFF_CHANGE]),
-            dct[diff.DIFF_NEW],
-            engine.plural("file", dct[diff.DIFF_NEW]),
-            dct[diff.DIFF_DEL],
-            engine.plural("file", dct[diff.DIFF_DEL]),
-            cls._print_size(dct[diff.DIFF_SIZE]),
-        )
-        return changes_msg
 
-    @classmethod
-    def _get_file_changes(cls, dct):
-        import dvc.repo.diff as diff
-
-        if (
-            dct.get(diff.DIFF_OLD_FILE)
-            and dct.get(diff.DIFF_NEW_FILE)
-            and dct[diff.DIFF_SIZE] == 0
-        ):
-            msg = "file size was not changed"
-        elif dct.get(diff.DIFF_NEW_FILE):
-            msg = "added file with size {}".format(
-                humanize.naturalsize(dct[diff.DIFF_SIZE])
-            )
-        elif dct.get(diff.DIFF_OLD_FILE):
-            msg = "deleted file with size {}".format(
-                humanize.naturalsize(abs(dct[diff.DIFF_SIZE]))
-            )
-        else:
-            msg = "file was modified, file size {}".format(
-                cls._print_size(dct[diff.DIFF_SIZE])
-            )
-        return msg
-
-    @classmethod
-    def _get_royal_changes(cls, dct):
-        import dvc.repo.diff as diff
-
-        if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
-            if dct.get("is_dir"):
-                return cls._get_dir_changes(dct)
-            else:
-                return cls._get_file_changes(dct)
-        return "size is ?"
-
-    @classmethod
-    def _show(cls, diff_dct):
-        import dvc.repo.diff as diff
-
-        msg = "dvc diff from {} to {}".format(
-            diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
-        )
-        if diff_dct.get(diff.DIFF_EQUAL):
-            logger.info(msg)
-            return
-        for dct in diff_dct[diff.DIFF_LIST]:
-            msg += "\n\ndiff for '{}'\n".format(dct[diff.DIFF_TARGET])
-            msg += cls._get_md5_string(
-                "-",
-                dct.get(diff.DIFF_OLD_FILE),
-                dct.get(diff.DIFF_OLD_CHECKSUM),
-            )
-            msg += cls._get_md5_string(
-                "+",
-                dct.get(diff.DIFF_NEW_FILE),
-                dct.get(diff.DIFF_NEW_CHECKSUM),
-            )
-            msg += "\n"
-            msg += cls._get_royal_changes(dct)
-        logger.info(msg)
-        return msg
+        return "\n\n".join(groups)
 
     def run(self):
         try:
-            msg = self.repo.diff(
-                self.args.a_ref, target=self.args.target, b_ref=self.args.b_ref
-            )
-            self._show(msg)
+            diff = self.repo.diff(self.args.a_ref, self.args.b_ref)
+
+            if not any(diff.values()):
+                return 0
+
+            if not self.args.checksums:
+                for _, entries in diff.items():
+                    for entry in entries:
+                        del entry["checksum"]
+
+            if self.args.show_json:
+                res = json.dumps(diff)
+            else:
+                res = self._format(diff)
+
+            logger.info(res)
+
         except DvcException:
-            msg = "failed to get 'diff {}'"
-            args = " ".join(
-                compact([self.args.target, self.args.a_ref, self.args.b_ref])
-            )
-            msg = msg.format(args)
-            logger.exception(msg)
+            logger.exception("failed to get diff")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     DIFF_DESCRIPTION = (
-        "Show diff of a data file or a directory that is under DVC control.\n"
-        "Some basic statistics summary, how many files were deleted/changed."
+        "Compare two versions of the DVC repository.\n"
+        "Shows the list of paths added, modified, or deleted"
     )
-    DIFF_HELP = "Show a diff of a DVC controlled data file or a directory."
     diff_parser = subparsers.add_parser(
         "diff",
         parents=[parent_parser],
         description=append_doc_link(DIFF_DESCRIPTION, "diff"),
-        help=DIFF_HELP,
+        help=DIFF_DESCRIPTION,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     diff_parser.add_argument(
-        "-t",
-        "--target",
+        "a_ref",
         help=(
-            "Source path to a data file or directory. Default None. "
-            "If not specified, compares all files and directories "
-            "that are under DVC control in the current working space."
+            "Git reference to the older version to compare "
+            "(defaults to HEAD)"
         ),
-    )
-    diff_parser.add_argument(
-        "a_ref", help="Git reference from which diff calculates"
+        nargs="?",
+        default="HEAD",
     )
     diff_parser.add_argument(
         "b_ref",
         help=(
-            "Git reference until which diff calculates, if omitted "
-            "diff shows the difference between current HEAD and a_ref"
+            "Git reference to the newer version to compare "
+            "(defaults to the current workspace)"
         ),
         nargs="?",
     )
+    diff_parser.add_argument(
+        "--show-json",
+        help="Format the output into a JSON",
+        action="store_true",
+        default=False,
+    )
+    diff_parser.add_argument(
+        "--checksums",
+        help="Display hash value for each entry",
+        action="store_true",
+        default=False,
+    )
     diff_parser.set_defaults(func=CmdDiff)

dvc/command/gc.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 import os
@@ -54,10 +52,10 @@ class CmdGC(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    GC_HELP = "Collect unused data from DVC cache or a remote storage."
+    GC_HELP = "Garbage collect unused objects from cache or remote storage."
     GC_DESCRIPTION = (
-        "Deletes all files in the cache or a remote which are not in\n"
-        "use by the specified git references (defaults to just HEAD)."
+        "Removes all files in the cache or a remote which are not in\n"
+        "use by the specified Git revisions (defaults to just HEAD)."
     )
     gc_parser = subparsers.add_parser(
         "gc",

dvc/command/get.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -12,7 +10,27 @@ logger = logging.getLogger(__name__)
 
 
 class CmdGet(CmdBaseNoRepo):
+    def _show_url(self):
+        from dvc.api import get_url
+
+        try:
+            url = get_url(
+                self.args.path, repo=self.args.url, rev=self.args.rev
+            )
+            logger.info(url)
+        except DvcException:
+            logger.exception("failed to show URL")
+            return 1
+
+        return 0
+
     def run(self):
+        if self.args.show_url:
+            return self._show_url()
+
+        return self._get_file_from_repo()
+
+    def _get_file_from_repo(self):
         from dvc.repo import Repo
 
         try:
@@ -33,7 +51,9 @@ class CmdGet(CmdBaseNoRepo):
 
 
 def add_parser(subparsers, parent_parser):
-    GET_HELP = "Download data from DVC repository."
+    GET_HELP = (
+        "Download file or directory from any DVC project or Git repository."
+    )
     get_parser = subparsers.add_parser(
         "get",
         parents=[parent_parser],
@@ -42,13 +62,23 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     get_parser.add_argument(
-        "url", help="URL of Git repository with DVC project to download from."
-    )
-    get_parser.add_argument("path", help="Path to data within DVC repository.")
-    get_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to put data to."
+        "url",
+        help="Location of DVC project or Git repository to download from",
     )
     get_parser.add_argument(
-        "--rev", nargs="?", help="DVC repository git revision."
+        "path",
+        help="Path to a file or directory within the project or repository",
+    )
+    get_parser.add_argument(
+        "-o", "--out", nargs="?", help="Destination path to download files to"
+    )
+    get_parser.add_argument(
+        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+    )
+    get_parser.add_argument(
+        "--show-url",
+        action="store_true",
+        help="Print the storage location (URL) the target data would be "
+        "downloaded from, and exit.",
     )
     get_parser.set_defaults(func=CmdGet)

dvc/command/get_url.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/imp.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -32,7 +30,8 @@ class CmdImport(CmdBase):
 
 def add_parser(subparsers, parent_parser):
     IMPORT_HELP = (
-        "Download data from DVC repository and take it under DVC control."
+        "Download file or directory from any DVC project or Git repository "
+        "into the workspace, and track it."
     )
 
     import_parser = subparsers.add_parser(
@@ -43,15 +42,17 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawTextHelpFormatter,
     )
     import_parser.add_argument(
-        "url", help="URL of Git repository with DVC project to download from."
+        "url",
+        help="Location of DVC project or Git repository to download from",
     )
     import_parser.add_argument(
-        "path", help="Path to data within DVC repository."
+        "path",
+        help="Path to a file or directory within the project or repository",
     )
     import_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to put data to."
+        "-o", "--out", nargs="?", help="Destination path to download files to"
     )
     import_parser.add_argument(
-        "--rev", nargs="?", help="DVC repository git revision."
+        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
     )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -44,12 +42,17 @@ def add_parser(subparsers, parent_parser):
         "url",
         help="Supported urls:\n"
         "/path/to/file\n"
+        "/path/to/directory\n"
         "C:\\\\path\\to\\file\n"
+        "C:\\\\path\\to\\directory\n"
         "https://example.com/path/to/file\n"
         "s3://bucket/path/to/file\n"
+        "s3://bucket/path/to/directory\n"
         "gs://bucket/path/to/file\n"
+        "gs://bucket/path/to/directory\n"
         "hdfs://example.com/path/to/file\n"
         "ssh://example.com:/path/to/file\n"
+        "ssh://example.com:/path/to/directory\n"
         "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(

dvc/command/init.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -21,7 +19,7 @@ class CmdInit(CmdBaseNoRepo):
             )
             self.config = self.repo.config
         except InitError:
-            logger.exception("failed to initiate dvc")
+            logger.exception("failed to initiate DVC")
             return 1
         return 0
 
@@ -45,7 +43,7 @@ def add_parser(subparsers, parent_parser):
         "--no-scm",
         action="store_true",
         default=False,
-        help="Initiate dvc in directory that is "
+        help="Initiate DVC in directory that is "
         "not tracked by any scm tool (e.g. git).",
     )
     init_parser.add_argument(

dvc/command/install.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -15,7 +13,7 @@ class CmdInstall(CmdBase):
         try:
             self.repo.install()
         except Exception:
-            logger.exception("failed to install dvc hooks")
+            logger.exception("failed to install DVC Git hooks")
             return 1
         return 0
 

dvc/command/lock.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/metrics.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -28,7 +26,12 @@ def show_metrics(metrics, all_branches=False, all_tags=False):
             logger.info("{branch}:".format(branch=branch))
 
         for fname, metric in val.items():
-            lines = metric if type(metric) is list else metric.splitlines()
+            if isinstance(metric, dict):
+                lines = list(metric.values())
+            elif isinstance(metric, list):
+                lines = metric
+            else:
+                lines = metric.splitlines()
 
             if len(lines) > 1:
                 logger.info("\t{fname}:".format(fname=fname))
@@ -102,6 +105,59 @@ class CmdMetricsRemove(CmdBase):
         return 0
 
 
+def _show_diff(diff):
+    from texttable import Texttable
+
+    if not diff:
+        return "No changes."
+
+    table = Texttable()
+
+    # remove borders to make it easier for users to copy stuff
+    table.set_chars(("", "", "", ""))
+    table.set_deco(0)
+
+    rows = [["Path", "Metric", "Value", "Change"]]
+    for fname, mdiff in diff.items():
+        for metric, change in mdiff.items():
+            rows.append(
+                [
+                    fname,
+                    metric,
+                    change["new"],
+                    change.get("diff", "diff not supported"),
+                ]
+            )
+    table.add_rows(rows)
+    return table.draw()
+
+
+class CmdMetricsDiff(CmdBase):
+    def run(self):
+        try:
+            diff = self.repo.metrics.diff(
+                a_ref=self.args.a_ref,
+                b_ref=self.args.b_ref,
+                targets=self.args.targets,
+                typ=self.args.type,
+                xpath=self.args.xpath,
+                recursive=self.args.recursive,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                logger.info(_show_diff(diff))
+
+        except DvcException:
+            logger.exception("failed to show metrics diff")
+            return 1
+
+        return 0
+
+
 def add_parser(subparsers, parent_parser):
     METRICS_HELP = "Commands to add, manage, collect and display metrics."
 
@@ -115,7 +171,7 @@ def add_parser(subparsers, parent_parser):
 
     metrics_subparsers = metrics_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc metrics CMD --help to display command-specific help.",
+        help="Use `dvc metrics CMD --help` to display command-specific help.",
     )
 
     fix_subparsers(metrics_subparsers)
@@ -216,3 +272,66 @@ def add_parser(subparsers, parent_parser):
     )
     metrics_remove_parser.add_argument("path", help="Path to a metric file.")
     metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
+
+    METRICS_DIFF_HELP = "Show a table of changes between metrics among "
+    "versions of the DVC repository."
+    metrics_diff_parser = metrics_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_DIFF_HELP, "metrics/diff"),
+        help=METRICS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_diff_parser.add_argument(
+        "a_ref",
+        nargs="?",
+        help=(
+            "Git reference to the older version to compare "
+            "(defaults to HEAD)"
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "b_ref",
+        nargs="?",
+        help=(
+            "Git reference to the newer version to compare "
+            "(defaults to the current workspace)"
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help=(
+            "Metric files or directories (see -R) to show diff for. "
+            "Shows diff for all metric files by default."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "-t",
+        "--type",
+        help=(
+            "Type of metrics (json/tsv/htsv/csv/hcsv). "
+            "It can be detected by the file extension automatically. "
+            "Unsupported types will be treated as raw."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
+    )
+    metrics_diff_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help=(
+            "If any target is a directory, recursively search and process "
+            "metric files."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    metrics_diff_parser.set_defaults(func=CmdMetricsDiff)

dvc/command/move.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/pipeline.py

@@ -1,14 +1,8 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.command.base import fix_subparsers
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.utils import relpath
-from dvc.utils.compat import str
 
 
 logger = logging.getLogger(__name__)
@@ -21,69 +15,66 @@ class CmdPipelineShow(CmdBase):
 
         stage = Stage.load(self.repo, target)
         G = self.repo.graph
-        stages = networkx.get_node_attributes(G, "stage")
-        node = relpath(stage.path, self.repo.root_dir)
-        nodes = networkx.dfs_postorder_nodes(G, node)
+        stages = networkx.dfs_postorder_nodes(G, stage)
 
         if locked:
-            nodes = [n for n in nodes if stages[n].locked]
+            stages = [s for s in stages if s.locked]
 
-        for n in nodes:
+        for stage in stages:
             if commands:
-                if stages[n].cmd is None:
+                if stage.cmd is None:
                     continue
-                logger.info(stages[n].cmd)
+                logger.info(stage.cmd)
             elif outs:
-                for out in stages[n].outs:
+                for out in stage.outs:
                     logger.info(str(out))
             else:
-                logger.info(n)
+                logger.info(stage.path_in_repo)
 
-    def __build_graph(self, target, commands, outs):
+    def _build_graph(self, target, commands, outs):
         import networkx
         from dvc.stage import Stage
         from dvc.repo.graph import get_pipeline
 
-        stage = Stage.load(self.repo, target)
-        node = relpath(stage.path, self.repo.root_dir)
+        target_stage = Stage.load(self.repo, target)
+        G = get_pipeline(self.repo.pipelines, target_stage)
 
-        G = get_pipeline(self.repo.pipelines, node)
-        stages = networkx.get_node_attributes(G, "stage")
-
-        nodes = []
-        for n in G:
-            stage = stages[n]
+        nodes = set()
+        for stage in networkx.dfs_preorder_nodes(G, target_stage):
             if commands:
                 if stage.cmd is None:
                     continue
-                nodes.append(stage.cmd)
+                nodes.add(stage.cmd)
             elif outs:
                 for out in stage.outs:
-                    nodes.append(str(out))
+                    nodes.add(str(out))
+                for dep in stage.deps:
+                    nodes.add(str(dep))
             else:
-                nodes.append(stage.relpath)
+                nodes.add(stage.relpath)
 
         edges = []
-        for e in G.edges():
-            from_stage = stages[e[0]]
-            to_stage = stages[e[1]]
-            if commands:
-                if to_stage.cmd is None:
-                    continue
-                edges.append((from_stage.cmd, to_stage.cmd))
-            elif outs:
-                for from_out in from_stage.outs:
-                    for to_out in to_stage.outs:
-                        edges.append((str(from_out), str(to_out)))
-            else:
-                edges.append((from_stage.relpath, to_stage.relpath))
 
-        return nodes, edges, networkx.is_tree(G)
+        if outs:
+            for stage in networkx.dfs_preorder_nodes(G, target_stage):
+                for dep in stage.deps:
+                    for out in stage.outs:
+                        edges.append((str(out), str(dep)))
+        else:
+            for from_stage, to_stage in networkx.dfs_edges(G, target_stage):
+                if commands:
+                    if to_stage.cmd is None:
+                        continue
+                    edges.append((from_stage.cmd, to_stage.cmd))
+                else:
+                    edges.append((from_stage.relpath, to_stage.relpath))
+
+        return list(nodes), edges, networkx.is_tree(G)
 
     def _show_ascii(self, target, commands, outs):
         from dvc.dagascii import draw
 
-        nodes, edges, _ = self.__build_graph(target, commands, outs)
+        nodes, edges, _ = self._build_graph(target, commands, outs)
 
         if not nodes:
             return
@@ -93,7 +84,7 @@ class CmdPipelineShow(CmdBase):
     def _show_dependencies_tree(self, target, commands, outs):
         from treelib import Tree
 
-        nodes, edges, is_tree = self.__build_graph(target, commands, outs)
+        nodes, edges, is_tree = self._build_graph(target, commands, outs)
         if not nodes:
             return
         if not is_tree:
@@ -114,18 +105,18 @@ class CmdPipelineShow(CmdBase):
             observe_list.pop(0)
         tree.show()
 
-    def __write_dot(self, target, commands, outs):
-        from dvc.utils.compat import StringIO
+    def _write_dot(self, target, commands, outs):
+        import io
         import networkx
         from networkx.drawing.nx_pydot import write_dot
 
-        _, edges, _ = self.__build_graph(target, commands, outs)
+        _, edges, _ = self._build_graph(target, commands, outs)
         edges = [edge[::-1] for edge in edges]
 
         simple_g = networkx.DiGraph()
         simple_g.add_edges_from(edges)
 
-        dot_file = StringIO()
+        dot_file = io.StringIO()
         write_dot(simple_g, dot_file)
         logger.info(dot_file.getvalue())
 
@@ -140,9 +131,7 @@ class CmdPipelineShow(CmdBase):
                         target, self.args.commands, self.args.outs
                     )
                 elif self.args.dot:
-                    self.__write_dot(
-                        target, self.args.commands, self.args.outs
-                    )
+                    self._write_dot(target, self.args.commands, self.args.outs)
                 elif self.args.tree:
                     self._show_dependencies_tree(
                         target, self.args.commands, self.args.outs
@@ -163,14 +152,11 @@ class CmdPipelineShow(CmdBase):
 
 class CmdPipelineList(CmdBase):
     def run(self):
-        import networkx
-
         pipelines = self.repo.pipelines
-        for p in pipelines:
-            stages = networkx.get_node_attributes(p, "stage")
-            for stage in stages:
-                logger.info(stage)
-            if len(stages) != 0:
+        for pipeline in pipelines:
+            for stage in pipeline:
+                logger.info(stage.relpath)
+            if len(pipeline) != 0:
                 logger.info("=" * 80)
         logger.info("{} pipelines total".format(len(pipelines)))
 
@@ -189,7 +175,7 @@ def add_parser(subparsers, parent_parser):
 
     pipeline_subparsers = pipeline_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc pipeline CMD --help for command-specific help.",
+        help="Use `dvc pipeline CMD --help` for command-specific help.",
     )
 
     fix_subparsers(pipeline_subparsers)

dvc/command/remote.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -7,18 +5,23 @@ from dvc.command.base import append_doc_link
 from dvc.command.base import fix_subparsers
 from dvc.command.config import CmdConfig
 from dvc.remote.config import RemoteConfig
+from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
 
 class CmdRemoteConfig(CmdConfig):
     def __init__(self, args):
-        super(CmdRemoteConfig, self).__init__(args)
+        super().__init__(args)
         self.remote_config = RemoteConfig(self.config)
 
 
 class CmdRemoteAdd(CmdRemoteConfig):
     def run(self):
+        if self.args.default:
+            logger.info(
+                "Setting '{}' as a default remote.".format(self.args.name)
+            )
         self.remote_config.add(
             self.args.name,
             self.args.url,
@@ -81,7 +84,7 @@ def add_parser(subparsers, parent_parser):
 
     remote_subparsers = remote_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc remote CMD --help for " "command-specific help.",
+        help="Use `dvc remote CMD --help` for " "command-specific help.",
     )
 
     fix_subparsers(remote_subparsers)
@@ -97,7 +100,9 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser.add_argument("name", help="Name of the remote")
     remote_add_parser.add_argument(
         "url",
-        help="URL. See full list of supported urls at " "man.dvc.org/remote",
+        help="URL. See full list of supported urls at {}".format(
+            format_link("https://man.dvc.org/remote")
+        ),
     )
     remote_add_parser.add_argument(
         "-d",

dvc/command/remove.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -20,7 +18,7 @@ class CmdRemove(CmdBase):
         if self.args.force:
             return False
 
-        msg = "Are you sure you want to remove {} with its outputs?".format(
+        msg = "Are you sure you want to remove '{}' with its outputs?".format(
             target
         )
 
@@ -38,7 +36,7 @@ class CmdRemove(CmdBase):
                 outs_only = self._is_outs_only(target)
                 self.repo.remove(target, outs_only=outs_only)
             except DvcException:
-                logger.exception("failed to remove {}".format(target))
+                logger.exception("failed to remove '{}'".format(target))
                 return 1
         return 0
 

dvc/command/repro.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 import os

dvc/command/root.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/run.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/status.py

@@ -1,9 +1,6 @@
-from __future__ import unicode_literals
-
 import logging
 
 from dvc.command.data_sync import CmdDataBase
-from dvc.utils.compat import str
 
 
 logger = logging.getLogger(__name__)

dvc/command/unprotect.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/update.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/version.py

@@ -1,11 +1,10 @@
-from __future__ import unicode_literals
-
 import argparse
 import itertools
 import logging
 import os
 import platform
 import uuid
+import pathlib
 
 try:
     import psutil
@@ -13,7 +12,6 @@ except ImportError:
     psutil = None
 
 from dvc.utils import is_binary, relpath
-from dvc.utils.compat import pathlib
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
 from dvc.exceptions import DvcException, NotDvcRepoError
@@ -27,23 +25,13 @@ class CmdVersion(CmdBaseNoRepo):
     def run(self):
         from dvc.repo import Repo
 
-        dvc_version = __version__
-        python_version = platform.python_version()
-        platform_type = platform.platform()
-        binary = is_binary()
-        info = (
-            "DVC version: {dvc_version}\n"
-            "Python version: {python_version}\n"
-            "Platform: {platform_type}\n"
-            "Binary: {binary}\n"
-            "Package: {package}\n"
-        ).format(
-            dvc_version=dvc_version,
-            python_version=python_version,
-            platform_type=platform_type,
-            binary=binary,
-            package=PKG,
-        )
+        info = [
+            "DVC version: {}".format(__version__),
+            "Python version: {}".format(platform.python_version()),
+            "Platform: {}".format(platform.platform()),
+            "Binary: {}".format(is_binary()),
+            "Package: {}".format(PKG),
+        ]
 
         try:
             repo = Repo()
@@ -54,9 +42,14 @@ class CmdVersion(CmdBaseNoRepo):
             # later decides to enable shared cache mode with
             # `dvc config cache.shared group`.
             if os.path.exists(repo.cache.local.cache_dir):
-                info += "Cache: {cache}\n".format(
-                    cache=self.get_linktype_support_info(repo)
+                info.append(
+                    "Cache: {}".format(self.get_linktype_support_info(repo))
                 )
+                if psutil:
+                    fs_type = self.get_fs_type(repo.cache.local.cache_dir)
+                    info.append(
+                        "Filesystem type (cache directory): {}".format(fs_type)
+                    )
             else:
                 logger.warning(
                     "Unable to detect supported link types, as cache "
@@ -66,19 +59,14 @@ class CmdVersion(CmdBaseNoRepo):
                     "check.".format(relpath(repo.cache.local.cache_dir))
                 )
 
-            if psutil:
-                info += (
-                    "Filesystem type (cache directory): {fs_cache}\n"
-                ).format(fs_cache=self.get_fs_type(repo.cache.local.cache_dir))
         except NotDvcRepoError:
             root_directory = os.getcwd()
 
         if psutil:
-            info += ("Filesystem type (workspace): {fs_root}").format(
-                fs_root=self.get_fs_type(os.path.abspath(root_directory))
-            )
+            fs_root = self.get_fs_type(os.path.abspath(root_directory))
+            info.append("Filesystem type (workspace): {}".format(fs_root))
 
-        logger.info(info)
+        logger.info("\n".join(info))
         return 0
 
     @staticmethod
@@ -98,9 +86,9 @@ class CmdVersion(CmdBaseNoRepo):
     @staticmethod
     def get_linktype_support_info(repo):
         links = {
-            "reflink": System.reflink,
-            "hardlink": System.hardlink,
-            "symlink": System.symlink,
+            "reflink": (System.reflink, None),
+            "hardlink": (System.hardlink, System.is_hardlink),
+            "symlink": (System.symlink, System.is_symlink),
         }
 
         fname = "." + str(uuid.uuid4())
@@ -110,18 +98,16 @@ class CmdVersion(CmdBaseNoRepo):
 
         cache = []
 
-        for name, link in links.items():
+        for name, (link, is_link) in links.items():
             try:
                 link(src, dst)
+                status = "supported"
+                if is_link and not is_link(dst):
+                    status = "broken"
                 os.unlink(dst)
-                supported = True
             except DvcException:
-                supported = False
-            cache.append(
-                "{name} - {supported}".format(
-                    name=name, supported=True if supported else False
-                )
-            )
+                status = "not supported"
+            cache.append("{name} - {status}".format(name=name, status=status))
         os.remove(src)
 
         return ", ".join(cache)

dvc/compat.py

@@ -0,0 +1,42 @@
+"""Python version compatibility code"""
+
+# Backport os.fspath() from Python 3.6
+try:
+    from os import fspath  # noqa: F821
+
+    fspath_py35 = lambda s: s  # noqa: E731
+except ImportError:
+
+    def fspath(path):
+        """Return the path representation of a path-like object.
+
+        If str or bytes is passed in, it is returned unchanged. Otherwise the
+        os.PathLike interface is used to get the path representation. If the
+        path representation is not str or bytes, TypeError is raised. If the
+        provided path is not str, bytes, or os.PathLike, TypeError is raised.
+        """
+        if isinstance(path, (str, bytes)):
+            return path
+
+        # Work from the object's type to match method resolution of other magic
+        # methods.
+        path_type = type(path)
+        try:
+            path_repr = path_type.__fspath__(path)
+        except AttributeError:
+            if hasattr(path_type, "__fspath__"):
+                raise
+            else:
+                raise TypeError(
+                    "expected str, bytes or os.PathLike object, "
+                    "not " + path_type.__name__
+                )
+        if isinstance(path_repr, (str, bytes)):
+            return path_repr
+        else:
+            raise TypeError(
+                "expected {}.__fspath__() to return str or bytes, "
+                "not {}".format(path_type.__name__, type(path_repr).__name__)
+            )
+
+    fspath_py35 = fspath

dvc/config.py

@@ -1,5 +1,4 @@
 """DVC config objects."""
-from __future__ import unicode_literals
 
 import copy
 import errno
@@ -13,36 +12,19 @@ from voluptuous import All, Any, Lower, Range, Coerce, Match
 
 from dvc.exceptions import DvcException
 from dvc.exceptions import NotDvcRepoError
-from dvc.utils.compat import open
-from dvc.utils.compat import str
 
 logger = logging.getLogger(__name__)
 
 
 class ConfigError(DvcException):
-    """DVC config exception.
+    """DVC config exception."""
 
-    Args:
-        msg (str): error message.
-        cause (Exception): optional exception that has caused this error.
-    """
-
-    def __init__(self, msg, cause=None):
-        super(ConfigError, self).__init__(
-            "config file error: {}".format(msg), cause=cause
-        )
+    def __init__(self, msg):
+        super().__init__("config file error: {}".format(msg))
 
 
 class NoRemoteError(ConfigError):
-    def __init__(self, command, cause=None):
-        msg = (
-            "no remote specified. Setup default remote with\n"
-            "    dvc config core.remote <name>\n"
-            "or use:\n"
-            "    dvc {} -r <name>\n".format(command)
-        )
-
-        super(NoRemoteError, self).__init__(msg, cause=cause)
+    pass
 
 
 def supported_cache_type(types):
@@ -82,11 +64,11 @@ def Choices(*choices):
         *choices: pass allowed values as arguments, or pass a list or
             tuple as a single argument
     """
-    return Any(*choices, msg="expected one of {}".format(",".join(choices)))
+    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
 
 
 class Config(object):  # pylint: disable=too-many-instance-attributes
-    """Class that manages configuration files for a dvc repo.
+    """Class that manages configuration files for a DVC repo.
 
     Args:
         dvc_dir (str): optional path to `.dvc` directory, that is used to
@@ -126,6 +108,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_CORE_INTERACTIVE = "interactive"
     SECTION_CORE_ANALYTICS = "analytics"
     SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
+    SECTION_CORE_HARDLINK_LOCK = "hardlink_lock"
 
     SECTION_CACHE = "cache"
     SECTION_CACHE_DIR = "dir"
@@ -161,6 +144,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
         Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
         SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
+        Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool,
     }
 
     # aws specific options
@@ -172,6 +156,10 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_AWS_USE_SSL = "use_ssl"
     SECTION_AWS_SSE = "sse"
     SECTION_AWS_ACL = "acl"
+    SECTION_AWS_GRANT_READ = "grant_read"
+    SECTION_AWS_GRANT_READ_ACP = "grant_read_acp"
+    SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp"
+    SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control"
 
     # gcp specific options
     SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH
@@ -190,6 +178,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret"
     SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file"
 
+    SECTION_REMOTE_CHECKSUM_JOBS = "checksum_jobs"
     SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
     SECTION_REMOTE_FMT = 'remote "{}"'
     SECTION_REMOTE_URL = "url"
@@ -201,6 +190,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_REMOTE_ASK_PASSWORD = "ask_password"
     SECTION_REMOTE_GSS_AUTH = "gss_auth"
     SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
+    SECTION_REMOTE_VERIFY = "verify"
     SECTION_REMOTE_SCHEMA = {
         Required(SECTION_REMOTE_URL): str,
         SECTION_AWS_REGION: str,
@@ -211,9 +201,14 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_USE_SSL, default=True): Bool,
         SECTION_AWS_SSE: str,
         SECTION_AWS_ACL: str,
+        SECTION_AWS_GRANT_READ: str,
+        SECTION_AWS_GRANT_READ_ACP: str,
+        SECTION_AWS_GRANT_WRITE_ACP: str,
+        SECTION_AWS_GRANT_FULL_CONTROL: str,
         SECTION_GCP_PROJECTNAME: str,
         SECTION_CACHE_TYPE: supported_cache_type,
         Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
+        SECTION_REMOTE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
         SECTION_REMOTE_USER: str,
         SECTION_REMOTE_PORT: Coerce(int),
         SECTION_REMOTE_KEY_FILE: str,
@@ -229,7 +224,8 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         SECTION_GDRIVE_CLIENT_SECRET: str,
         SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
         PRIVATE_CWD: str,
-        Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): Bool,
+        SECTION_REMOTE_NO_TRAVERSE: Bool,
+        SECTION_REMOTE_VERIFY: Bool,
     }
 
     SECTION_STATE = "state"
@@ -250,7 +246,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
 
     def __init__(self, dvc_dir=None, validate=True):
         self.dvc_dir = dvc_dir
-        self.validate = validate
+        self.should_validate = validate
 
         if not dvc_dir:
             try:
@@ -380,14 +376,10 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
             c = self._lower(c)
             self.config.merge(c)
 
-        if not self.validate:
+        if not self.should_validate:
             return
 
-        d = self.config.dict()
-        try:
-            d = self.COMPILED_SCHEMA(d)
-        except Invalid as exc:
-            raise ConfigError(str(exc), cause=exc)
+        d = self.validate(self.config)
         self.config = configobj.ConfigObj(d, write_empty_values=True)
 
     def save(self, config=None):
@@ -425,6 +417,12 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
                 raise
         config.write()
 
+    def validate(self, config):
+        try:
+            return self.COMPILED_SCHEMA(config.dict())
+        except Invalid as exc:
+            raise ConfigError(str(exc)) from exc
+
     def unset(self, section, opt=None, level=None, force=False):
         """Unsets specified option and/or section in the config.
 
@@ -489,6 +487,11 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
             )
 
         config[section][opt] = value
+
+        result = copy.deepcopy(self.config)
+        result.merge(config)
+        self.validate(result)
+
         self.save(config)
 
     def get(self, section, opt=None, level=None):

dvc/daemon.py

@@ -1,5 +1,4 @@
 """Launch `dvc daemon` command in a separate detached process."""
-from __future__ import unicode_literals
 
 import inspect
 import logging
@@ -10,7 +9,6 @@ from subprocess import Popen
 from dvc.env import DVC_DAEMON
 from dvc.utils import fix_env
 from dvc.utils import is_binary
-from dvc.utils.compat import cast_bytes_py2
 
 
 logger = logging.getLogger(__name__)
@@ -99,9 +97,7 @@ def daemon(args):
 
     env = fix_env()
     file_path = os.path.abspath(inspect.stack()[0][1])
-    env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2(
-        os.path.dirname(os.path.dirname(file_path))
-    )
-    env[cast_bytes_py2(DVC_DAEMON)] = cast_bytes_py2("1")
+    env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
+    env[DVC_DAEMON] = "1"
 
     _spawn(cmd, env)

dvc/dagascii.py

@@ -1,8 +1,9 @@
 """Draws DAG in ASCII."""
-from __future__ import print_function
-from __future__ import unicode_literals
 
+import logging
 import math
+import os
+import pydoc
 import sys
 
 from grandalf.graphs import Edge
@@ -12,6 +13,44 @@ from grandalf.layouts import SugiyamaLayout
 from grandalf.routing import EdgeViewer
 from grandalf.routing import route_with_lines
 
+from dvc.env import DVC_PAGER
+from dvc.utils import format_link
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_PAGER = "less"
+DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
+    DEFAULT_PAGER
+)
+
+
+def make_pager(cmd):
+    def pager(text):
+        return pydoc.tempfilepager(pydoc.plain(text), cmd)
+
+    return pager
+
+
+def find_pager():
+    if not sys.stdout.isatty():
+        return pydoc.plainpager
+
+    env_pager = os.getenv(DVC_PAGER)
+    if env_pager:
+        return make_pager(env_pager)
+
+    if os.system("({}) 2>{}".format(DEFAULT_PAGER, os.devnull)) == 0:
+        return make_pager(DEFAULT_PAGER_FORMATTED)
+
+    logger.warning(
+        "Unable to find `less` in the PATH. Check out "
+        "{} for more info.".format(
+            format_link("https://man.dvc.org/pipeline/show")
+        )
+    )
+    return pydoc.plainpager
+
 
 class VertexViewer(object):
     """Class to define vertex box boundaries that will be accounted for during
@@ -60,99 +99,10 @@ class AsciiCanvas(object):
 
     def draw(self):
         """Draws ASCII canvas on the screen."""
-        if sys.stdout.isatty():  # pragma: no cover
-            from asciimatics.screen import Screen
-
-            Screen.wrapper(self._do_draw)
-        else:
-            for line in self.canvas:
-                print("".join(line))
-
-    def _do_draw(self, screen):  # pragma: no cover
-        # pylint: disable=too-many-locals
-        # pylint: disable=too-many-branches, too-many-statements
-        from dvc.system import System
-        from asciimatics.event import KeyboardEvent
-
-        offset_x = 0
-        offset_y = 0
-        smaxrow, smaxcol = screen.dimensions
-        assert smaxrow > 1
-        assert smaxcol > 1
-        smaxrow -= 1
-        smaxcol -= 1
-
-        if self.lines + 1 > smaxrow:
-            max_y = self.lines + 1 - smaxrow
-        else:
-            max_y = 0
-
-        if self.cols + 1 > smaxcol:
-            max_x = self.cols + 1 - smaxcol
-        else:
-            max_x = 0
-
-        while True:
-            for y in range(smaxrow + 1):
-                y_index = offset_y + y
-                line = []
-                for x in range(smaxcol + 1):
-                    x_index = offset_x + x
-                    if (
-                        len(self.canvas) > y_index
-                        and len(self.canvas[y_index]) > x_index
-                    ):
-                        line.append(self.canvas[y_index][x_index])
-                    else:
-                        line.append(" ")
-                assert len(line) == (smaxcol + 1)
-                screen.print_at("".join(line), 0, y)
-
-            screen.refresh()
-
-            # NOTE: get_event() doesn't block by itself,
-            # so we have to do the blocking ourselves.
-            #
-            # NOTE: using this workaround while waiting for PR [1]
-            # to get merged and released. After that need to adjust
-            # asciimatics version requirements.
-            #
-            # [1] https://github.com/peterbrittain/asciimatics/pull/188
-            System.wait_for_input(self.TIMEOUT)
-
-            event = screen.get_event()
-            if not isinstance(event, KeyboardEvent):
-                continue
-
-            k = event.key_code
-            if k == screen.KEY_DOWN or k == ord("s"):
-                offset_y += 1
-            elif k == screen.KEY_PAGE_DOWN or k == ord("S"):
-                offset_y += smaxrow
-            elif k == screen.KEY_UP or k == ord("w"):
-                offset_y -= 1
-            elif k == screen.KEY_PAGE_UP or k == ord("W"):
-                offset_y -= smaxrow
-            elif k == screen.KEY_RIGHT or k == ord("d"):
-                offset_x += 1
-            elif k == ord("D"):
-                offset_x += smaxcol
-            elif k == screen.KEY_LEFT or k == ord("a"):
-                offset_x -= 1
-            elif k == ord("A"):
-                offset_x -= smaxcol
-            elif k == ord("q") or k == ord("Q"):
-                break
-
-            if offset_y > max_y:
-                offset_y = max_y
-            elif offset_y < 0:
-                offset_y = 0
-
-            if offset_x > max_x:
-                offset_x = max_x
-            elif offset_x < 0:
-                offset_x = 0
+        pager = find_pager()
+        lines = map("".join, self.canvas)
+        joined_lines = os.linesep.join(lines)
+        pager(joined_lines)
 
     def point(self, x, y, char):
         """Create a point on ASCII canvas.

dvc/data_cloud.py

@@ -1,5 +1,4 @@
 """Manages dvc remotes that user can use with push/pull/status commands."""
-from __future__ import unicode_literals
 
 import logging
 
@@ -40,7 +39,26 @@ class DataCloud(object):
         if remote:
             return self._init_remote(remote)
 
-        raise NoRemoteError(command)
+        has_non_default_remote = bool(
+            self.repo.config.list_options(
+                Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL
+            )
+        )
+
+        if has_non_default_remote:
+            error_msg = (
+                "no remote specified. Setup default remote with\n"
+                "    dvc remote default <remote name>\n"
+                "or use:\n"
+                "    dvc {} -r <remote name>".format(command)
+            )
+        else:
+            error_msg = (
+                "no remote specified. Create a default remote with\n"
+                "    dvc remote add -d <remote name> <remote url>"
+            )
+
+        raise NoRemoteError(error_msg)
 
     def _init_remote(self, remote):
         return Remote(self.repo, name=remote)
@@ -74,13 +92,25 @@ class DataCloud(object):
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
-        return self.repo.cache.local.pull(
-            cache,
-            jobs=jobs,
-            remote=self.get_remote(remote, "pull"),
-            show_checksums=show_checksums,
+        remote = self.get_remote(remote, "pull")
+        downloaded_items_num = self.repo.cache.local.pull(
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )
 
+        if not remote.verify:
+            self._save_pulled_checksums(cache)
+
+        return downloaded_items_num
+
+    def _save_pulled_checksums(self, cache):
+        for checksum in cache["local"].keys():
+            cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
+            if self.repo.cache.local.exists(cache_file):
+                # We can safely save here, as existing corrupted files will be
+                # removed upon status, while files corrupted during download
+                # will not be moved from tmp_file (see `RemoteBASE.download()`)
+                self.repo.state.save(cache_file, checksum)
+
     def status(self, cache, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
 

dvc/dependency/__init__.py

@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+from urllib.parse import urlparse
 
 import dvc.output as output
 from dvc.dependency.gs import DependencyGS
@@ -46,8 +46,6 @@ SCHEMA[DependencyREPO.PARAM_REPO] = DependencyREPO.REPO_SCHEMA
 
 
 def _get(stage, p, info):
-    from dvc.utils.compat import urlparse
-
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":

dvc/dependency/base.py

@@ -1,23 +1,21 @@
-from __future__ import unicode_literals
-
 from dvc.exceptions import DvcException
 
 
 class DependencyDoesNotExistError(DvcException):
     def __init__(self, path):
         msg = "dependency '{}' does not exist".format(path)
-        super(DependencyDoesNotExistError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class DependencyIsNotFileOrDirError(DvcException):
     def __init__(self, path):
         msg = "dependency '{}' is not a file or directory".format(path)
-        super(DependencyIsNotFileOrDirError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class DependencyIsStageFileError(DvcException):
     def __init__(self, path):
-        super(DependencyIsStageFileError, self).__init__(
+        super().__init__(
             "Stage file '{}' cannot be a dependency.".format(path)
         )
 

dvc/dependency/gs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.gs import OutputGS
 

dvc/dependency/hdfs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.hdfs import OutputHDFS
 

dvc/dependency/http.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP

dvc/dependency/https.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from .http import DependencyHTTP
 from dvc.remote.https import RemoteHTTPS
 

dvc/dependency/local.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.local import OutputLOCAL
 

dvc/dependency/repo.py

@@ -1,13 +1,10 @@
-from __future__ import unicode_literals
+import os
 
-import copy
-from contextlib import contextmanager
-
-from funcy import merge
+from voluptuous import Required
 
 from .local import DependencyLOCAL
-from dvc.external_repo import external_repo
-from dvc.utils.compat import str
+from dvc.exceptions import OutputNotFoundError
+from dvc.path_info import PathInfo
 
 
 class DependencyREPO(DependencyLOCAL):
@@ -16,11 +13,15 @@ class DependencyREPO(DependencyLOCAL):
     PARAM_REV = "rev"
     PARAM_REV_LOCK = "rev_lock"
 
-    REPO_SCHEMA = {PARAM_URL: str, PARAM_REV: str, PARAM_REV_LOCK: str}
+    REPO_SCHEMA = {
+        Required(PARAM_URL): str,
+        PARAM_REV: str,
+        PARAM_REV_LOCK: str,
+    }
 
     def __init__(self, def_repo, stage, *args, **kwargs):
         self.def_repo = def_repo
-        super(DependencyREPO, self).__init__(stage, *args, **kwargs)
+        super().__init__(stage, *args, **kwargs)
 
     def _parse_path(self, remote, path):
         return None
@@ -32,24 +33,33 @@ class DependencyREPO(DependencyLOCAL):
     @property
     def repo_pair(self):
         d = self.def_repo
-        return d[self.PARAM_URL], d[self.PARAM_REV_LOCK] or d[self.PARAM_REV]
+        rev = d.get(self.PARAM_REV_LOCK) or d.get(self.PARAM_REV)
+        return d[self.PARAM_URL], rev
 
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 
-    @contextmanager
-    def _make_repo(self, **overrides):
-        with external_repo(**merge(self.def_repo, overrides)) as repo:
-            yield repo
+    def _make_repo(self, *, locked=True):
+        from dvc.external_repo import external_repo
+
+        d = self.def_repo
+        rev = (d.get("rev_lock") if locked else None) or d.get("rev")
+        return external_repo(d["url"], rev=rev)
+
+    def _get_checksum(self, locked=True):
+        with self._make_repo(locked=locked) as repo:
+            try:
+                return repo.find_out_by_relpath(self.def_path).info["md5"]
+            except OutputNotFoundError:
+                path = PathInfo(os.path.join(repo.root_dir, self.def_path))
+                # We are polluting our repo cache with some dir listing here
+                return self.repo.cache.local.get_checksum(path)
 
     def status(self):
-        with self._make_repo() as repo:
-            current = repo.find_out_by_relpath(self.def_path).info
+        current_checksum = self._get_checksum(locked=True)
+        updated_checksum = self._get_checksum(locked=False)
 
-        with self._make_repo(rev_lock=None) as repo:
-            updated = repo.find_out_by_relpath(self.def_path).info
-
-        if current != updated:
+        if current_checksum != updated_checksum:
             return {str(self): "update available"}
 
         return {}
@@ -60,23 +70,16 @@ class DependencyREPO(DependencyLOCAL):
     def dumpd(self):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
-    def fetch(self):
-        with self._make_repo(
-            cache_dir=self.repo.cache.local.cache_dir
-        ) as repo:
-            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
-
-            out = repo.find_out_by_relpath(self.def_path)
-            with repo.state:
-                repo.cloud.pull(out.get_used_cache())
-
-        return out
-
     def download(self, to):
-        out = self.fetch()
-        to.info = copy.copy(out.info)
-        to.checkout()
+        with self._make_repo() as repo:
+            if self.def_repo.get(self.PARAM_REV_LOCK) is None:
+                self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+
+            if hasattr(repo, "cache"):
+                repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
+
+            repo.pull_to(self.def_path, to.path_info)
 
     def update(self):
-        with self._make_repo(rev_lock=None) as repo:
+        with self._make_repo(locked=False) as repo:
             self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()

dvc/dependency/s3.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.s3 import OutputS3
 

dvc/dependency/ssh.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.ssh import OutputSSH
 

dvc/env.py

@@ -1 +1,2 @@
 DVC_DAEMON = "DVC_DAEMON"
+DVC_PAGER = "DVC_PAGER"

dvc/exceptions.py

@@ -1,32 +1,10 @@
 """Exceptions raised by the dvc."""
-from __future__ import unicode_literals
 
-import traceback
-
-from dvc.utils import relpath
-from dvc.utils.compat import builtin_str
-from dvc.utils.compat import str
+from dvc.utils import relpath, format_link
 
 
 class DvcException(Exception):
-    """Base class for all dvc exceptions.
-
-    Args:
-        msg (unicode): message for this exception.
-        cause (Exception): optional cause exception.
-    """
-
-    def __init__(self, msg, cause=None):
-        # NOTE: unlike python 3, python 2 doesn't have built-in support
-        # for chained exceptions, so we are using our own implementation.
-        self.cause = cause
-        self.cause_tb = None
-        if cause:
-            try:
-                self.cause_tb = traceback.format_exc()
-            except AttributeError:  # pragma: no cover
-                pass
-        super(DvcException, self).__init__(msg)
+    """Base class for all dvc exceptions."""
 
 
 class OutputDuplicationError(DvcException):
@@ -39,17 +17,14 @@ class OutputDuplicationError(DvcException):
     """
 
     def __init__(self, output, stages):
-        assert isinstance(output, str) or isinstance(output, builtin_str)
-        assert isinstance(stages, list)
-        assert all(
-            isinstance(stage, str) or isinstance(stage, builtin_str)
-            for stage in stages
-        )
+        assert isinstance(output, str)
+        assert all(hasattr(stage, "relpath") for stage in stages)
         msg = (
             "file/directory '{}' is specified as an output in more than one "
-            "stage: {}"
-        ).format(output, "\n    ".join(stages))
-        super(OutputDuplicationError, self).__init__(msg)
+            "stage: {}\n"
+            "This is not allowed. Consider using a different output name."
+        ).format(output, "\n    ".join(s.relpath for s in stages))
+        super().__init__(msg)
 
 
 class OutputNotFoundError(DvcException):
@@ -62,8 +37,8 @@ class OutputNotFoundError(DvcException):
     def __init__(self, output, repo=None):
         self.output = output
         self.repo = repo
-        super(OutputNotFoundError, self).__init__(
-            "unable to find DVC-file with output '{path}'".format(
+        super().__init__(
+            "Unable to find DVC-file with output '{path}'".format(
                 path=relpath(self.output)
             )
         )
@@ -74,21 +49,17 @@ class StagePathAsOutputError(DvcException):
     an output of another stage.
 
     Args:
-        cwd (str): path to the directory.
-        fname (str): path to the DVC-file that has cwd specified as an
-            output.
+        stage (Stage): a stage that is in some other stages output
+        output (str): an output covering the stage above
     """
 
-    def __init__(self, wdir, fname):
-        assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
-        assert isinstance(fname, str) or isinstance(fname, builtin_str)
-        msg = (
-            "current working directory '{cwd}' is specified as an output in "
-            "'{fname}'. Use another CWD to prevent any data removal.".format(
-                cwd=wdir, fname=fname
+    def __init__(self, stage, output):
+        assert isinstance(output, str)
+        super().__init__(
+            "'{stage}' is within an output '{output}' of another stage".format(
+                stage=stage.relpath, output=output
             )
         )
-        super(StagePathAsOutputError, self).__init__(msg)
 
 
 class CircularDependencyError(DvcException):
@@ -100,13 +71,13 @@ class CircularDependencyError(DvcException):
     """
 
     def __init__(self, dependency):
-        assert isinstance(dependency, (str, builtin_str))
+        assert isinstance(dependency, str)
 
         msg = (
             "file/directory '{}' is specified as an output and as a "
             "dependency."
         )
-        super(CircularDependencyError, self).__init__(msg.format(dependency))
+        super().__init__(msg.format(dependency))
 
 
 class ArgumentDuplicationError(DvcException):
@@ -118,9 +89,8 @@ class ArgumentDuplicationError(DvcException):
     """
 
     def __init__(self, path):
-        assert isinstance(path, (str, builtin_str))
-        msg = "file '{}' is specified more than once."
-        super(ArgumentDuplicationError, self).__init__(msg.format(path))
+        assert isinstance(path, str)
+        super().__init__("file '{}' is specified more than once.".format(path))
 
 
 class MoveNotDataSourceError(DvcException):
@@ -138,11 +108,11 @@ class MoveNotDataSourceError(DvcException):
             "it by hand, or remove '{path}' and create a new one at the "
             "desired location."
         )
-        super(MoveNotDataSourceError, self).__init__(msg.format(path=path))
+        super().__init__(msg.format(path=path))
 
 
 class NotDvcRepoError(DvcException):
-    """Thrown if a directory is not a dvc repo.
+    """Thrown if a directory is not a DVC repo.
 
     Args:
         root (str): path to the directory.
@@ -150,56 +120,54 @@ class NotDvcRepoError(DvcException):
 
     def __init__(self, root):
         msg = (
-            "you are not inside of a dvc repository "
+            "you are not inside of a DVC repository "
             "(checked up to mount point '{}')"
         )
-        super(NotDvcRepoError, self).__init__(msg.format(root))
+        super().__init__(msg.format(root))
 
 
 class DvcParserError(DvcException):
     """Base class for CLI parser errors."""
 
     def __init__(self):
-        super(DvcParserError, self).__init__("parser error")
+        super().__init__("parser error")
 
 
 class CyclicGraphError(DvcException):
     def __init__(self, stages):
         assert isinstance(stages, list)
-        stages = "\n".join("\t- {}".format(stage) for stage in stages)
+        stages = "\n".join("\t- {}".format(stage.relpath) for stage in stages)
         msg = (
             "you've introduced a cycle in your pipeline that involves "
             "the following stages:"
             "\n"
             "{stages}".format(stages=stages)
         )
-        super(CyclicGraphError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class ConfirmRemoveError(DvcException):
     def __init__(self, path):
-        super(ConfirmRemoveError, self).__init__(
+        super().__init__(
             "unable to remove '{}' without a confirmation from the user. Use "
             "`-f` to force.".format(path)
         )
 
 
 class InitError(DvcException):
-    def __init__(self, msg):
-        super(InitError, self).__init__(msg)
+    pass
 
 
 class ReproductionError(DvcException):
-    def __init__(self, dvc_file_name, ex):
+    def __init__(self, dvc_file_name):
         self.path = dvc_file_name
-        msg = "failed to reproduce '{}'".format(dvc_file_name)
-        super(ReproductionError, self).__init__(msg, cause=ex)
+        super().__init__("failed to reproduce '{}'".format(dvc_file_name))
 
 
 class BadMetricError(DvcException):
     def __init__(self, paths):
-        super(BadMetricError, self).__init__(
-            "the following metrics do not exists, "
+        super().__init__(
+            "the following metrics do not exist, "
             "are not metric files or are malformed: {paths}".format(
                 paths=", ".join("'{}'".format(path) for path in paths)
             )
@@ -208,32 +176,31 @@ class BadMetricError(DvcException):
 
 class NoMetricsError(DvcException):
     def __init__(self):
-        super(NoMetricsError, self).__init__(
+        super().__init__(
             "no metric files in this repository. "
             "Use `dvc metrics add` to add a metric file to track."
         )
 
 
 class StageFileCorruptedError(DvcException):
-    def __init__(self, path, cause=None):
+    def __init__(self, path):
         path = relpath(path)
-        super(StageFileCorruptedError, self).__init__(
+        super().__init__(
             "unable to read DVC-file: {} "
-            "YAML file structure is corrupted".format(path),
-            cause=cause,
+            "YAML file structure is corrupted".format(path)
         )
 
 
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
-        super(RecursiveAddingWhileUsingFilename, self).__init__(
+        super().__init__(
             "cannot use `fname` with multiple targets or `-R|--recursive`"
         )
 
 
 class OverlappingOutputPathsError(DvcException):
     def __init__(self, out_1, out_2):
-        super(OverlappingOutputPathsError, self).__init__(
+        super().__init__(
             "Paths for outs:\n'{}'('{}')\n'{}'('{}')\noverlap. To avoid "
             "unpredictable behaviour, rerun command with non overlapping outs "
             "paths.".format(
@@ -246,57 +213,41 @@ class OverlappingOutputPathsError(DvcException):
 
 
 class CheckoutErrorSuggestGit(DvcException):
-    def __init__(self, target, cause):
-        super(CheckoutErrorSuggestGit, self).__init__(
-            "Did you mean `git checkout {}`?".format(target), cause=cause
-        )
+    def __init__(self, target):
+        super().__init__("Did you mean `git checkout {}`?".format(target))
 
 
 class ETagMismatchError(DvcException):
     def __init__(self, etag, cached_etag):
-        super(ETagMismatchError, self).__init__(
+        super().__init__(
             "ETag mismatch detected when copying file to cache! "
             "(expected: '{}', actual: '{}')".format(etag, cached_etag)
         )
 
 
 class FileMissingError(DvcException):
-    def __init__(self, path, cause=None):
-        super(FileMissingError, self).__init__(
-            "Can't find '{}' neither locally nor on remote".format(path),
-            cause=cause,
+    def __init__(self, path):
+        self.path = path
+        super().__init__(
+            "Can't find '{}' neither locally nor on remote".format(path)
         )
 
 
 class DvcIgnoreInCollectedDirError(DvcException):
     def __init__(self, ignore_dirname):
-        super(DvcIgnoreInCollectedDirError, self).__init__(
+        super().__init__(
             ".dvcignore file should not be in collected dir path: "
             "'{}'".format(ignore_dirname)
         )
 
 
-class UrlNotDvcRepoError(DvcException):
-    def __init__(self, url):
-        super(UrlNotDvcRepoError, self).__init__(
-            "URL '{}' is not a dvc repository.".format(url)
-        )
-
-
-class GetDVCFileError(DvcException):
-    def __init__(self):
-        super(GetDVCFileError, self).__init__(
-            "the given path is a DVC-file, you must specify a data file "
-            "or a directory"
-        )
-
-
 class GitHookAlreadyExistsError(DvcException):
     def __init__(self, hook_name):
-        super(GitHookAlreadyExistsError, self).__init__(
-            "Hook '{}' already exists. Please refer to "
-            "https://man.dvc.org/install "
-            "for more info.".format(hook_name)
+        super().__init__(
+            "Hook '{}' already exists. Please refer to {} for more "
+            "info.".format(
+                hook_name, format_link("https://man.dvc.org/install")
+            )
         )
 
 
@@ -304,7 +255,7 @@ class DownloadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super(DownloadError, self).__init__(
+        super().__init__(
             "{amount} files failed to download".format(amount=amount)
         )
 
@@ -313,7 +264,7 @@ class UploadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super(UploadError, self).__init__(
+        super().__init__(
             "{amount} files failed to upload".format(amount=amount)
         )
 
@@ -325,7 +276,7 @@ class CheckoutError(DvcException):
             "Checkout failed for following targets:\n {}\nDid you "
             "forget to fetch?".format("\n".join(targets))
         )
-        super(CheckoutError, self).__init__(m)
+        super().__init__(m)
 
 
 class CollectCacheError(DvcException):
@@ -334,14 +285,14 @@ class CollectCacheError(DvcException):
 
 class NoRemoteInExternalRepoError(DvcException):
     def __init__(self, url):
-        super(NoRemoteInExternalRepoError, self).__init__(
+        super().__init__(
             "No DVC remote is specified in target repository '{}'.".format(url)
         )
 
 
 class NoOutputInExternalRepoError(DvcException):
     def __init__(self, path, external_repo_path, external_repo_url):
-        super(NoOutputInExternalRepoError, self).__init__(
+        super().__init__(
             "Output '{}' not found in target repository '{}'".format(
                 relpath(path, external_repo_path), external_repo_url
             )
@@ -350,4 +301,28 @@ class NoOutputInExternalRepoError(DvcException):
 
 class HTTPError(DvcException):
     def __init__(self, code, reason):
-        super(HTTPError, self).__init__("'{} {}'".format(code, reason))
+        super().__init__("'{} {}'".format(code, reason))
+
+
+class PathMissingError(DvcException):
+    def __init__(self, path, repo):
+        msg = (
+            "The path '{}' does not exist in the target repository '{}'"
+            " neither as an output nor a git-handled file."
+        )
+        super().__init__(msg.format(path, repo))
+
+
+class RemoteCacheRequiredError(DvcException):
+    def __init__(self, path_info):
+        super().__init__(
+            (
+                "Current operation was unsuccessful because '{}' requires "
+                "existing cache on '{}' remote. See {} for information on how "
+                "to set up remote cache."
+            ).format(
+                path_info,
+                path_info.scheme,
+                format_link("https://man.dvc.org/config#cache"),
+            )
+        )

dvc/external_repo.py

@@ -1,28 +1,34 @@
-from __future__ import unicode_literals
-
+import logging
 import os
 import tempfile
 from contextlib import contextmanager
 from distutils.dir_util import copy_tree
+import threading
 
-from funcy import retry
+from funcy import retry, suppress, wrap_with, cached_property
 
-from dvc.config import NoRemoteError
+from dvc.compat import fspath
+from dvc.repo import Repo
+from dvc.config import Config, NoRemoteError, NotDvcRepoError
 from dvc.exceptions import NoRemoteInExternalRepoError
-from dvc.exceptions import NoOutputInExternalRepoError
-from dvc.exceptions import OutputNotFoundError
-from dvc.utils.fs import remove
+from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
+from dvc.exceptions import FileMissingError, PathMissingError
+from dvc.remote import RemoteConfig
+from dvc.utils.fs import remove, fs_copy
+from dvc.scm import SCM
 
 
-REPO_CACHE = {}
+logger = logging.getLogger(__name__)
 
 
 @contextmanager
-def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
-    from dvc.repo import Repo
+def external_repo(url, rev=None):
+    path = _cached_clone(url, rev)
+    try:
+        repo = ExternalRepo(path, url)
+    except NotDvcRepoError:
+        repo = ExternalGitRepo(path, url)
 
-    path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
-    repo = Repo(path)
     try:
         yield repo
     except NoRemoteError:
@@ -31,74 +37,198 @@ def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
         if exc.repo is repo:
             raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url)
         raise
-    repo.close()
-
-
-def _external_repo(url=None, rev=None, cache_dir=None):
-    from dvc.config import Config
-    from dvc.cache import CacheConfig
-    from dvc.repo import Repo
-
-    key = (url, rev, cache_dir)
-    if key in REPO_CACHE:
-        return REPO_CACHE[key]
-
-    new_path = tempfile.mkdtemp("dvc-erepo")
-
-    # Copy and adjust existing clone
-    if (url, None, None) in REPO_CACHE:
-        old_path = REPO_CACHE[url, None, None]
-
-        # This one unlike shutil.copytree() works with an existing dir
-        copy_tree(old_path, new_path)
-    else:
-        # Create a new clone
-        _clone_repo(url, new_path)
-
-        # Save clean clone dir so that we will have access to a default branch
-        clean_clone_path = tempfile.mkdtemp("dvc-erepo")
-        copy_tree(new_path, clean_clone_path)
-        REPO_CACHE[url, None, None] = clean_clone_path
-
-    # Adjust new clone/copy to fit rev and cache_dir
-
-    # Checkout needs to be done first because current branch might not be
-    # DVC repository
-    if rev is not None:
-        _git_checkout(new_path, rev)
-
-    repo = Repo(new_path)
-    try:
-        if cache_dir is not None:
-            cache_config = CacheConfig(repo.config)
-            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
+    except FileMissingError as exc:
+        raise PathMissingError(exc.path, url)
     finally:
-        # Need to close/reopen repo to force config reread
         repo.close()
-
-    REPO_CACHE[key] = new_path
-    return new_path
+        _remove(path)
 
 
-def _git_checkout(repo_path, revision):
-    from dvc.scm import Git
-
-    git = Git(repo_path)
-    try:
-        git.checkout(revision)
-    finally:
-        git.close()
+CLONES = {}
+CACHE_DIRS = {}
 
 
 def clean_repos():
     # Outside code should not see cache while we are removing
-    repo_paths = list(REPO_CACHE.values())
-    REPO_CACHE.clear()
+    paths = list(CLONES.values()) + list(CACHE_DIRS.values())
+    CLONES.clear()
+    CACHE_DIRS.clear()
 
-    for path in repo_paths:
+    for path in paths:
         _remove(path)
 
 
+class ExternalRepo(Repo):
+    def __init__(self, root_dir, url):
+        super().__init__(root_dir)
+        self.url = url
+        self._set_cache_dir()
+        self._set_upstream()
+
+    def pull_to(self, path, to_info):
+        try:
+            out = None
+            with suppress(OutputNotFoundError):
+                out = self.find_out_by_relpath(path)
+
+            if out and out.use_cache:
+                self._pull_cached(out, to_info)
+                return
+
+            # Git handled files can't have absolute path
+            if os.path.isabs(path):
+                raise FileNotFoundError
+
+            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+    def _pull_cached(self, out, to_info):
+        with self.state:
+            # Only pull unless all needed cache is present
+            if out.changed_cache():
+                self.cloud.pull(out.get_used_cache())
+
+            out.path_info = to_info
+            failed = out.checkout()
+            # This might happen when pull haven't really pulled all the files
+            if failed:
+                raise FileNotFoundError
+
+    @wrap_with(threading.Lock())
+    def _set_cache_dir(self):
+        try:
+            cache_dir = CACHE_DIRS[self.url]
+        except KeyError:
+            cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
+
+        self.cache.local.cache_dir = cache_dir
+
+    def _set_upstream(self):
+        # check if the URL is local and no default remote is present
+        # add default remote pointing to the original repo's cache location
+        if os.path.isdir(self.url):
+            rconfig = RemoteConfig(self.config)
+            if not rconfig.has_default():
+                src_repo = Repo(self.url)
+                try:
+                    rconfig.add(
+                        "auto-generated-upstream",
+                        src_repo.cache.local.cache_dir,
+                        default=True,
+                        level=Config.LEVEL_LOCAL,
+                    )
+                finally:
+                    src_repo.close()
+
+
+class ExternalGitRepo:
+    def __init__(self, root_dir, url):
+        self.root_dir = root_dir
+        self.url = url
+
+    @cached_property
+    def scm(self):
+        return SCM(self.root_dir)
+
+    def close(self):
+        if "scm" in self.__dict__:
+            self.scm.close()
+
+    def find_out_by_relpath(self, path):
+        raise OutputNotFoundError(path, self)
+
+    def pull_to(self, path, to_info):
+        try:
+            # Git handled files can't have absolute path
+            if os.path.isabs(path):
+                raise FileNotFoundError
+
+            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+    @contextmanager
+    def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
+        try:
+            abs_path = os.path.join(self.root_dir, path)
+            with open(abs_path, mode, encoding=encoding) as fd:
+                yield fd
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+
+@wrap_with(threading.Lock())
+def _cached_clone(url, rev):
+    """Clone an external git repo to a temporary directory.
+
+    Returns the path to a local temporary directory with the specified
+    revision checked out.
+    """
+    # Get or create a clean clone
+    clone_path = CLONES.get(url)
+    if clone_path:
+        # Do not pull for known shas, branches and tags might move
+        if not _is_known_sha(clone_path, rev):
+            _git_pull(clone_path)
+    else:
+        clone_path = tempfile.mkdtemp("dvc-clone")
+        _git_clone(url, clone_path)
+        CLONES[url] = clone_path
+
+    # Copy to a new dir to keep the clone clean
+    repo_path = tempfile.mkdtemp("dvc-erepo")
+    copy_tree(clone_path, repo_path)
+
+    # Check out the specified revision
+    if rev is not None:
+        _git_checkout(repo_path, rev)
+
+    return repo_path
+
+
+def _git_clone(url, path):
+    from dvc.scm.git import Git
+
+    git = Git.clone(url, path)
+    git.close()
+
+
+def _git_checkout(repo_path, rev):
+    from dvc.scm import Git
+
+    git = Git(repo_path)
+    try:
+        git.checkout(rev)
+    finally:
+        git.close()
+
+
+def _is_known_sha(repo_path, rev):
+    import git
+    from gitdb.exc import BadName
+
+    if not rev or not git.Repo.re_hexsha_shortened.search(rev):
+        return False
+
+    try:
+        git.Repo(repo_path).commit(rev)
+        return True
+    except BadName:
+        return False
+
+
+def _git_pull(repo_path):
+    import git
+
+    repo = git.Repo(repo_path)
+    try:
+        msg = repo.git.pull()
+        logger.debug("external repo: git pull: {}", msg)
+    finally:
+        repo.close()
+
+
 def _remove(path):
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
@@ -106,10 +236,3 @@ def _remove(path):
         os_retry(remove)(path)
     else:
         remove(path)
-
-
-def _clone_repo(url, path):
-    from dvc.scm.git import Git
-
-    git = Git.clone(url, path)
-    git.close()

dvc/ignore.py

@@ -1,11 +1,11 @@
-from __future__ import unicode_literals
-
 import logging
 import os
 
+from funcy import cached_property
 from pathspec import PathSpec
 from pathspec.patterns import GitWildMatchPattern
 
+from dvc.scm.tree import BaseTree
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
@@ -72,13 +72,12 @@ class DvcIgnoreDirs(DvcIgnore):
 
 
 class DvcIgnoreFilter(object):
-    def __init__(self, root_dir, tree):
+    def __init__(self, tree):
         self.tree = tree
         self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
-        self._update(root_dir)
-        for root, dirs, _ in self.tree.walk(root_dir, dvcignore=self):
-            for d in dirs:
-                self._update(os.path.join(root, d))
+        for root, dirs, files in self.tree.walk(self.tree.tree_root):
+            self._update(root)
+            dirs[:], files[:] = self(root, dirs, files)
 
     def _update(self, dirname):
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
@@ -90,3 +89,39 @@ class DvcIgnoreFilter(object):
             dirs, files = ignore(root, dirs, files)
 
         return dirs, files
+
+
+class CleanTree(BaseTree):
+    def __init__(self, tree):
+        self.tree = tree
+
+    @cached_property
+    def dvcignore(self):
+        return DvcIgnoreFilter(self.tree)
+
+    @property
+    def tree_root(self):
+        return self.tree.tree_root
+
+    def open(self, path, mode="r", encoding="utf-8"):
+        return self.tree.open(path, mode, encoding)
+
+    def exists(self, path):
+        return self.tree.exists(path)
+
+    def isdir(self, path):
+        return self.tree.isdir(path)
+
+    def isfile(self, path):
+        return self.tree.isfile(path)
+
+    def walk(self, top, topdown=True):
+        for root, dirs, files in self.tree.walk(top, topdown):
+            dirs[:], files[:] = self.dvcignore(root, dirs, files)
+
+            yield root, dirs, files
+
+    def walk_files(self, top):
+        for root, _, files in self.walk(top):
+            for file in files:
+                yield os.path.join(root, file)

dvc/istextfile.py

@@ -1,24 +1,10 @@
 """Use heuristics to guess if it is a text file or a binary file."""
-from __future__ import unicode_literals
-
-from dvc.utils.compat import is_py3
-from dvc.utils.compat import open
 
 # Based on https://eli.thegreenplace.net/2011/10/19/
 # perls-guess-if-file-is-text-or-binary-implemented-in-python
 
 
-# A function that takes an integer in the 8-bit range and returns
-# a single-character byte object in py3 / a single-character string
-# in py2.
-#
-def _int2byte(i):
-    if is_py3:
-        return bytes((i,))
-    return chr(i)
-
-
-TEXT_CHARS = b"".join(_int2byte(i) for i in range(32, 127)) + b"\n\r\t\f\b"
+TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
 
 
 def istextfile(fname, blocksize=512):

dvc/lock.py

@@ -1,17 +1,16 @@
 """Manages dvc lock file."""
-from __future__ import unicode_literals
 
 import hashlib
 import os
 import time
 from datetime import timedelta
 
-from funcy.py3 import lkeep
+import zc.lockfile
+import flufl.lock
 
 from dvc.exceptions import DvcException
-from dvc.utils import makedirs
-from dvc.utils.compat import is_py3
-
+from dvc.utils import format_link
+from dvc.progress import Tqdm
 
 DEFAULT_TIMEOUT = 5
 
@@ -23,122 +22,127 @@ FAILED_TO_LOCK_MESSAGE = (
 
 
 class LockError(DvcException):
-    """Thrown when unable to acquire the lock for dvc repo."""
+    """Thrown when unable to acquire the lock for DVC repo."""
 
 
-if is_py3:
-    import flufl.lock
+class Lock(object):
+    """Class for DVC repo lock.
 
-    class Lock(flufl.lock.Lock):
-        """Class for dvc repo lock.
+    Uses zc.lockfile as backend.
+    """
 
-        Args:
-            lockfile (str): the lock filename
-                in.
-            tmp_dir (str): a directory to store claim files.
-        """
+    def __init__(self, lockfile, friendly=False, **kwargs):
+        self._friendly = friendly
+        self.lockfile = lockfile
+        self._lock = None
 
-        def __init__(self, lockfile, tmp_dir=None):
-            import socket
+    @property
+    def files(self):
+        return [self.lockfile]
 
-            self._tmp_dir = tmp_dir
-            if self._tmp_dir is not None:
-                makedirs(self._tmp_dir, exist_ok=True)
-
-            # NOTE: this is basically Lock.__init__ copy-paste, except that
-            # instead of using `socket.getfqdn()` we use `socket.gethostname()`
-            # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
-            # anything, which is way too slow. `gethostname()` is actually a
-            # fallback for `getfqdn()` when it is not able to resolve a
-            # canonical hostname through network. The claimfile that uses
-            # `self._hostname` is still usable, as it uses `pid` and random
-            # number to generate the resulting lock file name, which is unique
-            # enough for our application.
-            #
-            # [1] https://github.com/iterative/dvc/issues/2582
-            self._hostname = socket.gethostname()
-
-            self._lockfile = lockfile
-            self._lifetime = timedelta(days=365)  # Lock for good by default
-            self._separator = flufl.lock.SEP
-            self._set_claimfile()
-            self._owned = True
-            self._retry_errnos = []
-
-        @property
-        def lockfile(self):
-            return self._lockfile
-
-        @property
-        def files(self):
-            return lkeep([self._lockfile, self._tmp_dir])
-
-        def lock(self):
-            try:
-                super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
-            except flufl.lock.TimeOutError:
-                raise LockError(FAILED_TO_LOCK_MESSAGE)
-
-        def _set_claimfile(self, pid=None):
-            super(Lock, self)._set_claimfile(pid)
-
-            if self._tmp_dir is not None:
-                # Under Windows file path length is limited so we hash it
-                filename = hashlib.md5(self._claimfile.encode()).hexdigest()
-                self._claimfile = os.path.join(
-                    self._tmp_dir, filename + ".lock"
-                )
-
-        # Fix for __del__ bug in flufl.lock [1] which is causing errors on
-        # Python shutdown [2].
-        # [1] https://gitlab.com/warsaw/flufl.lock/issues/7
-        # [2] https://github.com/iterative/dvc/issues/2573
-        def __del__(self):
-            try:
-                if self._owned:
-                    self.finalize()
-            except ImportError:
-                pass
-
-
-else:
-    import zc.lockfile
-
-    class Lock(object):
-        """Class for dvc repo lock.
-
-        Uses zc.lockfile as backend.
-        """
-
-        def __init__(self, lockfile, tmp_dir=None):
-            self.lockfile = lockfile
-            self._lock = None
-
-        @property
-        def files(self):
-            return [self.lockfile]
-
-        def _do_lock(self):
-            try:
+    def _do_lock(self):
+        try:
+            with Tqdm(
+                bar_format="{desc}",
+                disable=not self._friendly,
+                desc=(
+                    "If DVC froze, see `hardlink_lock` in {}".format(
+                        format_link("man.dvc.org/config#core")
+                    )
+                ),
+            ):
                 self._lock = zc.lockfile.LockFile(self.lockfile)
-            except zc.lockfile.LockError:
-                raise LockError(FAILED_TO_LOCK_MESSAGE)
-
-        def lock(self):
-            try:
-                self._do_lock()
-                return
-            except LockError:
-                time.sleep(DEFAULT_TIMEOUT)
+        except zc.lockfile.LockError:
+            raise LockError(FAILED_TO_LOCK_MESSAGE)
 
+    def lock(self):
+        try:
             self._do_lock()
+            return
+        except LockError:
+            time.sleep(DEFAULT_TIMEOUT)
 
-        def unlock(self):
-            self._lock.close()
-            self._lock = None
+        self._do_lock()
 
-        def __enter__(self):
-            self.lock()
+    def unlock(self):
+        self._lock.close()
+        self._lock = None
 
-        def __exit__(self, typ, value, tbck):
-            self.unlock()
+    @property
+    def is_locked(self):
+        return bool(self._lock)
+
+    def __enter__(self):
+        self.lock()
+
+    def __exit__(self, typ, value, tbck):
+        self.unlock()
+
+
+class HardlinkLock(flufl.lock.Lock):
+    """Class for DVC repo lock.
+
+    Args:
+        lockfile (str): the lock filename
+            in.
+        tmp_dir (str): a directory to store claim files.
+    """
+
+    def __init__(self, lockfile, tmp_dir=None, **kwargs):
+        import socket
+
+        self._tmp_dir = tmp_dir
+
+        # NOTE: this is basically Lock.__init__ copy-paste, except that
+        # instead of using `socket.getfqdn()` we use `socket.gethostname()`
+        # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
+        # anything, which is way too slow. `gethostname()` is actually a
+        # fallback for `getfqdn()` when it is not able to resolve a
+        # canonical hostname through network. The claimfile that uses
+        # `self._hostname` is still usable, as it uses `pid` and random
+        # number to generate the resulting lock file name, which is unique
+        # enough for our application.
+        #
+        # [1] https://github.com/iterative/dvc/issues/2582
+        self._hostname = socket.gethostname()
+
+        self._lockfile = lockfile
+        self._lifetime = timedelta(days=365)  # Lock for good by default
+        self._separator = flufl.lock.SEP
+        self._set_claimfile()
+        self._owned = True
+        self._retry_errnos = []
+
+    @property
+    def lockfile(self):
+        return self._lockfile
+
+    def lock(self):
+        try:
+            super().lock(timedelta(seconds=DEFAULT_TIMEOUT))
+        except flufl.lock.TimeOutError:
+            raise LockError(FAILED_TO_LOCK_MESSAGE)
+
+    def _set_claimfile(self, pid=None):
+        super()._set_claimfile(pid)
+
+        if self._tmp_dir is not None:
+            # Under Windows file path length is limited so we hash it
+            filename = hashlib.md5(self._claimfile.encode()).hexdigest()
+            self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
+
+    # Fix for __del__ bug in flufl.lock [1] which is causing errors on
+    # Python shutdown [2].
+    # [1] https://gitlab.com/warsaw/flufl.lock/issues/7
+    # [2] https://github.com/iterative/dvc/issues/2573
+    def __del__(self):
+        try:
+            if self._owned:
+                self.finalize()
+        except ImportError:
+            pass
+
+
+def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False):
+    cls = HardlinkLock if hardlink_lock else Lock
+    return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly)

dvc/logger.py

@@ -1,15 +1,12 @@
-"""Manages logging configuration for dvc repo."""
-from __future__ import unicode_literals
+"""Manages logging configuration for DVC repo."""
 
+import traceback
 import logging.config
 import logging.handlers
 
 import colorama
 
 from dvc.progress import Tqdm
-from dvc.utils.compat import RecursionError
-from dvc.utils.compat import str
-from dvc.utils.compat import StringIO
 
 
 FOOTER = (
@@ -26,7 +23,7 @@ FOOTER = (
 class LoggingException(Exception):
     def __init__(self, record):
         msg = "failed to log {}".format(str(record))
-        super(LoggingException, self).__init__(msg)
+        super().__init__(msg)
 
 
 class ExcludeErrorsFilter(logging.Filter):
@@ -61,11 +58,13 @@ class ColorFormatter(logging.Formatter):
     }
 
     def format(self, record):
+        msg = record.msg.format(*record.args) if record.args else record.msg
+
         if record.levelname == "INFO":
-            return record.msg
+            return msg
 
         if record.levelname == "ERROR" or record.levelname == "CRITICAL":
-            exception, stack_trace = self._parse_exc(record.exc_info)
+            exception, stack_trace = self._parse_exc(record)
 
             return (
                 "{color}{levelname}{nc}: {description}" "{stack_trace}\n"
@@ -73,8 +72,8 @@ class ColorFormatter(logging.Formatter):
                 color=self.color_code.get(record.levelname, ""),
                 nc=colorama.Fore.RESET,
                 levelname=record.levelname,
-                description=self._description(record.msg, exception),
-                msg=record.msg,
+                description=self._description(msg, exception),
+                msg=msg,
                 stack_trace=stack_trace,
             )
 
@@ -82,7 +81,7 @@ class ColorFormatter(logging.Formatter):
             color=self.color_code.get(record.levelname, ""),
             nc=colorama.Fore.RESET,
             levelname=record.levelname,
-            msg=record.msg,
+            msg=msg,
         )
 
     def _current_level(self):
@@ -104,34 +103,26 @@ class ColorFormatter(logging.Formatter):
         return description.format(message=message, exception=exception)
 
     def _walk_exc(self, exc_info):
-        import traceback
-
-        buffer = StringIO()
-
-        traceback.print_exception(*exc_info, file=buffer)
-
         exc = exc_info[1]
-        tb = buffer.getvalue()
 
         exc_list = [str(exc)]
-        tb_list = [tb]
 
-        # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info
-        while hasattr(exc, "cause") and exc.cause:
-            exc_list.append(str(exc.cause))
-            if hasattr(exc, "cause_tb") and exc.cause_tb:
-                tb_list.insert(0, str(exc.cause_tb))
-            exc = exc.cause
+        while hasattr(exc, "__cause__") and exc.__cause__:
+            exc_list.append(str(exc.__cause__))
+            exc = exc.__cause__
 
-        return exc_list, tb_list
+        return exc_list
 
-    def _parse_exc(self, exc_info):
-        if not exc_info:
+    def _parse_exc(self, record):
+        tb_only = getattr(record, "tb_only", False)
+
+        if not record.exc_info:
             return (None, "")
 
-        exc_list, tb_list = self._walk_exc(exc_info)
+        exc_list = self._walk_exc(record.exc_info)
+        tb = traceback.format_exception(*record.exc_info)
 
-        exception = ": ".join(exc_list)
+        exception = None if tb_only else ": ".join(exc_list)
 
         if self._current_level() == logging.DEBUG:
             stack_trace = (
@@ -140,7 +131,7 @@ class ColorFormatter(logging.Formatter):
                 red=colorama.Fore.RED,
                 nc=colorama.Fore.RESET,
                 line="-" * 60,
-                stack_trace="\n".join(tb_list),
+                stack_trace="".join(tb),
             )
         else:
             stack_trace = ""
@@ -150,7 +141,7 @@ class ColorFormatter(logging.Formatter):
 
 class LoggerHandler(logging.StreamHandler):
     def handleError(self, record):
-        super(LoggerHandler, self).handleError(record)
+        super().handleError(record)
         raise LoggingException(record)
 
     def emit(self, record):

dvc/main.py

@@ -1,22 +1,22 @@
 """Main entry point for dvc CLI."""
-from __future__ import unicode_literals
 
+import errno
 import logging
 
-from dvc.analytics import Analytics
+from dvc import analytics
 from dvc.cli import parse_args
 from dvc.config import ConfigError
 from dvc.exceptions import DvcParserError
 from dvc.exceptions import NotDvcRepoError
 from dvc.external_repo import clean_repos
 from dvc.logger import FOOTER
+from dvc.utils import format_link
 from dvc.remote.pool import close_pools
-from dvc.utils.compat import is_py2
-
 
 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
 # [2] https://bugs.python.org/issue29288
+
 "".encode("idna")
 
 
@@ -59,10 +59,14 @@ def main(argv=None):
     except DvcParserError:
         ret = 254
     except Exception as exc:  # pylint: disable=broad-except
-        if isinstance(exc, UnicodeError) and is_py2:
+        if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
             logger.exception(
-                "unicode is not supported in DVC for Python 2 "
-                "(end-of-life January 1, 2020), please upgrade to Python 3"
+                "too many open files, please visit "
+                "{} to see how to handle this "
+                "problem".format(
+                    format_link("https://error.dvc.org/many-files")
+                ),
+                extra={"tb_only": True},
             )
         else:
             logger.exception("unexpected error")
@@ -70,8 +74,9 @@ def main(argv=None):
     finally:
         logger.setLevel(outerLogLevel)
 
-        # Python 2 fails to close these clean occasionally and users see
-        # weird error messages, so we do it manually
+        # Closing pools by-hand to prevent wierd messages when closing SSH
+        # connections. See https://github.com/iterative/dvc/issues/3248 for
+        # more info.
         close_pools()
 
         # Remove cached repos in the end of the call, these are anonymous
@@ -81,6 +86,7 @@ def main(argv=None):
     if ret != 0:
         logger.info(FOOTER)
 
-    Analytics().send_cmd(cmd, args, ret)
+    if analytics.is_enabled():
+        analytics.collect_and_send_report(args, ret)
 
     return ret

dvc/output/__init__.py

@@ -1,5 +1,4 @@
-from __future__ import unicode_literals
-
+from urllib.parse import urlparse
 from voluptuous import Any, Required
 
 from dvc.output.base import OutputBase
@@ -13,8 +12,6 @@ from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.local import RemoteLOCAL
 from dvc.remote.s3 import RemoteS3
 from dvc.scheme import Schemes
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
 
 OUTS = [
     OutputHDFS,

dvc/output/base.py

@@ -1,17 +1,14 @@
-from __future__ import unicode_literals
-
 import logging
+from urllib.parse import urlparse
 from copy import copy
 
 from voluptuous import Any
 
 import dvc.prompt as prompt
 from dvc.cache import NamedCache
-from dvc.exceptions import CollectCacheError
+from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError
 from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteBASE
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -20,26 +17,24 @@ logger = logging.getLogger(__name__)
 class OutputDoesNotExistError(DvcException):
     def __init__(self, path):
         msg = "output '{}' does not exist".format(path)
-        super(OutputDoesNotExistError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class OutputIsNotFileOrDirError(DvcException):
     def __init__(self, path):
         msg = "output '{}' is not a file or directory".format(path)
-        super(OutputIsNotFileOrDirError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class OutputAlreadyTrackedError(DvcException):
     def __init__(self, path):
         msg = "output '{}' is already tracked by scm (e.g. git)".format(path)
-        super(OutputAlreadyTrackedError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class OutputIsStageFileError(DvcException):
     def __init__(self, path):
-        super(OutputIsStageFileError, self).__init__(
-            "Stage file '{}' cannot be an output.".format(path)
-        )
+        super().__init__("Stage file '{}' cannot be an output.".format(path))
 
 
 class OutputBase(object):
@@ -103,14 +98,9 @@ class OutputBase(object):
         self.persist = persist
         self.tags = None if self.IS_DEPENDENCY else (tags or {})
 
-        if self.use_cache and self.cache is None:
-            raise DvcException(
-                "no cache location setup for '{}' outputs.".format(
-                    self.REMOTE.scheme
-                )
-            )
-
         self.path_info = self._parse_path(remote, path)
+        if self.use_cache and self.cache is None:
+            raise RemoteCacheRequiredError(self.path_info)
 
     def _parse_path(self, remote, path):
         if remote:
@@ -282,10 +272,19 @@ class OutputBase(object):
     def download(self, to):
         self.remote.download(self.path_info, to.path_info)
 
-    def checkout(self, force=False, progress_callback=None, tag=None):
+    def checkout(
+        self,
+        force=False,
+        progress_callback=None,
+        tag=None,
+        relink=False,
+        filter_info=None,
+    ):
         if not self.use_cache:
             if progress_callback:
-                progress_callback(str(self.path_info), self.get_files_number())
+                progress_callback(
+                    str(self.path_info), self.get_files_number(filter_info)
+                )
             return None
 
         if tag:
@@ -298,6 +297,8 @@ class OutputBase(object):
             info,
             force=force,
             progress_callback=progress_callback,
+            relink=relink,
+            filter_info=filter_info,
         )
 
     def remove(self, ignore_remove=False):
@@ -321,17 +322,21 @@ class OutputBase(object):
         if self.scheme == "local" and self.use_scm_ignore:
             self.repo.scm.ignore(self.fspath)
 
-    def get_files_number(self):
+    def get_files_number(self, filter_info=None):
         if not self.use_cache:
             return 0
 
-        return self.cache.get_files_number(self.checksum)
+        return self.cache.get_files_number(
+            self.path_info, self.checksum, filter_info
+        )
 
     def unprotect(self):
         if self.exists:
             self.remote.unprotect(self.path_info)
 
-    def _collect_used_dir_cache(self, remote=None, force=False, jobs=None):
+    def _collect_used_dir_cache(
+        self, remote=None, force=False, jobs=None, filter_info=None
+    ):
         """Get a list of `info`s related to the given directory.
 
         - Pull the directory entry from the remote cache if it was changed.
@@ -380,8 +385,9 @@ class OutputBase(object):
 
         for entry in self.dir_cache:
             checksum = entry[self.remote.PARAM_CHECKSUM]
-            path_info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            cache.add(self.scheme, checksum, str(path_info))
+            info = self.path_info / entry[self.remote.PARAM_RELPATH]
+            if not filter_info or info.isin_or_eq(filter_info):
+                cache.add(self.scheme, checksum, str(info))
 
         return cache
 
@@ -397,10 +403,16 @@ class OutputBase(object):
         if not self.use_cache:
             return NamedCache()
 
+        if self.stage.is_repo_import:
+            cache = NamedCache()
+            (dep,) = self.stage.deps
+            cache.external[dep.repo_pair].add(dep.def_path)
+            return cache
+
         if not self.info:
             logger.warning(
                 "Output '{}'({}) is missing version info. Cache for it will "
-                "not be collected. Use dvc repro to get your pipeline up to "
+                "not be collected. Use `dvc repro` to get your pipeline up to "
                 "date.".format(self, self.stage)
             )
             return NamedCache()

dvc/output/gs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.s3 import OutputS3
 from dvc.remote.gs import RemoteGS
 

dvc/output/hdfs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.base import OutputBase
 from dvc.remote.hdfs import RemoteHDFS
 

dvc/output/local.py

@@ -1,16 +1,13 @@
-from __future__ import unicode_literals
-
 import logging
 import os
+from urllib.parse import urlparse
 
 from dvc.exceptions import DvcException
 from dvc.istextfile import istextfile
 from dvc.output.base import OutputBase
 from dvc.remote.local import RemoteLOCAL
 from dvc.utils import relpath
-from dvc.utils.compat import fspath_py35
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
+from dvc.compat import fspath_py35
 from dvc.utils.fs import path_isin
 
 
@@ -59,7 +56,7 @@ class OutputLOCAL(OutputBase):
         return def_scheme != "remote" and not os.path.isabs(self.def_path)
 
     def dumpd(self):
-        ret = super(OutputLOCAL, self).dumpd()
+        ret = super().dumpd()
         if self.is_in_repo:
             path = self.path_info.relpath(self.stage.wdir).as_posix()
         else:

dvc/output/s3.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.base import OutputBase
 from dvc.remote.s3 import RemoteS3
 

dvc/output/ssh.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.base import OutputBase
 from dvc.remote.ssh import RemoteSSH
 

dvc/path_info.py

@@ -1,29 +1,26 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 import os
 import posixpath
-import sys
+import pathlib
+from urllib.parse import urlparse
 
 from funcy import cached_property
 
 from dvc.utils import relpath
-from dvc.utils.compat import basestring
-from dvc.utils.compat import builtin_str
-from dvc.utils.compat import is_py2
-from dvc.utils.compat import pathlib
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
 
 
-# On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
-# which is lossy, thus we can't use that,
-# see https://github.com/mcmtroffaes/pathlib2/issues/56.
-if is_py2:
-    fs_encoding = "utf-8"
+class _BasePath(object):
+    def overlaps(self, other):
+        if isinstance(other, (str, bytes)):
+            other = self.__class__(other)
+        elif self.__class__ != other.__class__:
+            return False
+        return self.isin_or_eq(other) or other.isin(self)
+
+    def isin_or_eq(self, other):
+        return self == other or self.isin(other)
 
 
-class PathInfo(pathlib.PurePath):
+class PathInfo(pathlib.PurePath, _BasePath):
     # Use __slots__ in PathInfo objects following PurePath implementation.
     # This makes objects smaller and speeds up attribute access.
     # We don't add any fields so it's empty.
@@ -54,7 +51,7 @@ class PathInfo(pathlib.PurePath):
         return relpath(path)
 
     def __repr__(self):
-        return builtin_str("{}: '{}'").format(type(self).__name__, self)
+        return "{}: '{}'".format(type(self).__name__, self)
 
     # This permits passing it to file utils directly in Python 3.6+
     # With Python 2.7, Python 3.5+ we are stuck with path_info.fspath for now
@@ -71,7 +68,7 @@ class PathInfo(pathlib.PurePath):
         return self.__class__(relpath(self, other))
 
     def isin(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         elif self.__class__ != other.__class__:
             return False
@@ -79,34 +76,6 @@ class PathInfo(pathlib.PurePath):
         n = len(other._cparts)
         return len(self._cparts) > n and self._cparts[:n] == other._cparts
 
-    # pathlib2 uses bytes internally in Python 2, and we use unicode everywhere
-    # for paths in both pythons, thus we need this glue.
-    if is_py2:
-        __unicode__ = __str__
-
-        def __str__(self):
-            return self.__unicode__().encode(sys.getfilesystemencoding())
-
-        @classmethod
-        def _parse_args(cls, args):
-            args = [
-                a.encode(fs_encoding)
-                if isinstance(a, unicode)  # noqa: F821
-                else a
-                for a in args
-            ]
-            return super(PathInfo, cls)._parse_args(args)
-
-        @property
-        def name(self):
-            return super(PathInfo, self).name.decode(fs_encoding)
-
-        def __fspath__(self):  # noqa: F811
-            return pathlib.PurePath.__str__(self).decode(fs_encoding)
-
-        def with_name(self, name):
-            return pathlib.PurePath.with_name(self, name.encode(fs_encoding))
-
 
 class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
     pass
@@ -138,7 +107,7 @@ class _URLPathParents(object):
         return "<{}.parents>".format(self.src)
 
 
-class URLInfo(object):
+class URLInfo(_BasePath):
     DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22, "hdfs": 0}
 
     def __init__(self, url):
@@ -163,13 +132,13 @@ class URLInfo(object):
 
     def fill_parts(self, scheme, host, user, port, path):
         assert scheme != "remote"
-        assert isinstance(path, (basestring, _URLPathInfo))
+        assert isinstance(path, (str, bytes, _URLPathInfo))
 
         self.scheme, self.host, self.user = scheme, host, user
         self.port = int(port) if port else self.DEFAULT_PORTS.get(self.scheme)
 
         if isinstance(path, _URLPathInfo):
-            self._spath = builtin_str(path)
+            self._spath = str(path)
             self._path = path
         else:
             if path and path[0] != "/":
@@ -198,7 +167,7 @@ class URLInfo(object):
         return "{}: '{}'".format(type(self).__name__, self)
 
     def __eq__(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         return (
             self.__class__ == other.__class__
@@ -248,7 +217,7 @@ class URLInfo(object):
         return _URLPathParents(self)
 
     def relative_to(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         if self.__class__ != other.__class__:
             msg = "'{}' has incompatible class with '{}'".format(self, other)
@@ -259,7 +228,7 @@ class URLInfo(object):
         return self._path.relative_to(other._path)
 
     def isin(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         elif self.__class__ != other.__class__:
             return False

dvc/progress.py

@@ -1,11 +1,9 @@
-"""Manages progress bars for dvc repo."""
-from __future__ import print_function
+"""Manages progress bars for DVC repo."""
 
 import logging
 import sys
 from threading import RLock
 
-from funcy import merge
 from tqdm import tqdm
 
 from dvc.utils import env2bool
@@ -32,7 +30,10 @@ class Tqdm(tqdm):
     )
     BAR_FMT_NOTOTAL = (
         "{desc:{ncols_desc}.{ncols_desc}}{n_fmt}"
-        " [{elapsed}<??:??, {rate_fmt:>11}{postfix}]"
+        " [{elapsed}, {rate_fmt:>11}{postfix}]"
+    )
+    BYTES_DEFAULTS = dict(
+        unit="B", unit_scale=True, unit_divisor=1024, miniters=1
     )
 
     def __init__(
@@ -45,6 +46,7 @@ class Tqdm(tqdm):
         bar_format=None,
         bytes=False,  # pylint: disable=W0622
         file=None,
+        total=None,
         **kwargs
     ):
         """
@@ -59,12 +61,10 @@ class Tqdm(tqdm):
         kwargs  : anything accepted by `tqdm.tqdm()`
         """
         kwargs = kwargs.copy()
-        kwargs.setdefault("unit_scale", True)
         if bytes:
-            bytes_defaults = dict(
-                unit="B", unit_scale=True, unit_divisor=1024, miniters=1
-            )
-            kwargs = merge(bytes_defaults, kwargs)
+            kwargs = {**self.BYTES_DEFAULTS, **kwargs}
+        else:
+            kwargs.setdefault("unit_scale", total > 999 if total else True)
         if file is None:
             file = sys.stderr
         self.desc_persist = desc
@@ -78,13 +78,14 @@ class Tqdm(tqdm):
             and hasattr(file, "isatty")
         ):
             disable = not file.isatty()
-        super(Tqdm, self).__init__(
+        super().__init__(
             iterable=iterable,
             disable=disable,
             leave=leave,
             desc=desc,
             bar_format="!",
             lock_args=(False,),
+            total=total,
             **kwargs
         )
         if bar_format is None:
@@ -115,12 +116,16 @@ class Tqdm(tqdm):
     def close(self):
         if self.desc_persist is not None:
             self.set_description_str(self.desc_persist, refresh=False)
-        super(Tqdm, self).close()
+        # unknown/zero ETA
+        self.bar_format = self.bar_format.replace("<{remaining}", "")
+        # remove completed bar
+        self.bar_format = self.bar_format.replace("|{bar:10}|", " ")
+        super().close()
 
     @property
     def format_dict(self):
         """inject `ncols_desc` to fill the display width (`ncols`)"""
-        d = super(Tqdm, self).format_dict
+        d = super().format_dict
         ncols = d["ncols"] or 80
         ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1
         ncols_desc = max(ncols_desc, 0)

dvc/prompt.py

@@ -1,14 +1,9 @@
 """Manages user prompts."""
-from __future__ import print_function
-from __future__ import unicode_literals
 
 import logging
 import sys
 from getpass import getpass
 
-from dvc.utils.compat import input
-
-
 logger = logging.getLogger(__name__)
 
 

dvc/remote/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from .config import RemoteConfig
 from dvc.remote.azure import RemoteAZURE
 from dvc.remote.gdrive import RemoteGDrive

dvc/remote/azure.py

@@ -1,10 +1,8 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 import logging
 import os
 import re
 from datetime import datetime, timedelta
+from urllib.parse import urlparse
 import threading
 
 from funcy import cached_property, wrap_prop
@@ -14,7 +12,6 @@ from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
-from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -34,7 +31,7 @@ class RemoteAZURE(RemoteBASE):
     COPY_POLL_SECONDS = 5
 
     def __init__(self, repo, config):
-        super(RemoteAZURE, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL, "azure://")
 

dvc/remote/base.py

@@ -1,9 +1,9 @@
-from __future__ import unicode_literals
-
+import errno
 import itertools
 import json
 import logging
 import tempfile
+from urllib.parse import urlparse
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
 from functools import partial
@@ -14,23 +14,19 @@ from shortuuid import uuid
 
 import dvc.prompt as prompt
 from dvc.config import Config
-from dvc.exceptions import ConfirmRemoveError
-from dvc.exceptions import DvcException
-from dvc.exceptions import DvcIgnoreInCollectedDirError
+from dvc.exceptions import (
+    DvcException,
+    ConfirmRemoveError,
+    DvcIgnoreInCollectedDirError,
+    RemoteCacheRequiredError,
+)
 from dvc.ignore import DvcIgnore
-from dvc.path_info import PathInfo
-from dvc.path_info import URLInfo
+from dvc.path_info import PathInfo, URLInfo
 from dvc.progress import Tqdm
 from dvc.remote.slow_link_detection import slow_link_guard
 from dvc.state import StateNoop
-from dvc.utils import makedirs
-from dvc.utils import relpath
 from dvc.utils import tmp_fname
-from dvc.utils.compat import basestring
-from dvc.utils.compat import FileNotFoundError
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
-from dvc.utils.fs import move
+from dvc.utils.fs import move, makedirs
 from dvc.utils.http import open_url
 
 logger = logging.getLogger(__name__)
@@ -49,16 +45,9 @@ STATUS_MAP = {
 }
 
 
-class DataCloudError(DvcException):
-    """ Data Cloud exception """
-
-    def __init__(self, msg):
-        super(DataCloudError, self).__init__("Data sync error: {}".format(msg))
-
-
 class RemoteCmdError(DvcException):
     def __init__(self, remote, cmd, ret, err):
-        super(RemoteCmdError, self).__init__(
+        super().__init__(
             "{remote} command '{cmd}' finished with non-zero return code"
             " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
         )
@@ -66,8 +55,8 @@ class RemoteCmdError(DvcException):
 
 class RemoteActionNotImplemented(DvcException):
     def __init__(self, action, scheme):
-        m = "{} is not supported by {} remote".format(action, scheme)
-        super(RemoteActionNotImplemented, self).__init__(m)
+        m = "{} is not supported for {} remotes".format(action, scheme)
+        super().__init__(m)
 
 
 class RemoteMissingDepsError(DvcException):
@@ -75,10 +64,9 @@ class RemoteMissingDepsError(DvcException):
 
 
 class DirCacheError(DvcException):
-    def __init__(self, checksum, cause=None):
-        super(DirCacheError, self).__init__(
-            "Failed to load dir cache for checksum: '{}'.".format(checksum),
-            cause=cause,
+    def __init__(self, checksum):
+        super().__init__(
+            "Failed to load dir cache for hash value: '{}'.".format(checksum)
         )
 
 
@@ -92,6 +80,8 @@ class RemoteBASE(object):
     CHECKSUM_DIR_SUFFIX = ".dir"
     CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
     DEFAULT_CACHE_TYPES = ["copy"]
+    DEFAULT_NO_TRAVERSE = True
+    DEFAULT_VERIFY = False
 
     state = StateNoop()
 
@@ -99,14 +89,15 @@ class RemoteBASE(object):
         self.repo = repo
 
         self._check_requires(config)
-
-        core = config.get(Config.SECTION_CORE, {})
-        self.checksum_jobs = core.get(
-            Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
-        )
+        self.checksum_jobs = self._get_checksum_jobs(config)
 
         self.protected = False
-        self.no_traverse = config.get(Config.SECTION_REMOTE_NO_TRAVERSE, True)
+        self.no_traverse = config.get(
+            Config.SECTION_REMOTE_NO_TRAVERSE, self.DEFAULT_NO_TRAVERSE
+        )
+        self.verify = config.get(
+            Config.SECTION_REMOTE_VERIFY, self.DEFAULT_VERIFY
+        )
         self._dir_info = {}
 
         types = config.get(Config.SECTION_CACHE_TYPE, None)
@@ -155,6 +146,19 @@ class RemoteBASE(object):
         ).format(url, missing, " ".join(missing), self.scheme)
         raise RemoteMissingDepsError(msg)
 
+    def _get_checksum_jobs(self, config):
+        checksum_jobs = config.get(Config.SECTION_REMOTE_CHECKSUM_JOBS)
+        if checksum_jobs:
+            return checksum_jobs
+
+        if self.repo:
+            core = self.repo.config.config.get(Config.SECTION_CORE, {})
+            return core.get(
+                Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
+            )
+
+        return self.CHECKSUM_JOBS
+
     def __repr__(self):
         return "{class_name}: '{path_info}'".format(
             class_name=type(self).__name__,
@@ -163,7 +167,7 @@ class RemoteBASE(object):
 
     @classmethod
     def supported(cls, config):
-        if isinstance(config, basestring):
+        if isinstance(config, (str, bytes)):
             url = config
         else:
             url = config[Config.SECTION_REMOTE_URL]
@@ -187,7 +191,7 @@ class RemoteBASE(object):
                 tasks,
                 total=len(file_infos),
                 unit="md5",
-                desc="Computing hashes (only done once)",
+                desc="Computing file/dir hashes (only done once)",
             ) as tasks:
                 checksums = dict(zip(file_infos, tasks))
         return checksums
@@ -230,6 +234,9 @@ class RemoteBASE(object):
         return sorted(result, key=itemgetter(self.PARAM_RELPATH))
 
     def get_dir_checksum(self, path_info):
+        if not self.cache:
+            raise RemoteCacheRequiredError(path_info)
+
         dir_info = self._collect_dir(path_info)
         checksum, tmp_info = self._get_dir_info_checksum(dir_info)
         new_info = self.cache.checksum_to_path_info(checksum)
@@ -276,11 +283,13 @@ class RemoteBASE(object):
             with self.cache.open(path_info, "r") as fobj:
                 d = json.load(fobj)
         except (ValueError, FileNotFoundError) as exc:
-            raise DirCacheError(checksum, cause=exc)
+            raise DirCacheError(checksum) from exc
 
         if not isinstance(d, list):
-            msg = "dir cache file format error '{}' [skipping the file]"
-            logger.error(msg.format(relpath(path_info)))
+            logger.error(
+                "dir cache file format error '{}' [skipping the file]",
+                path_info,
+            )
             return []
 
         for info in d:
@@ -333,49 +342,48 @@ class RemoteBASE(object):
 
         A file is considered changed if:
             - It doesn't exist on the working directory (was unlinked)
-            - Checksum is not computed (saving a new file)
-            - The checkusm stored in the State is different from the given one
+            - Hash value is not computed (saving a new file)
+            - The hash value stored is different from the given one
             - There's no file in the cache
 
         Args:
             path_info: dict with path information.
-            checksum: expected checksum for this data.
+            checksum: expected hash value for this data.
 
         Returns:
             bool: True if data has changed, False otherwise.
         """
 
         logger.debug(
-            "checking if '{}'('{}') has changed.".format(
-                path_info, checksum_info
-            )
+            "checking if '{}'('{}') has changed.", path_info, checksum_info
         )
 
         if not self.exists(path_info):
-            logger.debug("'{}' doesn't exist.".format(path_info))
+            logger.debug("'{}' doesn't exist.", path_info)
             return True
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
         if checksum is None:
-            logger.debug("checksum for '{}' is missing.".format(path_info))
+            logger.debug("hash value for '{}' is missing.", path_info)
             return True
 
         if self.changed_cache(checksum):
             logger.debug(
-                "cache for '{}'('{}') has changed.".format(path_info, checksum)
+                "cache for '{}'('{}') has changed.", path_info, checksum
             )
             return True
 
         actual = self.get_checksum(path_info)
         if checksum != actual:
             logger.debug(
-                "checksum '{}'(actual '{}') for '{}' has changed.".format(
-                    checksum, actual, path_info
-                )
+                "hash value '{}' for '{}' has changed (actual '{}').",
+                checksum,
+                actual,
+                path_info,
             )
             return True
 
-        logger.debug("'{}' hasn't changed.".format(path_info))
+        logger.debug("'{}' hasn't changed.", path_info)
         return False
 
     def link(self, from_info, to_info):
@@ -388,18 +396,30 @@ class RemoteBASE(object):
 
         self._try_links(from_info, to_info, link_types)
 
+    def _verify_link(self, path_info, link_type):
+        if self.cache_type_confirmed:
+            return
+
+        is_link = getattr(self, "is_{}".format(link_type), None)
+        if is_link and not is_link(path_info):
+            self.remove(path_info)
+            raise DvcException("failed to verify {}".format(link_type))
+
+        self.cache_type_confirmed = True
+
     @slow_link_guard
     def _try_links(self, from_info, to_info, link_types):
         while link_types:
             link_method = getattr(self, link_types[0])
             try:
                 self._do_link(from_info, to_info, link_method)
-                self.cache_type_confirmed = True
+                self._verify_link(to_info, link_types[0])
                 return
 
             except DvcException as exc:
-                msg = "Cache type '{}' is not supported: {}"
-                logger.debug(msg.format(link_types[0], str(exc)))
+                logger.debug(
+                    "Cache type '{}' is not supported: {}", link_types[0], exc
+                )
                 del link_types[0]
 
         raise DvcException("no possible cache types left to try out.")
@@ -413,13 +433,13 @@ class RemoteBASE(object):
         if self.protected:
             self.protect(to_info)
 
-        msg = "Created {}'{}': {} -> {}".format(
+        logger.debug(
+            "Created {}'{}': {} -> {}",
             "protected " if self.protected else "",
             self.cache_types[0],
             from_info,
             to_info,
         )
-        logger.debug(msg)
 
     def _save_file(self, path_info, checksum, save_link=True):
         assert checksum
@@ -473,7 +493,9 @@ class RemoteBASE(object):
         cache_info = self.checksum_to_path_info(checksum)
         dir_info = self.get_dir_cache(checksum)
 
-        for entry in dir_info:
+        for entry in Tqdm(
+            dir_info, desc="Saving " + path_info.name, unit="file"
+        ):
             entry_info = path_info / entry[self.PARAM_RELPATH]
             entry_checksum = entry[self.PARAM_CHECKSUM]
             self._save_file(entry_info, entry_checksum, save_link=False)
@@ -521,12 +543,23 @@ class RemoteBASE(object):
 
     def _save(self, path_info, checksum):
         to_info = self.checksum_to_path_info(checksum)
-        logger.debug("Saving '{}' to '{}'.".format(path_info, to_info))
+        logger.debug("Saving '{}' to '{}'.", path_info, to_info)
         if self.isdir(path_info):
             self._save_dir(path_info, checksum)
             return
         self._save_file(path_info, checksum)
 
+    def _handle_transfer_exception(
+        self, from_info, to_info, exception, operation
+    ):
+        if isinstance(exception, OSError) and exception.errno == errno.EMFILE:
+            raise exception
+
+        logger.exception(
+            "failed to {} '{}' to '{}'", operation, from_info, to_info
+        )
+        return 1
+
     def upload(self, from_info, to_info, name=None, no_progress_bar=False):
         if not hasattr(self, "_upload"):
             raise RemoteActionNotImplemented("upload", self.scheme)
@@ -537,7 +570,7 @@ class RemoteBASE(object):
         if from_info.scheme != "local":
             raise NotImplementedError
 
-        logger.debug("Uploading '{}' to '{}'".format(from_info, to_info))
+        logger.debug("Uploading '{}' to '{}'", from_info, to_info)
 
         name = name or from_info.name
 
@@ -548,10 +581,10 @@ class RemoteBASE(object):
                 name=name,
                 no_progress_bar=no_progress_bar,
             )
-        except Exception:
-            msg = "failed to upload '{}' to '{}'"
-            logger.exception(msg.format(from_info, to_info))
-            return 1  # 1 fail
+        except Exception as e:
+            return self._handle_transfer_exception(
+                from_info, to_info, e, "upload"
+            )
 
         return 0
 
@@ -577,21 +610,58 @@ class RemoteBASE(object):
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
+        if self.isdir(from_info):
+            return self._download_dir(
+                from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+            )
+        return self._download_file(
+            from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+        )
 
+    def _download_dir(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        from_infos = list(self.walk_files(from_info))
+        to_infos = (
+            to_info / info.relative_to(from_info) for info in from_infos
+        )
+
+        with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
+            download_files = partial(
+                self._download_file,
+                name=name,
+                no_progress_bar=True,
+                file_mode=file_mode,
+                dir_mode=dir_mode,
+            )
+            futures = executor.map(download_files, from_infos, to_infos)
+            with Tqdm(
+                futures,
+                total=len(from_infos),
+                desc="Downloading directory",
+                unit="Files",
+                disable=no_progress_bar,
+            ) as futures:
+                return sum(futures)
+
+    def _download_file(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
+
+        logger.debug("Downloading '{}' to '{}'", from_info, to_info)
         name = name or to_info.name
 
-        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
         tmp_file = tmp_fname(to_info)
 
         try:
             self._download(
                 from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
             )
-        except Exception:
-            msg = "failed to download '{}' to '{}'"
-            logger.exception(msg.format(from_info, to_info))
-            return 1  # 1 fail
+        except Exception as e:
+            return self._handle_transfer_exception(
+                from_info, to_info, e, "download"
+            )
 
         move(tmp_file, to_info, mode=file_mode)
 
@@ -663,6 +733,8 @@ class RemoteBASE(object):
             if checksum in used:
                 continue
             path_info = self.checksum_to_path_info(checksum)
+            if self.is_dir_checksum(checksum):
+                self._remove_unpacked_dir(checksum)
             self.remove(path_info)
             removed = True
         return removed
@@ -683,9 +755,10 @@ class RemoteBASE(object):
         actual = self.get_checksum(cache_info)
 
         logger.debug(
-            "cache '{}' expected '{}' actual '{}'".format(
-                str(cache_info), checksum, actual
-            )
+            "cache '{}' expected '{}' actual '{}'",
+            cache_info,
+            checksum,
+            actual,
         )
 
         if not checksum or not actual:
@@ -695,29 +768,41 @@ class RemoteBASE(object):
             return False
 
         if self.exists(cache_info):
-            logger.warning("corrupted cache file '{}'.".format(cache_info))
+            logger.warning("corrupted cache file '{}'.", cache_info)
             self.remove(cache_info)
 
         return True
 
-    def _changed_dir_cache(self, checksum):
+    def _changed_dir_cache(self, checksum, path_info=None, filter_info=None):
         if self.changed_cache_file(checksum):
             return True
 
-        if not self._changed_unpacked_dir(checksum):
+        if not (path_info and filter_info) and not self._changed_unpacked_dir(
+            checksum
+        ):
             return False
 
         for entry in self.get_dir_cache(checksum):
             entry_checksum = entry[self.PARAM_CHECKSUM]
+
+            if path_info and filter_info:
+                entry_info = path_info / entry[self.PARAM_RELPATH]
+                if not entry_info.isin_or_eq(filter_info):
+                    continue
+
             if self.changed_cache_file(entry_checksum):
                 return True
 
-        self._update_unpacked_dir(checksum)
+        if not (path_info and filter_info):
+            self._update_unpacked_dir(checksum)
+
         return False
 
-    def changed_cache(self, checksum):
+    def changed_cache(self, checksum, path_info=None, filter_info=None):
         if self.is_dir_checksum(checksum):
-            return self._changed_dir_cache(checksum)
+            return self._changed_dir_cache(
+                checksum, path_info=path_info, filter_info=filter_info
+            )
         return self.changed_cache_file(checksum)
 
     def cache_exists(self, checksums, jobs=None, name=None):
@@ -793,8 +878,9 @@ class RemoteBASE(object):
         """The file is changed we need to checkout a new copy"""
         cache_info = self.checksum_to_path_info(checksum)
         if self.exists(path_info):
-            msg = "data '{}' exists. Removing before checkout."
-            logger.warning(msg.format(str(path_info)))
+            logger.warning(
+                "data '{}' exists. Removing before checkout.", path_info
+            )
             self.safe_remove(path_info, force=force)
 
         self.link(cache_info, path_info)
@@ -810,7 +896,13 @@ class RemoteBASE(object):
         pass
 
     def _checkout_dir(
-        self, path_info, checksum, force, progress_callback=None
+        self,
+        path_info,
+        checksum,
+        force,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
     ):
         # Create dir separately so that dir is created
         # even if there are no files in it
@@ -819,7 +911,7 @@ class RemoteBASE(object):
 
         dir_info = self.get_dir_cache(checksum)
 
-        logger.debug("Linking directory '{}'.".format(path_info))
+        logger.debug("Linking directory '{}'.", path_info)
 
         for entry in dir_info:
             relative_path = entry[self.PARAM_RELPATH]
@@ -827,10 +919,12 @@ class RemoteBASE(object):
             entry_cache_info = self.checksum_to_path_info(entry_checksum)
             entry_info = path_info / relative_path
 
+            if filter_info and not entry_info.isin_or_eq(filter_info):
+                continue
+
             entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
-            if self.changed(entry_info, entry_checksum_info):
-                if self.exists(entry_info):
-                    self.safe_remove(entry_info, force=force)
+            if relink or self.changed(entry_info, entry_checksum_info):
+                self.safe_remove(entry_info, force=force)
                 self.link(entry_cache_info, entry_info)
                 self.state.save(entry_info, entry_checksum)
             if progress_callback:
@@ -852,7 +946,13 @@ class RemoteBASE(object):
             self.safe_remove(path, force)
 
     def checkout(
-        self, path_info, checksum_info, force=False, progress_callback=None
+        self,
+        path_info,
+        checksum_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
     ):
         if path_info.scheme not in ["local", self.scheme]:
             raise NotImplementedError
@@ -862,55 +962,76 @@ class RemoteBASE(object):
         skip = False
         if not checksum:
             logger.warning(
-                "No checksum info found for '{}'. "
-                "It won't be created.".format(str(path_info))
+                "No file hash info found for '{}'. " "It won't be created.",
+                path_info,
             )
             self.safe_remove(path_info, force=force)
             failed = path_info
 
-        elif not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            logger.debug(msg.format(str(path_info)))
+        elif not relink and not self.changed(path_info, checksum_info):
+            logger.debug("Data '{}' didn't change.", path_info)
             skip = True
 
-        elif self.changed_cache(checksum):
-            msg = "Cache '{}' not found. File '{}' won't be created."
-            logger.warning(msg.format(checksum, str(path_info)))
+        elif self.changed_cache(
+            checksum, path_info=path_info, filter_info=filter_info
+        ):
+            logger.warning(
+                "Cache '{}' not found. File '{}' won't be created.",
+                checksum,
+                path_info,
+            )
             self.safe_remove(path_info, force=force)
             failed = path_info
 
         if failed or skip:
             if progress_callback:
                 progress_callback(
-                    str(path_info), self.get_files_number(checksum)
+                    str(path_info),
+                    self.get_files_number(
+                        self.path_info, checksum, filter_info
+                    ),
                 )
             return failed
 
-        msg = "Checking out '{}' with cache '{}'."
-        logger.debug(msg.format(str(path_info), checksum))
+        logger.debug("Checking out '{}' with cache '{}'.", path_info, checksum)
 
-        self._checkout(path_info, checksum, force, progress_callback)
-        return None
+        self._checkout(
+            path_info, checksum, force, progress_callback, relink, filter_info
+        )
 
     def _checkout(
-        self, path_info, checksum, force=False, progress_callback=None
+        self,
+        path_info,
+        checksum,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
     ):
         if not self.is_dir_checksum(checksum):
             return self._checkout_file(
                 path_info, checksum, force, progress_callback=progress_callback
             )
         return self._checkout_dir(
-            path_info, checksum, force, progress_callback=progress_callback
+            path_info, checksum, force, progress_callback, relink, filter_info
         )
 
-    def get_files_number(self, checksum):
+    def get_files_number(self, path_info, checksum, filter_info):
+        from funcy.py3 import ilen
+
         if not checksum:
             return 0
 
-        if self.is_dir_checksum(checksum):
+        if not self.is_dir_checksum(checksum):
+            return 1
+
+        if not filter_info:
             return len(self.get_dir_cache(checksum))
 
-        return 1
+        return ilen(
+            filter_info.isin_or_eq(path_info / entry[self.PARAM_CHECKSUM])
+            for entry in self.get_dir_cache(checksum)
+        )
 
     @staticmethod
     def unprotect(path_info):
@@ -929,3 +1050,6 @@ class RemoteBASE(object):
 
     def _update_unpacked_dir(self, checksum):
         pass
+
+    def _remove_unpacked_dir(self, checksum):
+        pass

dvc/remote/config.py

@@ -1,11 +1,10 @@
 import logging
 import os
 import posixpath
+from urllib.parse import urlparse
 
-from dvc.config import Config
-from dvc.config import ConfigError
+from dvc.config import Config, ConfigError
 from dvc.utils import relpath
-from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -34,7 +33,7 @@ class RemoteConfig(object):
 
         if settings is None:
             raise ConfigError(
-                "unable to find remote section '{}'".format(name)
+                "Unable to find remote section '{}'".format(name)
             )
 
         parsed = urlparse(settings["url"])
@@ -105,7 +104,6 @@ class RemoteConfig(object):
             level=level,
         )
         if default:
-            logger.info("Setting '{}' as a default remote.".format(name))
             self.config.set(
                 Config.SECTION_CORE,
                 Config.SECTION_CORE_REMOTE,
@@ -135,6 +133,9 @@ class RemoteConfig(object):
                 break
 
     def modify(self, name, option, value, level=None):
+        # try to get remote settings to verify that section exists
+        self.get_settings(name)
+
         self.config.set(
             Config.SECTION_REMOTE_FMT.format(name), option, value, level=level
         )
@@ -156,3 +157,7 @@ class RemoteConfig(object):
         return self.config.get(
             Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level
         )
+
+    def has_default(self):
+        core = self.config.config[Config.SECTION_CORE]
+        return bool(core.get(Config.SECTION_CORE_REMOTE))

dvc/remote/gdrive.py

@@ -1,9 +1,8 @@
-from __future__ import unicode_literals
-
 import os
 import posixpath
 import logging
 import threading
+import re
 
 from funcy import retry, compose, decorator, wrap_with
 from funcy.py3 import cat
@@ -14,31 +13,37 @@ from dvc.path_info import CloudURLInfo
 from dvc.remote.base import RemoteBASE
 from dvc.config import Config
 from dvc.exceptions import DvcException
-from dvc.utils import tmp_fname
+from dvc.utils import tmp_fname, format_link
 
 logger = logging.getLogger(__name__)
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
 
 
 class GDriveRetriableError(DvcException):
-    def __init__(self, msg):
-        super(GDriveRetriableError, self).__init__(msg)
+    pass
+
+
+class GDriveAccessTokenRefreshError(DvcException):
+    pass
+
+
+class GDriveMissedCredentialKeyError(DvcException):
+    pass
 
 
 @decorator
 def _wrap_pydrive_retriable(call):
-    from apiclient import errors
-    from pydrive.files import ApiRequestError
+    from pydrive2.files import ApiRequestError
 
     try:
         result = call()
-    except (ApiRequestError, errors.HttpError) as exception:
+    except ApiRequestError as exception:
         retry_codes = ["403", "500", "502", "503", "504"]
         if any(
             "HttpError {}".format(code) in str(exception)
             for code in retry_codes
         ):
-            raise GDriveRetriableError(msg="Google API request failed")
+            raise GDriveRetriableError("Google API request failed")
         raise
     return result
 
@@ -55,14 +60,26 @@ gdrive_retry = compose(
 class RemoteGDrive(RemoteBASE):
     scheme = Schemes.GDRIVE
     path_cls = CloudURLInfo
-    REQUIRES = {"pydrive": "pydrive"}
+    REQUIRES = {"pydrive2": "pydrive2"}
+    DEFAULT_NO_TRAVERSE = False
+    DEFAULT_VERIFY = True
+
     GDRIVE_USER_CREDENTIALS_DATA = "GDRIVE_USER_CREDENTIALS_DATA"
-    DEFAULT_USER_CREDENTIALS_FILE = ".dvc/tmp/gdrive-user-credentials.json"
+    DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"
 
     def __init__(self, repo, config):
-        super(RemoteGDrive, self).__init__(repo, config)
-        self.no_traverse = False
+        super().__init__(repo, config)
         self.path_info = self.path_cls(config[Config.SECTION_REMOTE_URL])
+
+        bucket = re.search(
+            "{}://(.*)".format(self.scheme),
+            config[Config.SECTION_REMOTE_URL],
+            re.IGNORECASE,
+        )
+        self.bucket = (
+            bucket.group(1).split("/")[0] if bucket else self.path_info.bucket
+        )
+
         self.config = config
         self.init_drive()
 
@@ -75,23 +92,27 @@ class RemoteGDrive(RemoteBASE):
             raise DvcException(
                 "Please specify Google Drive's client id and "
                 "secret in DVC's config. Learn more at "
-                "https://man.dvc.org/remote/add."
+                "{}.".format(format_link("https://man.dvc.org/remote/add"))
             )
         self.gdrive_user_credentials_path = (
-            tmp_fname(".dvc/tmp/")
+            tmp_fname(os.path.join(self.repo.tmp_dir, ""))
             if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
             else self.config.get(
                 Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE,
-                self.DEFAULT_USER_CREDENTIALS_FILE,
+                os.path.join(
+                    self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE
+                ),
             )
         )
+        self.remote_drive_id = None
 
     @gdrive_retry
     def gdrive_upload_file(
         self, args, no_progress_bar=True, from_file="", progress_name=""
     ):
+        parent = {"id": args["parent_id"]}
         item = self.drive.CreateFile(
-            {"title": args["title"], "parents": [{"id": args["parent_id"]}]}
+            {"title": args["title"], "parents": [parent]}
         )
 
         with open(from_file, "rb") as fobj:
@@ -114,7 +135,9 @@ class RemoteGDrive(RemoteBASE):
     def gdrive_download_file(
         self, file_id, to_file, progress_name, no_progress_bar
     ):
-        gdrive_file = self.drive.CreateFile({"id": file_id})
+        param = {"id": file_id}
+        # it does not create a file on the remote
+        gdrive_file = self.drive.CreateFile(param)
         bar_format = (
             "Donwloading {desc:{ncols_desc}.{ncols_desc}}... "
             + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
@@ -125,7 +148,12 @@ class RemoteGDrive(RemoteBASE):
             gdrive_file.GetContentFile(to_file)
 
     def gdrive_list_item(self, query):
-        file_list = self.drive.ListFile({"q": query, "maxResults": 1000})
+        param = {"q": query, "maxResults": 1000, "corpora": self.corpora}
+
+        if self.remote_drive_id:
+            param["driveId"] = self.remote_drive_id
+
+        file_list = self.drive.ListFile(param)
 
         # Isolate and decorate fetching of remote drive items in pages
         get_list = gdrive_retry(lambda: next(file_list, None))
@@ -137,7 +165,7 @@ class RemoteGDrive(RemoteBASE):
         cached_dirs = {}
         cached_ids = {}
         for dir1 in self.gdrive_list_item(
-            "'{}' in parents and trashed=false".format(self.root_id)
+            "'{}' in parents and trashed=false".format(self.remote_root_id)
         ):
             remote_path = posixpath.join(self.path_info.path, dir1["title"])
             cached_dirs.setdefault(remote_path, []).append(dir1["id"])
@@ -156,12 +184,20 @@ class RemoteGDrive(RemoteBASE):
             self.drive
         return self._cached_ids
 
+    @property
+    def corpora(self):
+        if not hasattr(self, "_corpora"):
+            self.drive
+        return self._corpora
+
     @property
     @wrap_with(threading.RLock())
     def drive(self):
+        from pydrive2.auth import RefreshError
+
         if not hasattr(self, "_gdrive"):
-            from pydrive.auth import GoogleAuth
-            from pydrive.drive import GoogleDrive
+            from pydrive2.auth import GoogleAuth
+            from pydrive2.drive import GoogleDrive
 
             if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
                 with open(
@@ -188,31 +224,53 @@ class RemoteGDrive(RemoteBASE):
             GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
             GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
                 "https://www.googleapis.com/auth/drive",
+                # drive.appdata grants access to appDataFolder GDrive directory
                 "https://www.googleapis.com/auth/drive.appdata",
             ]
 
             # Pass non existent settings path to force DEFAULT_SETTINGS loading
             gauth = GoogleAuth(settings_file="")
-            gauth.CommandLineAuth()
 
-            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
-                os.remove(self.gdrive_user_credentials_path)
+            try:
+                gauth.CommandLineAuth()
+            except RefreshError as exc:
+                raise GDriveAccessTokenRefreshError(
+                    "Google Drive's access token refreshment is failed"
+                ) from exc
+            except KeyError as exc:
+                raise GDriveMissedCredentialKeyError(
+                    "Google Drive's user credentials file '{}' "
+                    "misses value for key '{}'".format(
+                        self.gdrive_user_credentials_path, str(exc)
+                    )
+                )
+            # Handle pydrive2.auth.AuthenticationError and others auth failures
+            except Exception as exc:
+                raise DvcException(
+                    "Google Drive authentication failed"
+                ) from exc
+            finally:
+                if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                    os.remove(self.gdrive_user_credentials_path)
 
             self._gdrive = GoogleDrive(gauth)
 
-            self.root_id = self.get_remote_id(self.path_info, create=True)
+            if self.bucket != "root" and self.bucket != "appDataFolder":
+                self.remote_drive_id = self.get_remote_drive_id(self.bucket)
+            self._corpora = "drive" if self.remote_drive_id else "default"
+            self.remote_root_id = self.get_remote_id(
+                self.path_info, create=True
+            )
+
             self._cached_dirs, self._cached_ids = self.cache_root_dirs()
 
         return self._gdrive
 
     @gdrive_retry
     def create_remote_dir(self, parent_id, title):
+        parent = {"id": parent_id}
         item = self.drive.CreateFile(
-            {
-                "title": title,
-                "parents": [{"id": parent_id}],
-                "mimeType": FOLDER_MIME_TYPE,
-            }
+            {"title": title, "parents": [parent], "mimeType": FOLDER_MIME_TYPE}
         )
         item.Upload()
         return item
@@ -221,20 +279,39 @@ class RemoteGDrive(RemoteBASE):
     def get_remote_item(self, name, parents_ids):
         if not parents_ids:
             return None
-        query = " or ".join(
-            "'{}' in parents".format(parent_id) for parent_id in parents_ids
+        query = "({})".format(
+            " or ".join(
+                "'{}' in parents".format(parent_id)
+                for parent_id in parents_ids
+            )
         )
 
         query += " and trashed=false and title='{}'".format(name)
 
+        param = {
+            "q": query,
+            # Remote might contain items with duplicated titles
+            "maxResults": 1,
+            "corpora": self.corpora,
+        }
+
+        if self.remote_drive_id:
+            param["driveId"] = self.remote_drive_id
+
         # Limit found remote items count to 1 in response
-        item_list = self.drive.ListFile(
-            {"q": query, "maxResults": 1}
-        ).GetList()
+        item_list = self.drive.ListFile(param).GetList()
         return next(iter(item_list), None)
 
+    @gdrive_retry
+    def get_remote_drive_id(self, remote_id):
+        param = {"id": remote_id}
+        # it does not create a file on the remote
+        item = self.drive.CreateFile(param)
+        item.FetchMetadata("driveId")
+        return item.get("driveId", None)
+
     def resolve_remote_item_from_path(self, path_parts, create):
-        parents_ids = ["root"]
+        parents_ids = [self.bucket]
         current_path = ""
         for path_part in path_parts:
             current_path = posixpath.join(current_path, path_part)
@@ -256,6 +333,10 @@ class RemoteGDrive(RemoteBASE):
         return []
 
     def get_remote_id(self, path_info, create=False):
+        if not path_info.path and path_info.bucket:
+            # Case sensitive base path
+            return self.bucket
+
         remote_ids = self.get_remote_id_from_cache(path_info.path)
 
         if remote_ids:
@@ -291,8 +372,10 @@ class RemoteGDrive(RemoteBASE):
         if not self.cached_ids:
             return
 
-        query = " or ".join(
-            "'{}' in parents".format(dir_id) for dir_id in self.cached_ids
+        query = "({})".format(
+            " or ".join(
+                "'{}' in parents".format(dir_id) for dir_id in self.cached_ids
+            )
         )
 
         query += " and trashed=false"

dvc/remote/gs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals, division
-
 import logging
 from datetime import timedelta
 from functools import wraps
@@ -15,7 +13,6 @@ from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
-from dvc.utils.compat import FileNotFoundError  # skipcq: PYL-W0622
 
 logger = logging.getLogger(__name__)
 
@@ -76,7 +73,7 @@ class RemoteGS(RemoteBASE):
     PARAM_CHECKSUM = "md5"
 
     def __init__(self, repo, config):
-        super(RemoteGS, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL, "gs:///")
         self.path_info = self.path_cls(url)

dvc/remote/hdfs.py

@@ -1,14 +1,12 @@
-from __future__ import unicode_literals
-
-from dvc.utils.compat import FileNotFoundError, urlparse
 import io
 import logging
 import os
 import posixpath
 import re
+import subprocess
 from collections import deque
 from contextlib import closing, contextmanager
-import subprocess
+from urllib.parse import urlparse
 
 from .base import RemoteBASE, RemoteCmdError
 from .pool import get_connection
@@ -26,7 +24,7 @@ class RemoteHDFS(RemoteBASE):
     REQUIRES = {"pyarrow": "pyarrow"}
 
     def __init__(self, repo, config):
-        super(RemoteHDFS, self).__init__(repo, config)
+        super().__init__(repo, config)
         self.path_info = None
         url = config.get(Config.SECTION_REMOTE_URL)
         if not url:
@@ -100,8 +98,14 @@ class RemoteHDFS(RemoteBASE):
             # NOTE: this is how `hadoop fs -cp` works too: it copies through
             # your local machine.
             with hdfs.open(from_info.path, "rb") as from_fobj:
-                with hdfs.open(to_info.path, "wb") as to_fobj:
-                    to_fobj.upload(from_fobj)
+                tmp_info = to_info.parent / tmp_fname(to_info.name)
+                try:
+                    with hdfs.open(tmp_info.path, "wb") as tmp_fobj:
+                        tmp_fobj.upload(from_fobj)
+                    hdfs.rename(tmp_info.path, to_info.path)
+                except Exception:
+                    self.remove(tmp_info)
+                    raise
 
     def remove(self, path_info):
         if path_info.scheme != "hdfs":

dvc/remote/http.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import logging
 import threading
 
@@ -11,7 +9,6 @@ from dvc.exceptions import DvcException, HTTPError
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
-from dvc.utils.compat import open
 
 logger = logging.getLogger(__name__)
 
@@ -25,7 +22,7 @@ class RemoteHTTP(RemoteBASE):
     PARAM_CHECKSUM = "etag"
 
     def __init__(self, repo, config):
-        super(RemoteHTTP, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL)
         self.path_info = self.path_cls(url) if url else None
@@ -104,7 +101,22 @@ class RemoteHTTP(RemoteBASE):
         kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
 
         try:
-            return self._session.request(method, url, **kwargs)
+            res = self._session.request(method, url, **kwargs)
+
+            redirect_no_location = (
+                kwargs["allow_redirects"]
+                and res.status_code in (301, 302)
+                and "location" not in res.headers
+            )
+
+            if redirect_no_location:
+                # AWS s3 doesn't like to add a location header to its redirects
+                # from https://s3.amazonaws.com/<bucket name>/* type URLs.
+                # This should be treated as an error
+                raise requests.exceptions.RequestException
+
+            return res
+
         except requests.exceptions.RequestException:
             raise DvcException("could not perform a {} request".format(method))
 

dvc/remote/https.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from .http import RemoteHTTP
 from dvc.scheme import Schemes
 

dvc/remote/local.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import errno
 import logging
 import os
@@ -21,18 +19,15 @@ from dvc.remote.base import STATUS_MAP
 from dvc.remote.base import STATUS_MISSING
 from dvc.remote.base import STATUS_NEW
 from dvc.scheme import Schemes
+from dvc.scm.tree import is_working_tree
 from dvc.system import System
-from dvc.utils import copyfile
+from dvc.utils.fs import copyfile
 from dvc.utils import file_md5
-from dvc.utils import makedirs
 from dvc.utils import relpath
 from dvc.utils import tmp_fname
-from dvc.utils import walk_files
-from dvc.utils.compat import fspath_py35
-from dvc.utils.compat import open
-from dvc.utils.compat import str
-from dvc.utils.fs import move
-from dvc.utils.fs import remove
+from dvc.compat import fspath_py35
+from dvc.utils.fs import move, makedirs, remove
+from dvc.utils.fs import walk_files
 
 logger = logging.getLogger(__name__)
 
@@ -50,7 +45,7 @@ class RemoteLOCAL(RemoteBASE):
     SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
 
     def __init__(self, repo, config):
-        super(RemoteLOCAL, self).__init__(repo, config)
+        super().__init__(repo, config)
         self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
 
         shared = config.get(Config.SECTION_CACHE_SHARED)
@@ -66,7 +61,7 @@ class RemoteLOCAL(RemoteBASE):
             cwd = config[Config.PRIVATE_CWD]
             cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
 
-        self.path_info = PathInfo(cache_dir) if cache_dir else None
+        self.cache_dir = cache_dir
         self._dir_info = {}
 
     @property
@@ -77,13 +72,17 @@ class RemoteLOCAL(RemoteBASE):
     def cache_dir(self):
         return self.path_info.fspath if self.path_info else None
 
+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.path_info = PathInfo(value) if value else None
+
     @classmethod
     def supported(cls, config):
         return True
 
     def list_cache_paths(self):
         assert self.path_info is not None
-        return walk_files(self.path_info, None)
+        return walk_files(self.path_info)
 
     def get(self, md5):
         if not md5:
@@ -138,7 +137,9 @@ class RemoteLOCAL(RemoteBASE):
         return os.path.getsize(fspath_py35(path_info))
 
     def walk_files(self, path_info):
-        for fname in walk_files(path_info, self.repo.dvcignore):
+        assert is_working_tree(self.repo.tree)
+
+        for fname in self.repo.tree.walk_files(path_info):
             yield PathInfo(fname)
 
     def get_file_checksum(self, path_info):
@@ -164,14 +165,23 @@ class RemoteLOCAL(RemoteBASE):
 
         move(from_info, to_info, mode=mode)
 
-    @staticmethod
-    def copy(from_info, to_info):
-        System.copy(from_info, to_info)
+    def copy(self, from_info, to_info):
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        try:
+            System.copy(from_info, tmp_info)
+            os.rename(fspath_py35(tmp_info), fspath_py35(to_info))
+        except Exception:
+            self.remove(tmp_info)
+            raise
 
     @staticmethod
     def symlink(from_info, to_info):
         System.symlink(from_info, to_info)
 
+    @staticmethod
+    def is_symlink(path_info):
+        return System.is_symlink(path_info)
+
     def hardlink(self, from_info, to_info):
         # If there are a lot of empty files (which happens a lot in datasets),
         # and the cache type is `hardlink`, we might reach link limits and
@@ -198,6 +208,10 @@ class RemoteLOCAL(RemoteBASE):
 
         System.hardlink(from_info, to_info)
 
+    @staticmethod
+    def is_hardlink(path_info):
+        return System.is_hardlink(path_info)
+
     @staticmethod
     def reflink(from_info, to_info):
         System.reflink(from_info, to_info)
@@ -402,7 +416,7 @@ class RemoteLOCAL(RemoteBASE):
     def _unprotect_file(path):
         if System.is_symlink(path) or System.is_hardlink(path):
             logger.debug("Unprotecting '{}'".format(path))
-            tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
+            tmp = os.path.join(os.path.dirname(path), "." + uuid())
 
             # The operations order is important here - if some application
             # would access the file during the process of copyfile then it
@@ -422,7 +436,9 @@ class RemoteLOCAL(RemoteBASE):
         os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
 
     def _unprotect_dir(self, path):
-        for fname in walk_files(path, self.repo.dvcignore):
+        assert is_working_tree(self.repo.tree)
+
+        for fname in self.repo.tree.walk_files(path):
             RemoteLOCAL._unprotect_file(fname)
 
     def unprotect(self, path_info):
@@ -458,6 +474,10 @@ class RemoteLOCAL(RemoteBASE):
         info = self.checksum_to_path_info(checksum)
         return info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
 
+    def _remove_unpacked_dir(self, checksum):
+        path_info = self._get_unpacked_dir_path_info(checksum)
+        self.remove(path_info)
+
     def _path_info_changed(self, path_info):
         if self.exists(path_info) and self.state.get(path_info):
             return False

dvc/remote/oss.py

@@ -1,6 +1,3 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 import logging
 import os
 import threading
@@ -43,7 +40,7 @@ class RemoteOSS(RemoteBASE):
     COPY_POLL_SECONDS = 5
 
     def __init__(self, repo, config):
-        super(RemoteOSS, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL)
         self.path_info = self.path_cls(url) if url else None
@@ -69,9 +66,9 @@ class RemoteOSS(RemoteBASE):
     def oss_service(self):
         import oss2
 
-        logger.debug("URL {}".format(self.path_info))
-        logger.debug("key id {}".format(self.key_id))
-        logger.debug("key secret {}".format(self.key_secret))
+        logger.debug("URL: {}".format(self.path_info))
+        logger.debug("key id: {}".format(self.key_id))
+        logger.debug("key secret: {}".format(self.key_secret))
 
         auth = oss2.Auth(self.key_id, self.key_secret)
         bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)

dvc/remote/s3.py

@@ -1,14 +1,12 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
 
 import logging
 import os
 import threading
 
-from botocore.exceptions import ClientError
 from funcy import cached_property, wrap_prop
 
-from dvc.config import Config
+from dvc.config import Config, ConfigError
 from dvc.exceptions import DvcException
 from dvc.exceptions import ETagMismatchError
 from dvc.path_info import CloudURLInfo
@@ -26,7 +24,7 @@ class RemoteS3(RemoteBASE):
     PARAM_CHECKSUM = "etag"
 
     def __init__(self, repo, config):
-        super(RemoteS3, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL, "s3://")
         self.path_info = self.path_cls(url)
@@ -54,6 +52,8 @@ class RemoteS3(RemoteBASE):
         if self.acl:
             self.extra_args["ACL"] = self.acl
 
+        self._append_aws_grants_to_extra_args(config)
+
         shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
         if shared_creds:
             os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
@@ -87,8 +87,8 @@ class RemoteS3(RemoteBASE):
             obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
         except Exception as exc:
             raise DvcException(
-                "s3://{}/{} does not exist".format(bucket, path), exc
-            )
+                "s3://{}/{} does not exist".format(bucket, path)
+            ) from exc
         return obj
 
     @classmethod
@@ -156,6 +156,7 @@ class RemoteS3(RemoteBASE):
         # the same throughout the transfer, so it means that in order to
         # preserve etag, we need to transfer each part separately, so the
         # object is transfered in the same chunks as it was originally.
+        from boto3.s3.transfer import TransferConfig
 
         obj = cls.get_head_object(s3, from_info.bucket, from_info.path)
         etag = obj["ETag"].strip('"')
@@ -169,7 +170,13 @@ class RemoteS3(RemoteBASE):
             )
         else:
             source = {"Bucket": from_info.bucket, "Key": from_info.path}
-            s3.copy(source, to_info.bucket, to_info.path, ExtraArgs=extra_args)
+            s3.copy(
+                source,
+                to_info.bucket,
+                to_info.path,
+                ExtraArgs=extra_args,
+                Config=TransferConfig(multipart_threshold=size + 1),
+            )
 
         cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
         if etag != cached_etag:
@@ -194,11 +201,7 @@ class RemoteS3(RemoteBASE):
         }
         paginator = self.s3.get_paginator(self.list_objects_api)
         for page in paginator.paginate(**kwargs):
-            contents = page.get("Contents", None)
-            if not contents:
-                continue
-            for item in contents:
-                yield item
+            yield from page.get("Contents", ())
 
     def _list_paths(self, path_info, max_items=None):
         return (
@@ -209,6 +212,8 @@ class RemoteS3(RemoteBASE):
         return self._list_paths(self.path_info)
 
     def isfile(self, path_info):
+        from botocore.exceptions import ClientError
+
         if path_info.path.endswith("/"):
             return False
 
@@ -301,3 +306,34 @@ class RemoteS3(RemoteBASE):
                 continue
 
             yield path_info.replace(path=fname)
+
+    def _append_aws_grants_to_extra_args(self, config):
+        # Keys for extra_args can be one of the following list:
+        # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS
+        """
+          ALLOWED_UPLOAD_ARGS = [
+            'ACL', 'CacheControl', 'ContentDisposition', 'ContentEncoding',
+            'ContentLanguage', 'ContentType', 'Expires', 'GrantFullControl',
+            'GrantRead', 'GrantReadACP', 'GrantWriteACP', 'Metadata',
+            'RequestPayer', 'ServerSideEncryption', 'StorageClass',
+            'SSECustomerAlgorithm', 'SSECustomerKey', 'SSECustomerKeyMD5',
+            'SSEKMSKeyId', 'WebsiteRedirectLocation'
+          ]
+        """
+
+        grants = {
+            Config.SECTION_AWS_GRANT_FULL_CONTROL: "GrantFullControl",
+            Config.SECTION_AWS_GRANT_READ: "GrantRead",
+            Config.SECTION_AWS_GRANT_READ_ACP: "GrantReadACP",
+            Config.SECTION_AWS_GRANT_WRITE_ACP: "GrantWriteACP",
+        }
+
+        for grant_option, extra_args_key in grants.items():
+            if config.get(grant_option, ""):
+                if self.acl:
+                    raise ConfigError(
+                        "`acl` and `grant_*` AWS S3 config options "
+                        "are mutually exclusive"
+                    )
+
+                self.extra_args[extra_args_key] = config.get(grant_option)

dvc/remote/ssh/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import errno
 import getpass
 import io
@@ -9,6 +7,7 @@ import os
 import threading
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import closing, contextmanager
+from urllib.parse import urlparse
 
 from funcy import memoize, wrap_with
 
@@ -19,8 +18,6 @@ from dvc.remote.base import RemoteBASE
 from dvc.remote.pool import get_connection
 from dvc.scheme import Schemes
 from dvc.utils import to_chunks
-from dvc.utils.compat import StringIO
-from dvc.utils.compat import urlparse
 
 logger = logging.getLogger(__name__)
 
@@ -52,7 +49,7 @@ class RemoteSSH(RemoteBASE):
     DEFAULT_CACHE_TYPES = ["copy"]
 
     def __init__(self, repo, config):
-        super(RemoteSSH, self).__init__(repo, config)
+        super().__init__(repo, config)
         url = config.get(Config.SECTION_REMOTE_URL)
         if url:
             parsed = urlparse(url)
@@ -106,7 +103,7 @@ class RemoteSSH(RemoteBASE):
             ssh_config = paramiko.SSHConfig()
             with open(user_config_file) as f:
                 # For whatever reason parsing directly from f is unreliable
-                f_copy = StringIO(f.read())
+                f_copy = io.StringIO(f.read())
                 ssh_config.parse(f_copy)
             user_ssh_config = ssh_config.lookup(hostname)
         return user_ssh_config
@@ -178,7 +175,7 @@ class RemoteSSH(RemoteBASE):
             raise NotImplementedError
 
         with self.ssh(from_info) as ssh:
-            ssh.copy(from_info.path, to_info.path)
+            ssh.atomic_copy(from_info.path, to_info.path)
 
     def symlink(self, from_info, to_info):
         if not from_info.scheme == to_info.scheme == self.scheme:
@@ -263,8 +260,7 @@ class RemoteSSH(RemoteBASE):
     def list_cache_paths(self):
         with self.ssh(self.path_info) as ssh:
             # If we simply return an iterator then with above closes instantly
-            for path in ssh.walk_files(self.path_info.path):
-                yield path
+            yield from ssh.walk_files(self.path_info.path)
 
     def walk_files(self, path_info):
         with self.ssh(path_info) as ssh:

dvc/remote/ssh/connection.py

@@ -3,6 +3,7 @@ import logging
 import os
 import posixpath
 import stat
+from contextlib import suppress
 
 from funcy import cached_property
 
@@ -12,7 +13,6 @@ except ImportError:
     paramiko = None
 
 from dvc.utils import tmp_fname
-from dvc.utils.compat import ignore_file_not_found
 from dvc.progress import Tqdm
 from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteCmdError
@@ -60,13 +60,13 @@ class SSHConnection:
         self._ssh.close()
 
     def st_mode(self, path):
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
             return self.sftp.lstat(path).st_mode
 
         return 0
 
     def getsize(self, path):
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
             return self.sftp.lstat(path).st_size
 
         return 0
@@ -103,14 +103,13 @@ class SSHConnection:
         if tail:
             try:
                 self.sftp.mkdir(path)
-            except IOError as e:
+            except IOError as exc:
                 # Since paramiko errors are very vague we need to recheck
                 # whether it's because path already exists or something else
-                if e.errno == errno.EACCES or not self.exists(path):
+                if exc.errno == errno.EACCES or not self.exists(path):
                     raise DvcException(
-                        "unable to create remote directory '{}'".format(path),
-                        cause=e,
-                    )
+                        "unable to create remote directory '{}'".format(path)
+                    ) from exc
 
     def walk(self, directory, topdown=True):
         # NOTE: original os.walk() implementation [1] with default options was
@@ -123,9 +122,8 @@ class SSHConnection:
             raise DvcException(
                 "couldn't get the '{}' remote directory files list".format(
                     directory
-                ),
-                cause=exc,
-            )
+                )
+            ) from exc
 
         dirs = []
         nondirs = []
@@ -141,8 +139,7 @@ class SSHConnection:
 
         for dname in dirs:
             newpath = posixpath.join(directory, dname)
-            for entry in self.walk(newpath, topdown=topdown):
-                yield entry
+            yield from self.walk(newpath, topdown=topdown)
 
         if not topdown:
             yield directory, dirs, nondirs
@@ -153,22 +150,20 @@ class SSHConnection:
                 yield posixpath.join(root, fname)
 
     def _remove_file(self, path):
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
             self.sftp.remove(path)
 
     def _remove_dir(self, path):
         for root, dirs, files in self.walk(path, topdown=False):
             for fname in files:
-                path = posixpath.join(root, fname)
-                with ignore_file_not_found():
-                    self._remove_file(path)
+                with suppress(FileNotFoundError):
+                    self._remove_file(posixpath.join(root, fname))
 
             for dname in dirs:
-                path = posixpath.join(root, dname)
-                with ignore_file_not_found():
-                    self.sftp.rmdir(dname)
+                with suppress(FileNotFoundError):
+                    self.sftp.rmdir(posixpath.join(root, dname))
 
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
             self.sftp.rmdir(path)
 
     def remove(self, path):
@@ -195,10 +190,10 @@ class SSHConnection:
         try:
             self.sftp.rename(src, dst)
         except OSError:
-            self._atomic_copy(src, dst)
+            self.atomic_copy(src, dst)
             self.remove(src)
 
-    def _atomic_copy(self, src, dst):
+    def atomic_copy(self, src, dst):
         tmp = tmp_fname(dst)
 
         try:

dvc/repo/__init__.py

@@ -1,29 +1,24 @@
-from __future__ import unicode_literals
-
 import logging
 import os
 from contextlib import contextmanager
 from functools import wraps
 from itertools import chain
 
-from funcy import cached_property
+from dvc.ignore import CleanTree
+from dvc.compat import fspath_py35
+
+from funcy import cached_property, cat
 
-from .graph import check_acyclic
-from .graph import get_pipeline
-from .graph import get_pipelines
-from .graph import get_stages
 from dvc.config import Config
-from dvc.exceptions import FileMissingError
-from dvc.exceptions import NotDvcRepoError
-from dvc.exceptions import OutputNotFoundError
-from dvc.ignore import DvcIgnoreFilter
+from dvc.exceptions import (
+    FileMissingError,
+    NotDvcRepoError,
+    OutputNotFoundError,
+)
 from dvc.path_info import PathInfo
 from dvc.remote.base import RemoteActionNotImplemented
-from dvc.utils import relpath
 from dvc.utils.fs import path_isin
-from dvc.utils.compat import FileNotFoundError
-from dvc.utils.compat import fspath_py35
-from dvc.utils.compat import open as _open
+from .graph import check_acyclic, get_pipeline, get_pipelines
 
 
 logger = logging.getLogger(__name__)
@@ -69,13 +64,14 @@ class Repo(object):
 
     def __init__(self, root_dir=None):
         from dvc.state import State
-        from dvc.lock import Lock
+        from dvc.lock import make_lock
         from dvc.scm import SCM
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
         from dvc.repo.metrics import Metrics
         from dvc.scm.tree import WorkingTree
         from dvc.repo.tag import Tag
+        from dvc.utils.fs import makedirs
 
         root_dir = self.find_root(root_dir)
 
@@ -88,10 +84,17 @@ class Repo(object):
 
         self.tree = WorkingTree(self.root_dir)
 
-        self.lock = Lock(
+        self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
+        makedirs(self.tmp_dir, exist_ok=True)
+
+        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        self.lock = make_lock(
             os.path.join(self.dvc_dir, "lock"),
             tmp_dir=os.path.join(self.dvc_dir, "tmp"),
+            hardlink_lock=hardlink_lock,
+            friendly=True,
         )
+
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
         self.state = State(self, self.config.config)
@@ -116,13 +119,13 @@ class Repo(object):
 
     @tree.setter
     def tree(self, tree):
-        self._tree = tree
+        self._tree = tree if isinstance(tree, CleanTree) else CleanTree(tree)
         # Our graph cache is no longer valid, as it was based on the previous
         # tree.
         self._reset()
 
     def __repr__(self):
-        return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
+        return "{}: '{}'".format(self.__class__.__name__, self.root_dir)
 
     @classmethod
     def find_root(cls, root=None):
@@ -162,9 +165,8 @@ class Repo(object):
 
         flist = (
             [self.config.config_local_file, updater.updater_file]
+            + [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir]
             + self.state.files
-            + self.lock.files
-            + updater.lock.files
         )
 
         if path_isin(self.cache.local.cache_dir, self.root_dir):
@@ -183,32 +185,32 @@ class Repo(object):
         G = graph or self.graph
 
         if not target:
-            return get_stages(G)
+            return list(G)
 
         target = os.path.abspath(target)
 
         if recursive and os.path.isdir(target):
-            attrs = nx.get_node_attributes(G, "stage")
-            nodes = [node for node in nx.dfs_postorder_nodes(G)]
-
-            ret = []
-            for node in nodes:
-                stage = attrs[node]
-                if path_isin(stage.path, target):
-                    ret.append(stage)
-            return ret
+            stages = nx.dfs_postorder_nodes(G)
+            return [stage for stage in stages if path_isin(stage.path, target)]
 
         stage = Stage.load(self, target)
         if not with_deps:
             return [stage]
 
-        node = relpath(stage.path, self.root_dir)
-        pipeline = get_pipeline(get_pipelines(G), node)
+        pipeline = get_pipeline(get_pipelines(G), stage)
+        return list(nx.dfs_postorder_nodes(pipeline, stage))
 
-        return [
-            pipeline.node[n]["stage"]
-            for n in nx.dfs_postorder_nodes(pipeline, node)
-        ]
+    def collect_granular(self, target, *args, **kwargs):
+        if not target:
+            return [(stage, None) for stage in self.stages]
+
+        try:
+            (out,) = self.find_outs_by_path(target, strict=False)
+            filter_info = PathInfo(os.path.abspath(target))
+            return [(out.stage, filter_info)]
+        except OutputNotFoundError:
+            stages = self.collect(target, *args, **kwargs)
+            return [(stage, None) for stage in stages]
 
     def used_cache(
         self,
@@ -244,28 +246,24 @@ class Repo(object):
             all_tags=all_tags,
             all_commits=all_commits,
         ):
-            if targets:
-                stages = []
-                for target in targets:
-                    collected = self.collect(
-                        target, recursive=recursive, with_deps=with_deps
-                    )
-                    stages.extend(collected)
-            else:
-                stages = self.stages
+            targets = targets or [None]
 
-            for stage in stages:
-                if stage.is_repo_import:
-                    dep, = stage.deps
-                    cache.external[dep.repo_pair].add(dep.def_path)
-                    continue
+            pairs = cat(
+                self.collect_granular(
+                    target, recursive=recursive, with_deps=with_deps
+                )
+                for target in targets
+            )
 
-                for out in stage.outs:
-                    used_cache = out.get_used_cache(
-                        remote=remote, force=force, jobs=jobs
-                    )
-                    suffix = "({})".format(branch) if branch else ""
-                    cache.update(used_cache, suffix=suffix)
+            suffix = "({})".format(branch) if branch else ""
+            for stage, filter_info in pairs:
+                used_cache = stage.get_used_cache(
+                    remote=remote,
+                    force=force,
+                    jobs=jobs,
+                    filter_info=filter_info,
+                )
+                cache.update(used_cache, suffix=suffix)
 
         return cache
 
@@ -316,15 +314,15 @@ class Repo(object):
         )
 
         G = nx.DiGraph()
-        stages = stages or self.collect_stages()
+        stages = stages or self.stages
         stages = [stage for stage in stages if stage]
         outs = {}
 
         for stage in stages:
             for out in stage.outs:
                 if out.path_info in outs:
-                    stages = [stage.relpath, outs[out.path_info].stage.relpath]
-                    raise OutputDuplicationError(str(out), stages)
+                    dup_stages = [stage, outs[out.path_info].stage]
+                    raise OutputDuplicationError(str(out), dup_stages)
                 outs[out.path_info] = out
 
         for stage in stages:
@@ -337,27 +335,19 @@ class Repo(object):
             stage_path_info = PathInfo(stage.path)
             for p in chain([stage_path_info], stage_path_info.parents):
                 if p in outs:
-                    raise StagePathAsOutputError(stage.wdir, stage.relpath)
+                    raise StagePathAsOutputError(stage, str(outs[p]))
 
         for stage in stages:
-            node = relpath(stage.path, self.root_dir)
-
-            G.add_node(node, stage=stage)
+            G.add_node(stage)
 
             for dep in stage.deps:
                 if dep.path_info is None:
                     continue
 
-                for out in outs:
-                    if (
-                        out == dep.path_info
-                        or dep.path_info.isin(out)
-                        or out.isin(dep.path_info)
-                    ):
-                        dep_stage = outs[out].stage
-                        dep_node = relpath(dep_stage.path, self.root_dir)
-                        G.add_node(dep_node, stage=dep_stage)
-                        G.add_edge(node, dep_node)
+                for out_path_info, out in outs.items():
+                    if out_path_info.overlaps(dep.path_info):
+                        G.add_node(out.stage)
+                        G.add_edge(stage, out.stage)
 
         check_acyclic(G)
 
@@ -371,18 +361,8 @@ class Repo(object):
     def pipelines(self):
         return get_pipelines(self.graph)
 
-    @staticmethod
-    def _filter_out_dirs(dirs, outs, root_dir):
-        def filter_dirs(dname):
-            path = os.path.join(root_dir, dname)
-            for out in outs:
-                if path == os.path.normpath(out):
-                    return False
-            return True
-
-        return list(filter(filter_dirs, dirs))
-
-    def collect_stages(self):
+    @cached_property
+    def stages(self):
         """
         Walks down the root directory looking for Dvcfiles,
         skipping the directories that are related with
@@ -395,41 +375,37 @@ class Repo(object):
         from dvc.stage import Stage
 
         stages = []
-        outs = []
+        outs = set()
 
-        for root, dirs, files in self.tree.walk(
-            self.root_dir, dvcignore=self.dvcignore
-        ):
+        for root, dirs, files in self.tree.walk(self.root_dir):
             for fname in files:
                 path = os.path.join(root, fname)
                 if not Stage.is_valid_filename(path):
                     continue
                 stage = Stage.load(self, path)
-                for out in stage.outs:
-                    if out.scheme == "local":
-                        outs.append(out.fspath + out.sep)
                 stages.append(stage)
 
-            dirs[:] = self._filter_out_dirs(dirs, outs, root)
+                for out in stage.outs:
+                    if out.scheme == "local":
+                        outs.add(out.fspath)
+
+            dirs[:] = [d for d in dirs if os.path.join(root, d) not in outs]
 
         return stages
 
-    @cached_property
-    def stages(self):
-        return get_stages(self.graph)
-
-    def find_outs_by_path(self, path, outs=None, recursive=False):
+    def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
         if not outs:
             outs = [out for stage in self.stages for out in stage.outs]
 
         abs_path = os.path.abspath(path)
-        is_dir = self.tree.isdir(abs_path)
+        path_info = PathInfo(abs_path)
+        match = path_info.__eq__ if strict else path_info.isin_or_eq
 
         def func(out):
-            if out.scheme == "local" and out.fspath == abs_path:
+            if out.scheme == "local" and match(out.path_info):
                 return True
 
-            if is_dir and recursive and out.path_info.isin(abs_path):
+            if recursive and out.path_info.isin(path_info):
                 return True
 
             return False
@@ -442,7 +418,7 @@ class Repo(object):
 
     def find_out_by_relpath(self, relpath):
         path = os.path.join(self.root_dir, relpath)
-        out, = self.find_outs_by_path(path)
+        (out,) = self.find_outs_by_path(path)
         return out
 
     def is_dvc_internal(self, path):
@@ -450,29 +426,30 @@ class Repo(object):
         return self.DVC_DIR in path_parts
 
     @contextmanager
-    def open(self, path, remote=None, mode="r", encoding=None):
+    def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
         cause = None
         try:
-            out, = self.find_outs_by_path(path)
-        except OutputNotFoundError as e:
+            out = self.find_out_by_relpath(path)
+        except OutputNotFoundError as exc:
             out = None
-            cause = e
+            cause = exc
 
         if out and out.use_cache:
             try:
                 with self._open_cached(out, remote, mode, encoding) as fd:
                     yield fd
                 return
-            except FileNotFoundError as e:
-                raise FileMissingError(relpath(path, self.root_dir), cause=e)
+            except FileNotFoundError as exc:
+                raise FileMissingError(path) from exc
 
-        if self.tree.exists(path):
-            with self.tree.open(path, mode, encoding) as fd:
+        abs_path = os.path.join(self.root_dir, path)
+        if os.path.exists(abs_path):
+            with open(abs_path, mode=mode, encoding=encoding) as fd:
                 yield fd
             return
 
-        raise FileMissingError(relpath(path, self.root_dir), cause=cause)
+        raise FileMissingError(path) from cause
 
     def _open_cached(self, out, remote=None, mode="r", encoding=None):
         if out.isdir():
@@ -482,7 +459,7 @@ class Repo(object):
         cache_file = fspath_py35(cache_file)
 
         if os.path.exists(cache_file):
-            return _open(cache_file, mode=mode, encoding=encoding)
+            return open(cache_file, mode=mode, encoding=encoding)
 
         try:
             remote_obj = self.cloud.get_remote(remote)
@@ -493,11 +470,7 @@ class Repo(object):
                 cache_info = out.get_used_cache(remote=remote)
                 self.cloud.pull(cache_info, remote=remote)
 
-            return _open(cache_file, mode=mode, encoding=encoding)
-
-    @cached_property
-    def dvcignore(self):
-        return DvcIgnoreFilter(self.root_dir, self.tree)
+            return open(cache_file, mode=mode, encoding=encoding)
 
     def close(self):
         self.scm.close()

dvc/repo/add.py

@@ -1,18 +1,15 @@
-from __future__ import unicode_literals
-
 import logging
 import os
 
 import colorama
-from six import string_types
 
 from . import locked
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
-from dvc.progress import Tqdm
-from dvc.repo.scm_context import scm_context
-from dvc.stage import Stage
-from dvc.utils import LARGE_DIR_SIZE
-from dvc.utils import walk_files
+from ..exceptions import RecursiveAddingWhileUsingFilename
+from ..output.base import OutputDoesNotExistError
+from ..progress import Tqdm
+from ..repo.scm_context import scm_context
+from ..stage import Stage
+from ..utils import LARGE_DIR_SIZE
 
 logger = logging.getLogger(__name__)
 
@@ -23,11 +20,16 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
     if recursive and fname:
         raise RecursiveAddingWhileUsingFilename()
 
-    if isinstance(targets, string_types):
+    if isinstance(targets, str):
         targets = [targets]
 
     stages_list = []
-    with Tqdm(total=len(targets), desc="Add", unit="file", leave=True) as pbar:
+    num_targets = len(targets)
+    with Tqdm(total=num_targets, desc="Add", unit="file", leave=True) as pbar:
+        if num_targets == 1:
+            # clear unneeded top-level progress bar for single target
+            pbar.bar_format = "Adding..."
+            pbar.refresh()
         for target in targets:
             sub_targets = _find_all_targets(repo, target, recursive)
             pbar.total += len(sub_targets) - 1
@@ -37,7 +39,7 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
                     "You are adding a large directory '{target}' recursively,"
                     " consider tracking it as a whole instead.\n"
                     "{purple}HINT:{nc} Remove the generated DVC-file and then"
-                    " run {cyan}dvc add {target}{nc}".format(
+                    " run `{cyan}dvc add {target}{nc}`".format(
                         purple=colorama.Fore.MAGENTA,
                         cyan=colorama.Fore.CYAN,
                         nc=colorama.Style.RESET_ALL,
@@ -49,17 +51,29 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
 
             repo.check_modified_graph(stages)
 
-            for stage in stages:
-                stage.save()
+            with Tqdm(
+                total=len(stages),
+                desc="Processing",
+                unit="file",
+                disable=True if len(stages) == 1 else None,
+            ) as pbar_stages:
+                for stage in stages:
+                    try:
+                        stage.save()
+                    except OutputDoesNotExistError:
+                        pbar.n -= 1
+                        raise
 
-                if not no_commit:
-                    stage.commit()
+                    if not no_commit:
+                        stage.commit()
 
-                stage.dump()
+                    stage.dump()
+                    pbar_stages.update()
 
             stages_list += stages
-        # remove filled bar bit of progress, leaving stats
-        pbar.bar_format = pbar.BAR_FMT_DEFAULT.replace("|{bar:10}|", " ")
+
+        if num_targets == 1:  # restore bar format for stats
+            pbar.bar_format = pbar.BAR_FMT_DEFAULT
 
     return stages_list
 
@@ -68,7 +82,12 @@ def _find_all_targets(repo, target, recursive):
     if os.path.isdir(target) and recursive:
         return [
             fname
-            for fname in walk_files(target, repo.dvcignore)
+            for fname in Tqdm(
+                repo.tree.walk_files(target),
+                desc="Searching " + target,
+                bar_format=Tqdm.BAR_FMT_NOTOTAL,
+                unit="file",
+            )
             if not repo.is_dvc_internal(fname)
             if not Stage.is_stage_file(fname)
             if not repo.scm.belongs_to_scm(fname)
@@ -80,7 +99,12 @@ def _find_all_targets(repo, target, recursive):
 def _create_stages(repo, targets, fname, pbar=None):
     stages = []
 
-    for out in targets:
+    for out in Tqdm(
+        targets,
+        desc="Creating DVC-files",
+        disable=True if len(targets) < LARGE_DIR_SIZE else None,
+        unit="file",
+    ):
         stage = Stage.create(repo, outs=[out], add=True, fname=fname)
 
         if not stage:

dvc/repo/brancher.py

@@ -4,12 +4,7 @@ from dvc.scm.tree import WorkingTree
 
 
 def brancher(  # noqa: E302
-    self,
-    branches=None,
-    all_branches=False,
-    tags=None,
-    all_tags=False,
-    all_commits=False,
+    self, revs=None, all_branches=False, all_tags=False, all_commits=False
 ):
     """Generator that iterates over specified revisions.
 
@@ -25,12 +20,12 @@ def brancher(  # noqa: E302
             - empty string it there is no branches to iterate over
             - "Working Tree" if there are uncommitted changes in the SCM repo
     """
-    if not any([branches, all_branches, tags, all_tags, all_commits]):
+    if not any([revs, all_branches, all_tags, all_commits]):
         yield ""
         return
 
     saved_tree = self.tree
-    revs = []
+    revs = revs or []
 
     scm = self.scm
 
@@ -41,24 +36,15 @@ def brancher(  # noqa: E302
         revs = scm.list_all_commits()
     else:
         if all_branches:
-            branches = scm.list_branches()
+            revs.extend(scm.list_branches())
 
         if all_tags:
-            tags = scm.list_tags()
+            revs.extend(scm.list_tags())
 
-        if branches is not None:
-            revs.extend(branches)
-
-        if tags is not None:
-            revs.extend(tags)
-
-    # NOTE: it might be a good idea to wrap this loop in try/finally block
-    # to don't leave the tree on some unexpected branch after the
-    # `brancher()`, but this could cause problems on exception handling
-    # code which might expect the tree on which exception was raised to
-    # stay in place. This behavior is a subject to change.
-    for sha, names in group_by(scm.resolve_rev, revs).items():
-        self.tree = scm.get_tree(sha)
-        yield ", ".join(names)
-
-    self.tree = saved_tree
+    try:
+        if revs:
+            for sha, names in group_by(scm.resolve_rev, revs).items():
+                self.tree = scm.get_tree(sha)
+                yield ", ".join(names)
+    finally:
+        self.tree = saved_tree

dvc/repo/checkout.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import logging
 
 from dvc.exceptions import CheckoutError
@@ -20,42 +18,54 @@ def _cleanup_unused_links(repo):
     repo.state.remove_unused_links(used)
 
 
-def get_all_files_numbers(stages):
-    return sum(stage.get_all_files_number() for stage in stages)
+def get_all_files_numbers(pairs):
+    return sum(
+        stage.get_all_files_number(filter_info) for stage, filter_info in pairs
+    )
 
 
 def _checkout(
-    self, targets=None, with_deps=False, force=False, recursive=False
+    self,
+    targets=None,
+    with_deps=False,
+    force=False,
+    relink=False,
+    recursive=False,
 ):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    stages = set()
-
     if not targets:
         targets = [None]
         _cleanup_unused_links(self)
 
+    pairs = set()
     for target in targets:
         try:
-            new = self.collect(
-                target, with_deps=with_deps, recursive=recursive
+            pairs.update(
+                self.collect_granular(
+                    target, with_deps=with_deps, recursive=recursive
+                )
             )
-            stages.update(new)
         except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
             if not target:
                 raise
-            raise CheckoutErrorSuggestGit(target, exc)
+            raise CheckoutErrorSuggestGit(target) from exc
 
-    total = get_all_files_numbers(stages)
+    total = get_all_files_numbers(pairs)
     if total == 0:
         logger.info("Nothing to do")
     failed = []
     with Tqdm(
         total=total, unit="file", desc="Checkout", disable=total == 0
     ) as pbar:
-        for stage in stages:
+        for stage, filter_info in pairs:
             failed.extend(
-                stage.checkout(force=force, progress_callback=pbar.update_desc)
+                stage.checkout(
+                    force=force,
+                    progress_callback=pbar.update_desc,
+                    relink=relink,
+                    filter_info=filter_info,
+                )
             )
     if failed:
         raise CheckoutError(failed)

dvc/repo/destroy.py

@@ -1,8 +1,15 @@
 from dvc.utils.fs import remove
+from . import locked
 
 
-def destroy(self):
-    for stage in self.stages:
+@locked
+def _destroy_stages(repo):
+    for stage in repo.stages:
         stage.remove(remove_outs=False)
 
-    remove(self.dvc_dir)
+
+# NOTE: not locking `destroy`, as `remove` will need to delete `.dvc` dir,
+# which will cause issues on Windows, as `.dvc/lock` will be busy.
+def destroy(repo):
+    _destroy_stages(repo)
+    remove(repo.dvc_dir)

dvc/repo/diff.py

@@ -1,251 +1,76 @@
-from __future__ import unicode_literals
-
 import os
-from errno import ENOENT
 
-import dvc.logger as logger
-from . import locked
-from dvc.scm.base import FileNotInCommitError
-from dvc.scm.git import DIFF_A_REF
-from dvc.scm.git import DIFF_A_TREE
-from dvc.scm.git import DIFF_B_REF
-from dvc.scm.git import DIFF_B_TREE
-from dvc.scm.git import DIFF_EQUAL
-
-
-DIFF_TARGET = "target"
-DIFF_IS_DIR = "is_dir"
-DIFF_OLD_FILE = "old_file"
-DIFF_OLD_CHECKSUM = "old_checksum"
-DIFF_NEW_FILE = "new_file"
-DIFF_NEW_CHECKSUM = "new_checksum"
-DIFF_SIZE = "size_diff"
-DIFF_DEL = "del"
-DIFF_IDENT = "ident"
-DIFF_CHANGE = "changes"
-DIFF_NEW = "new"
-DIFF_MOVE = "moves"
-DIFF_LIST = "diffs"
-DIFF_SIZE_UNKNOWN = "?"
-DIFF_A_OUTPUT = "a_output"
-DIFF_B_OUTPUT = "b_output"
-DIFF_DELETED = "deleted_file"
-DIFF_IS_NEW = "created_file"
-
-
-def _file_not_exists(error, result):
-    if error.errno == ENOENT:
-        result.update({DIFF_SIZE: DIFF_SIZE_UNKNOWN})
-    else:
-        raise error
-
-
-def _extract_dir(self, dir_not_exists, output):
-    """Extract the content of dvc tree file
-    Args:
-        self(object) - Repo class instance
-        dir_not_exists(bool) - flag for directory existence
-        output(object) - OutputLOCAL class instance
-    Returns:
-        dict - dictionary with keys - paths to file in .dvc/cache
-                               values -checksums for that files
-    """
-    if not dir_not_exists:
-        lst = output.dir_cache
-        return {i["relpath"]: i["md5"] for i in lst}
-    return {}
-
-
-def _get_dir_info(dir_not_exists, a_output):
-    if not dir_not_exists:
-        return str(a_output), a_output.checksum
-    return "", ""
-
-
-def _ident_files(a_entries, b_entries):
-    keys = [
-        key for key in a_entries.keys() if b_entries.get(key) == a_entries[key]
-    ]
-    return len(keys)
-
-
-def _modified_files(self, a_entries, b_entries):
-    keys = [key for key in a_entries.keys() if key in b_entries]
-    diff_size = 0
-    modified_count = 0
-    for key in keys:
-        if a_entries[key] != b_entries[key]:
-            modified_count += 1
-            diff_size += os.path.getsize(
-                self.cache.local.get(b_entries[key])
-            ) - os.path.getsize(self.cache.local.get(a_entries[key]))
-    return modified_count, diff_size
-
-
-def _deleted_files(self, a_entries, b_entries):
-    diff_size = 0
-    deleted_count = 0
-    for key, value in a_entries.items():
-        if key not in b_entries:
-            deleted_count += 1
-            diff_size -= os.path.getsize(self.cache.local.get(a_entries[key]))
-    return deleted_count, diff_size
-
-
-def _new_files(self, a_entries, b_entries):
-    diff_size = 0
-    new_count = 0
-    for key, value in b_entries.items():
-        if key not in a_entries:
-            new_count += 1
-            diff_size += os.path.getsize(self.cache.local.get(b_entries[key]))
-    return new_count, diff_size
-
-
-def _get_tree_changes(self, a_entries, b_entries):
-    result = {
-        DIFF_DEL: 0,
-        DIFF_IDENT: 0,
-        DIFF_CHANGE: 0,
-        DIFF_NEW: 0,
-        DIFF_MOVE: 0,
-    }
-    result[DIFF_IDENT] = _ident_files(a_entries, b_entries)
-    result[DIFF_CHANGE], diff_size = _modified_files(
-        self, a_entries, b_entries
-    )
-    result[DIFF_SIZE] = diff_size
-    result[DIFF_DEL], diff_size = _deleted_files(self, a_entries, b_entries)
-    result[DIFF_SIZE] += diff_size
-    result[DIFF_NEW], diff_size = _new_files(self, a_entries, b_entries)
-    result[DIFF_SIZE] += diff_size
-    return result
-
-
-def _check_local_cache(a_out, is_checked):
-    if a_out is not None and a_out.scheme != "local":
-        is_checked.append(str(a_out))
-        return True
-    return False
-
-
-def _is_dir(path, a_outs, b_outs):
-    if a_outs.get(path):
-        return a_outs[path].is_dir_checksum
-    else:
-        return b_outs[path].is_dir_checksum
-
-
-def _get_diff_outs(self, diff_dct):
-    self.tree = diff_dct[DIFF_A_TREE]
-    a_outs = {str(out): out for st in self.stages for out in st.outs}
-    self.tree = diff_dct[DIFF_B_TREE]
-    b_outs = {str(out): out for st in self.stages for out in st.outs}
-    outs_paths = set(a_outs.keys())
-    outs_paths.update(b_outs.keys())
-    results = {}
-    non_local_cache = []
-    for path in outs_paths:
-        check1 = _check_local_cache(a_outs.get(path), non_local_cache)
-        check2 = _check_local_cache(b_outs.get(path), non_local_cache)
-        # skip files/directories with non-local cache for now
-        if check1 or check2:
-            continue
-        results[path] = {}
-        results[path][DIFF_A_OUTPUT] = a_outs.get(path)
-        results[path][DIFF_B_OUTPUT] = b_outs.get(path)
-        results[path][DIFF_IS_NEW] = path not in a_outs
-        results[path][DIFF_DELETED] = path not in b_outs
-        results[path][DIFF_IS_DIR] = _is_dir(path, a_outs, b_outs)
-    if non_local_cache:
-        logger.warning(
-            "Diff is not supported for non-local outputs. Ignoring: {}".format(
-                non_local_cache
-            )
-        )
-
-    return results
-
-
-def _diff_dir(self, target, diff_dct):
-    result = {DIFF_TARGET: target}
-    result[DIFF_IS_DIR] = True
-    a_entries, b_entries = {}, {}
-    try:
-        a_entries = _extract_dir(
-            self, diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
-        )
-        b_entries = _extract_dir(
-            self, diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
-        )
-        result[DIFF_OLD_FILE], result[DIFF_OLD_CHECKSUM] = _get_dir_info(
-            diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
-        )
-        result[DIFF_NEW_FILE], result[DIFF_NEW_CHECKSUM] = _get_dir_info(
-            diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
-        )
-        result.update(_get_tree_changes(self, a_entries, b_entries))
-    except IOError as e:
-        _file_not_exists(e, result)
-    return result
-
-
-def _diff_file(self, target, diff_dct):
-    result = {DIFF_TARGET: target}
-    size = 0
-    try:
-        if not diff_dct[DIFF_IS_NEW]:
-            result[DIFF_OLD_FILE] = target
-            result[DIFF_OLD_CHECKSUM] = diff_dct[DIFF_A_OUTPUT].checksum
-            size -= os.path.getsize(
-                self.cache.local.get(diff_dct[DIFF_A_OUTPUT].checksum)
-            )
-        if not diff_dct[DIFF_DELETED]:
-            result[DIFF_NEW_FILE] = target
-            result[DIFF_NEW_CHECKSUM] = diff_dct[DIFF_B_OUTPUT].checksum
-            size += os.path.getsize(
-                self.cache.local.get(diff_dct[DIFF_B_OUTPUT].checksum)
-            )
-        result[DIFF_SIZE] = size
-    except IOError as e:
-        _file_not_exists(e, result)
-    return result
-
-
-def _diff_royal(self, target, diff_dct):
-    if diff_dct[DIFF_IS_DIR]:
-        return _diff_dir(self, target, diff_dct)
-    return _diff_file(self, target, diff_dct)
+from dvc.exceptions import DvcException
+from dvc.repo import locked
+from dvc.scm.git import Git
 
 
 @locked
-def diff(self, a_ref, target=None, b_ref=None):
-    """Generates diff message string output
-
-    Args:
-        target(str) - file/directory to check diff of
-        a_ref(str) - first tag
-        (optional) b_ref(str) - second git tag
-
-    Returns:
-        string: string of output message with diff info
+def diff(self, a_ref="HEAD", b_ref=None):
     """
-    result = {}
-    diff_dct = self.scm.get_diff_trees(a_ref, b_ref=b_ref)
-    result[DIFF_A_REF] = diff_dct[DIFF_A_REF]
-    result[DIFF_B_REF] = diff_dct[DIFF_B_REF]
-    if diff_dct[DIFF_EQUAL]:
-        result[DIFF_EQUAL] = True
+    By default, it compares the working tree with the last commit's tree.
+
+    This implementation differs from `git diff` since DVC doesn't have
+    the concept of `index`, but it keeps the same interface, thus,
+    `dvc diff` would be the same as `dvc diff HEAD`.
+    """
+    if type(self.scm) is not Git:
+        raise DvcException("only supported for Git repositories")
+
+    def _paths_checksums():
+        """
+        A dictionary of checksums addressed by relpaths collected from
+        the current tree outputs.
+
+        Unpack directories to include their entries
+
+        To help distinguish between a directory and a file output,
+        the former one will come with a trailing slash in the path:
+
+            directory: "data/"
+            file:      "data"
+        """
+        result = {}
+
+        for stage in self.stages:
+            for output in stage.outs:
+                if not output.is_dir_checksum:
+                    result.update({str(output): output.checksum})
+                    continue
+
+                result.update({os.path.join(str(output), ""): output.checksum})
+
+                for entry in output.dir_cache:
+                    path = str(output.path_info / entry["relpath"])
+                    result.update({path: entry["md5"]})
+
         return result
-    result[DIFF_LIST] = []
-    diff_outs = _get_diff_outs(self, diff_dct)
-    if target is None:
-        result[DIFF_LIST] = [
-            _diff_royal(self, path, diff_outs[path]) for path in diff_outs
-        ]
-    elif target in diff_outs:
-        result[DIFF_LIST] = [_diff_royal(self, target, diff_outs[target])]
-    else:
-        msg = "Have not found file/directory '{}' in the commits"
-        raise FileNotInCommitError(msg.format(target))
-    return result
+
+    working_tree = self.tree
+    a_tree = self.scm.get_tree(a_ref)
+    b_tree = self.scm.get_tree(b_ref) if b_ref else working_tree
+
+    try:
+        self.tree = a_tree
+        old = _paths_checksums()
+
+        self.tree = b_tree
+        new = _paths_checksums()
+    finally:
+        self.tree = working_tree
+
+    # Compare paths between the old and new tree.
+    # set() efficiently converts dict keys to a set
+    added = sorted(set(new) - set(old))
+    deleted = sorted(set(old) - set(new))
+    modified = sorted(set(old) & set(new))
+
+    return {
+        "added": [{"path": path, "checksum": new[path]} for path in added],
+        "deleted": [{"path": path, "checksum": old[path]} for path in deleted],
+        "modified": [
+            {"path": path, "checksum": {"old": old[path], "new": new[path]}}
+            for path in modified
+            if old[path] != new[path]
+        ],
+    }