dvc: bump to 0.82.2

import-url: test: prevent in-repo testing

.travis.yml

@@ -13,8 +13,6 @@ stages:
   - check
   - test
   - build
-  - name: deploy
-    if: (branch = master OR tag is present) AND repo = iterative/dvc
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
@@ -35,20 +33,10 @@ jobs:
     install:
     script: ./scripts/ci/check_patch.sh
   # test jobs
-  - name: "2.7 on Windows"
+  - name: "3.7 on Windows"
     stage: test
     os: windows           # Windows 10.0.17134 N/A Build 17134
     language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-    - PY2=1
-  - name: "3.7 on Windows"
-    os: windows           # Windows 10.0.17134 N/A Build 17134
-    language: shell       # 'language: python' is an error on Travis CI Windows
-  - name: "2.7 on Linux"
-    language: python
-    python: 2.7
-    env:
-    - PY2=1
   - name: "3.5 on Linux"
     language: python
     python: 3.5
@@ -58,14 +46,9 @@ jobs:
   - name: "3.7 on Linux"
     language: python
     python: 3.7
-  - name: "2.7 on Mac"
+  - name: "3.7 on Mac"
     os: osx
-    osx_image: xcode9.4
-    env:
-    - PY2=1
-  - name: "3.6 on Mac"
-    os: osx
-    osx_image: xcode9.4
+    osx_image: xcode11.3
   # build jobs
   - name: "Mac pkg"
     stage: build
@@ -86,11 +69,13 @@ jobs:
     before_install:
     install:
     script: ./scripts/build_posix.sh
-  - name: "Pypi pkgs"
+  - name: "PyPI pkgs"
     language: python
     python: 3.7
     script: ./scripts/build_package.sh
   - name: Snapcraft snap
+    language: python
+    python: 3.7
     addons:
      snaps:
      - name: snapcraft
@@ -100,37 +85,14 @@ jobs:
        channel: stable
     env:
     - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}'
-    before_install:
+    install:
     - sudo /snap/bin/lxd.migrate -yes
     - sudo /snap/bin/lxd waitready
     - sudo /snap/bin/lxd init --auto
-    install:
     script:
-    - sudo snapcraft --use-lxd
+    - ./scripts/build_snap.sh
     after_failure:
     - sudo journalctl -u snapd
-  # deploy jobs
-  - name: "Pypi pkgs"
-    if: tag is present
-    stage: deploy
-    language: python
-    python: 3.7
-    script: ./scripts/build_package.sh
-    deploy:
-    - provider: pypi
-      user:
-        secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
-      password:
-        secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
-      skip_existing: true
-      file_glob: true
-      file:
-        - dist/dvc*.whl
-        - dist/dvc*.tar.gz
-      skip_cleanup: true
-      on:
-        tags: true
-        repo: iterative/dvc
 
 before_install:
   - bash ./scripts/ci/before_install.sh
@@ -142,7 +104,6 @@ script:
   - bash ./scripts/ci/script.sh
 deploy:
   - provider: releases
-    condition: "[[ $(./scripts/ci/deploy_condition.sh) == 'true' ]]"
     api_key:
       secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=
     file_glob: true
@@ -151,9 +112,34 @@ deploy:
       - dvc*.deb
       - dvc*.pkg
       - dvc*.exe
-      - dvc*.snap
     skip_cleanup: true
     on:
+      condition: "$(./scripts/ci/deploy_condition.sh dvc*.rpm dvc*.deb dvc*.pkg dvc*.exe)"
       tags: true
       repo: iterative/dvc
       stage: build
+  - provider: pypi
+    user:
+      secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
+    password:
+      secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
+    skip_existing: true
+    file_glob: true
+    file:
+      - dist/dvc*.whl
+      - dist/dvc*.tar.gz
+    skip_cleanup: true
+    on:
+      condition: "$(./scripts/ci/deploy_condition.sh dist/dvc*.whl dist/dvc*.tar.gz)"
+      tags: true
+      repo: iterative/dvc
+      stage: build
+  - provider: snap
+    snap: dvc_*.snap
+    channel: $SNAP_CHANNEL
+    skip_cleanup: true
+    on:
+      all_branches: true
+      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG)"
+      repo: iterative/dvc
+      stage: build

README.rst

@@ -10,9 +10,13 @@
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
-.. image:: https://travis-ci.com/iterative/dvc.svg?branch=master
+.. image:: https://img.shields.io/badge/release-ok-brightgreen
   :target: https://travis-ci.com/iterative/dvc
-  :alt: Travis
+  :alt: Release
+
+.. image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev
+  :target: https://travis-ci.com/iterative/dvc
+  :alt: Travis dev branch
 
 .. image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
   :target: https://codeclimate.com/github/iterative/dvc
@@ -153,17 +157,12 @@ Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
 Snap (Snapcraft)
 ----------------
 
-Download the latest ``dvc_*.snap`` from the
-GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
-
 .. code-block:: bash
 
-   snap install dvc_*.snap --dangerous --classic
+   snap install dvc --classic
 
-Once ``dvc`` is released on the snap store
-(`pending request <https://forum.snapcraft.io/t/classic-confinement-request-for-dvc/14124>`_)
-there will be no need to download ``dvc_*.snap`` or use ``--dangerous``
-(``snap install dvc --classic`` would suffice).
+This corresponds to the latest tagged release.
+Add ``--edge`` for the latest ``master`` version.
 
 Package
 -------

dvc/__init__.py

@@ -3,8 +3,6 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-from __future__ import unicode_literals
-
 import warnings
 
 import dvc.logger

dvc/__main__.py

@@ -1,6 +1,4 @@
-"""Main entry point for dvc command line tool."""
-from __future__ import unicode_literals
-
+"""Main entry point for DVC command line tool."""
 import sys
 
 from dvc.main import main

dvc/analytics.py

@@ -16,8 +16,8 @@ from dvc.exceptions import NotDvcRepoError
 from dvc.lock import Lock, LockError
 from dvc.repo import Repo
 from dvc.scm import SCM
-from dvc.utils import env2bool, is_binary, makedirs
-from dvc.utils.compat import str, FileNotFoundError
+from dvc.utils import env2bool, is_binary
+from dvc.utils.fs import makedirs
 
 
 logger = logging.getLogger(__name__)
@@ -164,4 +164,6 @@ def _find_or_create_user_id():
             return user_id
 
     except LockError:
-        logger.debug("Failed to acquire {lockfile}".format(lockfile=lockfile))
+        logger.debug(
+            "Failed to acquire '{lockfile}'".format(lockfile=lockfile)
+        )

dvc/api.py

@@ -1,27 +1,74 @@
+from builtins import open as builtin_open
+import importlib
 import os
-from contextlib import contextmanager
+import sys
+from contextlib import contextmanager, _GeneratorContextManager as GCM
+import threading
 
-try:
-    from contextlib import _GeneratorContextManager as GCM
-except ImportError:
-    from contextlib import GeneratorContextManager as GCM
+from funcy import wrap_with
+import ruamel.yaml
+from voluptuous import Schema, Required, Invalid
 
-from dvc.utils.compat import urlparse
 from dvc.repo import Repo
+from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.external_repo import external_repo
 
 
+SUMMON_FILE_SCHEMA = Schema(
+    {
+        Required("objects"): [
+            {
+                Required("name"): str,
+                "meta": dict,
+                Required("summon"): {
+                    Required("type"): str,
+                    "deps": [str],
+                    str: object,
+                },
+            }
+        ]
+    }
+)
+SUMMON_PYTHON_SCHEMA = Schema(
+    {
+        Required("type"): "python",
+        Required("call"): str,
+        "args": dict,
+        "deps": [str],
+    }
+)
+
+
+class SummonError(DvcException):
+    pass
+
+
+class UrlNotDvcRepoError(DvcException):
+    """Thrown if given url is not a DVC repository.
+
+    Args:
+        url (str): URL to the repository
+    """
+
+    def __init__(self, url):
+        super().__init__("'{}' is not a DVC repository.".format(url))
+
+
 def get_url(path, repo=None, rev=None, remote=None):
-    """Returns an url of a resource specified by path in repo"""
+    """
+    Returns the full URL to the data artifact specified by its `path` in a
+    `repo`.
+    NOTE: There is no guarantee that the file actually exists in that location.
+    """
     with _make_repo(repo, rev=rev) as _repo:
-        abspath = os.path.join(_repo.root_dir, path)
-        out, = _repo.find_outs_by_path(abspath)
+        _require_dvc(_repo)
+        out = _repo.find_out_by_relpath(path)
         remote_obj = _repo.cloud.get_remote(remote)
         return str(remote_obj.checksum_to_path_info(out.checksum))
 
 
 def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Opens a specified resource as a file descriptor"""
+    """Context manager to open a file artifact as a file object."""
     args = (path,)
     kwargs = {
         "repo": repo,
@@ -40,21 +87,20 @@ class _OpenContextManager(GCM):
 
     def __getattr__(self, name):
         raise AttributeError(
-            "dvc.api.open() should be used in a with statement"
+            "dvc.api.open() should be used in a with statement."
         )
 
 
 def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
     with _make_repo(repo, rev=rev) as _repo:
-        abspath = os.path.join(_repo.root_dir, path)
-        with _repo.open(
-            abspath, remote=remote, mode=mode, encoding=encoding
+        with _repo.open_by_relpath(
+            path, remote=remote, mode=mode, encoding=encoding
         ) as fd:
             yield fd
 
 
 def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Read a specified resource into string"""
+    """Returns the contents of a file artifact."""
     with open(
         path, repo=repo, rev=rev, remote=remote, mode=mode, encoding=encoding
     ) as fd:
@@ -62,10 +108,140 @@ def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
 
 
 @contextmanager
-def _make_repo(repo_url, rev=None):
-    if not repo_url or urlparse(repo_url).scheme == "":
-        assert rev is None, "Custom revision is not supported for local repo"
-        yield Repo(repo_url)
+def _make_repo(repo_url=None, rev=None):
+    repo_url = repo_url or os.getcwd()
+    if rev is None and os.path.exists(repo_url):
+        try:
+            yield Repo(repo_url)
+            return
+        except NotDvcRepoError:
+            pass  # fallthrough to external_repo
+    with external_repo(url=repo_url, rev=rev) as repo:
+        yield repo
+
+
+def summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml", args=None):
+    """Instantiate an object described in the `summon_file`."""
+    with prepare_summon(
+        name, repo=repo, rev=rev, summon_file=summon_file
+    ) as desc:
+        try:
+            summon_dict = SUMMON_PYTHON_SCHEMA(desc.obj["summon"])
+        except Invalid as exc:
+            raise SummonError(str(exc)) from exc
+
+        _args = {**summon_dict.get("args", {}), **(args or {})}
+        return _invoke_method(summon_dict["call"], _args, desc.repo.root_dir)
+
+
+@contextmanager
+def prepare_summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml"):
+    """Does a couple of things every summon needs as a prerequisite:
+    clones the repo, parses the summon file and pulls the deps.
+
+    Calling code is expected to complete the summon logic following
+    instructions stated in "summon" dict of the object spec.
+
+    Returns a SummonDesc instance, which contains references to a Repo object,
+    named object specification and resolved paths to deps.
+    """
+    with _make_repo(repo, rev=rev) as _repo:
+        _require_dvc(_repo)
+        try:
+            path = os.path.join(_repo.root_dir, summon_file)
+            obj = _get_object_spec(name, path)
+            yield SummonDesc(_repo, obj)
+        except SummonError as exc:
+            raise SummonError(
+                str(exc) + " at '{}' in '{}'".format(summon_file, repo)
+            ) from exc.__cause__
+
+
+class SummonDesc:
+    def __init__(self, repo, obj):
+        self.repo = repo
+        self.obj = obj
+        self._pull_deps()
+
+    @property
+    def deps(self):
+        return [os.path.join(self.repo.root_dir, d) for d in self._deps]
+
+    @property
+    def _deps(self):
+        return self.obj["summon"].get("deps", [])
+
+    def _pull_deps(self):
+        if not self._deps:
+            return
+
+        outs = [self.repo.find_out_by_relpath(d) for d in self._deps]
+
+        with self.repo.state:
+            for out in outs:
+                self.repo.cloud.pull(out.get_used_cache())
+                out.checkout()
+
+
+def _get_object_spec(name, path):
+    """
+    Given a summonable object's name, search for it on the given file
+    and return its description.
+    """
+    try:
+        with builtin_open(path, "r") as fobj:
+            content = SUMMON_FILE_SCHEMA(ruamel.yaml.safe_load(fobj.read()))
+            objects = [x for x in content["objects"] if x["name"] == name]
+
+        if not objects:
+            raise SummonError("No object with name '{}'".format(name))
+        elif len(objects) >= 2:
+            raise SummonError(
+                "More than one object with name '{}'".format(name)
+            )
+
+        return objects[0]
+
+    except FileNotFoundError as exc:
+        raise SummonError("Summon file not found") from exc
+    except ruamel.yaml.YAMLError as exc:
+        raise SummonError("Failed to parse summon file") from exc
+    except Invalid as exc:
+        raise SummonError(str(exc)) from exc
+
+
+@wrap_with(threading.Lock())
+def _invoke_method(call, args, path):
+    # XXX: Some issues with this approach:
+    #   * Import will pollute sys.modules
+    #   * sys.path manipulation is "theoretically" not needed,
+    #     but tests are failing for an unknown reason.
+    cwd = os.getcwd()
+
+    try:
+        os.chdir(path)
+        sys.path.insert(0, path)
+        method = _import_string(call)
+        return method(**args)
+    finally:
+        os.chdir(cwd)
+        sys.path.pop(0)
+
+
+def _import_string(import_name):
+    """Imports an object based on a string.
+    Useful to delay import to not load everything on startup.
+    Use dotted notaion in `import_name`, e.g. 'dvc.remote.gs.RemoteGS'.
+
+    :return: imported object
+    """
+    if "." in import_name:
+        module, obj = import_name.rsplit(".", 1)
     else:
-        with external_repo(url=repo_url, rev=rev) as repo:
-            yield repo
+        return importlib.import_module(import_name)
+    return getattr(importlib.import_module(module), obj)
+
+
+def _require_dvc(repo):
+    if not isinstance(repo, Repo):
+        raise UrlNotDvcRepoError(repo.url)

dvc/cache.py

@@ -1,13 +1,10 @@
-"""Manages cache of a dvc repo."""
-from __future__ import unicode_literals
-
+"""Manages cache of a DVC repo."""
 import os
 from collections import defaultdict
 
 from funcy import cached_property
 
 from dvc.config import Config
-from dvc.utils.compat import builtin_str
 
 
 class CacheConfig(object):
@@ -56,12 +53,12 @@ def _make_remote_property(name):
 
         return Remote(self.repo, name=remote)
 
-    getter.__name__ = builtin_str(name)
+    getter.__name__ = name
     return cached_property(getter)
 
 
 class Cache(object):
-    """Class that manages cache locations of a dvc repo.
+    """Class that manages cache locations of a DVC repo.
 
     Args:
         repo (dvc.repo.Repo): repo instance that this cache belongs to.

dvc/cli.py

@@ -1,7 +1,4 @@
-"""Command line interface for dvc."""
-from __future__ import print_function
-from __future__ import unicode_literals
-
+"""DVC command line interface"""
 import argparse
 import logging
 import sys
@@ -173,7 +170,7 @@ def parse_args(argv=None):
         title="Available Commands",
         metavar="COMMAND",
         dest="cmd",
-        help="Use dvc COMMAND --help for command-specific help.",
+        help="Use `dvc COMMAND --help` for command-specific help.",
     )
 
     fix_subparsers(subparsers)

dvc/command/add.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/base.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import logging
 
 
@@ -14,11 +12,8 @@ def fix_subparsers(subparsers):
         Args:
             subparsers: subparsers to fix.
     """
-    from dvc.utils.compat import is_py3
-
-    if is_py3:  # pragma: no cover
-        subparsers.required = True
-        subparsers.dest = "cmd"
+    subparsers.required = True
+    subparsers.dest = "cmd"
 
 
 def append_doc_link(help_message, path):

dvc/command/cache.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 
 from dvc.cache import CacheConfig
@@ -10,7 +8,7 @@ from dvc.command.config import CmdConfig
 
 class CmdCacheDir(CmdConfig):
     def __init__(self, args):
-        super(CmdCacheDir, self).__init__(args)
+        super().__init__(args)
         self.cache_config = CacheConfig(self.config)
 
     def run(self):
@@ -32,7 +30,8 @@ def add_parser(subparsers, parent_parser):
     )
 
     cache_subparsers = cache_parser.add_subparsers(
-        dest="cmd", help="Use dvc cache CMD --help for command-specific help."
+        dest="cmd",
+        help="Use `dvc cache CMD --help` for command-specific " "help.",
     )
 
     fix_subparsers(cache_subparsers)

dvc/command/checkout.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 
 from dvc.command.base import append_doc_link
@@ -28,19 +26,6 @@ def add_parser(subparsers, parent_parser):
         help=CHECKOUT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    checkout_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Do not prompt when removing working directory files.",
-    )
-    checkout_parser.add_argument(
-        "--relink",
-        action="store_true",
-        default=False,
-        help="Recreate links or copies from cache to workspace.",
-    )
     checkout_parser.add_argument(
         "-d",
         "--with-deps",
@@ -55,6 +40,19 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Checkout all subdirectories of the specified directory.",
     )
+    checkout_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Do not prompt when removing working directory files.",
+    )
+    checkout_parser.add_argument(
+        "--relink",
+        action="store_true",
+        default=False,
+        help="Recreate links or copies from cache to workspace.",
+    )
     checkout_parser.add_argument(
         "targets",
         nargs="*",

dvc/command/commit.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/config.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -13,7 +11,7 @@ logger = logging.getLogger(__name__)
 
 class CmdConfig(CmdBaseNoRepo):
     def __init__(self, args):
-        super(CmdConfig, self).__init__(args)
+        super().__init__(args)
 
         self.config = Config(validate=False)
 

dvc/command/daemon.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.command.base import CmdBaseNoRepo
 from dvc.command.base import fix_subparsers
 
@@ -46,7 +44,8 @@ def add_parser(subparsers, parent_parser):
     )
 
     daemon_subparsers = daemon_parser.add_subparsers(
-        dest="cmd", help="Use dvc daemon CMD --help for command-specific help."
+        dest="cmd",
+        help="Use `dvc daemon CMD --help` for command-specific " "help.",
     )
 
     fix_subparsers(daemon_subparsers)

dvc/command/data_sync.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/destroy.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/diff.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/gc.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 import os

dvc/command/get.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -12,7 +10,27 @@ logger = logging.getLogger(__name__)
 
 
 class CmdGet(CmdBaseNoRepo):
+    def _show_url(self):
+        from dvc.api import get_url
+
+        try:
+            url = get_url(
+                self.args.path, repo=self.args.url, rev=self.args.rev
+            )
+            logger.info(url)
+        except DvcException:
+            logger.exception("failed to show URL")
+            return 1
+
+        return 0
+
     def run(self):
+        if self.args.show_url:
+            return self._show_url()
+
+        return self._get_file_from_repo()
+
+    def _get_file_from_repo(self):
         from dvc.repo import Repo
 
         try:
@@ -33,7 +51,9 @@ class CmdGet(CmdBaseNoRepo):
 
 
 def add_parser(subparsers, parent_parser):
-    GET_HELP = "Download/copy files or directories from DVC repository."
+    GET_HELP = (
+        "Download file or directory from any DVC project or Git repository."
+    )
     get_parser = subparsers.add_parser(
         "get",
         parents=[parent_parser],
@@ -42,18 +62,23 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     get_parser.add_argument(
-        "url", help="URL of Git repository with DVC project to download from."
+        "url",
+        help="Location of DVC project or Git repository to download from",
     )
     get_parser.add_argument(
-        "path", help="Path to a file or directory within a DVC repository."
+        "path",
+        help="Path to a file or directory within the project or repository",
     )
     get_parser.add_argument(
-        "-o",
-        "--out",
-        nargs="?",
-        help="Destination path to copy/download files to.",
+        "-o", "--out", nargs="?", help="Destination path to download files to"
     )
     get_parser.add_argument(
-        "--rev", nargs="?", help="DVC repository git revision."
+        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
+    )
+    get_parser.add_argument(
+        "--show-url",
+        action="store_true",
+        help="Print the storage location (URL) the target data would be "
+        "downloaded from, and exit.",
     )
     get_parser.set_defaults(func=CmdGet)

dvc/command/get_url.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/imp.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -32,7 +30,8 @@ class CmdImport(CmdBase):
 
 def add_parser(subparsers, parent_parser):
     IMPORT_HELP = (
-        "Download data from DVC repository and take it under DVC control."
+        "Download file or directory from any DVC project or Git repository "
+        "into the workspace, and track it."
     )
 
     import_parser = subparsers.add_parser(
@@ -43,15 +42,17 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawTextHelpFormatter,
     )
     import_parser.add_argument(
-        "url", help="URL of Git repository with DVC project to download from."
+        "url",
+        help="Location of DVC project or Git repository to download from",
     )
     import_parser.add_argument(
-        "path", help="Path to data within DVC repository."
+        "path",
+        help="Path to a file or directory within the project or repository",
     )
     import_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to put data to."
+        "-o", "--out", nargs="?", help="Destination path to download files to"
     )
     import_parser.add_argument(
-        "--rev", nargs="?", help="DVC repository git revision."
+        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
     )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/init.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -21,7 +19,7 @@ class CmdInit(CmdBaseNoRepo):
             )
             self.config = self.repo.config
         except InitError:
-            logger.exception("failed to initiate dvc")
+            logger.exception("failed to initiate DVC")
             return 1
         return 0
 
@@ -45,7 +43,7 @@ def add_parser(subparsers, parent_parser):
         "--no-scm",
         action="store_true",
         default=False,
-        help="Initiate dvc in directory that is "
+        help="Initiate DVC in directory that is "
         "not tracked by any scm tool (e.g. git).",
     )
     init_parser.add_argument(

dvc/command/install.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -15,7 +13,7 @@ class CmdInstall(CmdBase):
         try:
             self.repo.install()
         except Exception:
-            logger.exception("failed to install dvc hooks")
+            logger.exception("failed to install DVC Git hooks")
             return 1
         return 0
 

dvc/command/lock.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/metrics.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -28,7 +26,12 @@ def show_metrics(metrics, all_branches=False, all_tags=False):
             logger.info("{branch}:".format(branch=branch))
 
         for fname, metric in val.items():
-            lines = metric if type(metric) is list else metric.splitlines()
+            if isinstance(metric, dict):
+                lines = list(metric.values())
+            elif isinstance(metric, list):
+                lines = metric
+            else:
+                lines = metric.splitlines()
 
             if len(lines) > 1:
                 logger.info("\t{fname}:".format(fname=fname))
@@ -102,6 +105,59 @@ class CmdMetricsRemove(CmdBase):
         return 0
 
 
+def _show_diff(diff):
+    from texttable import Texttable
+
+    if not diff:
+        return "No changes."
+
+    table = Texttable()
+
+    # remove borders to make it easier for users to copy stuff
+    table.set_chars(("", "", "", ""))
+    table.set_deco(0)
+
+    rows = [["Path", "Metric", "Value", "Change"]]
+    for fname, mdiff in diff.items():
+        for metric, change in mdiff.items():
+            rows.append(
+                [
+                    fname,
+                    metric,
+                    change["new"],
+                    change.get("diff", "diff not supported"),
+                ]
+            )
+    table.add_rows(rows)
+    return table.draw()
+
+
+class CmdMetricsDiff(CmdBase):
+    def run(self):
+        try:
+            diff = self.repo.metrics.diff(
+                a_ref=self.args.a_ref,
+                b_ref=self.args.b_ref,
+                targets=self.args.targets,
+                typ=self.args.type,
+                xpath=self.args.xpath,
+                recursive=self.args.recursive,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                logger.info(_show_diff(diff))
+
+        except DvcException:
+            logger.exception("failed to show metrics diff")
+            return 1
+
+        return 0
+
+
 def add_parser(subparsers, parent_parser):
     METRICS_HELP = "Commands to add, manage, collect and display metrics."
 
@@ -115,7 +171,7 @@ def add_parser(subparsers, parent_parser):
 
     metrics_subparsers = metrics_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc metrics CMD --help to display command-specific help.",
+        help="Use `dvc metrics CMD --help` to display command-specific help.",
     )
 
     fix_subparsers(metrics_subparsers)
@@ -216,3 +272,65 @@ def add_parser(subparsers, parent_parser):
     )
     metrics_remove_parser.add_argument("path", help="Path to a metric file.")
     metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
+
+    METRICS_DIFF_HELP = "Output metric values."
+    metrics_diff_parser = metrics_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_DIFF_HELP, "metrics/diff"),
+        help=METRICS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_diff_parser.add_argument(
+        "a_ref",
+        nargs="?",
+        help=(
+            "Git reference from which diff is calculated. "
+            "If omitted `HEAD`(latest commit) is used."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "b_ref",
+        nargs="?",
+        help=(
+            "Git reference to which diff is calculated. "
+            "If omitted current working tree is used."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help=(
+            "Metric files or directories (see -R) to show diff for. "
+            "Shows diff for all metric files by default."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "-t",
+        "--type",
+        help=(
+            "Type of metrics (json/tsv/htsv/csv/hcsv). "
+            "It can be detected by the file extension automatically. "
+            "Unsupported types will be treated as raw."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
+    )
+    metrics_diff_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help=(
+            "If any target is a directory, recursively search and process "
+            "metric files."
+        ),
+    )
+    metrics_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    metrics_diff_parser.set_defaults(func=CmdMetricsDiff)

dvc/command/move.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/pipeline.py

@@ -1,14 +1,8 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.command.base import fix_subparsers
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.utils import relpath
-from dvc.utils.compat import str
 
 
 logger = logging.getLogger(__name__)
@@ -21,23 +15,21 @@ class CmdPipelineShow(CmdBase):
 
         stage = Stage.load(self.repo, target)
         G = self.repo.graph
-        stages = networkx.get_node_attributes(G, "stage")
-        node = relpath(stage.path, self.repo.root_dir)
-        nodes = networkx.dfs_postorder_nodes(G, node)
+        stages = networkx.dfs_postorder_nodes(G, stage)
 
         if locked:
-            nodes = [n for n in nodes if stages[n].locked]
+            stages = [s for s in stages if s.locked]
 
-        for n in nodes:
+        for stage in stages:
             if commands:
-                if stages[n].cmd is None:
+                if stage.cmd is None:
                     continue
-                logger.info(stages[n].cmd)
+                logger.info(stage.cmd)
             elif outs:
-                for out in stages[n].outs:
+                for out in stage.outs:
                     logger.info(str(out))
             else:
-                logger.info(n)
+                logger.info(stage.path_in_repo)
 
     def __build_graph(self, target, commands, outs):
         import networkx
@@ -45,14 +37,10 @@ class CmdPipelineShow(CmdBase):
         from dvc.repo.graph import get_pipeline
 
         stage = Stage.load(self.repo, target)
-        node = relpath(stage.path, self.repo.root_dir)
-
-        G = get_pipeline(self.repo.pipelines, node)
-        stages = networkx.get_node_attributes(G, "stage")
+        G = get_pipeline(self.repo.pipelines, stage)
 
         nodes = []
-        for n in G:
-            stage = stages[n]
+        for stage in G:
             if commands:
                 if stage.cmd is None:
                     continue
@@ -64,9 +52,7 @@ class CmdPipelineShow(CmdBase):
                 nodes.append(stage.relpath)
 
         edges = []
-        for e in G.edges():
-            from_stage = stages[e[0]]
-            to_stage = stages[e[1]]
+        for from_stage, to_stage in G.edges():
             if commands:
                 if to_stage.cmd is None:
                     continue
@@ -115,7 +101,7 @@ class CmdPipelineShow(CmdBase):
         tree.show()
 
     def __write_dot(self, target, commands, outs):
-        from dvc.utils.compat import StringIO
+        import io
         import networkx
         from networkx.drawing.nx_pydot import write_dot
 
@@ -125,7 +111,7 @@ class CmdPipelineShow(CmdBase):
         simple_g = networkx.DiGraph()
         simple_g.add_edges_from(edges)
 
-        dot_file = StringIO()
+        dot_file = io.StringIO()
         write_dot(simple_g, dot_file)
         logger.info(dot_file.getvalue())
 
@@ -163,14 +149,11 @@ class CmdPipelineShow(CmdBase):
 
 class CmdPipelineList(CmdBase):
     def run(self):
-        import networkx
-
         pipelines = self.repo.pipelines
-        for p in pipelines:
-            stages = networkx.get_node_attributes(p, "stage")
-            for stage in stages:
-                logger.info(stage)
-            if len(stages) != 0:
+        for pipeline in pipelines:
+            for stage in pipeline:
+                logger.info(stage.relpath)
+            if len(pipeline) != 0:
                 logger.info("=" * 80)
         logger.info("{} pipelines total".format(len(pipelines)))
 
@@ -189,7 +172,7 @@ def add_parser(subparsers, parent_parser):
 
     pipeline_subparsers = pipeline_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc pipeline CMD --help for command-specific help.",
+        help="Use `dvc pipeline CMD --help` for command-specific help.",
     )
 
     fix_subparsers(pipeline_subparsers)

dvc/command/remote.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -13,7 +11,7 @@ logger = logging.getLogger(__name__)
 
 class CmdRemoteConfig(CmdConfig):
     def __init__(self, args):
-        super(CmdRemoteConfig, self).__init__(args)
+        super().__init__(args)
         self.remote_config = RemoteConfig(self.config)
 
 
@@ -85,7 +83,7 @@ def add_parser(subparsers, parent_parser):
 
     remote_subparsers = remote_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc remote CMD --help for " "command-specific help.",
+        help="Use `dvc remote CMD --help` for " "command-specific help.",
     )
 
     fix_subparsers(remote_subparsers)

dvc/command/remove.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 
@@ -20,7 +18,7 @@ class CmdRemove(CmdBase):
         if self.args.force:
             return False
 
-        msg = "Are you sure you want to remove {} with its outputs?".format(
+        msg = "Are you sure you want to remove '{}' with its outputs?".format(
             target
         )
 
@@ -38,7 +36,7 @@ class CmdRemove(CmdBase):
                 outs_only = self._is_outs_only(target)
                 self.repo.remove(target, outs_only=outs_only)
             except DvcException:
-                logger.exception("failed to remove {}".format(target))
+                logger.exception("failed to remove '{}'".format(target))
                 return 1
         return 0
 

dvc/command/repro.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 import os

dvc/command/root.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/run.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/status.py

@@ -1,9 +1,6 @@
-from __future__ import unicode_literals
-
 import logging
 
 from dvc.command.data_sync import CmdDataBase
-from dvc.utils.compat import str
 
 
 logger = logging.getLogger(__name__)

dvc/command/unprotect.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/update.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import argparse
 import logging
 

dvc/command/version.py

@@ -1,11 +1,10 @@
-from __future__ import unicode_literals
-
 import argparse
 import itertools
 import logging
 import os
 import platform
 import uuid
+import pathlib
 
 try:
     import psutil
@@ -13,7 +12,6 @@ except ImportError:
     psutil = None
 
 from dvc.utils import is_binary, relpath
-from dvc.utils.compat import pathlib
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
 from dvc.exceptions import DvcException, NotDvcRepoError
@@ -27,23 +25,13 @@ class CmdVersion(CmdBaseNoRepo):
     def run(self):
         from dvc.repo import Repo
 
-        dvc_version = __version__
-        python_version = platform.python_version()
-        platform_type = platform.platform()
-        binary = is_binary()
-        info = (
-            "DVC version: {dvc_version}\n"
-            "Python version: {python_version}\n"
-            "Platform: {platform_type}\n"
-            "Binary: {binary}\n"
-            "Package: {package}\n"
-        ).format(
-            dvc_version=dvc_version,
-            python_version=python_version,
-            platform_type=platform_type,
-            binary=binary,
-            package=PKG,
-        )
+        info = [
+            "DVC version: {}".format(__version__),
+            "Python version: {}".format(platform.python_version()),
+            "Platform: {}".format(platform.platform()),
+            "Binary: {}".format(is_binary()),
+            "Package: {}".format(PKG),
+        ]
 
         try:
             repo = Repo()
@@ -54,9 +42,14 @@ class CmdVersion(CmdBaseNoRepo):
             # later decides to enable shared cache mode with
             # `dvc config cache.shared group`.
             if os.path.exists(repo.cache.local.cache_dir):
-                info += "Cache: {cache}\n".format(
-                    cache=self.get_linktype_support_info(repo)
+                info.append(
+                    "Cache: {}".format(self.get_linktype_support_info(repo))
                 )
+                if psutil:
+                    fs_type = self.get_fs_type(repo.cache.local.cache_dir)
+                    info.append(
+                        "Filesystem type (cache directory): {}".format(fs_type)
+                    )
             else:
                 logger.warning(
                     "Unable to detect supported link types, as cache "
@@ -66,19 +59,14 @@ class CmdVersion(CmdBaseNoRepo):
                     "check.".format(relpath(repo.cache.local.cache_dir))
                 )
 
-            if psutil:
-                info += (
-                    "Filesystem type (cache directory): {fs_cache}\n"
-                ).format(fs_cache=self.get_fs_type(repo.cache.local.cache_dir))
         except NotDvcRepoError:
             root_directory = os.getcwd()
 
         if psutil:
-            info += ("Filesystem type (workspace): {fs_root}").format(
-                fs_root=self.get_fs_type(os.path.abspath(root_directory))
-            )
+            fs_root = self.get_fs_type(os.path.abspath(root_directory))
+            info.append("Filesystem type (workspace): {}".format(fs_root))
 
-        logger.info(info)
+        logger.info("\n".join(info))
         return 0
 
     @staticmethod
@@ -98,9 +86,9 @@ class CmdVersion(CmdBaseNoRepo):
     @staticmethod
     def get_linktype_support_info(repo):
         links = {
-            "reflink": System.reflink,
-            "hardlink": System.hardlink,
-            "symlink": System.symlink,
+            "reflink": (System.reflink, None),
+            "hardlink": (System.hardlink, System.is_hardlink),
+            "symlink": (System.symlink, System.is_symlink),
         }
 
         fname = "." + str(uuid.uuid4())
@@ -110,18 +98,16 @@ class CmdVersion(CmdBaseNoRepo):
 
         cache = []
 
-        for name, link in links.items():
+        for name, (link, is_link) in links.items():
             try:
                 link(src, dst)
+                status = "supported"
+                if is_link and not is_link(dst):
+                    status = "broken"
                 os.unlink(dst)
-                supported = True
             except DvcException:
-                supported = False
-            cache.append(
-                "{name} - {supported}".format(
-                    name=name, supported=True if supported else False
-                )
-            )
+                status = "not supported"
+            cache.append("{name} - {status}".format(name=name, status=status))
         os.remove(src)
 
         return ", ".join(cache)

dvc/compat.py

@@ -0,0 +1,42 @@
+"""Python version compatibility code"""
+
+# Backport os.fspath() from Python 3.6
+try:
+    from os import fspath  # noqa: F821
+
+    fspath_py35 = lambda s: s  # noqa: E731
+except ImportError:
+
+    def fspath(path):
+        """Return the path representation of a path-like object.
+
+        If str or bytes is passed in, it is returned unchanged. Otherwise the
+        os.PathLike interface is used to get the path representation. If the
+        path representation is not str or bytes, TypeError is raised. If the
+        provided path is not str, bytes, or os.PathLike, TypeError is raised.
+        """
+        if isinstance(path, (str, bytes)):
+            return path
+
+        # Work from the object's type to match method resolution of other magic
+        # methods.
+        path_type = type(path)
+        try:
+            path_repr = path_type.__fspath__(path)
+        except AttributeError:
+            if hasattr(path_type, "__fspath__"):
+                raise
+            else:
+                raise TypeError(
+                    "expected str, bytes or os.PathLike object, "
+                    "not " + path_type.__name__
+                )
+        if isinstance(path_repr, (str, bytes)):
+            return path_repr
+        else:
+            raise TypeError(
+                "expected {}.__fspath__() to return str or bytes, "
+                "not {}".format(path_type.__name__, type(path_repr).__name__)
+            )
+
+    fspath_py35 = fspath

dvc/config.py

@@ -1,5 +1,4 @@
 """DVC config objects."""
-from __future__ import unicode_literals
 
 import copy
 import errno
@@ -13,35 +12,27 @@ from voluptuous import All, Any, Lower, Range, Coerce, Match
 
 from dvc.exceptions import DvcException
 from dvc.exceptions import NotDvcRepoError
-from dvc.utils.compat import open, str
 
 logger = logging.getLogger(__name__)
 
 
 class ConfigError(DvcException):
-    """DVC config exception.
+    """DVC config exception."""
 
-    Args:
-        msg (str): error message.
-        cause (Exception): optional exception that has caused this error.
-    """
-
-    def __init__(self, msg, cause=None):
-        super(ConfigError, self).__init__(
-            "config file error: {}".format(msg), cause=cause
-        )
+    def __init__(self, msg):
+        super().__init__("config file error: {}".format(msg))
 
 
 class NoRemoteError(ConfigError):
-    def __init__(self, command, cause=None):
+    def __init__(self, command):
         msg = (
             "no remote specified. Setup default remote with\n"
-            "    dvc config core.remote <name>\n"
+            "    dvc remote default <name>\n"
             "or use:\n"
             "    dvc {} -r <name>\n".format(command)
         )
 
-        super(NoRemoteError, self).__init__(msg, cause=cause)
+        super().__init__(msg)
 
 
 def supported_cache_type(types):
@@ -81,11 +72,11 @@ def Choices(*choices):
         *choices: pass allowed values as arguments, or pass a list or
             tuple as a single argument
     """
-    return Any(*choices, msg="expected one of {}".format(",".join(choices)))
+    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
 
 
 class Config(object):  # pylint: disable=too-many-instance-attributes
-    """Class that manages configuration files for a dvc repo.
+    """Class that manages configuration files for a DVC repo.
 
     Args:
         dvc_dir (str): optional path to `.dvc` directory, that is used to
@@ -195,6 +186,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret"
     SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file"
 
+    SECTION_REMOTE_CHECKSUM_JOBS = "checksum_jobs"
     SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
     SECTION_REMOTE_FMT = 'remote "{}"'
     SECTION_REMOTE_URL = "url"
@@ -206,6 +198,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_REMOTE_ASK_PASSWORD = "ask_password"
     SECTION_REMOTE_GSS_AUTH = "gss_auth"
     SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
+    SECTION_REMOTE_VERIFY = "verify"
     SECTION_REMOTE_SCHEMA = {
         Required(SECTION_REMOTE_URL): str,
         SECTION_AWS_REGION: str,
@@ -223,6 +216,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         SECTION_GCP_PROJECTNAME: str,
         SECTION_CACHE_TYPE: supported_cache_type,
         Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
+        SECTION_REMOTE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
         SECTION_REMOTE_USER: str,
         SECTION_REMOTE_PORT: Coerce(int),
         SECTION_REMOTE_KEY_FILE: str,
@@ -238,7 +232,8 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         SECTION_GDRIVE_CLIENT_SECRET: str,
         SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
         PRIVATE_CWD: str,
-        Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): Bool,
+        SECTION_REMOTE_NO_TRAVERSE: Bool,
+        SECTION_REMOTE_VERIFY: Bool,
     }
 
     SECTION_STATE = "state"
@@ -396,7 +391,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         try:
             d = self.COMPILED_SCHEMA(d)
         except Invalid as exc:
-            raise ConfigError(str(exc), cause=exc)
+            raise ConfigError(str(exc)) from exc
         self.config = configobj.ConfigObj(d, write_empty_values=True)
 
     def save(self, config=None):

dvc/daemon.py

@@ -1,5 +1,4 @@
 """Launch `dvc daemon` command in a separate detached process."""
-from __future__ import unicode_literals
 
 import inspect
 import logging
@@ -10,7 +9,6 @@ from subprocess import Popen
 from dvc.env import DVC_DAEMON
 from dvc.utils import fix_env
 from dvc.utils import is_binary
-from dvc.utils.compat import cast_bytes_py2
 
 
 logger = logging.getLogger(__name__)
@@ -99,9 +97,7 @@ def daemon(args):
 
     env = fix_env()
     file_path = os.path.abspath(inspect.stack()[0][1])
-    env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2(
-        os.path.dirname(os.path.dirname(file_path))
-    )
-    env[cast_bytes_py2(DVC_DAEMON)] = cast_bytes_py2("1")
+    env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
+    env[DVC_DAEMON] = "1"
 
     _spawn(cmd, env)

dvc/dagascii.py

@@ -1,6 +1,4 @@
 """Draws DAG in ASCII."""
-from __future__ import print_function
-from __future__ import unicode_literals
 
 import logging
 import math

dvc/data_cloud.py

@@ -1,5 +1,4 @@
 """Manages dvc remotes that user can use with push/pull/status commands."""
-from __future__ import unicode_literals
 
 import logging
 
@@ -74,13 +73,25 @@ class DataCloud(object):
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
-        return self.repo.cache.local.pull(
-            cache,
-            jobs=jobs,
-            remote=self.get_remote(remote, "pull"),
-            show_checksums=show_checksums,
+        remote = self.get_remote(remote, "pull")
+        downloaded_items_num = self.repo.cache.local.pull(
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )
 
+        if not remote.verify:
+            self._save_pulled_checksums(cache)
+
+        return downloaded_items_num
+
+    def _save_pulled_checksums(self, cache):
+        for checksum in cache["local"].keys():
+            cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
+            if self.repo.cache.local.exists(cache_file):
+                # We can safely save here, as existing corrupted files will be
+                # removed upon status, while files corrupted during download
+                # will not be moved from tmp_file (see `RemoteBASE.download()`)
+                self.repo.state.save(cache_file, checksum)
+
     def status(self, cache, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
 

dvc/dependency/__init__.py

@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+from urllib.parse import urlparse
 
 import dvc.output as output
 from dvc.dependency.gs import DependencyGS
@@ -46,8 +46,6 @@ SCHEMA[DependencyREPO.PARAM_REPO] = DependencyREPO.REPO_SCHEMA
 
 
 def _get(stage, p, info):
-    from dvc.utils.compat import urlparse
-
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":

dvc/dependency/base.py

@@ -1,23 +1,21 @@
-from __future__ import unicode_literals
-
 from dvc.exceptions import DvcException
 
 
 class DependencyDoesNotExistError(DvcException):
     def __init__(self, path):
         msg = "dependency '{}' does not exist".format(path)
-        super(DependencyDoesNotExistError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class DependencyIsNotFileOrDirError(DvcException):
     def __init__(self, path):
         msg = "dependency '{}' is not a file or directory".format(path)
-        super(DependencyIsNotFileOrDirError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class DependencyIsStageFileError(DvcException):
     def __init__(self, path):
-        super(DependencyIsStageFileError, self).__init__(
+        super().__init__(
             "Stage file '{}' cannot be a dependency.".format(path)
         )
 

dvc/dependency/gs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.gs import OutputGS
 

dvc/dependency/hdfs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.hdfs import OutputHDFS
 

dvc/dependency/http.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP

dvc/dependency/https.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from .http import DependencyHTTP
 from dvc.remote.https import RemoteHTTPS
 

dvc/dependency/local.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.local import OutputLOCAL
 

dvc/dependency/repo.py

@@ -1,13 +1,10 @@
-from __future__ import unicode_literals
+import os
 
-import copy
-from contextlib import contextmanager
-
-from funcy import merge
+from voluptuous import Required
 
 from .local import DependencyLOCAL
-from dvc.external_repo import external_repo
-from dvc.utils.compat import str
+from dvc.exceptions import OutputNotFoundError
+from dvc.path_info import PathInfo
 
 
 class DependencyREPO(DependencyLOCAL):
@@ -16,11 +13,15 @@ class DependencyREPO(DependencyLOCAL):
     PARAM_REV = "rev"
     PARAM_REV_LOCK = "rev_lock"
 
-    REPO_SCHEMA = {PARAM_URL: str, PARAM_REV: str, PARAM_REV_LOCK: str}
+    REPO_SCHEMA = {
+        Required(PARAM_URL): str,
+        PARAM_REV: str,
+        PARAM_REV_LOCK: str,
+    }
 
     def __init__(self, def_repo, stage, *args, **kwargs):
         self.def_repo = def_repo
-        super(DependencyREPO, self).__init__(stage, *args, **kwargs)
+        super().__init__(stage, *args, **kwargs)
 
     def _parse_path(self, remote, path):
         return None
@@ -32,24 +33,33 @@ class DependencyREPO(DependencyLOCAL):
     @property
     def repo_pair(self):
         d = self.def_repo
-        return d[self.PARAM_URL], d[self.PARAM_REV_LOCK] or d[self.PARAM_REV]
+        rev = d.get(self.PARAM_REV_LOCK) or d.get(self.PARAM_REV)
+        return d[self.PARAM_URL], rev
 
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 
-    @contextmanager
-    def _make_repo(self, **overrides):
-        with external_repo(**merge(self.def_repo, overrides)) as repo:
-            yield repo
+    def _make_repo(self, *, locked=True):
+        from dvc.external_repo import external_repo
+
+        d = self.def_repo
+        rev = (d.get("rev_lock") if locked else None) or d.get("rev")
+        return external_repo(d["url"], rev=rev)
+
+    def _get_checksum(self, locked=True):
+        with self._make_repo(locked=locked) as repo:
+            try:
+                return repo.find_out_by_relpath(self.def_path).info["md5"]
+            except OutputNotFoundError:
+                path = PathInfo(os.path.join(repo.root_dir, self.def_path))
+                # We are polluting our repo cache with some dir listing here
+                return self.repo.cache.local.get_checksum(path)
 
     def status(self):
-        with self._make_repo() as repo:
-            current = repo.find_out_by_relpath(self.def_path).info
+        current_checksum = self._get_checksum(locked=True)
+        updated_checksum = self._get_checksum(locked=False)
 
-        with self._make_repo(rev_lock=None) as repo:
-            updated = repo.find_out_by_relpath(self.def_path).info
-
-        if current != updated:
+        if current_checksum != updated_checksum:
             return {str(self): "update available"}
 
         return {}
@@ -60,23 +70,16 @@ class DependencyREPO(DependencyLOCAL):
     def dumpd(self):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
-    def fetch(self):
-        with self._make_repo(
-            cache_dir=self.repo.cache.local.cache_dir
-        ) as repo:
-            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
-
-            out = repo.find_out_by_relpath(self.def_path)
-            with repo.state:
-                repo.cloud.pull(out.get_used_cache())
-
-        return out
-
     def download(self, to):
-        out = self.fetch()
-        to.info = copy.copy(out.info)
-        to.checkout()
+        with self._make_repo() as repo:
+            if self.def_repo.get(self.PARAM_REV_LOCK) is None:
+                self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+
+            if hasattr(repo, "cache"):
+                repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
+
+            repo.pull_to(self.def_path, to.path_info)
 
     def update(self):
-        with self._make_repo(rev_lock=None) as repo:
+        with self._make_repo(locked=False) as repo:
             self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()

dvc/dependency/s3.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.s3 import OutputS3
 

dvc/dependency/ssh.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.dependency.base import DependencyBase
 from dvc.output.ssh import OutputSSH
 

dvc/exceptions.py

@@ -1,32 +1,10 @@
 """Exceptions raised by the dvc."""
-from __future__ import unicode_literals
-
-import traceback
 
 from dvc.utils import relpath
-from dvc.utils.compat import builtin_str
-from dvc.utils.compat import str
 
 
 class DvcException(Exception):
-    """Base class for all dvc exceptions.
-
-    Args:
-        msg (unicode): message for this exception.
-        cause (Exception): optional cause exception.
-    """
-
-    def __init__(self, msg, cause=None):
-        # NOTE: unlike python 3, python 2 doesn't have built-in support
-        # for chained exceptions, so we are using our own implementation.
-        self.cause = cause
-        self.cause_tb = None
-        if cause:
-            try:
-                self.cause_tb = traceback.format_exc()
-            except AttributeError:  # pragma: no cover
-                pass
-        super(DvcException, self).__init__(msg)
+    """Base class for all dvc exceptions."""
 
 
 class OutputDuplicationError(DvcException):
@@ -39,18 +17,14 @@ class OutputDuplicationError(DvcException):
     """
 
     def __init__(self, output, stages):
-        assert isinstance(output, str) or isinstance(output, builtin_str)
-        assert isinstance(stages, list)
-        assert all(
-            isinstance(stage, str) or isinstance(stage, builtin_str)
-            for stage in stages
-        )
+        assert isinstance(output, str)
+        assert all(hasattr(stage, "relpath") for stage in stages)
         msg = (
             "file/directory '{}' is specified as an output in more than one "
             "stage: {}\n"
             "This is not allowed. Consider using a different output name."
-        ).format(output, "\n    ".join(stages))
-        super(OutputDuplicationError, self).__init__(msg)
+        ).format(output, "\n    ".join(s.relpath for s in stages))
+        super().__init__(msg)
 
 
 class OutputNotFoundError(DvcException):
@@ -63,8 +37,8 @@ class OutputNotFoundError(DvcException):
     def __init__(self, output, repo=None):
         self.output = output
         self.repo = repo
-        super(OutputNotFoundError, self).__init__(
-            "unable to find DVC-file with output '{path}'".format(
+        super().__init__(
+            "Unable to find DVC-file with output '{path}'".format(
                 path=relpath(self.output)
             )
         )
@@ -75,21 +49,17 @@ class StagePathAsOutputError(DvcException):
     an output of another stage.
 
     Args:
-        cwd (str): path to the directory.
-        fname (str): path to the DVC-file that has cwd specified as an
-            output.
+        stage (Stage): a stage that is in some other stages output
+        output (str): an output covering the stage above
     """
 
-    def __init__(self, wdir, fname):
-        assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
-        assert isinstance(fname, str) or isinstance(fname, builtin_str)
-        msg = (
-            "current working directory '{cwd}' is specified as an output in "
-            "'{fname}'. Use another CWD to prevent any data removal.".format(
-                cwd=wdir, fname=fname
+    def __init__(self, stage, output):
+        assert isinstance(output, str)
+        super().__init__(
+            "'{stage}' is within an output '{output}' of another stage".format(
+                stage=stage.relpath, output=output
             )
         )
-        super(StagePathAsOutputError, self).__init__(msg)
 
 
 class CircularDependencyError(DvcException):
@@ -101,13 +71,13 @@ class CircularDependencyError(DvcException):
     """
 
     def __init__(self, dependency):
-        assert isinstance(dependency, (str, builtin_str))
+        assert isinstance(dependency, str)
 
         msg = (
             "file/directory '{}' is specified as an output and as a "
             "dependency."
         )
-        super(CircularDependencyError, self).__init__(msg.format(dependency))
+        super().__init__(msg.format(dependency))
 
 
 class ArgumentDuplicationError(DvcException):
@@ -119,9 +89,8 @@ class ArgumentDuplicationError(DvcException):
     """
 
     def __init__(self, path):
-        assert isinstance(path, (str, builtin_str))
-        msg = "file '{}' is specified more than once."
-        super(ArgumentDuplicationError, self).__init__(msg.format(path))
+        assert isinstance(path, str)
+        super().__init__("file '{}' is specified more than once.".format(path))
 
 
 class MoveNotDataSourceError(DvcException):
@@ -139,11 +108,11 @@ class MoveNotDataSourceError(DvcException):
             "it by hand, or remove '{path}' and create a new one at the "
             "desired location."
         )
-        super(MoveNotDataSourceError, self).__init__(msg.format(path=path))
+        super().__init__(msg.format(path=path))
 
 
 class NotDvcRepoError(DvcException):
-    """Thrown if a directory is not a dvc repo.
+    """Thrown if a directory is not a DVC repo.
 
     Args:
         root (str): path to the directory.
@@ -151,56 +120,54 @@ class NotDvcRepoError(DvcException):
 
     def __init__(self, root):
         msg = (
-            "you are not inside of a dvc repository "
+            "you are not inside of a DVC repository "
             "(checked up to mount point '{}')"
         )
-        super(NotDvcRepoError, self).__init__(msg.format(root))
+        super().__init__(msg.format(root))
 
 
 class DvcParserError(DvcException):
     """Base class for CLI parser errors."""
 
     def __init__(self):
-        super(DvcParserError, self).__init__("parser error")
+        super().__init__("parser error")
 
 
 class CyclicGraphError(DvcException):
     def __init__(self, stages):
         assert isinstance(stages, list)
-        stages = "\n".join("\t- {}".format(stage) for stage in stages)
+        stages = "\n".join("\t- {}".format(stage.relpath) for stage in stages)
         msg = (
             "you've introduced a cycle in your pipeline that involves "
             "the following stages:"
             "\n"
             "{stages}".format(stages=stages)
         )
-        super(CyclicGraphError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class ConfirmRemoveError(DvcException):
     def __init__(self, path):
-        super(ConfirmRemoveError, self).__init__(
+        super().__init__(
             "unable to remove '{}' without a confirmation from the user. Use "
             "`-f` to force.".format(path)
         )
 
 
 class InitError(DvcException):
-    def __init__(self, msg):
-        super(InitError, self).__init__(msg)
+    pass
 
 
 class ReproductionError(DvcException):
-    def __init__(self, dvc_file_name, ex):
+    def __init__(self, dvc_file_name):
         self.path = dvc_file_name
-        msg = "failed to reproduce '{}'".format(dvc_file_name)
-        super(ReproductionError, self).__init__(msg, cause=ex)
+        super().__init__("failed to reproduce '{}'".format(dvc_file_name))
 
 
 class BadMetricError(DvcException):
     def __init__(self, paths):
-        super(BadMetricError, self).__init__(
-            "the following metrics do not exists, "
+        super().__init__(
+            "the following metrics do not exist, "
             "are not metric files or are malformed: {paths}".format(
                 paths=", ".join("'{}'".format(path) for path in paths)
             )
@@ -209,32 +176,31 @@ class BadMetricError(DvcException):
 
 class NoMetricsError(DvcException):
     def __init__(self):
-        super(NoMetricsError, self).__init__(
+        super().__init__(
             "no metric files in this repository. "
             "Use `dvc metrics add` to add a metric file to track."
         )
 
 
 class StageFileCorruptedError(DvcException):
-    def __init__(self, path, cause=None):
+    def __init__(self, path):
         path = relpath(path)
-        super(StageFileCorruptedError, self).__init__(
+        super().__init__(
             "unable to read DVC-file: {} "
-            "YAML file structure is corrupted".format(path),
-            cause=cause,
+            "YAML file structure is corrupted".format(path)
         )
 
 
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
-        super(RecursiveAddingWhileUsingFilename, self).__init__(
+        super().__init__(
             "cannot use `fname` with multiple targets or `-R|--recursive`"
         )
 
 
 class OverlappingOutputPathsError(DvcException):
     def __init__(self, out_1, out_2):
-        super(OverlappingOutputPathsError, self).__init__(
+        super().__init__(
             "Paths for outs:\n'{}'('{}')\n'{}'('{}')\noverlap. To avoid "
             "unpredictable behaviour, rerun command with non overlapping outs "
             "paths.".format(
@@ -247,46 +213,37 @@ class OverlappingOutputPathsError(DvcException):
 
 
 class CheckoutErrorSuggestGit(DvcException):
-    def __init__(self, target, cause):
-        super(CheckoutErrorSuggestGit, self).__init__(
-            "Did you mean `git checkout {}`?".format(target), cause=cause
-        )
+    def __init__(self, target):
+        super().__init__("Did you mean `git checkout {}`?".format(target))
 
 
 class ETagMismatchError(DvcException):
     def __init__(self, etag, cached_etag):
-        super(ETagMismatchError, self).__init__(
+        super().__init__(
             "ETag mismatch detected when copying file to cache! "
             "(expected: '{}', actual: '{}')".format(etag, cached_etag)
         )
 
 
 class FileMissingError(DvcException):
-    def __init__(self, path, cause=None):
-        super(FileMissingError, self).__init__(
-            "Can't find '{}' neither locally nor on remote".format(path),
-            cause=cause,
+    def __init__(self, path):
+        self.path = path
+        super().__init__(
+            "Can't find '{}' neither locally nor on remote".format(path)
         )
 
 
 class DvcIgnoreInCollectedDirError(DvcException):
     def __init__(self, ignore_dirname):
-        super(DvcIgnoreInCollectedDirError, self).__init__(
+        super().__init__(
             ".dvcignore file should not be in collected dir path: "
             "'{}'".format(ignore_dirname)
         )
 
 
-class UrlNotDvcRepoError(DvcException):
-    def __init__(self, url):
-        super(UrlNotDvcRepoError, self).__init__(
-            "URL '{}' is not a dvc repository.".format(url)
-        )
-
-
 class GitHookAlreadyExistsError(DvcException):
     def __init__(self, hook_name):
-        super(GitHookAlreadyExistsError, self).__init__(
+        super().__init__(
             "Hook '{}' already exists. Please refer to "
             "https://man.dvc.org/install "
             "for more info.".format(hook_name)
@@ -297,7 +254,7 @@ class DownloadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super(DownloadError, self).__init__(
+        super().__init__(
             "{amount} files failed to download".format(amount=amount)
         )
 
@@ -306,7 +263,7 @@ class UploadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super(UploadError, self).__init__(
+        super().__init__(
             "{amount} files failed to upload".format(amount=amount)
         )
 
@@ -318,7 +275,7 @@ class CheckoutError(DvcException):
             "Checkout failed for following targets:\n {}\nDid you "
             "forget to fetch?".format("\n".join(targets))
         )
-        super(CheckoutError, self).__init__(m)
+        super().__init__(m)
 
 
 class CollectCacheError(DvcException):
@@ -327,14 +284,14 @@ class CollectCacheError(DvcException):
 
 class NoRemoteInExternalRepoError(DvcException):
     def __init__(self, url):
-        super(NoRemoteInExternalRepoError, self).__init__(
+        super().__init__(
             "No DVC remote is specified in target repository '{}'.".format(url)
         )
 
 
 class NoOutputInExternalRepoError(DvcException):
     def __init__(self, path, external_repo_path, external_repo_url):
-        super(NoOutputInExternalRepoError, self).__init__(
+        super().__init__(
             "Output '{}' not found in target repository '{}'".format(
                 relpath(path, external_repo_path), external_repo_url
             )
@@ -343,4 +300,13 @@ class NoOutputInExternalRepoError(DvcException):
 
 class HTTPError(DvcException):
     def __init__(self, code, reason):
-        super(HTTPError, self).__init__("'{} {}'".format(code, reason))
+        super().__init__("'{} {}'".format(code, reason))
+
+
+class PathMissingError(DvcException):
+    def __init__(self, path, repo):
+        msg = (
+            "The path '{}' does not exist in the target repository '{}'"
+            " neither as an output nor a git-handled file."
+        )
+        super().__init__(msg.format(path, repo))

dvc/external_repo.py

@@ -1,29 +1,29 @@
-from __future__ import unicode_literals
-
 import os
 import tempfile
 from contextlib import contextmanager
 from distutils.dir_util import copy_tree
 
-from funcy import retry
+from funcy import retry, suppress, memoize, cached_property
 
-from dvc.config import NoRemoteError, ConfigError
+from dvc.compat import fspath
+from dvc.repo import Repo
+from dvc.config import Config, NoRemoteError, NotDvcRepoError
 from dvc.exceptions import NoRemoteInExternalRepoError
+from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
+from dvc.exceptions import FileMissingError, PathMissingError
 from dvc.remote import RemoteConfig
-from dvc.exceptions import NoOutputInExternalRepoError
-from dvc.exceptions import OutputNotFoundError
-from dvc.utils.fs import remove
-
-
-REPO_CACHE = {}
+from dvc.utils.fs import remove, fs_copy
+from dvc.scm import SCM
 
 
 @contextmanager
-def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
-    from dvc.repo import Repo
+def external_repo(url, rev=None):
+    path = _cached_clone(url, rev)
+    try:
+        repo = ExternalRepo(path, url)
+    except NotDvcRepoError:
+        repo = ExternalGitRepo(path, url)
 
-    path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
-    repo = Repo(path)
     try:
         yield repo
     except NoRemoteError:
@@ -32,26 +32,124 @@ def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
         if exc.repo is repo:
             raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url)
         raise
-    repo.close()
+    except FileMissingError as exc:
+        raise PathMissingError(exc.path, url)
+    finally:
+        repo.close()
 
 
-def _external_repo(url=None, rev=None, cache_dir=None):
-    from dvc.config import Config
-    from dvc.cache import CacheConfig
-    from dvc.repo import Repo
+def clean_repos():
+    # Outside code should not see cache while we are removing
+    repo_paths = list(_cached_clone.memory.values())
+    _cached_clone.memory.clear()
 
-    key = (url, rev, cache_dir)
-    if key in REPO_CACHE:
-        return REPO_CACHE[key]
+    for path in repo_paths:
+        _remove(path)
 
+
+class ExternalRepo(Repo):
+    def __init__(self, root_dir, url):
+        super().__init__(root_dir)
+        self.url = url
+        self._set_upstream()
+
+    def pull_to(self, path, to_info):
+        try:
+            out = None
+            with suppress(OutputNotFoundError):
+                out = self.find_out_by_relpath(path)
+
+            if out and out.use_cache:
+                self._pull_cached(out, to_info)
+                return
+
+            # Git handled files can't have absolute path
+            if os.path.isabs(path):
+                raise FileNotFoundError
+
+            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+    def _pull_cached(self, out, to_info):
+        with self.state:
+            # Only pull unless all needed cache is present
+            if out.changed_cache():
+                self.cloud.pull(out.get_used_cache())
+
+            out.path_info = to_info
+            failed = out.checkout()
+            # This might happen when pull haven't really pulled all the files
+            if failed:
+                raise FileNotFoundError
+
+    def _set_upstream(self):
+        # check if the URL is local and no default remote is present
+        # add default remote pointing to the original repo's cache location
+        if os.path.isdir(self.url):
+            rconfig = RemoteConfig(self.config)
+            if not rconfig.has_default():
+                src_repo = Repo(self.url)
+                try:
+                    rconfig.add(
+                        "auto-generated-upstream",
+                        src_repo.cache.local.cache_dir,
+                        default=True,
+                        level=Config.LEVEL_LOCAL,
+                    )
+                finally:
+                    src_repo.close()
+
+
+class ExternalGitRepo:
+    def __init__(self, root_dir, url):
+        self.root_dir = root_dir
+        self.url = url
+
+    @cached_property
+    def scm(self):
+        return SCM(self.root_dir)
+
+    def close(self):
+        if "scm" in self.__dict__:
+            self.scm.close()
+
+    def find_out_by_relpath(self, path):
+        raise OutputNotFoundError(path, self)
+
+    def pull_to(self, path, to_info):
+        try:
+            # Git handled files can't have absolute path
+            if os.path.isabs(path):
+                raise FileNotFoundError
+
+            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+    @contextmanager
+    def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
+        try:
+            abs_path = os.path.join(self.root_dir, path)
+            with open(abs_path, mode, encoding=encoding) as fd:
+                yield fd
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+
+@memoize
+def _cached_clone(url, rev):
+    """Clone an external git repo to a temporary directory.
+
+    Returns the path to a local temporary directory with the specified
+    revision checked out.
+    """
     new_path = tempfile.mkdtemp("dvc-erepo")
 
-    # Copy and adjust existing clone
-    if (url, None, None) in REPO_CACHE:
-        old_path = REPO_CACHE[url, None, None]
-
+    if url in _cached_clone.memory:
+        # Copy and an existing clean clone
         # This one unlike shutil.copytree() works with an existing dir
-        copy_tree(old_path, new_path)
+        copy_tree(_cached_clone.memory[url], new_path)
     else:
         # Create a new clone
         _clone_repo(url, new_path)
@@ -59,41 +157,12 @@ def _external_repo(url=None, rev=None, cache_dir=None):
         # Save clean clone dir so that we will have access to a default branch
         clean_clone_path = tempfile.mkdtemp("dvc-erepo")
         copy_tree(new_path, clean_clone_path)
-        REPO_CACHE[url, None, None] = clean_clone_path
+        _cached_clone.memory[url] = clean_clone_path
 
-    # Adjust new clone/copy to fit rev and cache_dir
-
-    # Checkout needs to be done first because current branch might not be
-    # DVC repository
+    # Check out the specified revision
     if rev is not None:
         _git_checkout(new_path, rev)
 
-    repo = Repo(new_path)
-    try:
-        # check if the URL is local and no default remote is present
-        # add default remote pointing to the original repo's cache location
-        if os.path.isdir(url):
-            rconfig = RemoteConfig(repo.config)
-            if not _default_remote_set(rconfig):
-                original_repo = Repo(url)
-                try:
-                    rconfig.add(
-                        "auto-generated-upstream",
-                        original_repo.cache.local.cache_dir,
-                        default=True,
-                        level=Config.LEVEL_LOCAL,
-                    )
-                finally:
-                    original_repo.close()
-
-        if cache_dir is not None:
-            cache_config = CacheConfig(repo.config)
-            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
-    finally:
-        # Need to close/reopen repo to force config reread
-        repo.close()
-
-    REPO_CACHE[key] = new_path
     return new_path
 
 
@@ -107,15 +176,6 @@ def _git_checkout(repo_path, revision):
         git.close()
 
 
-def clean_repos():
-    # Outside code should not see cache while we are removing
-    repo_paths = list(REPO_CACHE.values())
-    REPO_CACHE.clear()
-
-    for path in repo_paths:
-        _remove(path)
-
-
 def _remove(path):
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
@@ -130,19 +190,3 @@ def _clone_repo(url, path):
 
     git = Git.clone(url, path)
     git.close()
-
-
-def _default_remote_set(rconfig):
-    """
-    Checks if default remote config is present.
-    Args:
-        rconfig: a remote config
-
-    Returns:
-        True if the default remote config is set, else False
-    """
-    try:
-        rconfig.get_default()
-        return True
-    except ConfigError:
-        return False

dvc/ignore.py

@@ -1,11 +1,11 @@
-from __future__ import unicode_literals
-
 import logging
 import os
 
+from funcy import cached_property
 from pathspec import PathSpec
 from pathspec.patterns import GitWildMatchPattern
 
+from dvc.scm.tree import BaseTree
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
@@ -72,13 +72,12 @@ class DvcIgnoreDirs(DvcIgnore):
 
 
 class DvcIgnoreFilter(object):
-    def __init__(self, root_dir, tree):
+    def __init__(self, tree):
         self.tree = tree
         self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
-        self._update(root_dir)
-        for root, dirs, _ in self.tree.walk(root_dir, dvcignore=self):
-            for d in dirs:
-                self._update(os.path.join(root, d))
+        for root, dirs, files in self.tree.walk(self.tree.tree_root):
+            self._update(root)
+            dirs[:], files[:] = self(root, dirs, files)
 
     def _update(self, dirname):
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
@@ -90,3 +89,39 @@ class DvcIgnoreFilter(object):
             dirs, files = ignore(root, dirs, files)
 
         return dirs, files
+
+
+class CleanTree(BaseTree):
+    def __init__(self, tree):
+        self.tree = tree
+
+    @cached_property
+    def dvcignore(self):
+        return DvcIgnoreFilter(self.tree)
+
+    @property
+    def tree_root(self):
+        return self.tree.tree_root
+
+    def open(self, path, mode="r", encoding="utf-8"):
+        return self.tree.open(path, mode, encoding)
+
+    def exists(self, path):
+        return self.tree.exists(path)
+
+    def isdir(self, path):
+        return self.tree.isdir(path)
+
+    def isfile(self, path):
+        return self.tree.isfile(path)
+
+    def walk(self, top, topdown=True):
+        for root, dirs, files in self.tree.walk(top, topdown):
+            dirs[:], files[:] = self.dvcignore(root, dirs, files)
+
+            yield root, dirs, files
+
+    def walk_files(self, top):
+        for root, _, files in self.walk(top):
+            for file in files:
+                yield os.path.join(root, file)

dvc/istextfile.py

@@ -1,24 +1,10 @@
 """Use heuristics to guess if it is a text file or a binary file."""
-from __future__ import unicode_literals
-
-from dvc.utils.compat import is_py3
-from dvc.utils.compat import open
 
 # Based on https://eli.thegreenplace.net/2011/10/19/
 # perls-guess-if-file-is-text-or-binary-implemented-in-python
 
 
-# A function that takes an integer in the 8-bit range and returns
-# a single-character byte object in py3 / a single-character string
-# in py2.
-#
-def _int2byte(i):
-    if is_py3:
-        return bytes((i,))
-    return chr(i)
-
-
-TEXT_CHARS = b"".join(_int2byte(i) for i in range(32, 127)) + b"\n\r\t\f\b"
+TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
 
 
 def istextfile(fname, blocksize=512):

dvc/lock.py

@@ -1,5 +1,4 @@
 """Manages dvc lock file."""
-from __future__ import unicode_literals
 
 import hashlib
 import os
@@ -7,10 +6,10 @@ import time
 from datetime import timedelta
 
 import zc.lockfile
+import flufl.lock
 
 from dvc.exceptions import DvcException
 from dvc.utils import format_link
-from dvc.utils.compat import is_py3, is_py2
 from dvc.progress import Tqdm
 
 DEFAULT_TIMEOUT = 5
@@ -23,11 +22,11 @@ FAILED_TO_LOCK_MESSAGE = (
 
 
 class LockError(DvcException):
-    """Thrown when unable to acquire the lock for dvc repo."""
+    """Thrown when unable to acquire the lock for DVC repo."""
 
 
 class Lock(object):
-    """Class for dvc repo lock.
+    """Class for DVC repo lock.
 
     Uses zc.lockfile as backend.
     """
@@ -69,6 +68,10 @@ class Lock(object):
         self._lock.close()
         self._lock = None
 
+    @property
+    def is_locked(self):
+        return bool(self._lock)
+
     def __enter__(self):
         self.lock()
 
@@ -76,83 +79,70 @@ class Lock(object):
         self.unlock()
 
 
-if is_py3:
-    import flufl.lock
+class HardlinkLock(flufl.lock.Lock):
+    """Class for DVC repo lock.
 
-    class HardlinkLock(flufl.lock.Lock):
-        """Class for dvc repo lock.
+    Args:
+        lockfile (str): the lock filename
+            in.
+        tmp_dir (str): a directory to store claim files.
+    """
 
-        Args:
-            lockfile (str): the lock filename
-                in.
-            tmp_dir (str): a directory to store claim files.
-        """
+    def __init__(self, lockfile, tmp_dir=None, **kwargs):
+        import socket
 
-        def __init__(self, lockfile, tmp_dir=None, **kwargs):
-            import socket
+        self._tmp_dir = tmp_dir
 
-            self._tmp_dir = tmp_dir
+        # NOTE: this is basically Lock.__init__ copy-paste, except that
+        # instead of using `socket.getfqdn()` we use `socket.gethostname()`
+        # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
+        # anything, which is way too slow. `gethostname()` is actually a
+        # fallback for `getfqdn()` when it is not able to resolve a
+        # canonical hostname through network. The claimfile that uses
+        # `self._hostname` is still usable, as it uses `pid` and random
+        # number to generate the resulting lock file name, which is unique
+        # enough for our application.
+        #
+        # [1] https://github.com/iterative/dvc/issues/2582
+        self._hostname = socket.gethostname()
 
-            # NOTE: this is basically Lock.__init__ copy-paste, except that
-            # instead of using `socket.getfqdn()` we use `socket.gethostname()`
-            # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
-            # anything, which is way too slow. `gethostname()` is actually a
-            # fallback for `getfqdn()` when it is not able to resolve a
-            # canonical hostname through network. The claimfile that uses
-            # `self._hostname` is still usable, as it uses `pid` and random
-            # number to generate the resulting lock file name, which is unique
-            # enough for our application.
-            #
-            # [1] https://github.com/iterative/dvc/issues/2582
-            self._hostname = socket.gethostname()
+        self._lockfile = lockfile
+        self._lifetime = timedelta(days=365)  # Lock for good by default
+        self._separator = flufl.lock.SEP
+        self._set_claimfile()
+        self._owned = True
+        self._retry_errnos = []
 
-            self._lockfile = lockfile
-            self._lifetime = timedelta(days=365)  # Lock for good by default
-            self._separator = flufl.lock.SEP
-            self._set_claimfile()
-            self._owned = True
-            self._retry_errnos = []
+    @property
+    def lockfile(self):
+        return self._lockfile
 
-        @property
-        def lockfile(self):
-            return self._lockfile
+    def lock(self):
+        try:
+            super().lock(timedelta(seconds=DEFAULT_TIMEOUT))
+        except flufl.lock.TimeOutError:
+            raise LockError(FAILED_TO_LOCK_MESSAGE)
 
-        def lock(self):
-            try:
-                super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
-            except flufl.lock.TimeOutError:
-                raise LockError(FAILED_TO_LOCK_MESSAGE)
+    def _set_claimfile(self, pid=None):
+        super()._set_claimfile(pid)
 
-        def _set_claimfile(self, pid=None):
-            super(Lock, self)._set_claimfile(pid)
+        if self._tmp_dir is not None:
+            # Under Windows file path length is limited so we hash it
+            filename = hashlib.md5(self._claimfile.encode()).hexdigest()
+            self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
 
-            if self._tmp_dir is not None:
-                # Under Windows file path length is limited so we hash it
-                filename = hashlib.md5(self._claimfile.encode()).hexdigest()
-                self._claimfile = os.path.join(
-                    self._tmp_dir, filename + ".lock"
-                )
-
-        # Fix for __del__ bug in flufl.lock [1] which is causing errors on
-        # Python shutdown [2].
-        # [1] https://gitlab.com/warsaw/flufl.lock/issues/7
-        # [2] https://github.com/iterative/dvc/issues/2573
-        def __del__(self):
-            try:
-                if self._owned:
-                    self.finalize()
-            except ImportError:
-                pass
+    # Fix for __del__ bug in flufl.lock [1] which is causing errors on
+    # Python shutdown [2].
+    # [1] https://gitlab.com/warsaw/flufl.lock/issues/7
+    # [2] https://github.com/iterative/dvc/issues/2573
+    def __del__(self):
+        try:
+            if self._owned:
+                self.finalize()
+        except ImportError:
+            pass
 
 
 def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False):
-    if hardlink_lock and is_py2:
-        raise DvcException(
-            "Hardlink locks are not supported on Python <3.5. "
-            "See `hardlink_lock` in {}".format(
-                format_link("man.dvc.org/config#core")
-            )
-        )
-
     cls = HardlinkLock if hardlink_lock else Lock
     return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly)

dvc/logger.py

@@ -1,15 +1,12 @@
-"""Manages logging configuration for dvc repo."""
-from __future__ import unicode_literals
+"""Manages logging configuration for DVC repo."""
 
+import traceback
 import logging.config
 import logging.handlers
 
 import colorama
 
 from dvc.progress import Tqdm
-from dvc.utils.compat import RecursionError
-from dvc.utils.compat import str
-from dvc.utils.compat import StringIO
 
 
 FOOTER = (
@@ -26,7 +23,7 @@ FOOTER = (
 class LoggingException(Exception):
     def __init__(self, record):
         msg = "failed to log {}".format(str(record))
-        super(LoggingException, self).__init__(msg)
+        super().__init__(msg)
 
 
 class ExcludeErrorsFilter(logging.Filter):
@@ -61,11 +58,13 @@ class ColorFormatter(logging.Formatter):
     }
 
     def format(self, record):
+        msg = record.msg.format(*record.args) if record.args else record.msg
+
         if record.levelname == "INFO":
-            return record.msg
+            return msg
 
         if record.levelname == "ERROR" or record.levelname == "CRITICAL":
-            exception, stack_trace = self._parse_exc(record.exc_info)
+            exception, stack_trace = self._parse_exc(record)
 
             return (
                 "{color}{levelname}{nc}: {description}" "{stack_trace}\n"
@@ -73,8 +72,8 @@ class ColorFormatter(logging.Formatter):
                 color=self.color_code.get(record.levelname, ""),
                 nc=colorama.Fore.RESET,
                 levelname=record.levelname,
-                description=self._description(record.msg, exception),
-                msg=record.msg,
+                description=self._description(msg, exception),
+                msg=msg,
                 stack_trace=stack_trace,
             )
 
@@ -82,7 +81,7 @@ class ColorFormatter(logging.Formatter):
             color=self.color_code.get(record.levelname, ""),
             nc=colorama.Fore.RESET,
             levelname=record.levelname,
-            msg=record.msg,
+            msg=msg,
         )
 
     def _current_level(self):
@@ -104,34 +103,26 @@ class ColorFormatter(logging.Formatter):
         return description.format(message=message, exception=exception)
 
     def _walk_exc(self, exc_info):
-        import traceback
-
-        buffer = StringIO()
-
-        traceback.print_exception(*exc_info, file=buffer)
-
         exc = exc_info[1]
-        tb = buffer.getvalue()
 
         exc_list = [str(exc)]
-        tb_list = [tb]
 
-        # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info
-        while hasattr(exc, "cause") and exc.cause:
-            exc_list.append(str(exc.cause))
-            if hasattr(exc, "cause_tb") and exc.cause_tb:
-                tb_list.insert(0, str(exc.cause_tb))
-            exc = exc.cause
+        while hasattr(exc, "__cause__") and exc.__cause__:
+            exc_list.append(str(exc.__cause__))
+            exc = exc.__cause__
 
-        return exc_list, tb_list
+        return exc_list
 
-    def _parse_exc(self, exc_info):
-        if not exc_info:
+    def _parse_exc(self, record):
+        tb_only = getattr(record, "tb_only", False)
+
+        if not record.exc_info:
             return (None, "")
 
-        exc_list, tb_list = self._walk_exc(exc_info)
+        exc_list = self._walk_exc(record.exc_info)
+        tb = traceback.format_exception(*record.exc_info)
 
-        exception = ": ".join(exc_list)
+        exception = None if tb_only else ": ".join(exc_list)
 
         if self._current_level() == logging.DEBUG:
             stack_trace = (
@@ -140,7 +131,7 @@ class ColorFormatter(logging.Formatter):
                 red=colorama.Fore.RED,
                 nc=colorama.Fore.RESET,
                 line="-" * 60,
-                stack_trace="\n".join(tb_list),
+                stack_trace="".join(tb),
             )
         else:
             stack_trace = ""
@@ -150,7 +141,7 @@ class ColorFormatter(logging.Formatter):
 
 class LoggerHandler(logging.StreamHandler):
     def handleError(self, record):
-        super(LoggerHandler, self).handleError(record)
+        super().handleError(record)
         raise LoggingException(record)
 
     def emit(self, record):

dvc/main.py

@@ -1,6 +1,6 @@
 """Main entry point for dvc CLI."""
-from __future__ import unicode_literals
 
+import errno
 import logging
 
 from dvc import analytics
@@ -10,13 +10,12 @@ from dvc.exceptions import DvcParserError
 from dvc.exceptions import NotDvcRepoError
 from dvc.external_repo import clean_repos
 from dvc.logger import FOOTER
-from dvc.remote.pool import close_pools
-from dvc.utils.compat import is_py2
-
+from dvc.utils import format_link
 
 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
 # [2] https://bugs.python.org/issue29288
+
 "".encode("idna")
 
 
@@ -59,10 +58,14 @@ def main(argv=None):
     except DvcParserError:
         ret = 254
     except Exception as exc:  # pylint: disable=broad-except
-        if isinstance(exc, UnicodeError) and is_py2:
+        if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
             logger.exception(
-                "unicode is not supported in DVC for Python 2 "
-                "(end-of-life January 1, 2020), please upgrade to Python 3"
+                "too many open files, please visit "
+                "{} to see how to handle this "
+                "problem".format(
+                    format_link("https://error.dvc.org/many-files")
+                ),
+                extra={"tb_only": True},
             )
         else:
             logger.exception("unexpected error")
@@ -70,10 +73,6 @@ def main(argv=None):
     finally:
         logger.setLevel(outerLogLevel)
 
-        # Python 2 fails to close these clean occasionally and users see
-        # weird error messages, so we do it manually
-        close_pools()
-
         # Remove cached repos in the end of the call, these are anonymous
         # so won't be reused by any other subsequent run anyway.
         clean_repos()

dvc/output/__init__.py

@@ -1,5 +1,4 @@
-from __future__ import unicode_literals
-
+from urllib.parse import urlparse
 from voluptuous import Any, Required
 
 from dvc.output.base import OutputBase
@@ -13,8 +12,6 @@ from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.local import RemoteLOCAL
 from dvc.remote.s3 import RemoteS3
 from dvc.scheme import Schemes
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
 
 OUTS = [
     OutputHDFS,

dvc/output/base.py

@@ -1,6 +1,5 @@
-from __future__ import unicode_literals
-
 import logging
+from urllib.parse import urlparse
 from copy import copy
 
 from voluptuous import Any
@@ -10,8 +9,6 @@ from dvc.cache import NamedCache
 from dvc.exceptions import CollectCacheError
 from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteBASE
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -20,26 +17,24 @@ logger = logging.getLogger(__name__)
 class OutputDoesNotExistError(DvcException):
     def __init__(self, path):
         msg = "output '{}' does not exist".format(path)
-        super(OutputDoesNotExistError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class OutputIsNotFileOrDirError(DvcException):
     def __init__(self, path):
         msg = "output '{}' is not a file or directory".format(path)
-        super(OutputIsNotFileOrDirError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class OutputAlreadyTrackedError(DvcException):
     def __init__(self, path):
         msg = "output '{}' is already tracked by scm (e.g. git)".format(path)
-        super(OutputAlreadyTrackedError, self).__init__(msg)
+        super().__init__(msg)
 
 
 class OutputIsStageFileError(DvcException):
     def __init__(self, path):
-        super(OutputIsStageFileError, self).__init__(
-            "Stage file '{}' cannot be an output.".format(path)
-        )
+        super().__init__("Stage file '{}' cannot be an output.".format(path))
 
 
 class OutputBase(object):
@@ -283,11 +278,18 @@ class OutputBase(object):
         self.remote.download(self.path_info, to.path_info)
 
     def checkout(
-        self, force=False, progress_callback=None, tag=None, relink=False
+        self,
+        force=False,
+        progress_callback=None,
+        tag=None,
+        relink=False,
+        filter_info=None,
     ):
         if not self.use_cache:
             if progress_callback:
-                progress_callback(str(self.path_info), self.get_files_number())
+                progress_callback(
+                    str(self.path_info), self.get_files_number(filter_info)
+                )
             return None
 
         if tag:
@@ -301,6 +303,7 @@ class OutputBase(object):
             force=force,
             progress_callback=progress_callback,
             relink=relink,
+            filter_info=filter_info,
         )
 
     def remove(self, ignore_remove=False):
@@ -324,17 +327,21 @@ class OutputBase(object):
         if self.scheme == "local" and self.use_scm_ignore:
             self.repo.scm.ignore(self.fspath)
 
-    def get_files_number(self):
+    def get_files_number(self, filter_info=None):
         if not self.use_cache:
             return 0
 
-        return self.cache.get_files_number(self.checksum)
+        return self.cache.get_files_number(
+            self.path_info, self.checksum, filter_info
+        )
 
     def unprotect(self):
         if self.exists:
             self.remote.unprotect(self.path_info)
 
-    def _collect_used_dir_cache(self, remote=None, force=False, jobs=None):
+    def _collect_used_dir_cache(
+        self, remote=None, force=False, jobs=None, filter_info=None
+    ):
         """Get a list of `info`s related to the given directory.
 
         - Pull the directory entry from the remote cache if it was changed.
@@ -383,8 +390,9 @@ class OutputBase(object):
 
         for entry in self.dir_cache:
             checksum = entry[self.remote.PARAM_CHECKSUM]
-            path_info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            cache.add(self.scheme, checksum, str(path_info))
+            info = self.path_info / entry[self.remote.PARAM_RELPATH]
+            if not filter_info or info.isin_or_eq(filter_info):
+                cache.add(self.scheme, checksum, str(info))
 
         return cache
 
@@ -400,6 +408,12 @@ class OutputBase(object):
         if not self.use_cache:
             return NamedCache()
 
+        if self.stage.is_repo_import:
+            cache = NamedCache()
+            dep, = self.stage.deps
+            cache.external[dep.repo_pair].add(dep.def_path)
+            return cache
+
         if not self.info:
             logger.warning(
                 "Output '{}'({}) is missing version info. Cache for it will "

dvc/output/gs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.s3 import OutputS3
 from dvc.remote.gs import RemoteGS
 

dvc/output/hdfs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.base import OutputBase
 from dvc.remote.hdfs import RemoteHDFS
 

dvc/output/local.py

@@ -1,16 +1,13 @@
-from __future__ import unicode_literals
-
 import logging
 import os
+from urllib.parse import urlparse
 
 from dvc.exceptions import DvcException
 from dvc.istextfile import istextfile
 from dvc.output.base import OutputBase
 from dvc.remote.local import RemoteLOCAL
 from dvc.utils import relpath
-from dvc.utils.compat import fspath_py35
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
+from dvc.compat import fspath_py35
 from dvc.utils.fs import path_isin
 
 
@@ -59,7 +56,7 @@ class OutputLOCAL(OutputBase):
         return def_scheme != "remote" and not os.path.isabs(self.def_path)
 
     def dumpd(self):
-        ret = super(OutputLOCAL, self).dumpd()
+        ret = super().dumpd()
         if self.is_in_repo:
             path = self.path_info.relpath(self.stage.wdir).as_posix()
         else:

dvc/output/s3.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.base import OutputBase
 from dvc.remote.s3 import RemoteS3
 

dvc/output/ssh.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.output.base import OutputBase
 from dvc.remote.ssh import RemoteSSH
 

dvc/path_info.py

@@ -1,35 +1,23 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 import os
 import posixpath
-import sys
+import pathlib
+from urllib.parse import urlparse
 
 from funcy import cached_property
 
 from dvc.utils import relpath
-from dvc.utils.compat import basestring
-from dvc.utils.compat import builtin_str
-from dvc.utils.compat import is_py2
-from dvc.utils.compat import pathlib
-from dvc.utils.compat import str
-from dvc.utils.compat import urlparse
-
-
-# On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
-# which is lossy, thus we can't use that,
-# see https://github.com/mcmtroffaes/pathlib2/issues/56.
-if is_py2:
-    fs_encoding = "utf-8"
 
 
 class _BasePath(object):
     def overlaps(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         elif self.__class__ != other.__class__:
             return False
-        return self == other or self.isin(other) or other.isin(self)
+        return self.isin_or_eq(other) or other.isin(self)
+
+    def isin_or_eq(self, other):
+        return self == other or self.isin(other)
 
 
 class PathInfo(pathlib.PurePath, _BasePath):
@@ -63,7 +51,7 @@ class PathInfo(pathlib.PurePath, _BasePath):
         return relpath(path)
 
     def __repr__(self):
-        return builtin_str("{}: '{}'").format(type(self).__name__, self)
+        return "{}: '{}'".format(type(self).__name__, self)
 
     # This permits passing it to file utils directly in Python 3.6+
     # With Python 2.7, Python 3.5+ we are stuck with path_info.fspath for now
@@ -80,7 +68,7 @@ class PathInfo(pathlib.PurePath, _BasePath):
         return self.__class__(relpath(self, other))
 
     def isin(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         elif self.__class__ != other.__class__:
             return False
@@ -88,34 +76,6 @@ class PathInfo(pathlib.PurePath, _BasePath):
         n = len(other._cparts)
         return len(self._cparts) > n and self._cparts[:n] == other._cparts
 
-    # pathlib2 uses bytes internally in Python 2, and we use unicode everywhere
-    # for paths in both pythons, thus we need this glue.
-    if is_py2:
-        __unicode__ = __str__
-
-        def __str__(self):
-            return self.__unicode__().encode(sys.getfilesystemencoding())
-
-        @classmethod
-        def _parse_args(cls, args):
-            args = [
-                a.encode(fs_encoding)
-                if isinstance(a, unicode)  # noqa: F821
-                else a
-                for a in args
-            ]
-            return super(PathInfo, cls)._parse_args(args)
-
-        @property
-        def name(self):
-            return super(PathInfo, self).name.decode(fs_encoding)
-
-        def __fspath__(self):  # noqa: F811
-            return pathlib.PurePath.__str__(self).decode(fs_encoding)
-
-        def with_name(self, name):
-            return pathlib.PurePath.with_name(self, name.encode(fs_encoding))
-
 
 class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
     pass
@@ -172,13 +132,13 @@ class URLInfo(_BasePath):
 
     def fill_parts(self, scheme, host, user, port, path):
         assert scheme != "remote"
-        assert isinstance(path, (basestring, _URLPathInfo))
+        assert isinstance(path, (str, bytes, _URLPathInfo))
 
         self.scheme, self.host, self.user = scheme, host, user
         self.port = int(port) if port else self.DEFAULT_PORTS.get(self.scheme)
 
         if isinstance(path, _URLPathInfo):
-            self._spath = builtin_str(path)
+            self._spath = str(path)
             self._path = path
         else:
             if path and path[0] != "/":
@@ -207,7 +167,7 @@ class URLInfo(_BasePath):
         return "{}: '{}'".format(type(self).__name__, self)
 
     def __eq__(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         return (
             self.__class__ == other.__class__
@@ -257,7 +217,7 @@ class URLInfo(_BasePath):
         return _URLPathParents(self)
 
     def relative_to(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         if self.__class__ != other.__class__:
             msg = "'{}' has incompatible class with '{}'".format(self, other)
@@ -268,7 +228,7 @@ class URLInfo(_BasePath):
         return self._path.relative_to(other._path)
 
     def isin(self, other):
-        if isinstance(other, basestring):
+        if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         elif self.__class__ != other.__class__:
             return False

dvc/progress.py

@@ -1,5 +1,4 @@
-"""Manages progress bars for dvc repo."""
-from __future__ import print_function
+"""Manages progress bars for DVC repo."""
 
 import logging
 import sys
@@ -78,7 +77,7 @@ class Tqdm(tqdm):
             and hasattr(file, "isatty")
         ):
             disable = not file.isatty()
-        super(Tqdm, self).__init__(
+        super().__init__(
             iterable=iterable,
             disable=disable,
             leave=leave,
@@ -115,12 +114,12 @@ class Tqdm(tqdm):
     def close(self):
         if self.desc_persist is not None:
             self.set_description_str(self.desc_persist, refresh=False)
-        super(Tqdm, self).close()
+        super().close()
 
     @property
     def format_dict(self):
         """inject `ncols_desc` to fill the display width (`ncols`)"""
-        d = super(Tqdm, self).format_dict
+        d = super().format_dict
         ncols = d["ncols"] or 80
         ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1
         ncols_desc = max(ncols_desc, 0)

dvc/prompt.py

@@ -1,14 +1,9 @@
 """Manages user prompts."""
-from __future__ import print_function
-from __future__ import unicode_literals
 
 import logging
 import sys
 from getpass import getpass
 
-from dvc.utils.compat import input
-
-
 logger = logging.getLogger(__name__)
 
 

dvc/remote/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from .config import RemoteConfig
 from dvc.remote.azure import RemoteAZURE
 from dvc.remote.gdrive import RemoteGDrive

dvc/remote/azure.py

@@ -1,10 +1,8 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 import logging
 import os
 import re
 from datetime import datetime, timedelta
+from urllib.parse import urlparse
 import threading
 
 from funcy import cached_property, wrap_prop
@@ -14,7 +12,6 @@ from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
-from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -34,7 +31,7 @@ class RemoteAZURE(RemoteBASE):
     COPY_POLL_SECONDS = 5
 
     def __init__(self, repo, config):
-        super(RemoteAZURE, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL, "azure://")
 

dvc/remote/base.py

@@ -1,10 +1,9 @@
-from __future__ import unicode_literals
-from dvc.utils.compat import basestring, FileNotFoundError, str, urlparse
-
+import errno
 import itertools
 import json
 import logging
 import tempfile
+from urllib.parse import urlparse
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
 from functools import partial
@@ -25,8 +24,8 @@ from dvc.path_info import PathInfo, URLInfo
 from dvc.progress import Tqdm
 from dvc.remote.slow_link_detection import slow_link_guard
 from dvc.state import StateNoop
-from dvc.utils import makedirs, relpath, tmp_fname
-from dvc.utils.fs import move
+from dvc.utils import tmp_fname
+from dvc.utils.fs import move, makedirs
 from dvc.utils.http import open_url
 
 logger = logging.getLogger(__name__)
@@ -47,7 +46,7 @@ STATUS_MAP = {
 
 class RemoteCmdError(DvcException):
     def __init__(self, remote, cmd, ret, err):
-        super(RemoteCmdError, self).__init__(
+        super().__init__(
             "{remote} command '{cmd}' finished with non-zero return code"
             " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
         )
@@ -55,8 +54,8 @@ class RemoteCmdError(DvcException):
 
 class RemoteActionNotImplemented(DvcException):
     def __init__(self, action, scheme):
-        m = "{} is not supported by {} remote".format(action, scheme)
-        super(RemoteActionNotImplemented, self).__init__(m)
+        m = "{} is not supported for {} remotes".format(action, scheme)
+        super().__init__(m)
 
 
 class RemoteMissingDepsError(DvcException):
@@ -64,10 +63,9 @@ class RemoteMissingDepsError(DvcException):
 
 
 class DirCacheError(DvcException):
-    def __init__(self, checksum, cause=None):
-        super(DirCacheError, self).__init__(
-            "Failed to load dir cache for checksum: '{}'.".format(checksum),
-            cause=cause,
+    def __init__(self, checksum):
+        super().__init__(
+            "Failed to load dir cache for checksum: '{}'.".format(checksum)
         )
 
 
@@ -81,6 +79,8 @@ class RemoteBASE(object):
     CHECKSUM_DIR_SUFFIX = ".dir"
     CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
     DEFAULT_CACHE_TYPES = ["copy"]
+    DEFAULT_NO_TRAVERSE = True
+    DEFAULT_VERIFY = False
 
     state = StateNoop()
 
@@ -88,14 +88,15 @@ class RemoteBASE(object):
         self.repo = repo
 
         self._check_requires(config)
-
-        core = config.get(Config.SECTION_CORE, {})
-        self.checksum_jobs = core.get(
-            Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
-        )
+        self.checksum_jobs = self._get_checksum_jobs(config)
 
         self.protected = False
-        self.no_traverse = config.get(Config.SECTION_REMOTE_NO_TRAVERSE, True)
+        self.no_traverse = config.get(
+            Config.SECTION_REMOTE_NO_TRAVERSE, self.DEFAULT_NO_TRAVERSE
+        )
+        self.verify = config.get(
+            Config.SECTION_REMOTE_VERIFY, self.DEFAULT_VERIFY
+        )
         self._dir_info = {}
 
         types = config.get(Config.SECTION_CACHE_TYPE, None)
@@ -144,6 +145,19 @@ class RemoteBASE(object):
         ).format(url, missing, " ".join(missing), self.scheme)
         raise RemoteMissingDepsError(msg)
 
+    def _get_checksum_jobs(self, config):
+        checksum_jobs = config.get(Config.SECTION_REMOTE_CHECKSUM_JOBS)
+        if checksum_jobs:
+            return checksum_jobs
+
+        if self.repo:
+            core = self.repo.config.config.get(Config.SECTION_CORE, {})
+            return core.get(
+                Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
+            )
+
+        return self.CHECKSUM_JOBS
+
     def __repr__(self):
         return "{class_name}: '{path_info}'".format(
             class_name=type(self).__name__,
@@ -152,7 +166,7 @@ class RemoteBASE(object):
 
     @classmethod
     def supported(cls, config):
-        if isinstance(config, basestring):
+        if isinstance(config, (str, bytes)):
             url = config
         else:
             url = config[Config.SECTION_REMOTE_URL]
@@ -265,11 +279,13 @@ class RemoteBASE(object):
             with self.cache.open(path_info, "r") as fobj:
                 d = json.load(fobj)
         except (ValueError, FileNotFoundError) as exc:
-            raise DirCacheError(checksum, cause=exc)
+            raise DirCacheError(checksum) from exc
 
         if not isinstance(d, list):
-            msg = "dir cache file format error '{}' [skipping the file]"
-            logger.error(msg.format(relpath(path_info)))
+            logger.error(
+                "dir cache file format error '{}' [skipping the file]",
+                path_info,
+            )
             return []
 
         for info in d:
@@ -335,36 +351,35 @@ class RemoteBASE(object):
         """
 
         logger.debug(
-            "checking if '{}'('{}') has changed.".format(
-                path_info, checksum_info
-            )
+            "checking if '{}'('{}') has changed.", path_info, checksum_info
         )
 
         if not self.exists(path_info):
-            logger.debug("'{}' doesn't exist.".format(path_info))
+            logger.debug("'{}' doesn't exist.", path_info)
             return True
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
         if checksum is None:
-            logger.debug("checksum for '{}' is missing.".format(path_info))
+            logger.debug("checksum for '{}' is missing.", path_info)
             return True
 
         if self.changed_cache(checksum):
             logger.debug(
-                "cache for '{}'('{}') has changed.".format(path_info, checksum)
+                "cache for '{}'('{}') has changed.", path_info, checksum
             )
             return True
 
         actual = self.get_checksum(path_info)
         if checksum != actual:
             logger.debug(
-                "checksum '{}'(actual '{}') for '{}' has changed.".format(
-                    checksum, actual, path_info
-                )
+                "checksum '{}'(actual '{}') for '{}' has changed.",
+                checksum,
+                actual,
+                path_info,
             )
             return True
 
-        logger.debug("'{}' hasn't changed.".format(path_info))
+        logger.debug("'{}' hasn't changed.", path_info)
         return False
 
     def link(self, from_info, to_info):
@@ -377,18 +392,30 @@ class RemoteBASE(object):
 
         self._try_links(from_info, to_info, link_types)
 
+    def _verify_link(self, path_info, link_type):
+        if self.cache_type_confirmed:
+            return
+
+        is_link = getattr(self, "is_{}".format(link_type), None)
+        if is_link and not is_link(path_info):
+            self.remove(path_info)
+            raise DvcException("failed to verify {}".format(link_type))
+
+        self.cache_type_confirmed = True
+
     @slow_link_guard
     def _try_links(self, from_info, to_info, link_types):
         while link_types:
             link_method = getattr(self, link_types[0])
             try:
                 self._do_link(from_info, to_info, link_method)
-                self.cache_type_confirmed = True
+                self._verify_link(to_info, link_types[0])
                 return
 
             except DvcException as exc:
-                msg = "Cache type '{}' is not supported: {}"
-                logger.debug(msg.format(link_types[0], str(exc)))
+                logger.debug(
+                    "Cache type '{}' is not supported: {}", link_types[0], exc
+                )
                 del link_types[0]
 
         raise DvcException("no possible cache types left to try out.")
@@ -402,13 +429,13 @@ class RemoteBASE(object):
         if self.protected:
             self.protect(to_info)
 
-        msg = "Created {}'{}': {} -> {}".format(
+        logger.debug(
+            "Created {}'{}': {} -> {}",
             "protected " if self.protected else "",
             self.cache_types[0],
             from_info,
             to_info,
         )
-        logger.debug(msg)
 
     def _save_file(self, path_info, checksum, save_link=True):
         assert checksum
@@ -510,12 +537,23 @@ class RemoteBASE(object):
 
     def _save(self, path_info, checksum):
         to_info = self.checksum_to_path_info(checksum)
-        logger.debug("Saving '{}' to '{}'.".format(path_info, to_info))
+        logger.debug("Saving '{}' to '{}'.", path_info, to_info)
         if self.isdir(path_info):
             self._save_dir(path_info, checksum)
             return
         self._save_file(path_info, checksum)
 
+    def _handle_transfer_exception(
+        self, from_info, to_info, exception, operation
+    ):
+        if isinstance(exception, OSError) and exception.errno == errno.EMFILE:
+            raise exception
+
+        logger.exception(
+            "failed to {} '{}' to '{}'", operation, from_info, to_info
+        )
+        return 1
+
     def upload(self, from_info, to_info, name=None, no_progress_bar=False):
         if not hasattr(self, "_upload"):
             raise RemoteActionNotImplemented("upload", self.scheme)
@@ -526,7 +564,7 @@ class RemoteBASE(object):
         if from_info.scheme != "local":
             raise NotImplementedError
 
-        logger.debug("Uploading '{}' to '{}'".format(from_info, to_info))
+        logger.debug("Uploading '{}' to '{}'", from_info, to_info)
 
         name = name or from_info.name
 
@@ -537,10 +575,10 @@ class RemoteBASE(object):
                 name=name,
                 no_progress_bar=no_progress_bar,
             )
-        except Exception:
-            msg = "failed to upload '{}' to '{}'"
-            logger.exception(msg.format(from_info, to_info))
-            return 1  # 1 fail
+        except Exception as e:
+            return self._handle_transfer_exception(
+                from_info, to_info, e, "upload"
+            )
 
         return 0
 
@@ -605,7 +643,7 @@ class RemoteBASE(object):
     ):
         makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
 
-        logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
+        logger.debug("Downloading '{}' to '{}'", from_info, to_info)
         name = name or to_info.name
 
         tmp_file = tmp_fname(to_info)
@@ -614,10 +652,10 @@ class RemoteBASE(object):
             self._download(
                 from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
             )
-        except Exception:
-            msg = "failed to download '{}' to '{}'"
-            logger.exception(msg.format(from_info, to_info))
-            return 1  # 1 fail
+        except Exception as e:
+            return self._handle_transfer_exception(
+                from_info, to_info, e, "download"
+            )
 
         move(tmp_file, to_info, mode=file_mode)
 
@@ -689,6 +727,8 @@ class RemoteBASE(object):
             if checksum in used:
                 continue
             path_info = self.checksum_to_path_info(checksum)
+            if self.is_dir_checksum(checksum):
+                self._remove_unpacked_dir(checksum)
             self.remove(path_info)
             removed = True
         return removed
@@ -709,9 +749,10 @@ class RemoteBASE(object):
         actual = self.get_checksum(cache_info)
 
         logger.debug(
-            "cache '{}' expected '{}' actual '{}'".format(
-                str(cache_info), checksum, actual
-            )
+            "cache '{}' expected '{}' actual '{}'",
+            cache_info,
+            checksum,
+            actual,
         )
 
         if not checksum or not actual:
@@ -721,29 +762,41 @@ class RemoteBASE(object):
             return False
 
         if self.exists(cache_info):
-            logger.warning("corrupted cache file '{}'.".format(cache_info))
+            logger.warning("corrupted cache file '{}'.", cache_info)
             self.remove(cache_info)
 
         return True
 
-    def _changed_dir_cache(self, checksum):
+    def _changed_dir_cache(self, checksum, path_info=None, filter_info=None):
         if self.changed_cache_file(checksum):
             return True
 
-        if not self._changed_unpacked_dir(checksum):
+        if not (path_info and filter_info) and not self._changed_unpacked_dir(
+            checksum
+        ):
             return False
 
         for entry in self.get_dir_cache(checksum):
             entry_checksum = entry[self.PARAM_CHECKSUM]
+
+            if path_info and filter_info:
+                entry_info = path_info / entry[self.PARAM_RELPATH]
+                if not entry_info.isin_or_eq(filter_info):
+                    continue
+
             if self.changed_cache_file(entry_checksum):
                 return True
 
-        self._update_unpacked_dir(checksum)
+        if not (path_info and filter_info):
+            self._update_unpacked_dir(checksum)
+
         return False
 
-    def changed_cache(self, checksum):
+    def changed_cache(self, checksum, path_info=None, filter_info=None):
         if self.is_dir_checksum(checksum):
-            return self._changed_dir_cache(checksum)
+            return self._changed_dir_cache(
+                checksum, path_info=path_info, filter_info=filter_info
+            )
         return self.changed_cache_file(checksum)
 
     def cache_exists(self, checksums, jobs=None, name=None):
@@ -819,8 +872,9 @@ class RemoteBASE(object):
         """The file is changed we need to checkout a new copy"""
         cache_info = self.checksum_to_path_info(checksum)
         if self.exists(path_info):
-            msg = "data '{}' exists. Removing before checkout."
-            logger.warning(msg.format(str(path_info)))
+            logger.warning(
+                "data '{}' exists. Removing before checkout.", path_info
+            )
             self.safe_remove(path_info, force=force)
 
         self.link(cache_info, path_info)
@@ -836,7 +890,13 @@ class RemoteBASE(object):
         pass
 
     def _checkout_dir(
-        self, path_info, checksum, force, progress_callback=None, relink=False
+        self,
+        path_info,
+        checksum,
+        force,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
     ):
         # Create dir separately so that dir is created
         # even if there are no files in it
@@ -845,7 +905,7 @@ class RemoteBASE(object):
 
         dir_info = self.get_dir_cache(checksum)
 
-        logger.debug("Linking directory '{}'.".format(path_info))
+        logger.debug("Linking directory '{}'.", path_info)
 
         for entry in dir_info:
             relative_path = entry[self.PARAM_RELPATH]
@@ -853,6 +913,9 @@ class RemoteBASE(object):
             entry_cache_info = self.checksum_to_path_info(entry_checksum)
             entry_info = path_info / relative_path
 
+            if filter_info and not entry_info.isin_or_eq(filter_info):
+                continue
+
             entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
             if relink or self.changed(entry_info, entry_checksum_info):
                 self.safe_remove(entry_info, force=force)
@@ -883,6 +946,7 @@ class RemoteBASE(object):
         force=False,
         progress_callback=None,
         relink=False,
+        filter_info=None,
     ):
         if path_info.scheme not in ["local", self.scheme]:
             raise NotImplementedError
@@ -892,35 +956,42 @@ class RemoteBASE(object):
         skip = False
         if not checksum:
             logger.warning(
-                "No checksum info found for '{}'. "
-                "It won't be created.".format(str(path_info))
+                "No checksum info found for '{}'. " "It won't be created.",
+                path_info,
             )
             self.safe_remove(path_info, force=force)
             failed = path_info
 
         elif not relink and not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            logger.debug(msg.format(str(path_info)))
+            logger.debug("Data '{}' didn't change.", path_info)
             skip = True
 
-        elif self.changed_cache(checksum):
-            msg = "Cache '{}' not found. File '{}' won't be created."
-            logger.warning(msg.format(checksum, str(path_info)))
+        elif self.changed_cache(
+            checksum, path_info=path_info, filter_info=filter_info
+        ):
+            logger.warning(
+                "Cache '{}' not found. File '{}' won't be created.",
+                checksum,
+                path_info,
+            )
             self.safe_remove(path_info, force=force)
             failed = path_info
 
         if failed or skip:
             if progress_callback:
                 progress_callback(
-                    str(path_info), self.get_files_number(checksum)
+                    str(path_info),
+                    self.get_files_number(
+                        self.path_info, checksum, filter_info
+                    ),
                 )
             return failed
 
-        msg = "Checking out '{}' with cache '{}'."
-        logger.debug(msg.format(str(path_info), checksum))
+        logger.debug("Checking out '{}' with cache '{}'.", path_info, checksum)
 
-        self._checkout(path_info, checksum, force, progress_callback, relink)
-        return None
+        self._checkout(
+            path_info, checksum, force, progress_callback, relink, filter_info
+        )
 
     def _checkout(
         self,
@@ -929,23 +1000,32 @@ class RemoteBASE(object):
         force=False,
         progress_callback=None,
         relink=False,
+        filter_info=None,
     ):
         if not self.is_dir_checksum(checksum):
             return self._checkout_file(
                 path_info, checksum, force, progress_callback=progress_callback
             )
         return self._checkout_dir(
-            path_info, checksum, force, progress_callback, relink
+            path_info, checksum, force, progress_callback, relink, filter_info
         )
 
-    def get_files_number(self, checksum):
+    def get_files_number(self, path_info, checksum, filter_info):
+        from funcy.py3 import ilen
+
         if not checksum:
             return 0
 
-        if self.is_dir_checksum(checksum):
+        if not self.is_dir_checksum(checksum):
+            return 1
+
+        if not filter_info:
             return len(self.get_dir_cache(checksum))
 
-        return 1
+        return ilen(
+            filter_info.isin_or_eq(path_info / entry[self.PARAM_CHECKSUM])
+            for entry in self.get_dir_cache(checksum)
+        )
 
     @staticmethod
     def unprotect(path_info):
@@ -964,3 +1044,6 @@ class RemoteBASE(object):
 
     def _update_unpacked_dir(self, checksum):
         pass
+
+    def _remove_unpacked_dir(self, checksum):
+        pass

dvc/remote/config.py

@@ -1,11 +1,10 @@
 import logging
 import os
 import posixpath
+from urllib.parse import urlparse
 
-from dvc.config import Config
-from dvc.config import ConfigError
+from dvc.config import Config, ConfigError
 from dvc.utils import relpath
-from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -34,7 +33,7 @@ class RemoteConfig(object):
 
         if settings is None:
             raise ConfigError(
-                "unable to find remote section '{}'".format(name)
+                "Unable to find remote section '{}'".format(name)
             )
 
         parsed = urlparse(settings["url"])
@@ -134,6 +133,9 @@ class RemoteConfig(object):
                 break
 
     def modify(self, name, option, value, level=None):
+        # try to get remote settings to verify that section exists
+        self.get_settings(name)
+
         self.config.set(
             Config.SECTION_REMOTE_FMT.format(name), option, value, level=level
         )
@@ -155,3 +157,7 @@ class RemoteConfig(object):
         return self.config.get(
             Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level
         )
+
+    def has_default(self):
+        core = self.config.config[Config.SECTION_CORE]
+        return bool(core.get(Config.SECTION_CORE_REMOTE))

dvc/remote/gdrive.py

@@ -1,9 +1,8 @@
-from __future__ import unicode_literals
-
 import os
 import posixpath
 import logging
 import threading
+import re
 
 from funcy import retry, compose, decorator, wrap_with
 from funcy.py3 import cat
@@ -21,14 +20,21 @@ FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
 
 
 class GDriveRetriableError(DvcException):
-    def __init__(self, msg):
-        super(GDriveRetriableError, self).__init__(msg)
+    pass
+
+
+class GDriveAccessTokenRefreshError(DvcException):
+    pass
+
+
+class GDriveMissedCredentialKeyError(DvcException):
+    pass
 
 
 @decorator
 def _wrap_pydrive_retriable(call):
     from apiclient import errors
-    from pydrive.files import ApiRequestError
+    from pydrive2.files import ApiRequestError
 
     try:
         result = call()
@@ -38,7 +44,7 @@ def _wrap_pydrive_retriable(call):
             "HttpError {}".format(code) in str(exception)
             for code in retry_codes
         ):
-            raise GDriveRetriableError(msg="Google API request failed")
+            raise GDriveRetriableError("Google API request failed")
         raise
     return result
 
@@ -55,14 +61,26 @@ gdrive_retry = compose(
 class RemoteGDrive(RemoteBASE):
     scheme = Schemes.GDRIVE
     path_cls = CloudURLInfo
-    REQUIRES = {"pydrive": "pydrive"}
+    REQUIRES = {"pydrive2": "pydrive2"}
+    DEFAULT_NO_TRAVERSE = False
+    DEFAULT_VERIFY = True
+
     GDRIVE_USER_CREDENTIALS_DATA = "GDRIVE_USER_CREDENTIALS_DATA"
-    DEFAULT_USER_CREDENTIALS_FILE = ".dvc/tmp/gdrive-user-credentials.json"
+    DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"
 
     def __init__(self, repo, config):
-        super(RemoteGDrive, self).__init__(repo, config)
-        self.no_traverse = False
+        super().__init__(repo, config)
         self.path_info = self.path_cls(config[Config.SECTION_REMOTE_URL])
+
+        bucket = re.search(
+            "{}://(.*)".format(self.scheme),
+            config[Config.SECTION_REMOTE_URL],
+            re.IGNORECASE,
+        )
+        self.bucket = (
+            bucket.group(1).split("/")[0] if bucket else self.path_info.bucket
+        )
+
         self.config = config
         self.init_drive()
 
@@ -78,11 +96,13 @@ class RemoteGDrive(RemoteBASE):
                 "https://man.dvc.org/remote/add."
             )
         self.gdrive_user_credentials_path = (
-            tmp_fname(".dvc/tmp/")
+            tmp_fname(os.path.join(self.repo.tmp_dir, ""))
             if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
             else self.config.get(
                 Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE,
-                self.DEFAULT_USER_CREDENTIALS_FILE,
+                os.path.join(
+                    self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE
+                ),
             )
         )
 
@@ -137,7 +157,7 @@ class RemoteGDrive(RemoteBASE):
         cached_dirs = {}
         cached_ids = {}
         for dir1 in self.gdrive_list_item(
-            "'{}' in parents and trashed=false".format(self.root_id)
+            "'{}' in parents and trashed=false".format(self.remote_root_id)
         ):
             remote_path = posixpath.join(self.path_info.path, dir1["title"])
             cached_dirs.setdefault(remote_path, []).append(dir1["id"])
@@ -159,9 +179,11 @@ class RemoteGDrive(RemoteBASE):
     @property
     @wrap_with(threading.RLock())
     def drive(self):
+        from pydrive2.auth import RefreshError
+
         if not hasattr(self, "_gdrive"):
-            from pydrive.auth import GoogleAuth
-            from pydrive.drive import GoogleDrive
+            from pydrive2.auth import GoogleAuth
+            from pydrive2.drive import GoogleDrive
 
             if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
                 with open(
@@ -193,14 +215,34 @@ class RemoteGDrive(RemoteBASE):
 
             # Pass non existent settings path to force DEFAULT_SETTINGS loading
             gauth = GoogleAuth(settings_file="")
-            gauth.CommandLineAuth()
 
-            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
-                os.remove(self.gdrive_user_credentials_path)
+            try:
+                gauth.CommandLineAuth()
+            except RefreshError as exc:
+                raise GDriveAccessTokenRefreshError(
+                    "Google Drive's access token refreshment is failed"
+                ) from exc
+            except KeyError as exc:
+                raise GDriveMissedCredentialKeyError(
+                    "Google Drive's user credentials file '{}' "
+                    "misses value for key '{}'".format(
+                        self.gdrive_user_credentials_path, str(exc)
+                    )
+                )
+            # Handle pydrive2.auth.AuthenticationError and others auth failures
+            except Exception as exc:
+                raise DvcException(
+                    "Google Drive authentication failed"
+                ) from exc
+            finally:
+                if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                    os.remove(self.gdrive_user_credentials_path)
 
             self._gdrive = GoogleDrive(gauth)
 
-            self.root_id = self.get_remote_id(self.path_info, create=True)
+            self.remote_root_id = self.get_remote_id(
+                self.path_info, create=True
+            )
             self._cached_dirs, self._cached_ids = self.cache_root_dirs()
 
         return self._gdrive
@@ -221,8 +263,11 @@ class RemoteGDrive(RemoteBASE):
     def get_remote_item(self, name, parents_ids):
         if not parents_ids:
             return None
-        query = " or ".join(
-            "'{}' in parents".format(parent_id) for parent_id in parents_ids
+        query = "({})".format(
+            " or ".join(
+                "'{}' in parents".format(parent_id)
+                for parent_id in parents_ids
+            )
         )
 
         query += " and trashed=false and title='{}'".format(name)
@@ -234,7 +279,7 @@ class RemoteGDrive(RemoteBASE):
         return next(iter(item_list), None)
 
     def resolve_remote_item_from_path(self, path_parts, create):
-        parents_ids = ["root"]
+        parents_ids = [self.bucket]
         current_path = ""
         for path_part in path_parts:
             current_path = posixpath.join(current_path, path_part)
@@ -291,8 +336,10 @@ class RemoteGDrive(RemoteBASE):
         if not self.cached_ids:
             return
 
-        query = " or ".join(
-            "'{}' in parents".format(dir_id) for dir_id in self.cached_ids
+        query = "({})".format(
+            " or ".join(
+                "'{}' in parents".format(dir_id) for dir_id in self.cached_ids
+            )
         )
 
         query += " and trashed=false"

dvc/remote/gs.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals, division
-
 import logging
 from datetime import timedelta
 from functools import wraps
@@ -15,7 +13,6 @@ from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
-from dvc.utils.compat import FileNotFoundError  # skipcq: PYL-W0622
 
 logger = logging.getLogger(__name__)
 
@@ -76,7 +73,7 @@ class RemoteGS(RemoteBASE):
     PARAM_CHECKSUM = "md5"
 
     def __init__(self, repo, config):
-        super(RemoteGS, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL, "gs:///")
         self.path_info = self.path_cls(url)

dvc/remote/hdfs.py

@@ -1,14 +1,12 @@
-from __future__ import unicode_literals
-
-from dvc.utils.compat import FileNotFoundError, urlparse
 import io
 import logging
 import os
 import posixpath
 import re
+import subprocess
 from collections import deque
 from contextlib import closing, contextmanager
-import subprocess
+from urllib.parse import urlparse
 
 from .base import RemoteBASE, RemoteCmdError
 from .pool import get_connection
@@ -26,7 +24,7 @@ class RemoteHDFS(RemoteBASE):
     REQUIRES = {"pyarrow": "pyarrow"}
 
     def __init__(self, repo, config):
-        super(RemoteHDFS, self).__init__(repo, config)
+        super().__init__(repo, config)
         self.path_info = None
         url = config.get(Config.SECTION_REMOTE_URL)
         if not url:

dvc/remote/http.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import logging
 import threading
 
@@ -11,7 +9,6 @@ from dvc.exceptions import DvcException, HTTPError
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
-from dvc.utils.compat import open
 
 logger = logging.getLogger(__name__)
 
@@ -25,7 +22,7 @@ class RemoteHTTP(RemoteBASE):
     PARAM_CHECKSUM = "etag"
 
     def __init__(self, repo, config):
-        super(RemoteHTTP, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL)
         self.path_info = self.path_cls(url) if url else None
@@ -104,7 +101,22 @@ class RemoteHTTP(RemoteBASE):
         kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
 
         try:
-            return self._session.request(method, url, **kwargs)
+            res = self._session.request(method, url, **kwargs)
+
+            redirect_no_location = (
+                kwargs["allow_redirects"]
+                and res.status_code in (301, 302)
+                and "location" not in res.headers
+            )
+
+            if redirect_no_location:
+                # AWS s3 doesn't like to add a location header to its redirects
+                # from https://s3.amazonaws.com/<bucket name>/* type URLs.
+                # This should be treated as an error
+                raise requests.exceptions.RequestException
+
+            return res
+
         except requests.exceptions.RequestException:
             raise DvcException("could not perform a {} request".format(method))
 

dvc/remote/https.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from .http import RemoteHTTP
 from dvc.scheme import Schemes
 

dvc/remote/local.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import errno
 import logging
 import os
@@ -21,18 +19,15 @@ from dvc.remote.base import STATUS_MAP
 from dvc.remote.base import STATUS_MISSING
 from dvc.remote.base import STATUS_NEW
 from dvc.scheme import Schemes
+from dvc.scm.tree import is_working_tree
 from dvc.system import System
-from dvc.utils import copyfile
+from dvc.utils.fs import copyfile
 from dvc.utils import file_md5
-from dvc.utils import makedirs
 from dvc.utils import relpath
 from dvc.utils import tmp_fname
-from dvc.utils import walk_files
-from dvc.utils.compat import fspath_py35
-from dvc.utils.compat import open
-from dvc.utils.compat import str
-from dvc.utils.fs import move
-from dvc.utils.fs import remove
+from dvc.compat import fspath_py35
+from dvc.utils.fs import move, makedirs, remove
+from dvc.utils.fs import walk_files
 
 logger = logging.getLogger(__name__)
 
@@ -50,7 +45,7 @@ class RemoteLOCAL(RemoteBASE):
     SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
 
     def __init__(self, repo, config):
-        super(RemoteLOCAL, self).__init__(repo, config)
+        super().__init__(repo, config)
         self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
 
         shared = config.get(Config.SECTION_CACHE_SHARED)
@@ -66,7 +61,7 @@ class RemoteLOCAL(RemoteBASE):
             cwd = config[Config.PRIVATE_CWD]
             cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
 
-        self.path_info = PathInfo(cache_dir) if cache_dir else None
+        self.cache_dir = cache_dir
         self._dir_info = {}
 
     @property
@@ -77,13 +72,17 @@ class RemoteLOCAL(RemoteBASE):
     def cache_dir(self):
         return self.path_info.fspath if self.path_info else None
 
+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.path_info = PathInfo(value) if value else None
+
     @classmethod
     def supported(cls, config):
         return True
 
     def list_cache_paths(self):
         assert self.path_info is not None
-        return walk_files(self.path_info, None)
+        return walk_files(self.path_info)
 
     def get(self, md5):
         if not md5:
@@ -138,7 +137,9 @@ class RemoteLOCAL(RemoteBASE):
         return os.path.getsize(fspath_py35(path_info))
 
     def walk_files(self, path_info):
-        for fname in walk_files(path_info, self.repo.dvcignore):
+        assert is_working_tree(self.repo.tree)
+
+        for fname in self.repo.tree.walk_files(path_info):
             yield PathInfo(fname)
 
     def get_file_checksum(self, path_info):
@@ -177,6 +178,10 @@ class RemoteLOCAL(RemoteBASE):
     def symlink(from_info, to_info):
         System.symlink(from_info, to_info)
 
+    @staticmethod
+    def is_symlink(path_info):
+        return System.is_symlink(path_info)
+
     def hardlink(self, from_info, to_info):
         # If there are a lot of empty files (which happens a lot in datasets),
         # and the cache type is `hardlink`, we might reach link limits and
@@ -203,6 +208,10 @@ class RemoteLOCAL(RemoteBASE):
 
         System.hardlink(from_info, to_info)
 
+    @staticmethod
+    def is_hardlink(path_info):
+        return System.is_hardlink(path_info)
+
     @staticmethod
     def reflink(from_info, to_info):
         System.reflink(from_info, to_info)
@@ -407,7 +416,7 @@ class RemoteLOCAL(RemoteBASE):
     def _unprotect_file(path):
         if System.is_symlink(path) or System.is_hardlink(path):
             logger.debug("Unprotecting '{}'".format(path))
-            tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
+            tmp = os.path.join(os.path.dirname(path), "." + uuid())
 
             # The operations order is important here - if some application
             # would access the file during the process of copyfile then it
@@ -427,7 +436,9 @@ class RemoteLOCAL(RemoteBASE):
         os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
 
     def _unprotect_dir(self, path):
-        for fname in walk_files(path, self.repo.dvcignore):
+        assert is_working_tree(self.repo.tree)
+
+        for fname in self.repo.tree.walk_files(path):
             RemoteLOCAL._unprotect_file(fname)
 
     def unprotect(self, path_info):
@@ -463,6 +474,10 @@ class RemoteLOCAL(RemoteBASE):
         info = self.checksum_to_path_info(checksum)
         return info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
 
+    def _remove_unpacked_dir(self, checksum):
+        path_info = self._get_unpacked_dir_path_info(checksum)
+        self.remove(path_info)
+
     def _path_info_changed(self, path_info):
         if self.exists(path_info) and self.state.get(path_info):
             return False

dvc/remote/oss.py

@@ -1,6 +1,3 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 import logging
 import os
 import threading
@@ -43,7 +40,7 @@ class RemoteOSS(RemoteBASE):
     COPY_POLL_SECONDS = 5
 
     def __init__(self, repo, config):
-        super(RemoteOSS, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL)
         self.path_info = self.path_cls(url) if url else None
@@ -69,9 +66,9 @@ class RemoteOSS(RemoteBASE):
     def oss_service(self):
         import oss2
 
-        logger.debug("URL {}".format(self.path_info))
-        logger.debug("key id {}".format(self.key_id))
-        logger.debug("key secret {}".format(self.key_secret))
+        logger.debug("URL: {}".format(self.path_info))
+        logger.debug("key id: {}".format(self.key_id))
+        logger.debug("key secret: {}".format(self.key_secret))
 
         auth = oss2.Auth(self.key_id, self.key_secret)
         bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)

dvc/remote/s3.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
 
 import logging
 import os
@@ -25,7 +24,7 @@ class RemoteS3(RemoteBASE):
     PARAM_CHECKSUM = "etag"
 
     def __init__(self, repo, config):
-        super(RemoteS3, self).__init__(repo, config)
+        super().__init__(repo, config)
 
         url = config.get(Config.SECTION_REMOTE_URL, "s3://")
         self.path_info = self.path_cls(url)
@@ -88,8 +87,8 @@ class RemoteS3(RemoteBASE):
             obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
         except Exception as exc:
             raise DvcException(
-                "s3://{}/{} does not exist".format(bucket, path), exc
-            )
+                "s3://{}/{} does not exist".format(bucket, path)
+            ) from exc
         return obj
 
     @classmethod
@@ -157,6 +156,7 @@ class RemoteS3(RemoteBASE):
         # the same throughout the transfer, so it means that in order to
         # preserve etag, we need to transfer each part separately, so the
         # object is transfered in the same chunks as it was originally.
+        from boto3.s3.transfer import TransferConfig
 
         obj = cls.get_head_object(s3, from_info.bucket, from_info.path)
         etag = obj["ETag"].strip('"')
@@ -170,7 +170,13 @@ class RemoteS3(RemoteBASE):
             )
         else:
             source = {"Bucket": from_info.bucket, "Key": from_info.path}
-            s3.copy(source, to_info.bucket, to_info.path, ExtraArgs=extra_args)
+            s3.copy(
+                source,
+                to_info.bucket,
+                to_info.path,
+                ExtraArgs=extra_args,
+                Config=TransferConfig(multipart_threshold=size + 1),
+            )
 
         cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
         if etag != cached_etag:
@@ -195,11 +201,7 @@ class RemoteS3(RemoteBASE):
         }
         paginator = self.s3.get_paginator(self.list_objects_api)
         for page in paginator.paginate(**kwargs):
-            contents = page.get("Contents", None)
-            if not contents:
-                continue
-            for item in contents:
-                yield item
+            yield from page.get("Contents", ())
 
     def _list_paths(self, path_info, max_items=None):
         return (

dvc/remote/ssh/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import errno
 import getpass
 import io
@@ -9,6 +7,7 @@ import os
 import threading
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import closing, contextmanager
+from urllib.parse import urlparse
 
 from funcy import memoize, wrap_with
 
@@ -19,8 +18,6 @@ from dvc.remote.base import RemoteBASE
 from dvc.remote.pool import get_connection
 from dvc.scheme import Schemes
 from dvc.utils import to_chunks
-from dvc.utils.compat import StringIO
-from dvc.utils.compat import urlparse
 
 logger = logging.getLogger(__name__)
 
@@ -52,7 +49,7 @@ class RemoteSSH(RemoteBASE):
     DEFAULT_CACHE_TYPES = ["copy"]
 
     def __init__(self, repo, config):
-        super(RemoteSSH, self).__init__(repo, config)
+        super().__init__(repo, config)
         url = config.get(Config.SECTION_REMOTE_URL)
         if url:
             parsed = urlparse(url)
@@ -106,7 +103,7 @@ class RemoteSSH(RemoteBASE):
             ssh_config = paramiko.SSHConfig()
             with open(user_config_file) as f:
                 # For whatever reason parsing directly from f is unreliable
-                f_copy = StringIO(f.read())
+                f_copy = io.StringIO(f.read())
                 ssh_config.parse(f_copy)
             user_ssh_config = ssh_config.lookup(hostname)
         return user_ssh_config
@@ -263,8 +260,7 @@ class RemoteSSH(RemoteBASE):
     def list_cache_paths(self):
         with self.ssh(self.path_info) as ssh:
             # If we simply return an iterator then with above closes instantly
-            for path in ssh.walk_files(self.path_info.path):
-                yield path
+            yield from ssh.walk_files(self.path_info.path)
 
     def walk_files(self, path_info):
         with self.ssh(path_info) as ssh:

dvc/remote/ssh/connection.py

@@ -3,6 +3,7 @@ import logging
 import os
 import posixpath
 import stat
+from contextlib import suppress
 
 from funcy import cached_property
 
@@ -12,7 +13,6 @@ except ImportError:
     paramiko = None
 
 from dvc.utils import tmp_fname
-from dvc.utils.compat import ignore_file_not_found
 from dvc.progress import Tqdm
 from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteCmdError
@@ -60,13 +60,13 @@ class SSHConnection:
         self._ssh.close()
 
     def st_mode(self, path):
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
             return self.sftp.lstat(path).st_mode
 
         return 0
 
     def getsize(self, path):
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
             return self.sftp.lstat(path).st_size
 
         return 0
@@ -103,14 +103,13 @@ class SSHConnection:
         if tail:
             try:
                 self.sftp.mkdir(path)
-            except IOError as e:
+            except IOError as exc:
                 # Since paramiko errors are very vague we need to recheck
                 # whether it's because path already exists or something else
-                if e.errno == errno.EACCES or not self.exists(path):
+                if exc.errno == errno.EACCES or not self.exists(path):
                     raise DvcException(
-                        "unable to create remote directory '{}'".format(path),
-                        cause=e,
-                    )
+                        "unable to create remote directory '{}'".format(path)
+                    ) from exc
 
     def walk(self, directory, topdown=True):
         # NOTE: original os.walk() implementation [1] with default options was
@@ -123,9 +122,8 @@ class SSHConnection:
             raise DvcException(
                 "couldn't get the '{}' remote directory files list".format(
                     directory
-                ),
-                cause=exc,
-            )
+                )
+            ) from exc
 
         dirs = []
         nondirs = []
@@ -141,8 +139,7 @@ class SSHConnection:
 
         for dname in dirs:
             newpath = posixpath.join(directory, dname)
-            for entry in self.walk(newpath, topdown=topdown):
-                yield entry
+            yield from self.walk(newpath, topdown=topdown)
 
         if not topdown:
             yield directory, dirs, nondirs
@@ -153,22 +150,20 @@ class SSHConnection:
                 yield posixpath.join(root, fname)
 
     def _remove_file(self, path):
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
             self.sftp.remove(path)
 
     def _remove_dir(self, path):
         for root, dirs, files in self.walk(path, topdown=False):
             for fname in files:
-                path = posixpath.join(root, fname)
-                with ignore_file_not_found():
-                    self._remove_file(path)
+                with suppress(FileNotFoundError):
+                    self._remove_file(posixpath.join(root, fname))
 
             for dname in dirs:
-                path = posixpath.join(root, dname)
-                with ignore_file_not_found():
-                    self.sftp.rmdir(dname)
+                with suppress(FileNotFoundError):
+                    self.sftp.rmdir(posixpath.join(root, dname))
 
-        with ignore_file_not_found():
+        with suppress(FileNotFoundError):
             self.sftp.rmdir(path)
 
     def remove(self, path):

dvc/repo/__init__.py

@@ -1,29 +1,24 @@
-from __future__ import unicode_literals
-
 import logging
 import os
 from contextlib import contextmanager
 from functools import wraps
 from itertools import chain
 
-from funcy import cached_property
+from dvc.ignore import CleanTree
+from dvc.compat import fspath_py35
+
+from funcy import cached_property, cat
 
-from .graph import check_acyclic
-from .graph import get_pipeline
-from .graph import get_pipelines
-from .graph import get_stages
 from dvc.config import Config
-from dvc.exceptions import FileMissingError
-from dvc.exceptions import NotDvcRepoError
-from dvc.exceptions import OutputNotFoundError
-from dvc.ignore import DvcIgnoreFilter
+from dvc.exceptions import (
+    FileMissingError,
+    NotDvcRepoError,
+    OutputNotFoundError,
+)
 from dvc.path_info import PathInfo
 from dvc.remote.base import RemoteActionNotImplemented
-from dvc.utils import relpath
 from dvc.utils.fs import path_isin
-from dvc.utils.compat import FileNotFoundError
-from dvc.utils.compat import fspath_py35
-from dvc.utils.compat import open as _open
+from .graph import check_acyclic, get_pipeline, get_pipelines
 
 
 logger = logging.getLogger(__name__)
@@ -76,7 +71,7 @@ class Repo(object):
         from dvc.repo.metrics import Metrics
         from dvc.scm.tree import WorkingTree
         from dvc.repo.tag import Tag
-        from dvc.utils import makedirs
+        from dvc.utils.fs import makedirs
 
         root_dir = self.find_root(root_dir)
 
@@ -124,7 +119,7 @@ class Repo(object):
 
     @tree.setter
     def tree(self, tree):
-        self._tree = tree
+        self._tree = tree if isinstance(tree, CleanTree) else CleanTree(tree)
         # Our graph cache is no longer valid, as it was based on the previous
         # tree.
         self._reset()
@@ -190,32 +185,32 @@ class Repo(object):
         G = graph or self.graph
 
         if not target:
-            return get_stages(G)
+            return list(G)
 
         target = os.path.abspath(target)
 
         if recursive and os.path.isdir(target):
-            attrs = nx.get_node_attributes(G, "stage")
-            nodes = [node for node in nx.dfs_postorder_nodes(G)]
-
-            ret = []
-            for node in nodes:
-                stage = attrs[node]
-                if path_isin(stage.path, target):
-                    ret.append(stage)
-            return ret
+            stages = nx.dfs_postorder_nodes(G)
+            return [stage for stage in stages if path_isin(stage.path, target)]
 
         stage = Stage.load(self, target)
         if not with_deps:
             return [stage]
 
-        node = relpath(stage.path, self.root_dir)
-        pipeline = get_pipeline(get_pipelines(G), node)
+        pipeline = get_pipeline(get_pipelines(G), stage)
+        return list(nx.dfs_postorder_nodes(pipeline, stage))
 
-        return [
-            pipeline.node[n]["stage"]
-            for n in nx.dfs_postorder_nodes(pipeline, node)
-        ]
+    def collect_granular(self, target, *args, **kwargs):
+        if not target:
+            return [(stage, None) for stage in self.stages]
+
+        try:
+            out, = self.find_outs_by_path(target, strict=False)
+            filter_info = PathInfo(os.path.abspath(target))
+            return [(out.stage, filter_info)]
+        except OutputNotFoundError:
+            stages = self.collect(target, *args, **kwargs)
+            return [(stage, None) for stage in stages]
 
     def used_cache(
         self,
@@ -251,28 +246,24 @@ class Repo(object):
             all_tags=all_tags,
             all_commits=all_commits,
         ):
-            if targets:
-                stages = []
-                for target in targets:
-                    collected = self.collect(
-                        target, recursive=recursive, with_deps=with_deps
-                    )
-                    stages.extend(collected)
-            else:
-                stages = self.stages
+            targets = targets or [None]
 
-            for stage in stages:
-                if stage.is_repo_import:
-                    dep, = stage.deps
-                    cache.external[dep.repo_pair].add(dep.def_path)
-                    continue
+            pairs = cat(
+                self.collect_granular(
+                    target, recursive=recursive, with_deps=with_deps
+                )
+                for target in targets
+            )
 
-                for out in stage.outs:
-                    used_cache = out.get_used_cache(
-                        remote=remote, force=force, jobs=jobs
-                    )
-                    suffix = "({})".format(branch) if branch else ""
-                    cache.update(used_cache, suffix=suffix)
+            suffix = "({})".format(branch) if branch else ""
+            for stage, filter_info in pairs:
+                used_cache = stage.get_used_cache(
+                    remote=remote,
+                    force=force,
+                    jobs=jobs,
+                    filter_info=filter_info,
+                )
+                cache.update(used_cache, suffix=suffix)
 
         return cache
 
@@ -323,15 +314,15 @@ class Repo(object):
         )
 
         G = nx.DiGraph()
-        stages = stages or self.collect_stages()
+        stages = stages or self.stages
         stages = [stage for stage in stages if stage]
         outs = {}
 
         for stage in stages:
             for out in stage.outs:
                 if out.path_info in outs:
-                    stages = [stage.relpath, outs[out.path_info].stage.relpath]
-                    raise OutputDuplicationError(str(out), stages)
+                    dup_stages = [stage, outs[out.path_info].stage]
+                    raise OutputDuplicationError(str(out), dup_stages)
                 outs[out.path_info] = out
 
         for stage in stages:
@@ -344,23 +335,19 @@ class Repo(object):
             stage_path_info = PathInfo(stage.path)
             for p in chain([stage_path_info], stage_path_info.parents):
                 if p in outs:
-                    raise StagePathAsOutputError(stage.wdir, stage.relpath)
+                    raise StagePathAsOutputError(stage, str(outs[p]))
 
         for stage in stages:
-            node = relpath(stage.path, self.root_dir)
-
-            G.add_node(node, stage=stage)
+            G.add_node(stage)
 
             for dep in stage.deps:
                 if dep.path_info is None:
                     continue
 
-                for out in outs:
-                    if out.overlaps(dep.path_info):
-                        dep_stage = outs[out].stage
-                        dep_node = relpath(dep_stage.path, self.root_dir)
-                        G.add_node(dep_node, stage=dep_stage)
-                        G.add_edge(node, dep_node)
+                for out_path_info, out in outs.items():
+                    if out_path_info.overlaps(dep.path_info):
+                        G.add_node(out.stage)
+                        G.add_edge(stage, out.stage)
 
         check_acyclic(G)
 
@@ -374,18 +361,8 @@ class Repo(object):
     def pipelines(self):
         return get_pipelines(self.graph)
 
-    @staticmethod
-    def _filter_out_dirs(dirs, outs, root_dir):
-        def filter_dirs(dname):
-            path = os.path.join(root_dir, dname)
-            for out in outs:
-                if path == os.path.normpath(out):
-                    return False
-            return True
-
-        return list(filter(filter_dirs, dirs))
-
-    def collect_stages(self):
+    @cached_property
+    def stages(self):
         """
         Walks down the root directory looking for Dvcfiles,
         skipping the directories that are related with
@@ -398,41 +375,38 @@ class Repo(object):
         from dvc.stage import Stage
 
         stages = []
-        outs = []
+        outs = set()
 
-        for root, dirs, files in self.tree.walk(
-            self.root_dir, dvcignore=self.dvcignore
-        ):
+        for root, dirs, files in self.tree.walk(self.root_dir):
             for fname in files:
                 path = os.path.join(root, fname)
                 if not Stage.is_valid_filename(path):
                     continue
                 stage = Stage.load(self, path)
-                for out in stage.outs:
-                    if out.scheme == "local":
-                        outs.append(out.fspath + out.sep)
                 stages.append(stage)
 
-            dirs[:] = self._filter_out_dirs(dirs, outs, root)
+                for out in stage.outs:
+                    if out.scheme == "local":
+                        outs.add(out.fspath)
+
+            dirs[:] = [d for d in dirs if os.path.join(root, d) not in outs]
 
         return stages
 
-    @cached_property
-    def stages(self):
-        return get_stages(self.graph)
-
-    def find_outs_by_path(self, path, outs=None, recursive=False):
+    def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
         if not outs:
             outs = [out for stage in self.stages for out in stage.outs]
 
         abs_path = os.path.abspath(path)
+        path_info = PathInfo(abs_path)
         is_dir = self.tree.isdir(abs_path)
+        match = path_info.__eq__ if strict else path_info.isin_or_eq
 
         def func(out):
-            if out.scheme == "local" and out.fspath == abs_path:
+            if out.scheme == "local" and match(out.path_info):
                 return True
 
-            if is_dir and recursive and out.path_info.isin(abs_path):
+            if is_dir and recursive and out.path_info.isin(path_info):
                 return True
 
             return False
@@ -453,29 +427,30 @@ class Repo(object):
         return self.DVC_DIR in path_parts
 
     @contextmanager
-    def open(self, path, remote=None, mode="r", encoding=None):
+    def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
         cause = None
         try:
-            out, = self.find_outs_by_path(path)
-        except OutputNotFoundError as e:
+            out = self.find_out_by_relpath(path)
+        except OutputNotFoundError as exc:
             out = None
-            cause = e
+            cause = exc
 
         if out and out.use_cache:
             try:
                 with self._open_cached(out, remote, mode, encoding) as fd:
                     yield fd
                 return
-            except FileNotFoundError as e:
-                raise FileMissingError(relpath(path, self.root_dir), cause=e)
+            except FileNotFoundError as exc:
+                raise FileMissingError(path) from exc
 
-        if self.tree.exists(path):
-            with self.tree.open(path, mode, encoding) as fd:
+        abs_path = os.path.join(self.root_dir, path)
+        if os.path.exists(abs_path):
+            with open(abs_path, mode=mode, encoding=encoding) as fd:
                 yield fd
             return
 
-        raise FileMissingError(relpath(path, self.root_dir), cause=cause)
+        raise FileMissingError(path) from cause
 
     def _open_cached(self, out, remote=None, mode="r", encoding=None):
         if out.isdir():
@@ -485,7 +460,7 @@ class Repo(object):
         cache_file = fspath_py35(cache_file)
 
         if os.path.exists(cache_file):
-            return _open(cache_file, mode=mode, encoding=encoding)
+            return open(cache_file, mode=mode, encoding=encoding)
 
         try:
             remote_obj = self.cloud.get_remote(remote)
@@ -496,11 +471,7 @@ class Repo(object):
                 cache_info = out.get_used_cache(remote=remote)
                 self.cloud.pull(cache_info, remote=remote)
 
-            return _open(cache_file, mode=mode, encoding=encoding)
-
-    @cached_property
-    def dvcignore(self):
-        return DvcIgnoreFilter(self.root_dir, self.tree)
+            return open(cache_file, mode=mode, encoding=encoding)
 
     def close(self):
         self.scm.close()

dvc/repo/add.py

@@ -1,10 +1,7 @@
-from __future__ import unicode_literals
-
 import logging
 import os
 
 import colorama
-from six import string_types
 
 from . import locked
 from dvc.exceptions import RecursiveAddingWhileUsingFilename
@@ -12,7 +9,6 @@ from dvc.progress import Tqdm
 from dvc.repo.scm_context import scm_context
 from dvc.stage import Stage
 from dvc.utils import LARGE_DIR_SIZE
-from dvc.utils import walk_files
 
 logger = logging.getLogger(__name__)
 
@@ -23,7 +19,7 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
     if recursive and fname:
         raise RecursiveAddingWhileUsingFilename()
 
-    if isinstance(targets, string_types):
+    if isinstance(targets, str):
         targets = [targets]
 
     stages_list = []
@@ -37,7 +33,7 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
                     "You are adding a large directory '{target}' recursively,"
                     " consider tracking it as a whole instead.\n"
                     "{purple}HINT:{nc} Remove the generated DVC-file and then"
-                    " run {cyan}dvc add {target}{nc}".format(
+                    " run `{cyan}dvc add {target}{nc}`".format(
                         purple=colorama.Fore.MAGENTA,
                         cyan=colorama.Fore.CYAN,
                         nc=colorama.Style.RESET_ALL,
@@ -68,7 +64,7 @@ def _find_all_targets(repo, target, recursive):
     if os.path.isdir(target) and recursive:
         return [
             fname
-            for fname in walk_files(target, repo.dvcignore)
+            for fname in repo.tree.walk_files(target)
             if not repo.is_dvc_internal(fname)
             if not Stage.is_stage_file(fname)
             if not repo.scm.belongs_to_scm(fname)

dvc/repo/brancher.py

@@ -4,12 +4,7 @@ from dvc.scm.tree import WorkingTree
 
 
 def brancher(  # noqa: E302
-    self,
-    branches=None,
-    all_branches=False,
-    tags=None,
-    all_tags=False,
-    all_commits=False,
+    self, revs=None, all_branches=False, all_tags=False, all_commits=False
 ):
     """Generator that iterates over specified revisions.
 
@@ -25,12 +20,12 @@ def brancher(  # noqa: E302
             - empty string it there is no branches to iterate over
             - "Working Tree" if there are uncommitted changes in the SCM repo
     """
-    if not any([branches, all_branches, tags, all_tags, all_commits]):
+    if not any([revs, all_branches, all_tags, all_commits]):
         yield ""
         return
 
     saved_tree = self.tree
-    revs = []
+    revs = revs or []
 
     scm = self.scm
 
@@ -41,24 +36,15 @@ def brancher(  # noqa: E302
         revs = scm.list_all_commits()
     else:
         if all_branches:
-            branches = scm.list_branches()
+            revs.extend(scm.list_branches())
 
         if all_tags:
-            tags = scm.list_tags()
+            revs.extend(scm.list_tags())
 
-        if branches is not None:
-            revs.extend(branches)
-
-        if tags is not None:
-            revs.extend(tags)
-
-    # NOTE: it might be a good idea to wrap this loop in try/finally block
-    # to don't leave the tree on some unexpected branch after the
-    # `brancher()`, but this could cause problems on exception handling
-    # code which might expect the tree on which exception was raised to
-    # stay in place. This behavior is a subject to change.
-    for sha, names in group_by(scm.resolve_rev, revs).items():
-        self.tree = scm.get_tree(sha)
-        yield ", ".join(names)
-
-    self.tree = saved_tree
+    try:
+        if revs:
+            for sha, names in group_by(scm.resolve_rev, revs).items():
+                self.tree = scm.get_tree(sha)
+                yield ", ".join(names)
+    finally:
+        self.tree = saved_tree

dvc/repo/checkout.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import logging
 
 from dvc.exceptions import CheckoutError
@@ -20,8 +18,10 @@ def _cleanup_unused_links(repo):
     repo.state.remove_unused_links(used)
 
 
-def get_all_files_numbers(stages):
-    return sum(stage.get_all_files_number() for stage in stages)
+def get_all_files_numbers(pairs):
+    return sum(
+        stage.get_all_files_number(filter_info) for stage, filter_info in pairs
+    )
 
 
 def _checkout(
@@ -34,36 +34,37 @@ def _checkout(
 ):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    stages = set()
-
     if not targets:
         targets = [None]
         _cleanup_unused_links(self)
 
+    pairs = set()
     for target in targets:
         try:
-            new = self.collect(
-                target, with_deps=with_deps, recursive=recursive
+            pairs.update(
+                self.collect_granular(
+                    target, with_deps=with_deps, recursive=recursive
+                )
             )
-            stages.update(new)
         except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
             if not target:
                 raise
-            raise CheckoutErrorSuggestGit(target, exc)
+            raise CheckoutErrorSuggestGit(target) from exc
 
-    total = get_all_files_numbers(stages)
+    total = get_all_files_numbers(pairs)
     if total == 0:
         logger.info("Nothing to do")
     failed = []
     with Tqdm(
         total=total, unit="file", desc="Checkout", disable=total == 0
     ) as pbar:
-        for stage in stages:
+        for stage, filter_info in pairs:
             failed.extend(
                 stage.checkout(
                     force=force,
                     progress_callback=pbar.update_desc,
                     relink=relink,
+                    filter_info=filter_info,
                 )
             )
     if failed:

dvc/repo/destroy.py

@@ -1,8 +1,15 @@
 from dvc.utils.fs import remove
+from . import locked
 
 
-def destroy(self):
-    for stage in self.stages:
+@locked
+def _destroy_stages(repo):
+    for stage in repo.stages:
         stage.remove(remove_outs=False)
 
-    remove(self.dvc_dir)
+
+# NOTE: not locking `destroy`, as `remove` will need to delete `.dvc` dir,
+# which will cause issues on Windows, as `.dvc/lock` will be busy.
+def destroy(repo):
+    _destroy_stages(repo)
+    remove(repo.dvc_dir)

dvc/repo/diff.py

@@ -1,16 +1,11 @@
-from __future__ import unicode_literals
-
 import os
 from errno import ENOENT
 
-import dvc.logger as logger
-from . import locked
+from dvc import logger
 from dvc.scm.base import FileNotInCommitError
-from dvc.scm.git import DIFF_A_REF
-from dvc.scm.git import DIFF_A_TREE
-from dvc.scm.git import DIFF_B_REF
-from dvc.scm.git import DIFF_B_TREE
+from dvc.scm.git import DIFF_A_REF, DIFF_B_REF, DIFF_A_TREE, DIFF_B_TREE
 from dvc.scm.git import DIFF_EQUAL
+from . import locked
 
 
 DIFF_TARGET = "target"

dvc/repo/fetch.py

@@ -1,12 +1,9 @@
-from __future__ import unicode_literals
-
 import logging
 
 from dvc.cache import NamedCache
 from dvc.config import NoRemoteError
 from dvc.exceptions import DownloadError
 from dvc.exceptions import OutputNotFoundError
-from dvc.external_repo import external_repo
 from dvc.scm.base import CloneError
 
 
@@ -72,11 +69,13 @@ def _fetch(
 
 
 def _fetch_external(self, repo_url, repo_rev, files):
-    failed = 0
+    from dvc.external_repo import external_repo
 
-    cache_dir = self.cache.local.cache_dir
+    failed = 0
     try:
-        with external_repo(repo_url, repo_rev, cache_dir=cache_dir) as repo:
+        with external_repo(repo_url, repo_rev) as repo:
+            repo.cache.local.cache_dir = self.cache.local.cache_dir
+
             with repo.state:
                 cache = NamedCache()
                 for name in files:

dvc/repo/gc.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import logging
 
 from . import locked
@@ -12,7 +10,7 @@ logger = logging.getLogger(__name__)
 def _do_gc(typ, func, clist):
     removed = func(clist)
     if not removed:
-        logger.info("No unused {} cache to remove.".format(typ))
+        logger.info("No unused '{}' cache to remove.".format(typ))
 
 
 @locked
@@ -28,7 +26,7 @@ def gc(
     jobs=None,
     repos=None,
 ):
-    from dvc.utils.compat import ExitStack
+    from contextlib import ExitStack
     from dvc.repo import Repo
 
     all_repos = []

dvc/repo/get.py

@@ -1,44 +1,29 @@
 import logging
 import os
-import shutil
-from dvc.utils.compat import FileNotFoundError
 
 import shortuuid
 
-from dvc.exceptions import (
-    DvcException,
-    NotDvcRepoError,
-    OutputNotFoundError,
-    UrlNotDvcRepoError,
-)
-from dvc.external_repo import external_repo
-from dvc.path_info import PathInfo
+from dvc.exceptions import DvcException
 from dvc.stage import Stage
 from dvc.utils import resolve_output
 from dvc.utils.fs import remove
+from dvc.path_info import PathInfo
 
 logger = logging.getLogger(__name__)
 
 
 class GetDVCFileError(DvcException):
     def __init__(self):
-        super(GetDVCFileError, self).__init__(
+        super().__init__(
             "the given path is a DVC-file, you must specify a data file "
             "or a directory"
         )
 
 
-class PathMissingError(DvcException):
-    def __init__(self, path, repo):
-        msg = (
-            "The path '{}' does not exist in the target repository '{}'"
-            " neighther as an output nor a git-handled file."
-        )
-        super(PathMissingError, self).__init__(msg.format(path, repo))
-
-
 @staticmethod
 def get(url, path, out=None, rev=None):
+    from dvc.external_repo import external_repo
+
     out = resolve_output(path, out)
 
     if Stage.is_valid_filename(out):
@@ -52,52 +37,21 @@ def get(url, path, out=None, rev=None):
     dpath = os.path.dirname(os.path.abspath(out))
     tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid()))
     try:
-        with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo:
-            # Try any links possible to avoid data duplication.
-            #
-            # Not using symlink, because we need to remove cache after we are
-            # done, and to make that work we would have to copy data over
-            # anyway before removing the cache, so we might just copy it
-            # right away.
-            #
-            # Also, we can't use theoretical "move" link type here, because
-            # the same cache file might be used a few times in a directory.
-            repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]
+        with external_repo(url=url, rev=rev) as repo:
+            if hasattr(repo, "cache"):
+                repo.cache.local.cache_dir = tmp_dir
 
-            try:
-                output = repo.find_out_by_relpath(path)
-            except OutputNotFoundError:
-                output = None
+                # Try any links possible to avoid data duplication.
+                #
+                # Not using symlink, because we need to remove cache after we
+                # are done, and to make that work we would have to copy data
+                # over anyway before removing the cache, so we might just copy
+                # it right away.
+                #
+                # Also, we can't use theoretical "move" link type here, because
+                # the same cache file might be used a few times in a directory.
+                repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]
 
-            if output and output.use_cache:
-                _get_cached(repo, output, out)
-            else:
-                # Either an uncached out with absolute path or a user error
-                if os.path.isabs(path):
-                    raise FileNotFoundError
-
-                _copy(os.path.join(repo.root_dir, path), out)
-
-    except (OutputNotFoundError, FileNotFoundError):
-        raise PathMissingError(path, url)
-    except NotDvcRepoError:
-        raise UrlNotDvcRepoError(url)
+            repo.pull_to(path, PathInfo(out))
     finally:
         remove(tmp_dir)
-
-
-def _get_cached(repo, output, out):
-    with repo.state:
-        repo.cloud.pull(output.get_used_cache())
-        output.path_info = PathInfo(os.path.abspath(out))
-        failed = output.checkout()
-        # This might happen when pull haven't really pulled all the files
-        if failed:
-            raise FileNotFoundError
-
-
-def _copy(src, dst):
-    if os.path.isdir(src):
-        shutil.copytree(src, dst)
-    else:
-        shutil.copy2(src, dst)

dvc/repo/graph.py

@@ -1,6 +1,3 @@
-from __future__ import unicode_literals
-
-
 def check_acyclic(graph):
     import networkx as nx
     from dvc.exceptions import CyclicGraphError
@@ -28,9 +25,3 @@ def get_pipelines(G):
     import networkx as nx
 
     return [G.subgraph(c).copy() for c in nx.weakly_connected_components(G)]
-
-
-def get_stages(G):
-    import networkx
-
-    return list(networkx.get_node_attributes(G, "stage").values())

dvc/repo/metrics/__init__.py

@@ -21,3 +21,8 @@ class Metrics(object):
         from dvc.repo.metrics.remove import remove
 
         return remove(self.repo, *args, **kwargs)
+
+    def diff(self, *args, **kwargs):
+        from .diff import diff
+
+        return diff(self.repo, *args, **kwargs)

dvc/repo/metrics/diff.py

@@ -0,0 +1,98 @@
+import json
+from collections import defaultdict
+
+from flatten_json import flatten
+
+from dvc.exceptions import NoMetricsError
+
+
+def _parse(raw):
+    if isinstance(raw, (dict, list, int, float)):
+        return raw
+
+    assert isinstance(raw, str)
+    try:
+        return json.loads(raw)
+    except json.JSONDecodeError:
+        return raw
+
+
+def _diff_vals(old, new):
+    if (
+        isinstance(new, list)
+        and isinstance(old, list)
+        and len(old) == len(new) == 1
+    ):
+        return _diff_vals(old[0], new[0])
+
+    if old == new:
+        return {}
+
+    res = {"old": old, "new": new}
+    if isinstance(new, (int, float)) and isinstance(old, (int, float)):
+        res["diff"] = new - old
+    return res
+
+
+def _diff_dicts(old_dict, new_dict):
+    old_default = None
+    new_default = None
+
+    if isinstance(new_dict, dict):
+        new = flatten(new_dict, ".")
+    else:
+        new = defaultdict(lambda: "not a dict")
+        new_default = "unable to parse"
+
+    if isinstance(old_dict, dict):
+        old = flatten(old_dict, ".")
+    else:
+        old = defaultdict(lambda: "not a dict")
+        old_default = "unable to parse"
+
+    res = defaultdict(dict)
+
+    xpaths = set(old.keys())
+    xpaths.update(set(new.keys()))
+    for xpath in xpaths:
+        old_val = old.get(xpath, old_default)
+        new_val = new.get(xpath, new_default)
+        val_diff = _diff_vals(old_val, new_val)
+        if val_diff:
+            res[xpath] = val_diff
+    return dict(res)
+
+
+def _diff(old_raw, new_raw):
+    old = _parse(old_raw)
+    new = _parse(new_raw)
+
+    if isinstance(new, dict) or isinstance(old, dict):
+        return _diff_dicts(old, new)
+
+    return {"": _diff_vals(old, new)}
+
+
+def _get_metrics(repo, *args, rev=None, **kwargs):
+    try:
+        metrics = repo.metrics.show(
+            *args, **kwargs, revs=[rev] if rev else None
+        )
+        return metrics[rev or ""]
+    except NoMetricsError:
+        return {}
+
+
+def diff(repo, *args, a_ref=None, b_ref=None, **kwargs):
+    old = _get_metrics(repo, *args, **kwargs, rev=(a_ref or "HEAD"))
+    new = _get_metrics(repo, *args, **kwargs, rev=b_ref)
+
+    paths = set(old.keys())
+    paths.update(set(new.keys()))
+
+    res = defaultdict(dict)
+    for path in paths:
+        path_diff = _diff(old[path], new[path])
+        if path_diff:
+            res[path] = path_diff
+    return dict(res)

dvc/repo/metrics/modify.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from dvc.exceptions import DvcException
 from dvc.repo import locked
 

dvc/repo/metrics/show.py

@@ -1,20 +1,15 @@
-from __future__ import unicode_literals
-
 import csv
 import errno
 import json
 import logging
 import os
+import io
 
 from jsonpath_ng.ext import parse
 
 from dvc.exceptions import NoMetricsError
 from dvc.exceptions import OutputNotFoundError
 from dvc.repo import locked
-from dvc.utils.compat import builtin_str
-from dvc.utils.compat import csv_reader
-from dvc.utils.compat import open
-from dvc.utils.compat import StringIO
 
 NO_METRICS_FILE_AT_REFERENCE_WARNING = (
     "Metrics file '{}' does not exist at the reference '{}'."
@@ -25,7 +20,7 @@ logger = logging.getLogger(__name__)
 
 def _read_metric_json(fd, json_path):
     parser = parse(json_path)
-    return [x.value for x in parser.find(json.load(fd))]
+    return {str(x.full_path): x.value for x in parser.find(json.load(fd))}
 
 
 def _get_values(row):
@@ -50,7 +45,7 @@ def _read_metric_hxsv(fd, hxsv_path, delimiter):
     row = indices[0]
     row = int(row) if row else None
     col = indices[1] if len(indices) > 1 and indices[1] else None
-    reader = list(csv.DictReader(fd, delimiter=builtin_str(delimiter)))
+    reader = list(csv.DictReader(fd, delimiter=delimiter))
     return _do_read_metric_xsv(reader, row, col)
 
 
@@ -59,7 +54,7 @@ def _read_metric_xsv(fd, xsv_path, delimiter):
     row = indices[0]
     row = int(row) if row else None
     col = int(indices[1]) if len(indices) > 1 and indices[1] else None
-    reader = list(csv.reader(fd, delimiter=builtin_str(delimiter)))
+    reader = list(csv.reader(fd, delimiter=delimiter))
     return _do_read_metric_xsv(reader, row, col)
 
 
@@ -104,7 +99,7 @@ def _format_csv(content, delimiter):
         "0.67528    0.289545        testing\n"
         "0.671502   0.297848        validation\n"
     """
-    reader = csv_reader(StringIO(content), delimiter=builtin_str(delimiter))
+    reader = csv.reader(io.StringIO(content), delimiter=delimiter)
     rows = [row for row in reader]
     max_widths = [max(map(len, column)) for column in zip(*rows)]
 
@@ -130,10 +125,10 @@ def _format_output(content, typ):
         str: Content in a raw or tabular format.
     """
 
-    if "csv" in str(typ):
+    if "csv" in typ:
         return _format_csv(content, delimiter=",")
 
-    if "tsv" in str(typ):
+    if "tsv" in typ:
         return _format_csv(content, delimiter="\t")
 
     return content
@@ -271,6 +266,7 @@ def show(
     all_branches=False,
     all_tags=False,
     recursive=False,
+    revs=None,
 ):
     res = {}
     found = set()
@@ -279,7 +275,9 @@ def show(
         # Iterate once to call `_collect_metrics` on all the stages
         targets = [None]
 
-    for branch in repo.brancher(all_branches=all_branches, all_tags=all_tags):
+    for branch in repo.brancher(
+        revs=revs, all_branches=all_branches, all_tags=all_tags
+    ):
         metrics = {}
 
         for target in targets:
@@ -298,6 +296,15 @@ def show(
     if not res and not any(targets):
         raise NoMetricsError()
 
+    # Hide working tree metrics if they are the same as in the active branch
+    try:
+        active_branch = repo.scm.active_branch()
+    except TypeError:
+        pass  # Detached head
+    else:
+        if res.get("working tree") == res.get(active_branch):
+            res.pop("working tree", None)
+
     missing = set(targets) - found
 
     if missing:

dvc/repo/pull.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import logging
 
 from dvc.repo import locked

dvc/repo/push.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from . import locked