dvc: bump to 0.77.2

remote: make `copy` links atomic for local/hdfs/ssh

.deepsource.toml

@@ -5,7 +5,7 @@ test_patterns = [
 ]
 
 exclude_patterns = [
-  
+  'dvc/utils/compat.py'
 ]
 
 [[analyzers]]

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,9 +1,8 @@
-* [ ] Have you followed the guidelines in our
-      [Contributing document](https://dvc.org/doc/user-guide/contributing)?
+* [ ] ❗ Have you followed the guidelines in the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) list?
 
-* [ ] Does your PR affect documented changes or does it add new functionality
-      that should be documented? If yes, have you created a PR for
-      [dvc.org](https://github.com/iterative/dvc.org) documenting it or at
-      least opened an issue for it? If so, please add a link to it.
+* [ ] 📖 Check this box if this PR **does not** require [documentation](https://dvc.org/doc) updates, or if it does **and** you have created a separate PR in [dvc.org](https://github.com/iterative/dvc.org) with such updates (or at least opened an issue about it in that repo). Please link below to your PR (or issue) in the [dvc.org](https://github.com/iterative/dvc.org) repo.
+
+* [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect everything to be addressed. Please review them carefully and fix those that actually improve code or fix bugs.
+
+Thank you for the contribution - we'll try to review it as soon as possible. 🙏
 
------

.github/release-drafter.yml

@@ -0,0 +1,5 @@
+# Config for https://github.com/apps/release-drafter
+branches:
+  - master
+template: |
+  $CHANGES

.gitignore

@@ -4,6 +4,7 @@ neatlynx/__pycache__
 cache
 *.pyc
 .env/
+.env2.7/
 
 cache
 .dvc.conf.lock

.pylintrc

@@ -139,7 +139,9 @@ disable=print-statement,
         deprecated-sys-function,
         exception-escape,
         comprehension-escape,
-        useless-object-inheritance
+        useless-object-inheritance,
+        missing-docstring,
+        bad-continuation
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -314,7 +316,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=100
+max-line-length=79
 
 # Maximum number of lines in a module.
 max-module-lines=1000

.restyled.yaml

@@ -0,0 +1,11 @@
+---
+restylers:
+  - name: black
+    image: restyled/restyler-black:v19.3b0
+    command:
+    - black
+    arguments: []
+    include:
+    - "**/*.py"
+    interpreters:
+    - python

.travis.yml

@@ -2,7 +2,6 @@ cache:
   directories:
     - $HOME/.cache/pip
     - $HOME/.pyenv
-sudo: true
 dist: xenial
 env:
   global:
@@ -15,18 +14,20 @@ stages:
   - test
   - build
   - name: deploy
-    if: tag is present
+    if: (branch = master OR tag is present) AND repo = iterative/dvc
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
-# Note that we specify language as ruby-2.4.0 to help us satisfy
+# Note that we specify language as ruby-2.4.3 to help us satisfy
 # fpm dependencies, as travis runs el capitan with an old ruby
 # version, that causes fpm to crash.
+#
+# Each job will use the previous job's `stage` tag if not specified.
 jobs:
   include:
   # patch formatting checks
   - name: "Check patch formatting"
-    stage: "check"
+    stage: check
     os: linux
     language: python
     python: 3.7
@@ -41,35 +42,28 @@ jobs:
     env:
     - PY2=1
   - name: "3.7 on Windows"
-    stage: test
     os: windows           # Windows 10.0.17134 N/A Build 17134
     language: shell       # 'language: python' is an error on Travis CI Windows
   - name: "2.7 on Linux"
-    stage: test
     language: python
     python: 2.7
     env:
     - PY2=1
   - name: "3.5 on Linux"
-    stage: test
     language: python
     python: 3.5
   - name: "3.6 on Linux"
-    stage: test
     language: python
     python: 3.6
   - name: "3.7 on Linux"
-    stage: test
     language: python
     python: 3.7
   - name: "2.7 on Mac"
-    stage: test
     os: osx
     osx_image: xcode9.4
     env:
     - PY2=1
   - name: "3.6 on Mac"
-    stage: test
     os: osx
     osx_image: xcode9.4
   # build jobs
@@ -78,29 +72,46 @@ jobs:
     os: osx
     osx_image: xcode9.4
     language: ruby
-    rvm: 2.4.0
+    rvm: 2.4.3
     install:
     script: ./scripts/build_posix.sh
   - name: "Windows pkg"
-    stage: build
     os: windows
     language: shell
     install:
     script: ./scripts/build_windows.cmd
   - name: "Linux pkgs"
-    stage: build
     language: python
     python: 3.7
     before_install:
     install:
     script: ./scripts/build_posix.sh
   - name: "Pypi pkgs"
-    stage: build
     language: python
     python: 3.7
     script: ./scripts/build_package.sh
+  - name: Snapcraft snap
+    addons:
+     snaps:
+     - name: snapcraft
+       channel: stable
+       confinement: classic
+     - name: lxd
+       channel: stable
+    env:
+    - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}'
+    before_install:
+    - sudo /snap/bin/lxd.migrate -yes
+    - sudo /snap/bin/lxd waitready
+    - sudo /snap/bin/lxd init --auto
+    install:
+    script:
+    - sudo snapcraft --use-lxd
+    after_failure:
+    - sudo journalctl -u snapd
   # deploy jobs
   - name: "Pypi pkgs"
+    if: tag is present
     stage: deploy
     language: python
     python: 3.7
@@ -140,6 +151,7 @@ deploy:
       - dvc*.deb
       - dvc*.pkg
       - dvc*.exe
+      - dvc*.snap
     skip_cleanup: true
     on:
       tags: true

CONTRIBUTING.md

@@ -1 +1 @@
-### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing#contributing).
+### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing/core).

README.rst

@@ -30,25 +30,30 @@
   :target: https://anaconda.org/conda-forge/dvc
   :alt: Conda-forge
 
+.. image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
+  :target: https://snapcraft.io/dvc
+  :alt: Snapcraft
+
 |
 
 **Data Version Control** or **DVC** is an **open-source** tool for data science and machine
 learning projects. Key features:
 
-#. simple **command line** Git-like experience. Does not require installing and maintaining
-   any databases. Does not depend on any proprietary online services;
+#. Simple **command line** Git-like experience. Does not require installing and maintaining
+   any databases. Does not depend on any proprietary online services.
 
-#. it manages and versions **datasets** and **machine learning models**. Data is saved in
-   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS or even local HDD RAID;
+#. Management and versioning of **datasets** and **machine learning models**. Data is saved in
+   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
 
-#. it makes projects **reproducible** and **shareable**, it helps answering question how
-   the model was build;
+#. Makes projects **reproducible** and **shareable**; helping to answer questions about how
+   a model was built.
 
-#. it helps manage experiments with Git tags or branches and **metrics** tracking;
+#. Helps manage experiments with Git tags/branches and **metrics** tracking.
 
-It aims to replace tools like Excel and Docs that are being commonly used as a knowledge repo and
-a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
-data file suffixes and prefixes.
+**DVC** aims to replace spreadsheet and document sharing tools (such as Excel or Google Docs)
+which are being used frequently as both knowledge repositories and team ledgers.
+DVC also replaces both ad-hoc scripts to track, move, and deploy different model versions;
+as well as ad-hoc data file suffixes and prefixes.
 
 .. contents:: **Contents**
   :backlinks: none
@@ -56,34 +61,33 @@ data file suffixes and prefixes.
 How DVC works
 =============
 
-We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ to better understand what DVC
-is and how does it fit your scenarios.
+We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ guide to better understand what DVC
+is and how it can fit your scenarios.
 
-The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-lfs to be precise) + ``makefiles``
+The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-LFS to be precise) & Makefiles
 made right and tailored specifically for ML and Data Science scenarios.
 
-#. ``Git/Git-lfs`` part - DVC helps you storing and sharing data artifacts, models. It connects them with your
-   Git repository.
-#. ``Makefiles`` part - DVC describes how one data or model artifact was build from another data.
+#. ``Git/Git-LFS`` part - DVC helps store and share data artifacts and models, connecting them with a Git repository.
+#. ``Makefile``\ s part - DVC describes how one data or model artifact was built from other data and code.
 
-DVC usually runs along with Git. Git is used as usual to store and version code and DVC meta-files. DVC helps
-to store data and model files seamlessly out of Git while preserving almost the same user experience as if they
-were stored in Git itself. To store and share data files cache DVC supports remotes - any cloud (S3, Azure,
+DVC usually runs along with Git. Git is used as usual to store and version code (including DVC meta-files). DVC helps
+to store data and model files seamlessly out of Git, while preserving almost the same user experience as if they
+were stored in Git itself. To store and share the data cache, DVC supports multiple remotes - any cloud (S3, Azure,
 Google Cloud, etc) or any on-premise network storage (via SSH, for example).
 
 .. image:: https://dvc.org/static/img/flow.gif
    :target: https://dvc.org/static/img/flow.gif
    :alt: how_dvc_works
 
-DVC pipelines (aka computational graph) feature connects code and data together. In a very explicit way you can
-specify, run, and save information that a certain command with certain dependencies needs to be run to produce
-a model. See the quick start section below or check `Get Started <https://dvc.org/doc/get-started>`_ tutorial to
-learn more.
+The DVC pipelines (computational graph) feature connects code and data together. It is possible to explicitly
+specify all steps required to produce a model: input dependencies including data, commands to run,
+and output information to be saved. See the quick start section below or
+the `Get Started <https://dvc.org/doc/get-started>`_ tutorial to learn more.
 
 Quick start
 ===========
 
-Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full version. Common workflow commands include:
+Please read `Get Started <https://dvc.org/doc/get-started>`_ guide for a full version. Common workflow commands include:
 
 +-----------------------------------+-------------------------------------------------------------------+
 | Step                              | Command                                                           |
@@ -108,8 +112,8 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 Installation
 ============
 
-Read this `instruction <https://dvc.org/doc/get-started/install>`_ to get more details. There are four
-options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package:
+There are four options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package.
+Full instructions are `available here <https://dvc.org/doc/get-started/install>`_.
 
 pip (PyPI)
 ----------
@@ -120,8 +124,8 @@ pip (PyPI)
 
 Depending on the remote storage type you plan to use to keep and share your data, you might need to specify
 one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all`` to include them all.
-The command should look like this: ``pip install dvc[s3]`` - it installs the ``boto3`` library along with
-DVC to support the AWS S3 storage.
+The command should look like this: ``pip install dvc[s3]`` (in this case AWS S3 dependencies such as ``boto3``
+will be installed automatically).
 
 To install the development version, run:
 
@@ -134,14 +138,9 @@ Homebrew
 
 .. code-block:: bash
 
-   brew install iterative/homebrew-dvc/dvc
+   brew install dvc
 
-or:
 
-.. code-block:: bash
-
-   brew cask install iterative/homebrew-dvc/dvc
-   
 Conda (Anaconda)
 ----------------
 
@@ -149,13 +148,28 @@ Conda (Anaconda)
 
    conda install -c conda-forge dvc
 
-Currently, it supports only python version 2.7, 3.6 and 3.7.
+Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
+
+Snap (Snapcraft)
+----------------
+
+Download the latest ``dvc_*.snap`` from the
+GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+
+.. code-block:: bash
+
+   snap install dvc_*.snap --dangerous --classic
+
+Once ``dvc`` is released on the snap store
+(`pending request <https://forum.snapcraft.io/t/classic-confinement-request-for-dvc/14124>`_)
+there will be no need to download ``dvc_*.snap`` or use ``--dangerous``
+(``snap install dvc --classic`` would suffice).
 
 Package
 -------
 
-Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
-GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+Self-contained packages for Linux, Windows, and Mac are available. The latest version of the packages
+can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
 
 Ubuntu / Debian (deb)
 ^^^^^^^^^^^^^^^^^^^^^
@@ -173,36 +187,27 @@ Fedora / CentOS (rpm)
    sudo yum update
    sudo yum install dvc
 
-Arch Linux (AUR)
-^^^^^^^^^^^^^^^^
-*Unofficial package*, any inquiries regarding the AUR package,
-`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
+Comparison to related technologies
+==================================
 
-.. code-block:: bash
-
-   yay -S dvc
-
-Related technologies
-====================
-
-#. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (that you
-   don't want to see in your Git repository) in a local key-value store and uses file hardlinks/symlinks instead of the
-   copying actual files.
+#. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (which should
+   not be in a Git repository) in a local key-value store, and uses file hardlinks/symlinks instead of
+   copying/duplicating files.
 
 #. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with any remote storage (S3, Google Cloud, Azure, SSH,
-   etc). DVC utilizes reflinks or hardlinks to avoid copy operation on checkouts which makes much more efficient for
-   large data files.
+   etc). DVC also uses reflinks or hardlinks to avoid copy operations on checkouts; thus handling large data files
+   much more efficiently.
 
-#. *Makefile* (and its analogues). DVC tracks dependencies (DAG).
+#. *Makefile* (and analogues including ad-hoc scripts) - DVC tracks dependencies (in a directed acyclic graph).
 
-#. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_. DVC is a workflow
+#. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_ - DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-#. `DAGsHub <https://dagshub.com/>`_ Is a Github equivalent for DVC - pushing your Git+DVC based repo to DAGsHub will give you a high level dashboard of your project, including DVC pipeline and metrics visualizations, as well as links to DVC managed files if they are in cloud storage.
+#. `DAGsHub <https://dagshub.com/>`_ - This is a Github equivalent for DVC. Pushing Git+DVC based repositories to DAGsHub will produce in a high level project dashboard; including DVC pipelines and metrics visualizations, as well as links to any DVC-managed files present in cloud storage.
 
 Contributing
 ============
-Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/>`_ for more
+Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
@@ -247,5 +252,5 @@ Copyright
 
 This project is distributed under the Apache license version 2.0 (see the LICENSE file in the project root).
 
-By submitting a pull request for this project, you agree to license your contribution under the Apache license version
+By submitting a pull request to this project, you agree to license your contribution under the Apache license version
 2.0 to this project.

debian/changelog

@@ -0,0 +1,5 @@
+dvc (0.75.b1) experimental; urgency=low
+
+  * First native Debian package version.
+
+ -- Dmitry Petrov <[email protected]>  Sat, 07 Dec 2019 09:49:28 +0300

debian/compat

@@ -0,0 +1 @@
+10

debian/control

@@ -0,0 +1,33 @@
+Source: dvc
+Maintainer: Dmitry Petrov <[email protected]>
+Section: python
+Priority: optional
+Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
+Standards-Version: 4.0.0
+Homepage: http://dvc.org
+Vcs-Git: https://github.com/iterative/dvc.git
+Vcs-browser: https://github.com/iterative/dvc
+
+Package: dvc
+Architecture: all
+Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
+Recommends: python3-arrow (=0.14.0),
+            python3-paramiko (>=2.5.0),
+            python3-gssapi (>= 2.5.0),
+            python3-boto3 (>=1.9.86),
+            python3-pydrive (=1.3.1),
+            python3-azure-storage-blob (=12.1.0),
+            python3-oss2 (=2.6.1),
+            python3-google-cloud-storage (=1.19.0)
+Description: Git for data scientists - manage your code and data together
+ Data Version Control or DVC is an open-source tool for data science and machine learning projects.
+ .
+ Key features:
+ .
+ 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
+ .
+ 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
+ .
+ 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
+ .
+ 4. Helps manage experiments with Git tags/branches and metrics tracking.

debian/copyright

@@ -0,0 +1,11 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: DVC
+Upstream-Contact: Dmitry Petrov <[email protected]>
+Source: https://github.com/iterative/dvc
+
+Files: *
+Copyright: Dmitry Petrov <[email protected]>
+License: Apache 2.0
+ On Debian systems, the full text of the Apache 2.0
+ License can be found in the file
+ `/usr/share/common-licenses/Apache-2.0`.

debian/rules

@@ -0,0 +1,6 @@
+#!/usr/bin/make -f
+
+export PYBUILD_NAME=dvc
+%:
+	dh $@ --with python3 --buildsystem=pybuild
+

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

debian/watch

@@ -0,0 +1,4 @@
+# please also check http://pypi.debian.net/dvc/watch
+version=3
+opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
+http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

dvc/__init__.py

@@ -3,12 +3,12 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-
 from __future__ import unicode_literals
 
-from dvc.version import __version__  # noqa: F401
 import warnings
+
 import dvc.logger
+from dvc.version import __version__  # noqa: F401
 
 
 dvc.logger.setup()

dvc/__main__.py

@@ -1,5 +1,4 @@
 """Main entry point for dvc command line tool."""
-
 from __future__ import unicode_literals
 
 import sys
@@ -7,4 +6,4 @@ import sys
 from dvc.main import main
 
 if __name__ == "__main__":
-    main(sys.argv[1:])
+    sys.exit(main(sys.argv[1:]))

dvc/analytics.py

@@ -1,283 +1,167 @@
-"""Collect and send usage analytics"""
-
-from __future__ import unicode_literals
-
-from dvc.utils.compat import str
-
-import os
 import json
-import errno
 import logging
+import os
+import platform
+import requests
+import sys
+import tempfile
+import uuid
+
+import distro
 
 from dvc import __version__
+from dvc.config import Config, to_bool
+from dvc.daemon import daemon
+from dvc.exceptions import NotDvcRepoError
+from dvc.lock import Lock, LockError
+from dvc.repo import Repo
+from dvc.scm import SCM
+from dvc.utils import env2bool, is_binary, makedirs
+from dvc.utils.compat import str, FileNotFoundError
 
 
 logger = logging.getLogger(__name__)
 
 
-class Analytics(object):
-    """Class for collecting and sending usage analytics.
-
-    Args:
-        info (dict): optional existing analytics report.
+def collect_and_send_report(args=None, return_code=None):
     """
+    Collect information from the runtime/environment and the command
+    being executed into a report and send it over the network.
 
-    URL = "https://analytics.dvc.org"
-    TIMEOUT_POST = 5
+    To prevent analytics from blocking the execution of the main thread,
+    sending the report is done in a separate process.
 
-    USER_ID_FILE = "user_id"
+    The inter-process communication happens through a file containing the
+    report as a JSON, where the _collector_ generates it and the _sender_
+    removes it after sending it.
+    """
+    report = _runtime_info()
 
-    PARAM_DVC_VERSION = "dvc_version"
-    PARAM_USER_ID = "user_id"
-    PARAM_SYSTEM_INFO = "system_info"
+    # Include command execution information on the report only when available.
+    if args and hasattr(args, "func"):
+        report.update({"cmd_class": args.func.__name__})
 
-    PARAM_OS = "os"
+    if return_code is not None:
+        report.update({"cmd_return_code": return_code})
 
-    PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major"
-    PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor"
-    PARAM_WINDOWS_VERSION_BUILD = "windows_version_build"
-    PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack"
+    with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
+        json.dump(report, fobj)
+        daemon(["analytics", fobj.name])
 
-    PARAM_MAC_VERSION = "mac_version"
 
-    PARAM_LINUX_DISTRO = "linux_distro"
-    PARAM_LINUX_DISTRO_VERSION = "linux_distro_version"
-    PARAM_LINUX_DISTRO_LIKE = "linux_distro_like"
+def is_enabled():
+    if env2bool("DVC_TEST"):
+        return False
 
-    PARAM_SCM_CLASS = "scm_class"
-    PARAM_IS_BINARY = "is_binary"
-    PARAM_CMD_CLASS = "cmd_class"
-    PARAM_CMD_RETURN_CODE = "cmd_return_code"
+    enabled = to_bool(
+        Config(validate=False)
+        .config.get(Config.SECTION_CORE, {})
+        .get(Config.SECTION_CORE_ANALYTICS, "true")
+    )
 
-    def __init__(self, info=None):
-        from dvc.config import Config
-        from dvc.lock import Lock
+    logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
 
-        if info is None:
-            info = {}
+    return enabled
 
-        self.info = info
 
-        cdir = Config.get_global_config_dir()
-        try:
-            os.makedirs(cdir)
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                raise
+def send(report):
+    """
+    Side effect: Removes the report after sending it.
 
-        self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
-        self.user_id_file_lock = Lock(self.user_id_file + ".lock")
+    The report is generated and stored in a temporary file, see:
+    `collect_and_send_report`. Sending happens on another process,
+    thus, the need of removing such file afterwards.
+    """
+    url = "https://analytics.dvc.org"
+    headers = {"content-type": "application/json"}
 
-    @staticmethod
-    def load(path):
-        """Loads analytics report from json file specified by path.
+    with open(report, "rb") as fobj:
+        requests.post(url, data=fobj, headers=headers, timeout=5)
 
-        Args:
-            path (str): path to json file with analytics report.
-        """
-        with open(path, "r") as fobj:
-            analytics = Analytics(info=json.load(fobj))
-        os.unlink(path)
-        return analytics
+    os.remove(report)
 
-    def _write_user_id(self):
-        import uuid
 
-        with open(self.user_id_file, "w+") as fobj:
-            user_id = str(uuid.uuid4())
-            info = {self.PARAM_USER_ID: user_id}
-            json.dump(info, fobj)
+def _scm_in_use():
+    try:
+        scm = SCM(root_dir=Repo.find_root())
+        return type(scm).__name__
+    except NotDvcRepoError:
+        pass
+
+
+def _runtime_info():
+    """
+    Gather information from the environment where DVC runs to fill a report.
+    """
+    return {
+        "dvc_version": __version__,
+        "is_binary": is_binary(),
+        "scm_class": _scm_in_use(),
+        "system_info": _system_info(),
+        "user_id": _find_or_create_user_id(),
+    }
+
+
+def _system_info():
+    system = platform.system()
+
+    if system == "Windows":
+        version = sys.getwindowsversion()
+
+        return {
+            "os": "windows",
+            "windows_version_build": version.build,
+            "windows_version_major": version.major,
+            "windows_version_minor": version.minor,
+            "windows_version_service_pack": version.service_pack,
+        }
+
+    if system == "Darwin":
+        return {"os": "mac", "mac_version": platform.mac_ver()[0]}
+
+    if system == "Linux":
+        return {
+            "os": "linux",
+            "linux_distro": distro.id(),
+            "linux_distro_like": distro.like(),
+            "linux_distro_version": distro.version(),
+        }
+
+    # We don't collect data for any other system.
+    raise NotImplementedError
+
+
+def _find_or_create_user_id():
+    """
+    The user's ID is stored on a file under the global config directory.
+
+    The file should contain a JSON with a "user_id" key:
+
+        {"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"}
+
+    IDs are generated randomly with UUID.
+    """
+    config_dir = Config.get_global_config_dir()
+    fname = os.path.join(config_dir, "user_id")
+    lockfile = os.path.join(config_dir, "user_id.lock")
+
+    # Since the `fname` and `lockfile` are under the global config,
+    # we need to make sure such directory exist already.
+    makedirs(config_dir, exist_ok=True)
+
+    try:
+        with Lock(lockfile):
+            try:
+                with open(fname, "r") as fobj:
+                    user_id = json.load(fobj)["user_id"]
+
+            except (FileNotFoundError, ValueError, KeyError):
+                user_id = str(uuid.uuid4())
+
+                with open(fname, "w") as fobj:
+                    json.dump({"user_id": user_id}, fobj)
+
             return user_id
 
-    def _read_user_id(self):
-        if not os.path.exists(self.user_id_file):
-            return None
-
-        with open(self.user_id_file, "r") as fobj:
-            try:
-                info = json.load(fobj)
-            except ValueError as exc:
-                logger.debug("Failed to load user_id: {}".format(exc))
-                return None
-
-            return info[self.PARAM_USER_ID]
-
-    def _get_user_id(self):
-        from dvc.lock import LockError
-
-        try:
-            with self.user_id_file_lock:
-                user_id = self._read_user_id()
-                if user_id is None:
-                    user_id = self._write_user_id()
-                return user_id
-        except LockError:
-            msg = "Failed to acquire '{}'"
-            logger.debug(msg.format(self.user_id_file_lock.lockfile))
-
-    def _collect_windows(self):
-        import sys
-
-        version = sys.getwindowsversion()  # pylint: disable=no-member
-        info = {}
-        info[self.PARAM_OS] = "windows"
-        info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
-        info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
-        info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
-        info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack
-        return info
-
-    def _collect_darwin(self):
-        import platform
-
-        info = {}
-        info[self.PARAM_OS] = "mac"
-        info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
-        return info
-
-    def _collect_linux(self):
-        import distro
-
-        info = {}
-        info[self.PARAM_OS] = "linux"
-        info[self.PARAM_LINUX_DISTRO] = distro.id()
-        info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
-        info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
-        return info
-
-    def _collect_system_info(self):
-        import platform
-
-        system = platform.system()
-
-        if system == "Windows":
-            return self._collect_windows()
-
-        if system == "Darwin":
-            return self._collect_darwin()
-
-        if system == "Linux":
-            return self._collect_linux()
-
-        raise NotImplementedError
-
-    def collect(self):
-        """Collect analytics report."""
-        from dvc.scm import SCM
-        from dvc.utils import is_binary
-        from dvc.repo import Repo
-        from dvc.exceptions import NotDvcRepoError
-
-        self.info[self.PARAM_DVC_VERSION] = __version__
-        self.info[self.PARAM_IS_BINARY] = is_binary()
-        self.info[self.PARAM_USER_ID] = self._get_user_id()
-
-        self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
-
-        try:
-            scm = SCM(root_dir=Repo.find_root())
-            self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
-        except NotDvcRepoError:
-            pass
-
-    def collect_cmd(self, args, ret):
-        """Collect analytics info from a CLI command."""
-        from dvc.command.daemon import CmdDaemonAnalytics
-
-        assert isinstance(ret, int) or ret is None
-
-        if ret is not None:
-            self.info[self.PARAM_CMD_RETURN_CODE] = ret
-
-        if args is not None and hasattr(args, "func"):
-            assert args.func != CmdDaemonAnalytics
-            self.info[self.PARAM_CMD_CLASS] = args.func.__name__
-
-    def dump(self):
-        """Save analytics report to a temporary file.
-
-        Returns:
-            str: path to the temporary file that contains the analytics report.
-        """
-        import tempfile
-
-        with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
-            json.dump(self.info, fobj)
-            return fobj.name
-
-    @staticmethod
-    def _is_enabled_config(config):
-        from dvc.config import Config
-
-        core = config.config.get(Config.SECTION_CORE, {})
-        return core.get(Config.SECTION_CORE_ANALYTICS, True)
-
-    @staticmethod
-    def _get_current_config():
-        from dvc.config import Config
-        from dvc.repo import Repo
-        from dvc.exceptions import NotDvcRepoError
-
-        try:
-            dvc_dir = Repo.find_dvc_dir()
-            config = Config(dvc_dir)
-        except NotDvcRepoError:
-            config = Config(validate=False)
-        return config
-
-    @staticmethod
-    def is_enabled(cmd=None):
-        from dvc.command.daemon import CmdDaemonBase
-
-        if os.getenv("DVC_TEST"):
-            return False
-
-        if isinstance(cmd, CmdDaemonBase):
-            return False
-
-        config = (
-            Analytics._get_current_config()
-            if cmd is None or not hasattr(cmd, "config")
-            else cmd.config
-        )
-
-        assert config is not None
-
-        enabled = Analytics._is_enabled_config(config)
-        logger.debug(
-            "Analytics is {}.".format("enabled" if enabled else "disabled")
-        )
-        return enabled
-
-    @staticmethod
-    def send_cmd(cmd, args, ret):
-        """Collect and send analytics for CLI command.
-
-        Args:
-            args (list): parsed args for the CLI command.
-            ret (int): return value of the CLI command.
-        """
-        from dvc.daemon import daemon
-
-        if not Analytics.is_enabled(cmd):
-            return
-
-        analytics = Analytics()
-        analytics.collect_cmd(args, ret)
-        daemon(["analytics", analytics.dump()])
-
-    def send(self):
-        """Collect and send analytics."""
-        import requests
-
-        if not self.is_enabled():
-            return
-
-        self.collect()
-
-        logger.debug("Sending analytics: {}".format(self.info))
-
-        try:
-            requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
-        except requests.exceptions.RequestException as exc:
-            logger.debug("Failed to send analytics: {}".format(str(exc)))
+    except LockError:
+        logger.debug("Failed to acquire {lockfile}".format(lockfile=lockfile))

dvc/api.py

@@ -1,5 +1,5 @@
-from contextlib import contextmanager
 import os
+from contextlib import contextmanager
 
 try:
     from contextlib import _GeneratorContextManager as GCM

dvc/cache.py

@@ -1,12 +1,13 @@
 """Manages cache of a dvc repo."""
-
 from __future__ import unicode_literals
 
 import os
+from collections import defaultdict
+
 from funcy import cached_property
 
-from dvc.utils.compat import builtin_str
 from dvc.config import Config
+from dvc.utils.compat import builtin_str
 
 
 class CacheConfig(object):
@@ -103,3 +104,36 @@ class Cache(object):
     ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
     hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
     azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
+
+
+class NamedCache(object):
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(set))
+        self.external = defaultdict(set)
+
+    @classmethod
+    def make(cls, scheme, checksum, name):
+        cache = cls()
+        cache.add(scheme, checksum, name)
+        return cache
+
+    def __getitem__(self, key):
+        return self._items[key]
+
+    def add(self, scheme, checksum, name):
+        self._items[scheme][checksum].add(name)
+
+    def add_external(self, url, rev, path):
+        self.external[url, rev].add(path)
+
+    def update(self, cache, suffix=""):
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme]
+            for checksum, names in src.items():
+                if suffix:
+                    dst[checksum].update(n + suffix for n in names)
+                else:
+                    dst[checksum].update(names)
+
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)

dvc/cli.py

@@ -1,42 +1,41 @@
 """Command line interface for dvc."""
-
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import sys
 import argparse
 import logging
+import sys
 
-from dvc.command.base import fix_subparsers
-import dvc.command.init as init
+import dvc.command.add as add
+import dvc.command.cache as cache
+import dvc.command.checkout as checkout
+import dvc.command.commit as commit
+import dvc.command.config as config
+import dvc.command.daemon as daemon
+import dvc.command.data_sync as data_sync
+import dvc.command.destroy as destroy
+import dvc.command.diff as diff
+import dvc.command.gc as gc
 import dvc.command.get as get
 import dvc.command.get_url as get_url
-import dvc.command.destroy as destroy
-import dvc.command.remove as remove
-import dvc.command.move as move
-import dvc.command.unprotect as unprotect
-import dvc.command.run as run
-import dvc.command.repro as repro
-import dvc.command.data_sync as data_sync
-import dvc.command.gc as gc
-import dvc.command.add as add
 import dvc.command.imp as imp
 import dvc.command.imp_url as imp_url
-import dvc.command.config as config
-import dvc.command.checkout as checkout
-import dvc.command.remote as remote
-import dvc.command.cache as cache
-import dvc.command.metrics as metrics
+import dvc.command.init as init
 import dvc.command.install as install
-import dvc.command.root as root
 import dvc.command.lock as lock
+import dvc.command.metrics as metrics
+import dvc.command.move as move
 import dvc.command.pipeline as pipeline
-import dvc.command.daemon as daemon
-import dvc.command.commit as commit
+import dvc.command.remote as remote
+import dvc.command.remove as remove
+import dvc.command.repro as repro
+import dvc.command.root as root
+import dvc.command.run as run
 import dvc.command.tag as tag
-import dvc.command.diff as diff
-import dvc.command.version as version
+import dvc.command.unprotect as unprotect
 import dvc.command.update as update
+import dvc.command.version as version
+from dvc.command.base import fix_subparsers
 from dvc.exceptions import DvcParserError
 
 
@@ -106,7 +105,7 @@ def get_parent_parser():
     """Create instances of a parser containing common arguments shared among
     all the commands.
 
-    When overwritting `-q` or `-v`, you need to instantiate a new object
+    When overwriting `-q` or `-v`, you need to instantiate a new object
     in order to prevent some weird behavior.
     """
     parent_parser = argparse.ArgumentParser(add_help=False)
@@ -144,6 +143,19 @@ def parse_args(argv=None):
         description=desc,
         parents=[parent_parser],
         formatter_class=argparse.RawTextHelpFormatter,
+        add_help=False,
+    )
+
+    # NOTE: We are doing this to capitalize help message.
+    # Unfortunately, there is no easier and clearer way to do it,
+    # as adding this argument in get_parent_parser() either in
+    # log_level_group or on parent_parser itself will cause unexpected error.
+    parser.add_argument(
+        "-h",
+        "--help",
+        action="help",
+        default=argparse.SUPPRESS,
+        help="Show this help message and exit.",
     )
 
     # NOTE: On some python versions action='version' prints to stderr

dvc/command/add.py

@@ -3,8 +3,10 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import RecursiveAddingWhileUsingFilename
 
 
 logger = logging.getLogger(__name__)
@@ -14,49 +16,50 @@ class CmdAdd(CmdBase):
     def run(self):
         try:
             if len(self.args.targets) > 1 and self.args.file:
-                raise DvcException("can't use '--file' with multiple targets")
+                raise RecursiveAddingWhileUsingFilename()
 
-            for target in self.args.targets:
-                self.repo.add(
-                    target,
-                    recursive=self.args.recursive,
-                    no_commit=self.args.no_commit,
-                    fname=self.args.file,
-                )
+            self.repo.add(
+                self.args.targets,
+                recursive=self.args.recursive,
+                no_commit=self.args.no_commit,
+                fname=self.args.file,
+            )
 
         except DvcException:
-            logger.exception("failed to add file")
+            logger.exception("")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    ADD_HELP = "Take data files or directories under DVC control."
+    ADD_HELP = "Track data files or directories with DVC."
 
-    add_parser = subparsers.add_parser(
+    parser = subparsers.add_parser(
         "add",
         parents=[parent_parser],
         description=append_doc_link(ADD_HELP, "add"),
         help=ADD_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    add_parser.add_argument(
+    parser.add_argument(
         "-R",
         "--recursive",
         action="store_true",
         default=False,
-        help="Recursively add each file under the directory.",
+        help="Recursively add files under directory targets.",
     )
-    add_parser.add_argument(
+    parser.add_argument(
         "--no-commit",
         action="store_true",
         default=False,
         help="Don't put files/directories into cache.",
     )
-    add_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC-file it generates."
+    parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
     )
-    add_parser.add_argument(
+    parser.add_argument(
         "targets", nargs="+", help="Input files/directories to add."
     )
-    add_parser.set_defaults(func=CmdAdd)
+    parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import colorama
 import logging
 
 
@@ -23,15 +22,13 @@ def fix_subparsers(subparsers):
 
 
 def append_doc_link(help_message, path):
+    from dvc.utils import format_link
+
     if not path:
         return help_message
     doc_base = "https://man.dvc.org/"
-    return "{message}\ndocumentation: {blue}{base}{path}{nc}".format(
-        message=help_message,
-        base=doc_base,
-        path=path,
-        blue=colorama.Fore.CYAN,
-        nc=colorama.Fore.RESET,
+    return "{message}\nDocumentation: {link}".format(
+        message=help_message, link=format_link(doc_base + path)
     )
 
 
@@ -43,7 +40,8 @@ class CmdBase(object):
         self.repo = Repo()
         self.config = self.repo.config
         self.args = args
-        updater = Updater(self.repo.dvc_dir)
+        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
         updater.check()
 
     @property
@@ -57,7 +55,7 @@ class CmdBase(object):
 
     # Abstract methods that have to be implemented by any inheritance class
     def run(self):
-        pass
+        raise NotImplementedError
 
 
 class CmdBaseNoRepo(CmdBase):

dvc/command/cache.py

@@ -2,9 +2,10 @@ from __future__ import unicode_literals
 
 import argparse
 
-from dvc.command.base import fix_subparsers, append_doc_link
-from dvc.command.config import CmdConfig
 from dvc.cache import CacheConfig
+from dvc.command.base import append_doc_link
+from dvc.command.base import fix_subparsers
+from dvc.command.config import CmdConfig
 
 
 class CmdCacheDir(CmdConfig):

dvc/command/checkout.py

@@ -2,21 +2,19 @@ from __future__ import unicode_literals
 
 import argparse
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 
 
 class CmdCheckout(CmdBase):
     def run(self):
-        if not self.args.targets:
-            self.repo.checkout(force=self.args.force)
-        else:
-            for target in self.args.targets:
-                self.repo.checkout(
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    force=self.args.force,
-                    recursive=self.args.recursive,
-                )
+        self.repo.checkout(
+            targets=self.args.targets,
+            with_deps=self.args.with_deps,
+            force=self.args.force,
+            relink=self.args.relink,
+            recursive=self.args.recursive,
+        )
         return 0
 
 
@@ -37,6 +35,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Do not prompt when removing working directory files.",
     )
+    checkout_parser.add_argument(
+        "--relink",
+        action="store_true",
+        default=False,
+        help="Recreate links or copies from cache to workspace.",
+    )
     checkout_parser.add_argument(
         "-d",
         "--with-deps",

dvc/command/commit.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/config.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 from dvc.config import Config
 
 
@@ -28,6 +29,14 @@ class CmdConfig(CmdBaseNoRepo):
                 section, opt, self.args.value, level=self.args.level
             )
 
+        is_write = self.args.unset or self.args.value is not None
+        if is_write and self.args.name == "cache.type":
+            logger.warning(
+                "You have changed the 'cache.type' option. This doesn't update"
+                " any existing workspace file links, but it can be done with:"
+                "\n             dvc checkout --relink"
+            )
+
         return 0
 
 

dvc/command/daemon.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.command.base import CmdBaseNoRepo
+from dvc.command.base import fix_subparsers
 
 
 class CmdDaemonBase(CmdBaseNoRepo):
@@ -11,11 +12,16 @@ class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
         from dvc.repo import Repo
+        from dvc.config import Config
         from dvc.updater import Updater
 
         root_dir = Repo.find_root()
         dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
-        updater = Updater(dvc_dir)
+        config = Config(dvc_dir, validate=False)
+        hardlink_lock = config.config.get("core", {}).get(
+            "hardlink_lock", False
+        )
+        updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
         updater.fetch(detach=False)
 
         return 0
@@ -23,10 +29,9 @@ class CmdDaemonUpdater(CmdDaemonBase):
 
 class CmdDaemonAnalytics(CmdDaemonBase):
     def run(self):
-        from dvc.analytics import Analytics
+        from dvc import analytics
 
-        analytics = Analytics.load(self.args.target)
-        analytics.send()
+        analytics.send(self.args.target)
 
         return 0
 

dvc/command/data_sync.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 

dvc/command/destroy.py

@@ -4,7 +4,8 @@ import argparse
 import logging
 
 import dvc.prompt as prompt
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 
@@ -24,7 +25,7 @@ class CmdDestroy(CmdBase):
             if not self.args.force and not prompt.confirm(statement):
                 raise DvcException(
                     "cannot destroy without a confirmation from the user."
-                    " Use '-f' to force."
+                    " Use `-f` to force."
                 )
 
             self.repo.destroy()

dvc/command/diff.py

@@ -1,15 +1,15 @@
 from __future__ import unicode_literals
 
 import argparse
+import logging
 
 import humanize
 import inflect
-import logging
 from funcy import compact
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
-import dvc.repo.diff as diff
 
 
 logger = logging.getLogger(__name__)
@@ -37,6 +37,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_dir_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         engine = inflect.engine()
         changes_msg = (
             "{} {} untouched, {} {} modified, {} {} added, "
@@ -57,6 +59,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_file_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         if (
             dct.get(diff.DIFF_OLD_FILE)
             and dct.get(diff.DIFF_NEW_FILE)
@@ -79,6 +83,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_royal_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
             if dct.get("is_dir"):
                 return cls._get_dir_changes(dct)
@@ -88,6 +94,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _show(cls, diff_dct):
+        import dvc.repo.diff as diff
+
         msg = "dvc diff from {} to {}".format(
             diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
         )

dvc/command/gc.py

@@ -1,11 +1,12 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
-import dvc.prompt as prompt
 import logging
+import os
 
-from dvc.command.base import CmdBase, append_doc_link
+import dvc.prompt as prompt
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 
 
 logger = logging.getLogger(__name__)
@@ -13,17 +14,19 @@ logger = logging.getLogger(__name__)
 
 class CmdGC(CmdBase):
     def run(self):
-        msg = "this will remove all cache except the cache that is used in "
-        if not self.args.all_branches and not self.args.all_tags:
-            msg += "the current git branch"
-        elif self.args.all_branches and not self.args.all_tags:
-            msg += "all git branches"
-        elif not self.args.all_branches and self.args.all_tags:
-            msg += "all git tags"
-        else:
-            msg += "all git branches and all git tags"
+        msg = "This will remove all cache except items used in "
 
-        if self.args.repos is not None and len(self.args.repos) > 0:
+        msg += "the working tree"
+        if self.args.all_commits:
+            msg += " and all git commits"
+        elif self.args.all_branches and self.args.all_tags:
+            msg += " and all git branches and tags"
+        elif self.args.all_branches:
+            msg += " and all git branches"
+        elif self.args.all_tags:
+            msg += " and all git tags"
+
+        if self.args.repos:
             msg += " of the current and the following repos:"
 
             for repo_path in self.args.repos:
@@ -40,6 +43,7 @@ class CmdGC(CmdBase):
         self.repo.gc(
             all_branches=self.args.all_branches,
             all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
             cloud=self.args.cloud,
             remote=self.args.remote,
             force=self.args.force,
@@ -76,6 +80,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Keep data files for all git tags.",
     )
+    gc_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     gc_parser.add_argument(
         "-c",
         "--cloud",

dvc/command/get.py

@@ -3,9 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
+from .base import append_doc_link
+from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
-from .base import CmdBaseNoRepo, append_doc_link
 
 
 logger = logging.getLogger(__name__)
@@ -13,6 +13,8 @@ logger = logging.getLogger(__name__)
 
 class CmdGet(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         try:
             Repo.get(
                 self.args.url,
@@ -31,7 +33,7 @@ class CmdGet(CmdBaseNoRepo):
 
 
 def add_parser(subparsers, parent_parser):
-    GET_HELP = "Download data from DVC repository."
+    GET_HELP = "Download/copy files or directories from DVC repository."
     get_parser = subparsers.add_parser(
         "get",
         parents=[parent_parser],
@@ -42,9 +44,14 @@ def add_parser(subparsers, parent_parser):
     get_parser.add_argument(
         "url", help="URL of Git repository with DVC project to download from."
     )
-    get_parser.add_argument("path", help="Path to data within DVC repository.")
     get_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to put data to."
+        "path", help="Path to a file or directory within a DVC repository."
+    )
+    get_parser.add_argument(
+        "-o",
+        "--out",
+        nargs="?",
+        help="Destination path to copy/download files to.",
     )
     get_parser.add_argument(
         "--rev", nargs="?", help="DVC repository git revision."

dvc/command/get_url.py

@@ -3,9 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
+from .base import append_doc_link
+from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
-from .base import CmdBaseNoRepo, append_doc_link
 
 
 logger = logging.getLogger(__name__)
@@ -13,6 +13,8 @@ logger = logging.getLogger(__name__)
 
 class CmdGetUrl(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         try:
             Repo.get_url(self.args.url, out=self.args.out)
             return 0

dvc/command/imp.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/imp_url.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)
@@ -19,7 +20,7 @@ class CmdImportUrl(CmdBase):
         except DvcException:
             logger.exception(
                 "failed to import {}. You could also try downloading "
-                "it manually and adding it with `dvc add` command.".format(
+                "it manually, and adding it with `dvc add`.".format(
                     self.args.url
                 )
             )
@@ -43,12 +44,17 @@ def add_parser(subparsers, parent_parser):
         "url",
         help="Supported urls:\n"
         "/path/to/file\n"
+        "/path/to/directory\n"
         "C:\\\\path\\to\\file\n"
+        "C:\\\\path\\to\\directory\n"
         "https://example.com/path/to/file\n"
         "s3://bucket/path/to/file\n"
+        "s3://bucket/path/to/directory\n"
         "gs://bucket/path/to/file\n"
+        "gs://bucket/path/to/directory\n"
         "hdfs://example.com/path/to/file\n"
         "ssh://example.com:/path/to/file\n"
+        "ssh://example.com:/path/to/directory\n"
         "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(

dvc/command/init.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 
 
 logger = logging.getLogger(__name__)
@@ -53,7 +54,7 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
         help=(
-            "Overwrite existing '.dvc' directory. "
+            "Overwrite existing '.dvc/' directory. "
             "This operation removes local cache."
         ),
     )

dvc/command/install.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 
 
 logger = logging.getLogger(__name__)

dvc/command/lock.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/metrics.py

@@ -3,8 +3,11 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.exceptions import DvcException, BadMetricError
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
+from dvc.exceptions import BadMetricError
+from dvc.exceptions import DvcException
 
 
 logger = logging.getLogger(__name__)

dvc/command/move.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/pipeline.py

@@ -1,13 +1,14 @@
 from __future__ import unicode_literals
 
-from dvc.utils import relpath
-from dvc.utils.compat import str
-
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.utils import relpath
+from dvc.utils.compat import str
 
 
 logger = logging.getLogger(__name__)
@@ -19,7 +20,7 @@ class CmdPipelineShow(CmdBase):
         from dvc.stage import Stage
 
         stage = Stage.load(self.repo, target)
-        G = self.repo.graph()[0]
+        G = self.repo.graph
         stages = networkx.get_node_attributes(G, "stage")
         node = relpath(stage.path, self.repo.root_dir)
         nodes = networkx.dfs_postorder_nodes(G, node)
@@ -41,20 +42,16 @@ class CmdPipelineShow(CmdBase):
     def __build_graph(self, target, commands, outs):
         import networkx
         from dvc.stage import Stage
+        from dvc.repo.graph import get_pipeline
 
         stage = Stage.load(self.repo, target)
         node = relpath(stage.path, self.repo.root_dir)
 
-        pipelines = list(
-            filter(lambda g: node in g.nodes(), self.repo.pipelines())
-        )
-
-        assert len(pipelines) == 1
-        G = pipelines[0]
+        G = get_pipeline(self.repo.pipelines, node)
         stages = networkx.get_node_attributes(G, "stage")
 
         nodes = []
-        for n in G.nodes():
+        for n in G:
             stage = stages[n]
             if commands:
                 if stage.cmd is None:
@@ -168,7 +165,7 @@ class CmdPipelineList(CmdBase):
     def run(self):
         import networkx
 
-        pipelines = self.repo.pipelines()
+        pipelines = self.repo.pipelines
         for p in pipelines:
             stages = networkx.get_node_attributes(p, "stage")
             for stage in stages:

dvc/command/remote.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link, fix_subparsers
+from dvc.command.base import append_doc_link
+from dvc.command.base import fix_subparsers
 from dvc.command.config import CmdConfig
 from dvc.remote.config import RemoteConfig
 
@@ -18,6 +19,10 @@ class CmdRemoteConfig(CmdConfig):
 
 class CmdRemoteAdd(CmdRemoteConfig):
     def run(self):
+        if self.args.default:
+            logger.info(
+                "Setting '{}' as a default remote.".format(self.args.name)
+            )
         self.remote_config.add(
             self.args.name,
             self.args.url,
@@ -47,9 +52,13 @@ class CmdRemoteModify(CmdRemoteConfig):
 
 class CmdRemoteDefault(CmdRemoteConfig):
     def run(self):
-        self.remote_config.set_default(
-            self.args.name, unset=self.args.unset, level=self.args.level
-        )
+        if self.args.name is None and not self.args.unset:
+            name = self.remote_config.get_default(level=self.args.level)
+            print(name)
+        else:
+            self.remote_config.set_default(
+                self.args.name, unset=self.args.unset, level=self.args.level
+            )
         return 0
 
 

dvc/command/remove.py

@@ -4,8 +4,9 @@ import argparse
 import logging
 
 import dvc.prompt as prompt
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)
@@ -28,7 +29,7 @@ class CmdRemove(CmdBase):
 
         raise DvcException(
             "Cannot purge without a confirmation from the user."
-            " Use '-f' to force."
+            " Use `-f` to force."
         )
 
     def run(self):

dvc/command/repro.py

@@ -1,10 +1,11 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
 import logging
+import os
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
 from dvc.exceptions import DvcException

dvc/command/root.py

@@ -3,8 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
@@ -12,6 +12,8 @@ logger = logging.getLogger(__name__)
 
 class CmdRoot(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         logger.info(relpath(Repo.find_root()))
         return 0
 

dvc/command/run.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 
@@ -27,9 +28,9 @@ class CmdRun(CmdBase):
             ]
         ):  # pragma: no cover
             logger.error(
-                "too few arguments. Specify at least one: '-d', '-o', '-O',"
-                " '-m', '-M', '--outs-persist', '--outs-persist-no-cache',"
-                " 'command'."
+                "too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
+                "`-m`, `-M`, `--outs-persist`, `--outs-persist-no-cache`, "
+                "`command`."
             )
             return 1
 

dvc/command/tag.py

@@ -1,8 +1,10 @@
 import logging
 
-from dvc.utils import to_yaml_string
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.utils import to_yaml_string
 
 
 logger = logging.getLogger(__name__)

dvc/command/unprotect.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/update.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/version.py

@@ -1,55 +1,70 @@
 from __future__ import unicode_literals
 
+import argparse
+import itertools
+import logging
 import os
 import platform
-import argparse
-import logging
 import uuid
-import itertools
 
 try:
     import psutil
 except ImportError:
     psutil = None
 
-from dvc.utils import is_binary
+from dvc.utils import is_binary, relpath
 from dvc.utils.compat import pathlib
-from dvc.repo import Repo
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.system import System
-
+from dvc.utils.pkg import PKG
 
 logger = logging.getLogger(__name__)
 
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         dvc_version = __version__
         python_version = platform.python_version()
         platform_type = platform.platform()
         binary = is_binary()
-
         info = (
             "DVC version: {dvc_version}\n"
             "Python version: {python_version}\n"
             "Platform: {platform_type}\n"
             "Binary: {binary}\n"
+            "Package: {package}\n"
         ).format(
             dvc_version=dvc_version,
             python_version=python_version,
             platform_type=platform_type,
             binary=binary,
+            package=PKG,
         )
 
         try:
             repo = Repo()
             root_directory = repo.root_dir
 
-            info += "Cache: {cache}\n".format(
-                cache=self.get_linktype_support_info(repo)
-            )
+            # cache_dir might not exist yet (e.g. after `dvc init`), and we
+            # can't auto-create it, as it might cause issues if the user
+            # later decides to enable shared cache mode with
+            # `dvc config cache.shared group`.
+            if os.path.exists(repo.cache.local.cache_dir):
+                info += "Cache: {cache}\n".format(
+                    cache=self.get_linktype_support_info(repo)
+                )
+            else:
+                logger.warning(
+                    "Unable to detect supported link types, as cache "
+                    "directory '{}' doesn't exist. It is usually auto-created "
+                    "by commands such as `dvc add/fetch/pull/run/import`, "
+                    "but you could create it manually to enable this "
+                    "check.".format(relpath(repo.cache.local.cache_dir))
+                )
 
             if psutil:
                 info += (
@@ -78,7 +93,7 @@ class CmdVersion(CmdBaseNoRepo):
         for parent in itertools.chain([path], path.parents):
             if parent in partition:
                 return partition[parent]
-        return ("unkown", "none")
+        return ("unknown", "none")
 
     @staticmethod
     def get_linktype_support_info(repo):

dvc/config.py

@@ -1,18 +1,19 @@
 """DVC config objects."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, open
-
-import os
-import re
 import copy
 import errno
-import configobj
 import logging
+import os
+import re
 
-from schema import Schema, Optional, And, Use, Regex, SchemaError
-from dvc.exceptions import DvcException, NotDvcRepoError
+import configobj
+from voluptuous import Schema, Required, Optional, Invalid
+from voluptuous import All, Any, Lower, Range, Coerce, Match
+
+from dvc.exceptions import DvcException
+from dvc.exceptions import NotDvcRepoError
+from dvc.utils.compat import open, str
 
 logger = logging.getLogger(__name__)
 
@@ -44,82 +45,43 @@ class NoRemoteError(ConfigError):
 
 
 def supported_cache_type(types):
-    """Checks if link type config option has a valid value.
+    """Checks if link type config option consists only of valid values.
 
     Args:
         types (list/string): type(s) of links that dvc should try out.
     """
+    if types is None:
+        return None
     if isinstance(types, str):
         types = [typ.strip() for typ in types.split(",")]
-    for typ in types:
-        if typ not in ["reflink", "hardlink", "symlink", "copy"]:
-            return False
-    return True
+
+    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
+    if unsupported:
+        raise Invalid(
+            "Unsupported cache type(s): {}".format(", ".join(unsupported))
+        )
+
+    return types
 
 
-def is_bool(val):
-    """Checks that value is a boolean.
-
-    Args:
-        val (str): string value verify.
-
-    Returns:
-        bool: True if value stands for boolean, False otherwise.
-    """
-    return val.lower() in ["true", "false"]
+# Checks that value is either true or false and converts it to bool
+Bool = All(
+    Lower,
+    Any("true", "false"),
+    lambda v: v == "true",
+    msg="expected true or false",
+)
+to_bool = Schema(Bool)
 
 
-def to_bool(val):
-    """Converts value to boolean.
-
-    Args:
-        val (str): string to convert to boolean.
-
-    Returns:
-        bool: True if value.lower() == 'true', False otherwise.
-    """
-    return val.lower() == "true"
-
-
-def is_whole(val):
-    """Checks that value is a whole integer.
-
-    Args:
-        val (str): number string to verify.
-
-    Returns:
-        bool: True if val is a whole number, False otherwise.
-    """
-    return int(val) >= 0
-
-
-def is_percent(val):
-    """Checks that value is a percent.
-
-    Args:
-        val (str): number string to verify.
-
-    Returns:
-        bool: True if 0<=value<=100, False otherwise.
-    """
-    return int(val) >= 0 and int(val) <= 100
-
-
-class Choices(object):
+def Choices(*choices):
     """Checks that value belongs to the specified set of values
 
     Args:
         *choices: pass allowed values as arguments, or pass a list or
             tuple as a single argument
     """
-
-    def __init__(self, *choices):
-        if len(choices) == 1 and isinstance(choices[0], (list, tuple)):
-            choices = choices[0]
-        self.choices = choices
-
-    def __call__(self, value):
-        return value in self.choices
+    return Any(*choices, msg="expected one of {}".format(",".join(choices)))
 
 
 class Config(object):  # pylint: disable=too-many-instance-attributes
@@ -147,33 +109,30 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
+    CREDENTIALPATH = "credentialpath"
+
     LEVEL_LOCAL = 0
     LEVEL_REPO = 1
     LEVEL_GLOBAL = 2
     LEVEL_SYSTEM = 3
 
-    BOOL_SCHEMA = And(str, is_bool, Use(to_bool))
-
     SECTION_CORE = "core"
     SECTION_CORE_LOGLEVEL = "loglevel"
-    SECTION_CORE_LOGLEVEL_SCHEMA = And(
-        Use(str.lower), Choices("info", "debug", "warning", "error")
+    SECTION_CORE_LOGLEVEL_SCHEMA = All(
+        Lower, Choices("info", "debug", "warning", "error")
     )
     SECTION_CORE_REMOTE = "remote"
-    SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA
     SECTION_CORE_INTERACTIVE = "interactive"
     SECTION_CORE_ANALYTICS = "analytics"
-    SECTION_CORE_ANALYTICS_SCHEMA = BOOL_SCHEMA
     SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_CORE_CHECKSUM_JOBS_SCHEMA = And(Use(int), lambda x: x > 0)
+    SECTION_CORE_HARDLINK_LOCK = "hardlink_lock"
 
     SECTION_CACHE = "cache"
     SECTION_CACHE_DIR = "dir"
     SECTION_CACHE_TYPE = "type"
-    SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
     SECTION_CACHE_PROTECTED = "protected"
     SECTION_CACHE_SHARED = "shared"
-    SECTION_CACHE_SHARED_SCHEMA = And(Use(str.lower), Choices("group"))
+    SECTION_CACHE_SHARED_SCHEMA = All(Lower, Choices("group"))
     SECTION_CACHE_LOCAL = "local"
     SECTION_CACHE_S3 = "s3"
     SECTION_CACHE_GS = "gs"
@@ -182,40 +141,31 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_CACHE_AZURE = "azure"
     SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
     SECTION_CACHE_SCHEMA = {
-        Optional(SECTION_CACHE_LOCAL): str,
-        Optional(SECTION_CACHE_S3): str,
-        Optional(SECTION_CACHE_GS): str,
-        Optional(SECTION_CACHE_HDFS): str,
-        Optional(SECTION_CACHE_SSH): str,
-        Optional(SECTION_CACHE_AZURE): str,
-        Optional(SECTION_CACHE_DIR): str,
-        Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
-        Optional(SECTION_CACHE_SHARED): SECTION_CACHE_SHARED_SCHEMA,
-        Optional(PRIVATE_CWD): str,
-        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): BOOL_SCHEMA,
+        SECTION_CACHE_LOCAL: str,
+        SECTION_CACHE_S3: str,
+        SECTION_CACHE_GS: str,
+        SECTION_CACHE_HDFS: str,
+        SECTION_CACHE_SSH: str,
+        SECTION_CACHE_AZURE: str,
+        SECTION_CACHE_DIR: str,
+        SECTION_CACHE_TYPE: supported_cache_type,
+        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
+        SECTION_CACHE_SHARED: SECTION_CACHE_SHARED_SCHEMA,
+        PRIVATE_CWD: str,
+        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): Bool,
     }
 
     SECTION_CORE_SCHEMA = {
-        Optional(SECTION_CORE_LOGLEVEL): And(
-            str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA
-        ),
-        Optional(SECTION_CORE_REMOTE, default=""): And(str, Use(str.lower)),
-        Optional(
-            SECTION_CORE_INTERACTIVE, default=False
-        ): SECTION_CORE_INTERACTIVE_SCHEMA,
-        Optional(
-            SECTION_CORE_ANALYTICS, default=True
-        ): SECTION_CORE_ANALYTICS_SCHEMA,
-        Optional(
-            SECTION_CORE_CHECKSUM_JOBS, default=None
-        ): SECTION_CORE_CHECKSUM_JOBS_SCHEMA,
+        SECTION_CORE_LOGLEVEL: SECTION_CORE_LOGLEVEL_SCHEMA,
+        SECTION_CORE_REMOTE: Lower,
+        Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
+        Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
+        SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
+        Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool,
     }
 
-    # backward compatibility
-    SECTION_AWS = "aws"
-    SECTION_AWS_STORAGEPATH = "storagepath"
-    SECTION_AWS_CREDENTIALPATH = "credentialpath"
+    # aws specific options
+    SECTION_AWS_CREDENTIALPATH = CREDENTIALPATH
     SECTION_AWS_ENDPOINT_URL = "endpointurl"
     SECTION_AWS_LIST_OBJECTS = "listobjects"
     SECTION_AWS_REGION = "region"
@@ -223,39 +173,28 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_AWS_USE_SSL = "use_ssl"
     SECTION_AWS_SSE = "sse"
     SECTION_AWS_ACL = "acl"
-    SECTION_AWS_SCHEMA = {
-        SECTION_AWS_STORAGEPATH: str,
-        Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE): str,
-        Optional(SECTION_AWS_CREDENTIALPATH): str,
-        Optional(SECTION_AWS_ENDPOINT_URL): str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
-        Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
-        Optional(SECTION_AWS_SSE): str,
-        Optional(SECTION_AWS_ACL): str,
-    }
+    SECTION_AWS_GRANT_READ = "grant_read"
+    SECTION_AWS_GRANT_READ_ACP = "grant_read_acp"
+    SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp"
+    SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control"
 
-    # backward compatibility
-    SECTION_GCP = "gcp"
-    SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
-    SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH
+    # gcp specific options
+    SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH
     SECTION_GCP_PROJECTNAME = "projectname"
-    SECTION_GCP_SCHEMA = {
-        SECTION_GCP_STORAGEPATH: str,
-        Optional(SECTION_GCP_PROJECTNAME): str,
-    }
-
-    # backward compatibility
-    SECTION_LOCAL = "local"
-    SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
-    SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str}
 
+    # azure specific option
     SECTION_AZURE_CONNECTION_STRING = "connection_string"
+
     # Alibabacloud oss options
     SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
     SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
     SECTION_OSS_ENDPOINT = "oss_endpoint"
 
+    # GDrive options
+    SECTION_GDRIVE_CLIENT_ID = "gdrive_client_id"
+    SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret"
+    SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file"
+
     SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
     SECTION_REMOTE_FMT = 'remote "{}"'
     SECTION_REMOTE_URL = "url"
@@ -268,51 +207,55 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_REMOTE_GSS_AUTH = "gss_auth"
     SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
     SECTION_REMOTE_SCHEMA = {
-        SECTION_REMOTE_URL: str,
-        Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE): str,
-        Optional(SECTION_AWS_CREDENTIALPATH): str,
-        Optional(SECTION_AWS_ENDPOINT_URL): str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
-        Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
-        Optional(SECTION_AWS_SSE): str,
-        Optional(SECTION_AWS_ACL): str,
-        Optional(SECTION_GCP_PROJECTNAME): str,
-        Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
-        Optional(SECTION_REMOTE_USER): str,
-        Optional(SECTION_REMOTE_PORT): Use(int),
-        Optional(SECTION_REMOTE_KEY_FILE): str,
-        Optional(SECTION_REMOTE_TIMEOUT): Use(int),
-        Optional(SECTION_REMOTE_PASSWORD): str,
-        Optional(SECTION_REMOTE_ASK_PASSWORD): BOOL_SCHEMA,
-        Optional(SECTION_REMOTE_GSS_AUTH): BOOL_SCHEMA,
-        Optional(SECTION_AZURE_CONNECTION_STRING): str,
-        Optional(SECTION_OSS_ACCESS_KEY_ID): str,
-        Optional(SECTION_OSS_ACCESS_KEY_SECRET): str,
-        Optional(SECTION_OSS_ENDPOINT): str,
-        Optional(PRIVATE_CWD): str,
-        Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): BOOL_SCHEMA,
+        Required(SECTION_REMOTE_URL): str,
+        SECTION_AWS_REGION: str,
+        SECTION_AWS_PROFILE: str,
+        SECTION_AWS_CREDENTIALPATH: str,
+        SECTION_AWS_ENDPOINT_URL: str,
+        Optional(SECTION_AWS_LIST_OBJECTS, default=False): Bool,
+        Optional(SECTION_AWS_USE_SSL, default=True): Bool,
+        SECTION_AWS_SSE: str,
+        SECTION_AWS_ACL: str,
+        SECTION_AWS_GRANT_READ: str,
+        SECTION_AWS_GRANT_READ_ACP: str,
+        SECTION_AWS_GRANT_WRITE_ACP: str,
+        SECTION_AWS_GRANT_FULL_CONTROL: str,
+        SECTION_GCP_PROJECTNAME: str,
+        SECTION_CACHE_TYPE: supported_cache_type,
+        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
+        SECTION_REMOTE_USER: str,
+        SECTION_REMOTE_PORT: Coerce(int),
+        SECTION_REMOTE_KEY_FILE: str,
+        SECTION_REMOTE_TIMEOUT: Coerce(int),
+        SECTION_REMOTE_PASSWORD: str,
+        SECTION_REMOTE_ASK_PASSWORD: Bool,
+        SECTION_REMOTE_GSS_AUTH: Bool,
+        SECTION_AZURE_CONNECTION_STRING: str,
+        SECTION_OSS_ACCESS_KEY_ID: str,
+        SECTION_OSS_ACCESS_KEY_SECRET: str,
+        SECTION_OSS_ENDPOINT: str,
+        SECTION_GDRIVE_CLIENT_ID: str,
+        SECTION_GDRIVE_CLIENT_SECRET: str,
+        SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
+        PRIVATE_CWD: str,
+        Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): Bool,
     }
 
     SECTION_STATE = "state"
     SECTION_STATE_ROW_LIMIT = "row_limit"
     SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
     SECTION_STATE_SCHEMA = {
-        Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
-        Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
+        SECTION_STATE_ROW_LIMIT: All(Coerce(int), Range(1)),
+        SECTION_STATE_ROW_CLEANUP_QUOTA: All(Coerce(int), Range(0, 100)),
     }
 
     SCHEMA = {
         Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
-        Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
+        Match(SECTION_REMOTE_REGEX): SECTION_REMOTE_SCHEMA,
         Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
         Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
-        # backward compatibility
-        Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
-        Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
-        Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
     }
+    COMPILED_SCHEMA = Schema(SCHEMA)
 
     def __init__(self, dvc_dir=None, validate=True):
         self.dvc_dir = dvc_dir
@@ -451,9 +394,9 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
 
         d = self.config.dict()
         try:
-            d = Schema(self.SCHEMA).validate(d)
-        except SchemaError as exc:
-            raise ConfigError("config format error", cause=exc)
+            d = self.COMPILED_SCHEMA(d)
+        except Invalid as exc:
+            raise ConfigError(str(exc), cause=exc)
         self.config = configobj.ConfigObj(d, write_empty_values=True)
 
     def save(self, config=None):

dvc/daemon.py

@@ -1,15 +1,15 @@
 """Launch `dvc daemon` command in a separate detached process."""
-
 from __future__ import unicode_literals
 
-import os
-import sys
 import inspect
 import logging
+import os
+import sys
 from subprocess import Popen
 
 from dvc.env import DVC_DAEMON
-from dvc.utils import is_binary, fix_env
+from dvc.utils import fix_env
+from dvc.utils import is_binary
 from dvc.utils.compat import cast_bytes_py2
 
 
@@ -39,7 +39,7 @@ def _spawn_windows(cmd, env):
 
 def _spawn_posix(cmd, env):
     # NOTE: using os._exit instead of sys.exit, because dvc built
-    # with PyInstaller has trouble with SystemExit exeption and throws
+    # with PyInstaller has trouble with SystemExit exception and throws
     # errors such as "[26338] Failed to execute script __main__"
     try:
         pid = os.fork()

dvc/dagascii.py

@@ -1,14 +1,55 @@
 """Draws DAG in ASCII."""
-
-from __future__ import unicode_literals
 from __future__ import print_function
+from __future__ import unicode_literals
 
-import sys
+import logging
 import math
+import os
+import pydoc
+import sys
 
-from grandalf.graphs import Vertex, Edge, Graph
+from grandalf.graphs import Edge
+from grandalf.graphs import Graph
+from grandalf.graphs import Vertex
 from grandalf.layouts import SugiyamaLayout
-from grandalf.routing import route_with_lines, EdgeViewer
+from grandalf.routing import EdgeViewer
+from grandalf.routing import route_with_lines
+
+from dvc.env import DVC_PAGER
+
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_PAGER = "less"
+DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
+    DEFAULT_PAGER
+)
+
+
+def make_pager(cmd):
+    def pager(text):
+        return pydoc.tempfilepager(pydoc.plain(text), cmd)
+
+    return pager
+
+
+def find_pager():
+    if not sys.stdout.isatty():
+        return pydoc.plainpager
+
+    env_pager = os.getenv(DVC_PAGER)
+    if env_pager:
+        return make_pager(env_pager)
+
+    if os.system("({}) 2>{}".format(DEFAULT_PAGER, os.devnull)) == 0:
+        return make_pager(DEFAULT_PAGER_FORMATTED)
+
+    logger.warning(
+        "Unable to find `less` in the PATH. Check out "
+        "man.dvc.org/doc/command-reference/pipeline/show for more info."
+    )
+    return pydoc.plainpager
 
 
 class VertexViewer(object):
@@ -58,99 +99,10 @@ class AsciiCanvas(object):
 
     def draw(self):
         """Draws ASCII canvas on the screen."""
-        if sys.stdout.isatty():  # pragma: no cover
-            from asciimatics.screen import Screen
-
-            Screen.wrapper(self._do_draw)
-        else:
-            for line in self.canvas:
-                print("".join(line))
-
-    def _do_draw(self, screen):  # pragma: no cover
-        # pylint: disable=too-many-locals
-        # pylint: disable=too-many-branches, too-many-statements
-        from dvc.system import System
-        from asciimatics.event import KeyboardEvent
-
-        offset_x = 0
-        offset_y = 0
-        smaxrow, smaxcol = screen.dimensions
-        assert smaxrow > 1
-        assert smaxcol > 1
-        smaxrow -= 1
-        smaxcol -= 1
-
-        if self.lines + 1 > smaxrow:
-            max_y = self.lines + 1 - smaxrow
-        else:
-            max_y = 0
-
-        if self.cols + 1 > smaxcol:
-            max_x = self.cols + 1 - smaxcol
-        else:
-            max_x = 0
-
-        while True:
-            for y in range(smaxrow + 1):
-                y_index = offset_y + y
-                line = []
-                for x in range(smaxcol + 1):
-                    x_index = offset_x + x
-                    if (
-                        len(self.canvas) > y_index
-                        and len(self.canvas[y_index]) > x_index
-                    ):
-                        line.append(self.canvas[y_index][x_index])
-                    else:
-                        line.append(" ")
-                assert len(line) == (smaxcol + 1)
-                screen.print_at("".join(line), 0, y)
-
-            screen.refresh()
-
-            # NOTE: get_event() doesn't block by itself,
-            # so we have to do the blocking ourselves.
-            #
-            # NOTE: using this workaround while waiting for PR [1]
-            # to get merged and released. After that need to adjust
-            # asciimatics version requirements.
-            #
-            # [1] https://github.com/peterbrittain/asciimatics/pull/188
-            System.wait_for_input(self.TIMEOUT)
-
-            event = screen.get_event()
-            if not isinstance(event, KeyboardEvent):
-                continue
-
-            k = event.key_code
-            if k == screen.KEY_DOWN or k == ord("s"):
-                offset_y += 1
-            elif k == screen.KEY_PAGE_DOWN or k == ord("S"):
-                offset_y += smaxrow
-            elif k == screen.KEY_UP or k == ord("w"):
-                offset_y -= 1
-            elif k == screen.KEY_PAGE_UP or k == ord("W"):
-                offset_y -= smaxrow
-            elif k == screen.KEY_RIGHT or k == ord("d"):
-                offset_x += 1
-            elif k == ord("D"):
-                offset_x += smaxcol
-            elif k == screen.KEY_LEFT or k == ord("a"):
-                offset_x -= 1
-            elif k == ord("A"):
-                offset_x -= smaxcol
-            elif k == ord("q") or k == ord("Q"):
-                break
-
-            if offset_y > max_y:
-                offset_y = max_y
-            elif offset_y < 0:
-                offset_y = 0
-
-            if offset_x > max_x:
-                offset_x = max_x
-            elif offset_x < 0:
-                offset_x = 0
+        pager = find_pager()
+        lines = map("".join, self.canvas)
+        joined_lines = os.linesep.join(lines)
+        pager(joined_lines)
 
     def point(self, x, y, char):
         """Create a point on ASCII canvas.

dvc/data_cloud.py

@@ -1,10 +1,10 @@
-"""Manages dvc remotes that user can use with push/pull/status comamnds."""
-
+"""Manages dvc remotes that user can use with push/pull/status commands."""
 from __future__ import unicode_literals
 
 import logging
 
-from dvc.config import Config, NoRemoteError
+from dvc.config import Config
+from dvc.config import NoRemoteError
 from dvc.remote import Remote
 
 
@@ -45,11 +45,11 @@ class DataCloud(object):
     def _init_remote(self, remote):
         return Remote(self.repo, name=remote)
 
-    def push(self, targets, jobs=None, remote=None, show_checksums=False):
+    def push(self, cache, jobs=None, remote=None, show_checksums=False):
         """Push data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to push to the cloud.
+            cache (NamedCache): named checksums to push to the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to push to.
                 By default remote from core.remote config option is used.
@@ -57,17 +57,17 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.push(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "push"),
             show_checksums=show_checksums,
         )
 
-    def pull(self, targets, jobs=None, remote=None, show_checksums=False):
+    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
         """Pull data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to pull from the cloud.
+            cache (NamedCache): named checksums to pull from the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
                 By default remote from core.remote config option is used.
@@ -75,25 +75,25 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.pull(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "pull"),
             show_checksums=show_checksums,
         )
 
-    def status(self, targets, jobs=None, remote=None, show_checksums=False):
+    def status(self, cache, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to check status for.
+            cache (NamedCache): named checksums to check status for.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to compare
-                targets to. By default remote from core.remote config option
+                cache to. By default remote from core.remote config option
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
         remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            targets, jobs=jobs, remote=remote, show_checksums=show_checksums
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )

dvc/dependency/__init__.py

@@ -1,21 +1,17 @@
 from __future__ import unicode_literals
 
-import schema
-
-from dvc.scheme import Schemes
-
 import dvc.output as output
-from dvc.output.base import OutputBase
-from dvc.dependency.s3 import DependencyS3
 from dvc.dependency.gs import DependencyGS
-from dvc.dependency.local import DependencyLOCAL
 from dvc.dependency.hdfs import DependencyHDFS
-from dvc.dependency.ssh import DependencySSH
 from dvc.dependency.http import DependencyHTTP
 from dvc.dependency.https import DependencyHTTPS
-from .repo import DependencyREPO
-
+from dvc.dependency.local import DependencyLOCAL
+from dvc.dependency.s3 import DependencyS3
+from dvc.dependency.ssh import DependencySSH
+from dvc.output.base import OutputBase
 from dvc.remote import Remote
+from dvc.scheme import Schemes
+from .repo import DependencyREPO
 
 
 DEPS = [
@@ -44,9 +40,9 @@ DEP_MAP = {
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
 # output is a metric file and how to parse it, see `-M` flag for `dvc run`).
 SCHEMA = output.SCHEMA.copy()
-del SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)]
-del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)]
-SCHEMA[schema.Optional(DependencyREPO.PARAM_REPO)] = DependencyREPO.REPO_SCHEMA
+del SCHEMA[OutputBase.PARAM_CACHE]
+del SCHEMA[OutputBase.PARAM_METRIC]
+SCHEMA[DependencyREPO.PARAM_REPO] = DependencyREPO.REPO_SCHEMA
 
 
 def _get(stage, p, info):

dvc/dependency/gs.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.gs import OutputGS
 from dvc.dependency.base import DependencyBase
+from dvc.output.gs import OutputGS
 
 
 class DependencyGS(DependencyBase, OutputGS):

dvc/dependency/hdfs.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.hdfs import OutputHDFS
 from dvc.dependency.base import DependencyBase
+from dvc.output.hdfs import OutputHDFS
 
 
 class DependencyHDFS(DependencyBase, OutputHDFS):

dvc/dependency/http.py

@@ -1,8 +1,8 @@
 from __future__ import unicode_literals
 
+from dvc.dependency.base import DependencyBase
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP
-from dvc.dependency.base import DependencyBase
 
 
 class DependencyHTTP(DependencyBase, OutputBase):

dvc/dependency/https.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.remote.https import RemoteHTTPS
 from .http import DependencyHTTP
+from dvc.remote.https import RemoteHTTPS
 
 
 class DependencyHTTPS(DependencyHTTP):

dvc/dependency/repo.py

@@ -1,15 +1,13 @@
 from __future__ import unicode_literals
 
 import copy
-
-from funcy import merge
-from schema import Optional
 from contextlib import contextmanager
 
-from dvc.external_repo import external_repo
-from dvc.utils.compat import str
+from funcy import merge
 
 from .local import DependencyLOCAL
+from dvc.external_repo import external_repo
+from dvc.utils.compat import str
 
 
 class DependencyREPO(DependencyLOCAL):
@@ -18,11 +16,7 @@ class DependencyREPO(DependencyLOCAL):
     PARAM_REV = "rev"
     PARAM_REV_LOCK = "rev_lock"
 
-    REPO_SCHEMA = {
-        Optional(PARAM_URL): str,
-        Optional(PARAM_REV): str,
-        Optional(PARAM_REV_LOCK): str,
-    }
+    REPO_SCHEMA = {PARAM_URL: str, PARAM_REV: str, PARAM_REV_LOCK: str}
 
     def __init__(self, def_repo, stage, *args, **kwargs):
         self.def_repo = def_repo
@@ -35,6 +29,11 @@ class DependencyREPO(DependencyLOCAL):
     def is_in_repo(self):
         return False
 
+    @property
+    def repo_pair(self):
+        d = self.def_repo
+        return d[self.PARAM_URL], d[self.PARAM_REV_LOCK] or d[self.PARAM_REV]
+
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 

dvc/dependency/s3.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.s3 import OutputS3
 from dvc.dependency.base import DependencyBase
+from dvc.output.s3 import OutputS3
 
 
 class DependencyS3(DependencyBase, OutputS3):

dvc/dependency/ssh.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.ssh import OutputSSH
 from dvc.dependency.base import DependencyBase
+from dvc.output.ssh import OutputSSH
 
 
 class DependencySSH(DependencyBase, OutputSSH):

dvc/env.py

@@ -1 +1,2 @@
 DVC_DAEMON = "DVC_DAEMON"
+DVC_PAGER = "DVC_PAGER"

dvc/exceptions.py

@@ -1,12 +1,11 @@
 """Exceptions raised by the dvc."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, builtin_str
-
 import traceback
 
 from dvc.utils import relpath
+from dvc.utils.compat import builtin_str
+from dvc.utils.compat import str
 
 
 class DvcException(Exception):
@@ -47,8 +46,9 @@ class OutputDuplicationError(DvcException):
             for stage in stages
         )
         msg = (
-            "file/directory '{}' is specified as an output in more than one"
-            "stage: {}"
+            "file/directory '{}' is specified as an output in more than one "
+            "stage: {}\n"
+            "This is not allowed. Consider using a different output name."
         ).format(output, "\n    ".join(stages))
         super(OutputDuplicationError, self).__init__(msg)
 
@@ -60,10 +60,12 @@ class OutputNotFoundError(DvcException):
         output (unicode): path to the file/directory.
     """
 
-    def __init__(self, output):
+    def __init__(self, output, repo=None):
+        self.output = output
+        self.repo = repo
         super(OutputNotFoundError, self).__init__(
             "unable to find DVC-file with output '{path}'".format(
-                path=relpath(output)
+                path=relpath(self.output)
             )
         )
 
@@ -82,8 +84,8 @@ class StagePathAsOutputError(DvcException):
         assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
         assert isinstance(fname, str) or isinstance(fname, builtin_str)
         msg = (
-            "current working directory '{cwd}' is specified as an output in"
-            " '{fname}'. Use another CWD to prevent any data removal.".format(
+            "current working directory '{cwd}' is specified as an output in "
+            "'{fname}'. Use another CWD to prevent any data removal.".format(
                 cwd=wdir, fname=fname
             )
         )
@@ -167,8 +169,8 @@ class CyclicGraphError(DvcException):
         assert isinstance(stages, list)
         stages = "\n".join("\t- {}".format(stage) for stage in stages)
         msg = (
-            "you've introduced a cycle in your pipeline that involves"
-            " the following stages:"
+            "you've introduced a cycle in your pipeline that involves "
+            "the following stages:"
             "\n"
             "{stages}".format(stages=stages)
         )
@@ -179,7 +181,7 @@ class ConfirmRemoveError(DvcException):
     def __init__(self, path):
         super(ConfirmRemoveError, self).__init__(
             "unable to remove '{}' without a confirmation from the user. Use "
-            "'-f' to force.".format(path)
+            "`-f` to force.".format(path)
         )
 
 
@@ -209,7 +211,7 @@ class NoMetricsError(DvcException):
     def __init__(self):
         super(NoMetricsError, self).__init__(
             "no metric files in this repository. "
-            "Use 'dvc metrics add' to add a metric file to track."
+            "Use `dvc metrics add` to add a metric file to track."
         )
 
 
@@ -226,7 +228,7 @@ class StageFileCorruptedError(DvcException):
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
         super(RecursiveAddingWhileUsingFilename, self).__init__(
-            "using fname with recursive is not allowed."
+            "cannot use `fname` with multiple targets or `-R|--recursive`"
         )
 
 
@@ -244,17 +246,10 @@ class OverlappingOutputPathsError(DvcException):
         )
 
 
-class TargetNotDirectoryError(DvcException):
-    def __init__(self, path):
-        super(TargetNotDirectoryError, self).__init__(
-            "Target: {} is not a directory".format(path)
-        )
-
-
 class CheckoutErrorSuggestGit(DvcException):
     def __init__(self, target, cause):
         super(CheckoutErrorSuggestGit, self).__init__(
-            "Did you mean 'git checkout {}'?".format(target), cause=cause
+            "Did you mean `git checkout {}`?".format(target), cause=cause
         )
 
 
@@ -266,10 +261,11 @@ class ETagMismatchError(DvcException):
         )
 
 
-class OutputFileMissingError(DvcException):
-    def __init__(self, path):
-        super(OutputFileMissingError, self).__init__(
-            "Can't find {} neither locally nor on remote".format(path)
+class FileMissingError(DvcException):
+    def __init__(self, path, cause=None):
+        super(FileMissingError, self).__init__(
+            "Can't find '{}' neither locally nor on remote".format(path),
+            cause=cause,
         )
 
 
@@ -288,14 +284,6 @@ class UrlNotDvcRepoError(DvcException):
         )
 
 
-class GetDVCFileError(DvcException):
-    def __init__(self):
-        super(GetDVCFileError, self).__init__(
-            "the given path is a DVC-file, you must specify a data file "
-            "or a directory"
-        )
-
-
 class GitHookAlreadyExistsError(DvcException):
     def __init__(self, hook_name):
         super(GitHookAlreadyExistsError, self).__init__(
@@ -331,3 +319,28 @@ class CheckoutError(DvcException):
             "forget to fetch?".format("\n".join(targets))
         )
         super(CheckoutError, self).__init__(m)
+
+
+class CollectCacheError(DvcException):
+    pass
+
+
+class NoRemoteInExternalRepoError(DvcException):
+    def __init__(self, url):
+        super(NoRemoteInExternalRepoError, self).__init__(
+            "No DVC remote is specified in target repository '{}'.".format(url)
+        )
+
+
+class NoOutputInExternalRepoError(DvcException):
+    def __init__(self, path, external_repo_path, external_repo_url):
+        super(NoOutputInExternalRepoError, self).__init__(
+            "Output '{}' not found in target repository '{}'".format(
+                relpath(path, external_repo_path), external_repo_url
+            )
+        )
+
+
+class HTTPError(DvcException):
+    def __init__(self, code, reason):
+        super(HTTPError, self).__init__("'{} {}'".format(code, reason))

dvc/external_repo.py

@@ -1,50 +1,148 @@
 from __future__ import unicode_literals
 
 import os
-import logging
 import tempfile
+from contextlib import contextmanager
+from distutils.dir_util import copy_tree
 
 from funcy import retry
-from contextlib import contextmanager
 
-from dvc.config import Config
-from dvc.cache import CacheConfig
-from dvc.utils import remove
+from dvc.config import NoRemoteError, ConfigError
+from dvc.exceptions import NoRemoteInExternalRepoError
+from dvc.remote import RemoteConfig
+from dvc.exceptions import NoOutputInExternalRepoError
+from dvc.exceptions import OutputNotFoundError
+from dvc.utils.fs import remove
 
 
-logger = logging.getLogger(__name__)
-
-
-def _clone(cache_dir=None, url=None, rev=None, rev_lock=None):
-    from dvc.repo import Repo
-
-    _path = tempfile.mkdtemp("dvc-repo")
-
-    repo = Repo.clone(url, _path, rev=(rev_lock or rev))
-
-    try:
-        if cache_dir:
-            cache_config = CacheConfig(repo.config)
-            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
-    finally:
-        repo.close()
-
-    return Repo(_path)
-
-
-def _remove(repo):
-    repo.close()
-
-    if os.name == "nt":
-        # git.exe may hang for a while not permitting to remove temp dir
-        os_retry = retry(5, errors=OSError, timeout=0.1)
-        os_retry(remove)(repo.root_dir)
-    else:
-        remove(repo.root_dir)
+REPO_CACHE = {}
 
 
 @contextmanager
-def external_repo(**kwargs):
-    repo = _clone(**kwargs)
-    yield repo
-    _remove(repo)
+def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
+    from dvc.repo import Repo
+
+    path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
+    repo = Repo(path)
+    try:
+        yield repo
+    except NoRemoteError:
+        raise NoRemoteInExternalRepoError(url)
+    except OutputNotFoundError as exc:
+        if exc.repo is repo:
+            raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url)
+        raise
+    repo.close()
+
+
+def _external_repo(url=None, rev=None, cache_dir=None):
+    from dvc.config import Config
+    from dvc.cache import CacheConfig
+    from dvc.repo import Repo
+
+    key = (url, rev, cache_dir)
+    if key in REPO_CACHE:
+        return REPO_CACHE[key]
+
+    new_path = tempfile.mkdtemp("dvc-erepo")
+
+    # Copy and adjust existing clone
+    if (url, None, None) in REPO_CACHE:
+        old_path = REPO_CACHE[url, None, None]
+
+        # This one unlike shutil.copytree() works with an existing dir
+        copy_tree(old_path, new_path)
+    else:
+        # Create a new clone
+        _clone_repo(url, new_path)
+
+        # Save clean clone dir so that we will have access to a default branch
+        clean_clone_path = tempfile.mkdtemp("dvc-erepo")
+        copy_tree(new_path, clean_clone_path)
+        REPO_CACHE[url, None, None] = clean_clone_path
+
+    # Adjust new clone/copy to fit rev and cache_dir
+
+    # Checkout needs to be done first because current branch might not be
+    # DVC repository
+    if rev is not None:
+        _git_checkout(new_path, rev)
+
+    repo = Repo(new_path)
+    try:
+        # check if the URL is local and no default remote is present
+        # add default remote pointing to the original repo's cache location
+        if os.path.isdir(url):
+            rconfig = RemoteConfig(repo.config)
+            if not _default_remote_set(rconfig):
+                original_repo = Repo(url)
+                try:
+                    rconfig.add(
+                        "auto-generated-upstream",
+                        original_repo.cache.local.cache_dir,
+                        default=True,
+                        level=Config.LEVEL_LOCAL,
+                    )
+                finally:
+                    original_repo.close()
+
+        if cache_dir is not None:
+            cache_config = CacheConfig(repo.config)
+            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
+    finally:
+        # Need to close/reopen repo to force config reread
+        repo.close()
+
+    REPO_CACHE[key] = new_path
+    return new_path
+
+
+def _git_checkout(repo_path, revision):
+    from dvc.scm import Git
+
+    git = Git(repo_path)
+    try:
+        git.checkout(revision)
+    finally:
+        git.close()
+
+
+def clean_repos():
+    # Outside code should not see cache while we are removing
+    repo_paths = list(REPO_CACHE.values())
+    REPO_CACHE.clear()
+
+    for path in repo_paths:
+        _remove(path)
+
+
+def _remove(path):
+    if os.name == "nt":
+        # git.exe may hang for a while not permitting to remove temp dir
+        os_retry = retry(5, errors=OSError, timeout=0.1)
+        os_retry(remove)(path)
+    else:
+        remove(path)
+
+
+def _clone_repo(url, path):
+    from dvc.scm.git import Git
+
+    git = Git.clone(url, path)
+    git.close()
+
+
+def _default_remote_set(rconfig):
+    """
+    Checks if default remote config is present.
+    Args:
+        rconfig: a remote config
+
+    Returns:
+        True if the default remote config is set, else False
+    """
+    try:
+        rconfig.get_default()
+        return True
+    except ConfigError:
+        return False

dvc/ignore.py

@@ -2,13 +2,12 @@ from __future__ import unicode_literals
 
 import logging
 import os
+
 from pathspec import PathSpec
 from pathspec.patterns import GitWildMatchPattern
 
 from dvc.utils import relpath
 
-from dvc.utils.compat import open
-
 logger = logging.getLogger(__name__)
 
 
@@ -20,13 +19,13 @@ class DvcIgnore(object):
 
 
 class DvcIgnorePatterns(DvcIgnore):
-    def __init__(self, ignore_file_path):
+    def __init__(self, ignore_file_path, tree):
         assert os.path.isabs(ignore_file_path)
 
         self.ignore_file_path = ignore_file_path
         self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
 
-        with open(ignore_file_path, encoding="utf-8") as fobj:
+        with tree.open(ignore_file_path, encoding="utf-8") as fobj:
             self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
 
     def __call__(self, root, dirs, files):
@@ -47,6 +46,9 @@ class DvcIgnorePatterns(DvcIgnore):
         return hash(self.ignore_file_path)
 
     def __eq__(self, other):
+        if not isinstance(other, DvcIgnorePatterns):
+            return NotImplemented
+
         return self.ignore_file_path == other.ignore_file_path
 
 
@@ -59,19 +61,29 @@ class DvcIgnoreDirs(DvcIgnore):
 
         return dirs, files
 
+    def __hash__(self):
+        return hash(tuple(self.basenames))
+
+    def __eq__(self, other):
+        if not isinstance(other, DvcIgnoreDirs):
+            return NotImplemented
+
+        return self.basenames == other.basenames
+
 
 class DvcIgnoreFilter(object):
-    def __init__(self, root_dir):
+    def __init__(self, root_dir, tree):
+        self.tree = tree
         self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
         self._update(root_dir)
-        for root, dirs, _ in os.walk(root_dir):
+        for root, dirs, _ in self.tree.walk(root_dir, dvcignore=self):
             for d in dirs:
                 self._update(os.path.join(root, d))
 
     def _update(self, dirname):
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if os.path.exists(ignore_file_path):
-            self.ignores.add(DvcIgnorePatterns(ignore_file_path))
+        if self.tree.exists(ignore_file_path):
+            self.ignores.add(DvcIgnorePatterns(ignore_file_path, self.tree))
 
     def __call__(self, root, dirs, files):
         for ignore in self.ignores:

dvc/istextfile.py

@@ -1,8 +1,8 @@
 """Use heuristics to guess if it is a text file or a binary file."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import is_py3, open
+from dvc.utils.compat import is_py3
+from dvc.utils.compat import open
 
 # Based on https://eli.thegreenplace.net/2011/10/19/
 # perls-guess-if-file-is-text-or-binary-implemented-in-python

dvc/lock.py

@@ -1,30 +1,85 @@
 """Manages dvc lock file."""
-
 from __future__ import unicode_literals
 
-import os
 import hashlib
+import os
 import time
 from datetime import timedelta
 
-from funcy.py3 import lkeep
+import zc.lockfile
 
 from dvc.exceptions import DvcException
-from dvc.utils import makedirs
-from dvc.utils.compat import is_py3
-
+from dvc.utils import format_link
+from dvc.utils.compat import is_py3, is_py2
+from dvc.progress import Tqdm
 
 DEFAULT_TIMEOUT = 5
 
+FAILED_TO_LOCK_MESSAGE = (
+    "cannot perform the command because another DVC process seems to be "
+    "running on this project. If that is not the case, manually remove "
+    "`.dvc/lock` and try again."
+)
+
 
 class LockError(DvcException):
     """Thrown when unable to acquire the lock for dvc repo."""
 
 
+class Lock(object):
+    """Class for dvc repo lock.
+
+    Uses zc.lockfile as backend.
+    """
+
+    def __init__(self, lockfile, friendly=False, **kwargs):
+        self._friendly = friendly
+        self.lockfile = lockfile
+        self._lock = None
+
+    @property
+    def files(self):
+        return [self.lockfile]
+
+    def _do_lock(self):
+        try:
+            with Tqdm(
+                bar_format="{desc}",
+                disable=not self._friendly,
+                desc=(
+                    "If DVC froze, see `hardlink_lock` in {}".format(
+                        format_link("man.dvc.org/config#core")
+                    )
+                ),
+            ):
+                self._lock = zc.lockfile.LockFile(self.lockfile)
+        except zc.lockfile.LockError:
+            raise LockError(FAILED_TO_LOCK_MESSAGE)
+
+    def lock(self):
+        try:
+            self._do_lock()
+            return
+        except LockError:
+            time.sleep(DEFAULT_TIMEOUT)
+
+        self._do_lock()
+
+    def unlock(self):
+        self._lock.close()
+        self._lock = None
+
+    def __enter__(self):
+        self.lock()
+
+    def __exit__(self, typ, value, tbck):
+        self.unlock()
+
+
 if is_py3:
     import flufl.lock
 
-    class Lock(flufl.lock.Lock):
+    class HardlinkLock(flufl.lock.Lock):
         """Class for dvc repo lock.
 
         Args:
@@ -33,30 +88,40 @@ if is_py3:
             tmp_dir (str): a directory to store claim files.
         """
 
-        def __init__(self, lockfile, tmp_dir=None):
-            lifetime = timedelta(days=365)  # Lock for good by default
-            self._tmp_dir = tmp_dir
-            if self._tmp_dir is not None:
-                makedirs(self._tmp_dir, exist_ok=True)
+        def __init__(self, lockfile, tmp_dir=None, **kwargs):
+            import socket
 
-            super(Lock, self).__init__(lockfile, lifetime=lifetime)
+            self._tmp_dir = tmp_dir
+
+            # NOTE: this is basically Lock.__init__ copy-paste, except that
+            # instead of using `socket.getfqdn()` we use `socket.gethostname()`
+            # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
+            # anything, which is way too slow. `gethostname()` is actually a
+            # fallback for `getfqdn()` when it is not able to resolve a
+            # canonical hostname through network. The claimfile that uses
+            # `self._hostname` is still usable, as it uses `pid` and random
+            # number to generate the resulting lock file name, which is unique
+            # enough for our application.
+            #
+            # [1] https://github.com/iterative/dvc/issues/2582
+            self._hostname = socket.gethostname()
+
+            self._lockfile = lockfile
+            self._lifetime = timedelta(days=365)  # Lock for good by default
+            self._separator = flufl.lock.SEP
+            self._set_claimfile()
+            self._owned = True
+            self._retry_errnos = []
 
         @property
         def lockfile(self):
             return self._lockfile
 
-        @property
-        def files(self):
-            return lkeep([self._lockfile, self._tmp_dir])
-
         def lock(self):
             try:
                 super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
             except flufl.lock.TimeOutError:
-                raise LockError(
-                    "cannot perform the cmd since DVC is busy and "
-                    "locked. Please retry the cmd later."
-                )
+                raise LockError(FAILED_TO_LOCK_MESSAGE)
 
         def _set_claimfile(self, pid=None):
             super(Lock, self)._set_claimfile(pid)
@@ -68,48 +133,26 @@ if is_py3:
                     self._tmp_dir, filename + ".lock"
                 )
 
-
-else:
-    import zc.lockfile
-
-    class Lock(object):
-        """Class for dvc repo lock.
-
-        Uses zc.lockfile as backend.
-        """
-
-        def __init__(self, lockfile, tmp_dir=None):
-            self.lockfile = lockfile
-            self._lock = None
-
-        @property
-        def files(self):
-            return [self.lockfile]
-
-        def _do_lock(self):
+        # Fix for __del__ bug in flufl.lock [1] which is causing errors on
+        # Python shutdown [2].
+        # [1] https://gitlab.com/warsaw/flufl.lock/issues/7
+        # [2] https://github.com/iterative/dvc/issues/2573
+        def __del__(self):
             try:
-                self._lock = zc.lockfile.LockFile(self.lockfile)
-            except zc.lockfile.LockError:
-                raise LockError(
-                    "cannot perform the cmd since DVC is busy and "
-                    "locked. Please retry the cmd later."
-                )
+                if self._owned:
+                    self.finalize()
+            except ImportError:
+                pass
 
-        def lock(self):
-            try:
-                self._do_lock()
-                return
-            except LockError:
-                time.sleep(DEFAULT_TIMEOUT)
 
-            self._do_lock()
+def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False):
+    if hardlink_lock and is_py2:
+        raise DvcException(
+            "Hardlink locks are not supported on Python <3.5. "
+            "See `hardlink_lock` in {}".format(
+                format_link("man.dvc.org/config#core")
+            )
+        )
 
-        def unlock(self):
-            self._lock.close()
-            self._lock = None
-
-        def __enter__(self):
-            self.lock()
-
-        def __exit__(self, typ, value, tbck):
-            self.unlock()
+    cls = HardlinkLock if hardlink_lock else Lock
+    return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly)

dvc/logger.py

@@ -1,15 +1,27 @@
 """Manages logging configuration for dvc repo."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, StringIO
-from dvc.progress import Tqdm
-
-import logging
-import logging.handlers
 import logging.config
+import logging.handlers
+
 import colorama
 
+from dvc.progress import Tqdm
+from dvc.utils.compat import RecursionError
+from dvc.utils.compat import str
+from dvc.utils.compat import StringIO
+
+
+FOOTER = (
+    "\n{yellow}Having any troubles?{nc}"
+    " Hit us up at {blue}https://dvc.org/support{nc},"
+    " we are always happy to help!"
+).format(
+    blue=colorama.Fore.BLUE,
+    nc=colorama.Fore.RESET,
+    yellow=colorama.Fore.YELLOW,
+)
+
 
 class LoggingException(Exception):
     def __init__(self, record):
@@ -19,7 +31,12 @@ class LoggingException(Exception):
 
 class ExcludeErrorsFilter(logging.Filter):
     def filter(self, record):
-        return record.levelno < logging.ERROR
+        return record.levelno < logging.WARNING
+
+
+class ExcludeInfoFilter(logging.Filter):
+    def filter(self, record):
+        return record.levelno < logging.INFO
 
 
 class ColorFormatter(logging.Formatter):
@@ -32,7 +49,7 @@ class ColorFormatter(logging.Formatter):
     __ https://pypi.python.org/pypi/colorama
 
     For records containing `exc_info`, it will use a custom `_walk_exc` to
-    retrieve the whole tracebak.
+    retrieve the whole traceback.
     """
 
     color_code = {
@@ -43,16 +60,6 @@ class ColorFormatter(logging.Formatter):
         "CRITICAL": colorama.Fore.RED,
     }
 
-    footer = (
-        "{yellow}Having any troubles?{nc}"
-        " Hit us up at {blue}https://dvc.org/support{nc},"
-        " we are always happy to help!"
-    ).format(
-        blue=colorama.Fore.BLUE,
-        nc=colorama.Fore.RESET,
-        yellow=colorama.Fore.YELLOW,
-    )
-
     def format(self, record):
         if record.levelname == "INFO":
             return record.msg
@@ -61,10 +68,7 @@ class ColorFormatter(logging.Formatter):
             exception, stack_trace = self._parse_exc(record.exc_info)
 
             return (
-                "{color}{levelname}{nc}: {description}"
-                "{stack_trace}\n"
-                "\n"
-                "{footer}"
+                "{color}{levelname}{nc}: {description}" "{stack_trace}\n"
             ).format(
                 color=self.color_code.get(record.levelname, ""),
                 nc=colorama.Fore.RESET,
@@ -72,7 +76,6 @@ class ColorFormatter(logging.Formatter):
                 description=self._description(record.msg, exception),
                 msg=record.msg,
                 stack_trace=stack_trace,
-                footer=self.footer,
             )
 
         return "{color}{levelname}{nc}: {msg}".format(
@@ -151,7 +154,7 @@ class LoggerHandler(logging.StreamHandler):
         raise LoggingException(record)
 
     def emit(self, record):
-        """Write to Tqdm's stream so as to not break progressbars"""
+        """Write to Tqdm's stream so as to not break progress-bars"""
         try:
             msg = self.format(record)
             Tqdm.write(
@@ -170,19 +173,29 @@ def setup(level=logging.INFO):
     logging.config.dictConfig(
         {
             "version": 1,
-            "filters": {"exclude_errors": {"()": ExcludeErrorsFilter}},
+            "filters": {
+                "exclude_errors": {"()": ExcludeErrorsFilter},
+                "exclude_info": {"()": ExcludeInfoFilter},
+            },
             "formatters": {"color": {"()": ColorFormatter}},
             "handlers": {
-                "console": {
+                "console_info": {
                     "class": "dvc.logger.LoggerHandler",
-                    "level": "DEBUG",
+                    "level": "INFO",
                     "formatter": "color",
                     "stream": "ext://sys.stdout",
                     "filters": ["exclude_errors"],
                 },
+                "console_debug": {
+                    "class": "dvc.logger.LoggerHandler",
+                    "level": "DEBUG",
+                    "formatter": "color",
+                    "stream": "ext://sys.stdout",
+                    "filters": ["exclude_info"],
+                },
                 "console_errors": {
                     "class": "dvc.logger.LoggerHandler",
-                    "level": "ERROR",
+                    "level": "WARNING",
                     "formatter": "color",
                     "stream": "ext://sys.stderr",
                 },
@@ -190,15 +203,27 @@ def setup(level=logging.INFO):
             "loggers": {
                 "dvc": {
                     "level": level,
-                    "handlers": ["console", "console_errors"],
+                    "handlers": [
+                        "console_info",
+                        "console_debug",
+                        "console_errors",
+                    ],
                 },
                 "paramiko": {
                     "level": logging.CRITICAL,
-                    "handlers": ["console", "console_errors"],
+                    "handlers": [
+                        "console_info",
+                        "console_debug",
+                        "console_errors",
+                    ],
                 },
                 "flufl.lock": {
                     "level": logging.CRITICAL,
-                    "handlers": ["console", "console_errors"],
+                    "handlers": [
+                        "console_info",
+                        "console_debug",
+                        "console_errors",
+                    ],
                 },
             },
         }

dvc/main.py

@@ -1,15 +1,23 @@
 """Main entry point for dvc CLI."""
-
 from __future__ import unicode_literals
 
 import logging
 
+from dvc import analytics
 from dvc.cli import parse_args
-from dvc.lock import LockError
 from dvc.config import ConfigError
-from dvc.analytics import Analytics
-from dvc.exceptions import NotDvcRepoError, DvcParserError
+from dvc.exceptions import DvcParserError
+from dvc.exceptions import NotDvcRepoError
+from dvc.external_repo import clean_repos
+from dvc.logger import FOOTER
 from dvc.remote.pool import close_pools
+from dvc.utils.compat import is_py2
+
+
+# Workaround for CPython bug. See [1] and [2] for more info.
+# [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
+# [2] https://bugs.python.org/issue29288
+"".encode("idna")
 
 
 logger = logging.getLogger("dvc")
@@ -39,9 +47,6 @@ def main(argv=None):
 
         cmd = args.func(args)
         ret = cmd.run()
-    except LockError:
-        logger.exception("failed to lock before running a command")
-        ret = 250
     except ConfigError:
         logger.exception("configuration error")
         ret = 251
@@ -53,15 +58,30 @@ def main(argv=None):
         ret = 253
     except DvcParserError:
         ret = 254
-    except Exception:  # pylint: disable=broad-except
-        logger.exception("unexpected error")
+    except Exception as exc:  # pylint: disable=broad-except
+        if isinstance(exc, UnicodeError) and is_py2:
+            logger.exception(
+                "unicode is not supported in DVC for Python 2 "
+                "(end-of-life January 1, 2020), please upgrade to Python 3"
+            )
+        else:
+            logger.exception("unexpected error")
         ret = 255
     finally:
         logger.setLevel(outerLogLevel)
+
         # Python 2 fails to close these clean occasionally and users see
         # weird error messages, so we do it manually
         close_pools()
 
-    Analytics().send_cmd(cmd, args, ret)
+        # Remove cached repos in the end of the call, these are anonymous
+        # so won't be reused by any other subsequent run anyway.
+        clean_repos()
+
+    if ret != 0:
+        logger.info(FOOTER)
+
+    if analytics.is_enabled():
+        analytics.collect_and_send_report(args, ret)
 
     return ret

dvc/output/__init__.py

@@ -1,21 +1,20 @@
 from __future__ import unicode_literals
 
-import schema
-
-from dvc.scheme import Schemes
-from dvc.utils.compat import urlparse, str
+from voluptuous import Any, Required
 
 from dvc.output.base import OutputBase
-from dvc.output.s3 import OutputS3
 from dvc.output.gs import OutputGS
-from dvc.output.local import OutputLOCAL
 from dvc.output.hdfs import OutputHDFS
+from dvc.output.local import OutputLOCAL
+from dvc.output.s3 import OutputS3
 from dvc.output.ssh import OutputSSH
-
 from dvc.remote import Remote
-from dvc.remote.s3 import RemoteS3
 from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.local import RemoteLOCAL
+from dvc.remote.s3 import RemoteS3
+from dvc.scheme import Schemes
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
 
 OUTS = [
     OutputHDFS,
@@ -42,19 +41,19 @@ OUTS_MAP = {
 # so when a few types of outputs share the same name, we only need
 # specify it once.
 CHECKSUM_SCHEMA = {
-    schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None),
-    schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None),
-    schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
+    RemoteLOCAL.PARAM_CHECKSUM: Any(str, None),
+    RemoteS3.PARAM_CHECKSUM: Any(str, None),
+    RemoteHDFS.PARAM_CHECKSUM: Any(str, None),
 }
 
-TAGS_SCHEMA = {schema.Optional(str): CHECKSUM_SCHEMA}
+TAGS_SCHEMA = {str: CHECKSUM_SCHEMA}
 
 SCHEMA = CHECKSUM_SCHEMA.copy()
-SCHEMA[OutputBase.PARAM_PATH] = str
-SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)] = bool
-SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)] = OutputBase.METRIC_SCHEMA
-SCHEMA[schema.Optional(OutputBase.PARAM_TAGS)] = TAGS_SCHEMA
-SCHEMA[schema.Optional(OutputBase.PARAM_PERSIST)] = bool
+SCHEMA[Required(OutputBase.PARAM_PATH)] = str
+SCHEMA[OutputBase.PARAM_CACHE] = bool
+SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA
+SCHEMA[OutputBase.PARAM_TAGS] = TAGS_SCHEMA
+SCHEMA[OutputBase.PARAM_PERSIST] = bool
 
 
 def _get(stage, p, info, cache, metric, persist=False, tags=None):

dvc/output/base.py

@@ -3,12 +3,15 @@ from __future__ import unicode_literals
 import logging
 from copy import copy
 
-from schema import Or, Optional
+from voluptuous import Any
 
 import dvc.prompt as prompt
+from dvc.cache import NamedCache
+from dvc.exceptions import CollectCacheError
 from dvc.exceptions import DvcException
-from dvc.utils.compat import str, urlparse
 from dvc.remote.base import RemoteBASE
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -51,12 +54,12 @@ class OutputBase(object):
     PARAM_METRIC_XPATH = "xpath"
     PARAM_PERSIST = "persist"
 
-    METRIC_SCHEMA = Or(
+    METRIC_SCHEMA = Any(
         None,
         bool,
         {
-            Optional(PARAM_METRIC_TYPE): Or(str, None),
-            Optional(PARAM_METRIC_XPATH): Or(str, None),
+            PARAM_METRIC_TYPE: Any(str, None),
+            PARAM_METRIC_XPATH: Any(str, None),
         },
     )
 
@@ -167,12 +170,7 @@ class OutputBase(object):
         return self.remote.exists(self.path_info)
 
     def changed_checksum(self):
-        return (
-            self.checksum
-            != self.remote.save_info(self.path_info)[
-                self.remote.PARAM_CHECKSUM
-            ]
-        )
+        return self.checksum != self.remote.get_checksum(self.path_info)
 
     def changed_cache(self):
         if not self.use_cache or not self.checksum:
@@ -284,9 +282,12 @@ class OutputBase(object):
     def download(self, to):
         self.remote.download(self.path_info, to.path_info)
 
-    def checkout(self, force=False, progress_callback=None, tag=None):
+    def checkout(
+        self, force=False, progress_callback=None, tag=None, relink=False
+    ):
         if not self.use_cache:
-            progress_callback(str(self.path_info), self.get_files_number())
+            if progress_callback:
+                progress_callback(str(self.path_info), self.get_files_number())
             return None
 
         if tag:
@@ -299,6 +300,7 @@ class OutputBase(object):
             info,
             force=force,
             progress_callback=progress_callback,
+            relink=relink,
         )
 
     def remove(self, ignore_remove=False):
@@ -333,7 +335,7 @@ class OutputBase(object):
             self.remote.unprotect(self.path_info)
 
     def _collect_used_dir_cache(self, remote=None, force=False, jobs=None):
-        """Get a list of `info`s retaled to the given directory.
+        """Get a list of `info`s related to the given directory.
 
         - Pull the directory entry from the remote cache if it was changed.
 
@@ -345,25 +347,19 @@ class OutputBase(object):
             $ echo "bar" > directory/bar
             $ dvc add directory
 
-            It will return something similar to the following list:
+            It will return a NamedCache like:
 
-            [
-                { 'path': 'directory/foo', 'md5': 'c157a79031e1', ... },
-                { 'path': 'directory/bar', 'md5': 'd3b07384d113', ... },
-            ]
+            nc = NamedCache()
+            nc.add(self.scheme, 'c157a79031e1', 'directory/foo')
+            nc.add(self.scheme, 'd3b07384d113', 'directory/bar')
         """
 
-        ret = []
+        cache = NamedCache()
 
         if self.cache.changed_cache_file(self.checksum):
             try:
                 self.repo.cloud.pull(
-                    [
-                        {
-                            self.remote.PARAM_CHECKSUM: self.checksum,
-                            "name": str(self),
-                        }
-                    ],
+                    NamedCache.make("local", self.checksum, str(self)),
                     jobs=jobs,
                     remote=remote,
                     show_checksums=False,
@@ -378,20 +374,19 @@ class OutputBase(object):
                 "Would you like to continue? Use '-f' to force."
             )
             if not force and not prompt.confirm(msg.format(self.path_info)):
-                raise DvcException(
+                raise CollectCacheError(
                     "unable to fully collect used cache"
                     " without cache for directory '{}'".format(self)
                 )
             else:
-                return ret
+                return cache
 
         for entry in self.dir_cache:
-            info = copy(entry)
+            checksum = entry[self.remote.PARAM_CHECKSUM]
             path_info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            info["name"] = str(path_info)
-            ret.append(info)
+            cache.add(self.scheme, checksum, str(path_info))
 
-        return ret
+        return cache
 
     def get_used_cache(self, **kwargs):
         """Get a dumpd of the given `out`, with an entry including the branch.
@@ -403,7 +398,7 @@ class OutputBase(object):
         """
 
         if not self.use_cache:
-            return []
+            return NamedCache()
 
         if not self.info:
             logger.warning(
@@ -411,14 +406,14 @@ class OutputBase(object):
                 "not be collected. Use dvc repro to get your pipeline up to "
                 "date.".format(self, self.stage)
             )
-            return []
+            return NamedCache()
 
-        ret = [{self.remote.PARAM_CHECKSUM: self.checksum, "name": str(self)}]
+        ret = NamedCache.make(self.scheme, self.checksum, str(self))
 
         if not self.is_dir_checksum:
             return ret
 
-        ret.extend(self._collect_used_dir_cache(**kwargs))
+        ret.update(self._collect_used_dir_cache(**kwargs))
 
         return ret
 

dvc/output/gs.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.remote.gs import RemoteGS
 from dvc.output.s3 import OutputS3
+from dvc.remote.gs import RemoteGS
 
 
 class OutputGS(OutputS3):

dvc/output/local.py

@@ -1,13 +1,17 @@
 from __future__ import unicode_literals
 
-import os
 import logging
+import os
 
-from dvc.utils.compat import urlparse, fspath_py35, str
-from dvc.istextfile import istextfile
 from dvc.exceptions import DvcException
-from dvc.remote.local import RemoteLOCAL
+from dvc.istextfile import istextfile
 from dvc.output.base import OutputBase
+from dvc.remote.local import RemoteLOCAL
+from dvc.utils import relpath
+from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
+from dvc.utils.fs import path_isin
 
 
 logger = logging.getLogger(__name__)
@@ -26,7 +30,7 @@ class OutputLOCAL(OutputBase):
             # so we should expect both posix and windows style paths.
             # PathInfo accepts both, i.e. / works everywhere, \ only on win.
             #
-            # FIXME: if we have Windows path containig / or posix one with \
+            # FIXME: if we have Windows path containing / or posix one with \
             # then we have #2059 bug and can't really handle that.
             p = self.REMOTE.path_cls(path)
             if not p.is_absolute():
@@ -36,7 +40,14 @@ class OutputLOCAL(OutputBase):
         return self.REMOTE.path_cls(abs_p)
 
     def __str__(self):
-        return str(self.path_info)
+        if not self.is_in_repo:
+            return str(self.def_path)
+
+        cur_dir = os.getcwd()
+        if path_isin(cur_dir, self.repo.root_dir):
+            return relpath(self.path_info, cur_dir)
+
+        return relpath(self.path_info, self.repo.root_dir)
 
     @property
     def fspath(self):

dvc/output/s3.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.remote.s3 import RemoteS3
 from dvc.output.base import OutputBase
+from dvc.remote.s3 import RemoteS3
 
 
 class OutputS3(OutputBase):

dvc/path_info.py

@@ -1,25 +1,38 @@
-from __future__ import unicode_literals, absolute_import
+from __future__ import absolute_import
+from __future__ import unicode_literals
 
-import sys
 import os
 import posixpath
+import sys
 
 from funcy import cached_property
 
-from dvc.utils.compat import str, builtin_str, basestring, is_py2
-from dvc.utils.compat import pathlib, urlparse
+from dvc.utils import relpath
+from dvc.utils.compat import basestring
+from dvc.utils.compat import builtin_str
+from dvc.utils.compat import is_py2
+from dvc.utils.compat import pathlib
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
 
 
 # On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
 # which is lossy, thus we can't use that,
 # see https://github.com/mcmtroffaes/pathlib2/issues/56.
-from dvc.utils import relpath
-
 if is_py2:
     fs_encoding = "utf-8"
 
 
-class PathInfo(pathlib.PurePath):
+class _BasePath(object):
+    def overlaps(self, other):
+        if isinstance(other, basestring):
+            other = self.__class__(other)
+        elif self.__class__ != other.__class__:
+            return False
+        return self == other or self.isin(other) or other.isin(self)
+
+
+class PathInfo(pathlib.PurePath, _BasePath):
     # Use __slots__ in PathInfo objects following PurePath implementation.
     # This makes objects smaller and speeds up attribute access.
     # We don't add any fields so it's empty.
@@ -39,7 +52,7 @@ class PathInfo(pathlib.PurePath):
     def as_posix(self):
         f = self._flavour
         # Unlike original implementation [1] that uses `str()` we actually need
-        # to use `fspath`, because we've overriden `__str__` method to return
+        # to use `fspath`, because we've overridden `__str__` method to return
         # relative paths, which will break original `as_posix`.
         #
         # [1] https://github.com/python/cpython/blob/v3.7.0/Lib/pathlib.py#L692
@@ -112,52 +125,80 @@ class PosixPathInfo(PathInfo, pathlib.PurePosixPath):
     pass
 
 
+class _URLPathInfo(PosixPathInfo):
+    def __str__(self):
+        return self.__fspath__()
+
+    __unicode__ = __str__
+
+
 class _URLPathParents(object):
-    def __init__(self, pathcls, scheme, netloc, path):
-        self._scheme = scheme
-        self._netloc = netloc
-        self._parents = path.parents
-        self._pathcls = pathcls
+    def __init__(self, src):
+        self.src = src
+        self._parents = self.src._path.parents
 
     def __len__(self):
         return len(self._parents)
 
     def __getitem__(self, idx):
-        return self._pathcls.from_parts(
-            scheme=self._scheme,
-            netloc=self._netloc,
-            path=self._parents[idx].fspath,
-        )
+        return self.src.replace(path=self._parents[idx])
 
     def __repr__(self):
-        return "<{}.parents>".format(self._pathcls.__name__)
+        return "<{}.parents>".format(self.src)
 
 
-class URLInfo(object):
+class URLInfo(_BasePath):
     DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22, "hdfs": 0}
 
     def __init__(self, url):
-        self.parsed = urlparse(url)
-        assert self.parsed.scheme != "remote"
+        p = urlparse(url)
+        assert not p.query and not p.params and not p.fragment
+        assert p.password is None
+
+        self.fill_parts(p.scheme, p.hostname, p.username, p.port, p.path)
 
     @classmethod
     def from_parts(
-        cls, scheme=None, netloc=None, host=None, user=None, port=None, path=""
+        cls, scheme=None, host=None, user=None, port=None, path="", netloc=None
     ):
-        assert scheme and (bool(host) ^ bool(netloc))
+        assert bool(host) ^ bool(netloc)
 
-        if netloc is None:
-            netloc = host
-            if user:
-                netloc = user + "@" + host
-            if port:
-                netloc += ":" + str(port)
-        return cls("{}://{}{}".format(scheme, netloc, path))
+        if netloc is not None:
+            return cls("{}://{}{}".format(scheme, netloc, path))
+
+        obj = cls.__new__(cls)
+        obj.fill_parts(scheme, host, user, port, path)
+        return obj
+
+    def fill_parts(self, scheme, host, user, port, path):
+        assert scheme != "remote"
+        assert isinstance(path, (basestring, _URLPathInfo))
+
+        self.scheme, self.host, self.user = scheme, host, user
+        self.port = int(port) if port else self.DEFAULT_PORTS.get(self.scheme)
+
+        if isinstance(path, _URLPathInfo):
+            self._spath = builtin_str(path)
+            self._path = path
+        else:
+            if path and path[0] != "/":
+                path = "/" + path
+            self._spath = path
+
+    @property
+    def _base_parts(self):
+        return (self.scheme, self.host, self.user, self.port)
+
+    @property
+    def parts(self):
+        return self._base_parts + self._path.parts
+
+    def replace(self, path=None):
+        return self.from_parts(*self._base_parts, path=path)
 
     @cached_property
     def url(self):
-        p = self.parsed
-        return "{}://{}{}".format(p.scheme, self.netloc, p.path)
+        return "{}://{}{}".format(self.scheme, self.netloc, self._spath)
 
     def __str__(self):
         return self.url
@@ -170,92 +211,60 @@ class URLInfo(object):
             other = self.__class__(other)
         return (
             self.__class__ == other.__class__
-            and self.scheme == other.scheme
-            and self.netloc == other.netloc
+            and self._base_parts == other._base_parts
             and self._path == other._path
         )
 
     def __hash__(self):
-        return hash(self.url)
+        return hash(self.parts)
 
     def __div__(self, other):
-        p = self.parsed
-        new_path = posixpath.join(p.path, str(other))
-        if not new_path.startswith("/"):
-            new_path = "/" + new_path
-        new_url = "{}://{}{}".format(p.scheme, p.netloc, new_path)
-        return self.__class__(new_url)
+        return self.replace(path=posixpath.join(self._spath, other))
 
     __truediv__ = __div__
 
-    def __getattr__(self, name):
-        # When deepcopy is called, it creates and object without __init__,
-        # self.parsed is not initialized and it causes infinite recursion.
-        # More on this special casing here:
-        # https://stackoverflow.com/a/47300262/298182
-        if name.startswith("__"):
-            raise AttributeError(name)
-        return getattr(self.parsed, name)
-
-    @cached_property
-    def netloc(self):
-        p = self.parsed
-        netloc = p.hostname
-        if p.username:
-            netloc = p.username + "@" + netloc
-        if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme):
-            netloc += ":" + str(p.port)
-        return netloc
-
     @property
-    def port(self):
-        return self.parsed.port or self.DEFAULT_PORTS.get(self.parsed.scheme)
-
-    @property
-    def host(self):
-        return self.parsed.hostname
-
-    @property
-    def user(self):
-        return self.parsed.username
+    def path(self):
+        return self._spath
 
     @cached_property
     def _path(self):
-        return PosixPathInfo(self.parsed.path)
+        return _URLPathInfo(self._spath)
 
     @property
     def name(self):
         return self._path.name
 
-    @property
-    def parts(self):
-        return (self.scheme, self.netloc) + self._path.parts
+    @cached_property
+    def netloc(self):
+        netloc = self.host
+        if self.user:
+            netloc = self.user + "@" + netloc
+        if self.port and int(self.port) != self.DEFAULT_PORTS.get(self.scheme):
+            netloc += ":" + str(self.port)
+        return netloc
 
     @property
     def bucket(self):
-        return self.parsed.netloc
+        return self.netloc
 
     @property
     def parent(self):
-        return self.from_parts(
-            scheme=self.scheme,
-            netloc=self.parsed.netloc,
-            path=self._path.parent.fspath,
-        )
+        return self.replace(path=self._path.parent)
 
     @property
     def parents(self):
-        return _URLPathParents(
-            type(self), self.scheme, self.parsed.netloc, self._path
-        )
+        return _URLPathParents(self)
 
     def relative_to(self, other):
-        if isinstance(other, str):
-            other = URLInfo(other)
-        if self.scheme != other.scheme or self.netloc != other.netloc:
-            raise ValueError(
-                "'{}' does not start with '{}'".format(self, other)
-            )
+        if isinstance(other, basestring):
+            other = self.__class__(other)
+        if self.__class__ != other.__class__:
+            msg = "'{}' has incompatible class with '{}'".format(self, other)
+            raise ValueError(msg)
+        if self._base_parts != other._base_parts:
+            msg = "'{}' does not start with '{}'".format(self, other)
+            raise ValueError(msg)
         return self._path.relative_to(other._path)
 
     def isin(self, other):
@@ -263,14 +272,12 @@ class URLInfo(object):
             other = self.__class__(other)
         elif self.__class__ != other.__class__:
             return False
-        return (
-            self.scheme == other.scheme
-            and self.netloc == other.netloc
-            and self._path.isin(other._path)
+        return self._base_parts == other._base_parts and self._path.isin(
+            other._path
         )
 
 
 class CloudURLInfo(URLInfo):
     @property
     def path(self):
-        return self.parsed.path.lstrip("/")
+        return self._spath.lstrip("/")

dvc/progress.py

@@ -1,33 +1,17 @@
 """Manages progress bars for dvc repo."""
 from __future__ import print_function
+
 import logging
-from tqdm import tqdm
-from concurrent.futures import ThreadPoolExecutor
+import sys
+from threading import RLock
+
 from funcy import merge
+from tqdm import tqdm
+
+from dvc.utils import env2bool
 
 logger = logging.getLogger(__name__)
-
-
-class TqdmThreadPoolExecutor(ThreadPoolExecutor):
-    """
-    Ensure worker progressbars are cleared away properly.
-    """
-
-    def __enter__(self):
-        """
-        Creates a blank initial dummy progress bar if needed so that workers
-        are forced to create "nested" bars.
-        """
-        blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False)
-        if blank_bar.pos > 0:
-            # already nested - don't need a placeholder bar
-            blank_bar.close()
-        self.bar = blank_bar
-        return super(TqdmThreadPoolExecutor, self).__enter__()
-
-    def __exit__(self, *a, **k):
-        super(TqdmThreadPoolExecutor, self).__exit__(*a, **k)
-        self.bar.close()
+tqdm.set_lock(RLock())
 
 
 class Tqdm(tqdm):
@@ -36,12 +20,18 @@ class Tqdm(tqdm):
     """
 
     BAR_FMT_DEFAULT = (
-        "{percentage:3.0f}%|{bar:10}|"
-        "{desc:{ncols_desc}.{ncols_desc}}{n}/{total}"
+        "{percentage:3.0f}% {desc}|{bar}|"
+        "{n_fmt}/{total_fmt}"
+        " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
+    )
+    # nested bars should have fixed bar widths to align nicely
+    BAR_FMT_DEFAULT_NESTED = (
+        "{percentage:3.0f}%|{bar:10}|{desc:{ncols_desc}.{ncols_desc}}"
+        "{n_fmt}/{total_fmt}"
         " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
     )
     BAR_FMT_NOTOTAL = (
-        "{desc:{ncols_desc}.{ncols_desc}}{n}"
+        "{desc:{ncols_desc}.{ncols_desc}}{n_fmt}"
         " [{elapsed}<??:??, {rate_fmt:>11}{postfix}]"
     )
 
@@ -54,6 +44,7 @@ class Tqdm(tqdm):
         leave=False,
         bar_format=None,
         bytes=False,  # pylint: disable=W0622
+        file=None,
         **kwargs
     ):
         """
@@ -62,6 +53,9 @@ class Tqdm(tqdm):
         desc  : persists after `close()`
         level  : effective logging level for determining `disable`;
             used only if `disable` is unspecified
+        disable  : If (default: None), will be determined by logging level.
+            May be overridden to `True` due to non-TTY status.
+            Skip override by specifying env var `DVC_IGNORE_ISATTY`.
         kwargs  : anything accepted by `tqdm.tqdm()`
         """
         kwargs = kwargs.copy()
@@ -71,20 +65,35 @@ class Tqdm(tqdm):
                 unit="B", unit_scale=True, unit_divisor=1024, miniters=1
             )
             kwargs = merge(bytes_defaults, kwargs)
+        if file is None:
+            file = sys.stderr
         self.desc_persist = desc
+        # auto-disable based on `logger.level`
         if disable is None:
             disable = logger.getEffectiveLevel() > level
+        # auto-disable based on TTY
+        if (
+            not disable
+            and not env2bool("DVC_IGNORE_ISATTY")
+            and hasattr(file, "isatty")
+        ):
+            disable = not file.isatty()
         super(Tqdm, self).__init__(
             iterable=iterable,
             disable=disable,
             leave=leave,
             desc=desc,
             bar_format="!",
+            lock_args=(False,),
             **kwargs
         )
         if bar_format is None:
             if self.__len__():
-                self.bar_format = self.BAR_FMT_DEFAULT
+                self.bar_format = (
+                    self.BAR_FMT_DEFAULT_NESTED
+                    if self.pos
+                    else self.BAR_FMT_DEFAULT
+                )
             else:
                 self.bar_format = self.BAR_FMT_NOTOTAL
         else:
@@ -114,5 +123,11 @@ class Tqdm(tqdm):
         d = super(Tqdm, self).format_dict
         ncols = d["ncols"] or 80
         ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1
-        d["ncols_desc"] = max(ncols_desc, 0)
+        ncols_desc = max(ncols_desc, 0)
+        if ncols_desc:
+            d["ncols_desc"] = ncols_desc
+        else:
+            # work-around for zero-width description
+            d["ncols_desc"] = 1
+            d["prefix"] = ""
         return d

dvc/prompt.py

@@ -1,10 +1,9 @@
 """Manages user prompts."""
-
-from __future__ import unicode_literals
 from __future__ import print_function
+from __future__ import unicode_literals
 
-import sys
 import logging
+import sys
 from getpass import getpass
 
 from dvc.utils.compat import input

dvc/remote/__init__.py

@@ -1,20 +1,21 @@
 from __future__ import unicode_literals
 
+from .config import RemoteConfig
 from dvc.remote.azure import RemoteAZURE
+from dvc.remote.gdrive import RemoteGDrive
 from dvc.remote.gs import RemoteGS
 from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
-from dvc.remote.ssh import RemoteSSH
 from dvc.remote.http import RemoteHTTP
 from dvc.remote.https import RemoteHTTPS
+from dvc.remote.local import RemoteLOCAL
 from dvc.remote.oss import RemoteOSS
-
-from .config import RemoteConfig
+from dvc.remote.s3 import RemoteS3
+from dvc.remote.ssh import RemoteSSH
 
 
 REMOTES = [
     RemoteAZURE,
+    RemoteGDrive,
     RemoteGS,
     RemoteHDFS,
     RemoteHTTP,

dvc/remote/azure.py

@@ -1,20 +1,20 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+import logging
 import os
 import re
-import logging
 from datetime import datetime, timedelta
+import threading
 
-from funcy import cached_property
+from funcy import cached_property, wrap_prop
 
-from dvc.scheme import Schemes
-
-from dvc.utils.compat import urlparse
-from dvc.progress import Tqdm
 from dvc.config import Config
-from dvc.remote.base import RemoteBASE
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.scheme import Schemes
+from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -64,6 +64,7 @@ class RemoteAZURE(RemoteBASE):
             else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
         )
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def blob_service(self):
         from azure.storage.blob import BlockBlobService

dvc/remote/base.py

@@ -1,18 +1,17 @@
 from __future__ import unicode_literals
+from dvc.utils.compat import basestring, FileNotFoundError, str, urlparse
 
-from dvc.ignore import DvcIgnore
-from dvc.utils.compat import str, basestring, urlparse, FileNotFoundError
-
+import itertools
 import json
 import logging
 import tempfile
-import itertools
-from operator import itemgetter
-from multiprocessing import cpu_count
-from functools import partial
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
-from dvc.remote.slow_link_detection import slow_link_guard
+from functools import partial
+from multiprocessing import cpu_count
+from operator import itemgetter
+
+from shortuuid import uuid
 
 import dvc.prompt as prompt
 from dvc.config import Config
@@ -21,22 +20,22 @@ from dvc.exceptions import (
     ConfirmRemoveError,
     DvcIgnoreInCollectedDirError,
 )
-from dvc.progress import Tqdm, TqdmThreadPoolExecutor
-from dvc.utils import LARGE_DIR_SIZE, tmp_fname, move, relpath, makedirs
-from dvc.state import StateNoop
+from dvc.ignore import DvcIgnore
 from dvc.path_info import PathInfo, URLInfo
+from dvc.progress import Tqdm
+from dvc.remote.slow_link_detection import slow_link_guard
+from dvc.state import StateNoop
+from dvc.utils import makedirs, relpath, tmp_fname
+from dvc.utils.fs import move
 from dvc.utils.http import open_url
 
-
 logger = logging.getLogger(__name__)
 
-
 STATUS_OK = 1
 STATUS_MISSING = 2
 STATUS_NEW = 3
 STATUS_DELETED = 4
 
-
 STATUS_MAP = {
     # (local_exists, remote_exists)
     (True, True): STATUS_OK,
@@ -46,13 +45,6 @@ STATUS_MAP = {
 }
 
 
-class DataCloudError(DvcException):
-    """ Data Cloud exception """
-
-    def __init__(self, msg):
-        super(DataCloudError, self).__init__("Data sync error: {}".format(msg))
-
-
 class RemoteCmdError(DvcException):
     def __init__(self, remote, cmd, ret, err):
         super(RemoteCmdError, self).__init__(
@@ -178,31 +170,25 @@ class RemoteBASE(object):
 
     def _calculate_checksums(self, file_infos):
         file_infos = list(file_infos)
-        with TqdmThreadPoolExecutor(
-            max_workers=self.checksum_jobs
-        ) as executor:
+        with ThreadPoolExecutor(max_workers=self.checksum_jobs) as executor:
             tasks = executor.map(self.get_file_checksum, file_infos)
-
-            if len(file_infos) > LARGE_DIR_SIZE:
-                logger.info(
-                    (
-                        "Computing md5 for a large number of files. "
-                        "This is only done once."
-                    )
-                )
-                tasks = Tqdm(tasks, total=len(file_infos), unit="md5")
-            checksums = dict(zip(file_infos, tasks))
+            with Tqdm(
+                tasks,
+                total=len(file_infos),
+                unit="md5",
+                desc="Computing hashes (only done once)",
+            ) as tasks:
+                checksums = dict(zip(file_infos, tasks))
         return checksums
 
     def _collect_dir(self, path_info):
-
         file_infos = set()
-        for root, _dirs, files in self.walk(path_info):
 
-            if DvcIgnore.DVCIGNORE_FILE in files:
-                raise DvcIgnoreInCollectedDirError(root)
+        for fname in self.walk_files(path_info):
+            if DvcIgnore.DVCIGNORE_FILE == fname.name:
+                raise DvcIgnoreInCollectedDirError(fname.parent)
 
-            file_infos.update(path_info / root / fname for fname in files)
+            file_infos.add(fname)
 
         checksums = {fi: self.state.get(fi) for fi in file_infos}
         not_in_state = {
@@ -298,6 +284,8 @@ class RemoteBASE(object):
         return checksum.endswith(cls.CHECKSUM_DIR_SUFFIX)
 
     def get_checksum(self, path_info):
+        assert path_info.scheme == self.scheme
+
         if not self.exists(path_info):
             return None
 
@@ -327,7 +315,6 @@ class RemoteBASE(object):
         return checksum
 
     def save_info(self, path_info):
-        assert path_info.scheme == self.scheme
         return {self.PARAM_CHECKSUM: self.get_checksum(path_info)}
 
     def changed(self, path_info, checksum_info):
@@ -368,7 +355,7 @@ class RemoteBASE(object):
             )
             return True
 
-        actual = self.save_info(path_info)[self.PARAM_CHECKSUM]
+        actual = self.get_checksum(path_info)
         if checksum != actual:
             logger.debug(
                 "checksum '{}'(actual '{}') for '{}' has changed.".format(
@@ -429,10 +416,16 @@ class RemoteBASE(object):
         cache_info = self.checksum_to_path_info(checksum)
         if self.changed_cache(checksum):
             self.move(path_info, cache_info)
+            self.link(cache_info, path_info)
+        elif self.iscopy(path_info) and self._cache_is_copy(path_info):
+            # Default relink procedure involves unneeded copy
+            if self.protected:
+                self.protect(path_info)
+            else:
+                self.unprotect(path_info)
         else:
             self.remove(path_info)
-
-        self.link(cache_info, path_info)
+            self.link(cache_info, path_info)
 
         if save_link:
             self.state.save_link(path_info)
@@ -443,6 +436,28 @@ class RemoteBASE(object):
         self.state.save(path_info, checksum)
         self.state.save(cache_info, checksum)
 
+    def _cache_is_copy(self, path_info):
+        """Checks whether cache uses copies."""
+        if self.cache_type_confirmed:
+            return self.cache_types[0] == "copy"
+
+        if set(self.cache_types) <= {"copy"}:
+            return True
+
+        workspace_file = path_info.with_name("." + uuid())
+        test_cache_file = self.path_info / ".cache_type_test_file"
+        if not self.exists(test_cache_file):
+            with self.open(test_cache_file, "wb") as fobj:
+                fobj.write(bytes(1))
+        try:
+            self.link(test_cache_file, workspace_file)
+        finally:
+            self.remove(workspace_file)
+            self.remove(test_cache_file)
+
+        self.cache_type_confirmed = True
+        return self.cache_types[0] == "copy"
+
     def _save_dir(self, path_info, checksum):
         cache_info = self.checksum_to_path_info(checksum)
         dir_info = self.get_dir_cache(checksum)
@@ -471,7 +486,12 @@ class RemoteBASE(object):
         """
         return False
 
-    def walk(self, path_info):
+    def iscopy(self, path_info):
+        """Check if this file is an independent copy."""
+        return False  # We can't be sure by default
+
+    def walk_files(self, path_info):
+        """Return a generator with `PathInfo`s to all the files"""
         raise NotImplementedError
 
     @staticmethod
@@ -486,15 +506,11 @@ class RemoteBASE(object):
             )
 
         checksum = checksum_info[self.PARAM_CHECKSUM]
-        if not self.changed_cache(checksum):
-            self._checkout(path_info, checksum)
-            return
-
         self._save(path_info, checksum)
 
     def _save(self, path_info, checksum):
         to_info = self.checksum_to_path_info(checksum)
-        logger.info("Saving '{}' to '{}'.".format(path_info, to_info))
+        logger.debug("Saving '{}' to '{}'.".format(path_info, to_info))
         if self.isdir(path_info):
             self._save_dir(path_info, checksum)
             return
@@ -550,11 +566,48 @@ class RemoteBASE(object):
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
+        if self.isdir(from_info):
+            return self._download_dir(
+                from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+            )
+        return self._download_file(
+            from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+        )
 
+    def _download_dir(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        from_infos = list(self.walk_files(from_info))
+        to_infos = (
+            to_info / info.relative_to(from_info) for info in from_infos
+        )
+
+        with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
+            download_files = partial(
+                self._download_file,
+                name=name,
+                no_progress_bar=True,
+                file_mode=file_mode,
+                dir_mode=dir_mode,
+            )
+            futures = executor.map(download_files, from_infos, to_infos)
+            with Tqdm(
+                futures,
+                total=len(from_infos),
+                desc="Downloading directory",
+                unit="Files",
+                disable=no_progress_bar,
+            ) as futures:
+                return sum(futures)
+
+    def _download_file(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
+
+        logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
         name = name or to_info.name
 
-        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
         tmp_file = tmp_fname(to_info)
 
         try:
@@ -625,14 +678,11 @@ class RemoteBASE(object):
                 # We ignore all the non-cache looking files
                 pass
 
-    def gc(self, cinfos):
-        used = self.extract_used_local_checksums(cinfos)
+    def gc(self, named_cache):
+        used = self.extract_used_local_checksums(named_cache)
 
         if self.scheme != "":
-            used |= {
-                info[self.PARAM_CHECKSUM]
-                for info in cinfos.get(self.scheme, [])
-            }
+            used.update(named_cache[self.scheme])
 
         removed = False
         for checksum in self.all():
@@ -764,45 +814,21 @@ class RemoteBASE(object):
         self.remove(path_info)
 
     def _checkout_file(
-        self,
-        path_info,
-        checksum,
-        force,
-        progress_callback=None,
-        save_link=True,
+        self, path_info, checksum, force, progress_callback=None
     ):
-        # NOTE: In case if path_info is already cached and path_info's
-        # link type matches cache link type, we would like to avoid
-        # relinking.
-        if self.changed(
-            path_info, {self.PARAM_CHECKSUM: checksum}
-        ) or not self._link_matches(path_info):
+        """The file is changed we need to checkout a new copy"""
+        cache_info = self.checksum_to_path_info(checksum)
+        if self.exists(path_info):
+            msg = "data '{}' exists. Removing before checkout."
+            logger.warning(msg.format(str(path_info)))
             self.safe_remove(path_info, force=force)
 
-            cache_info = self.checksum_to_path_info(checksum)
-            self.link(cache_info, path_info)
-
-            if save_link:
-                self.state.save_link(path_info)
-
-            self.state.save(path_info, checksum)
-        else:
-            # NOTE: performing (un)protection costs us +/- the same as checking
-            # if path_info is protected. Instead of implementing logic,
-            # just (un)protect according to self.protected.
-            if self.protected:
-                self.protect(path_info)
-            else:
-                # NOTE dont allow copy, because we checked before that link
-                # type matches cache, and we don't want data duplication
-                self.unprotect(path_info, allow_copy=False)
-
+        self.link(cache_info, path_info)
+        self.state.save_link(path_info)
+        self.state.save(path_info, checksum)
         if progress_callback:
             progress_callback(str(path_info))
 
-    def _link_matches(self, path_info):
-        return True
-
     def makedirs(self, path_info):
         """Optional: Implement only if the remote needs to create
         directories before copying/linking/moving data
@@ -810,7 +836,7 @@ class RemoteBASE(object):
         pass
 
     def _checkout_dir(
-        self, path_info, checksum, force, progress_callback=None
+        self, path_info, checksum, force, progress_callback=None, relink=False
     ):
         # Create dir separately so that dir is created
         # even if there are no files in it
@@ -824,14 +850,16 @@ class RemoteBASE(object):
         for entry in dir_info:
             relative_path = entry[self.PARAM_RELPATH]
             entry_checksum = entry[self.PARAM_CHECKSUM]
+            entry_cache_info = self.checksum_to_path_info(entry_checksum)
             entry_info = path_info / relative_path
-            self._checkout_file(
-                entry_info,
-                entry_checksum,
-                force,
-                progress_callback,
-                save_link=False,
-            )
+
+            entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
+            if relink or self.changed(entry_info, entry_checksum_info):
+                self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+                self.state.save(entry_info, entry_checksum)
+            if progress_callback:
+                progress_callback(str(entry_info))
 
         self._remove_redundant_files(path_info, dir_info, force)
 
@@ -839,11 +867,7 @@ class RemoteBASE(object):
         self.state.save(path_info, checksum)
 
     def _remove_redundant_files(self, path_info, dir_info, force):
-        existing_files = set(
-            path_info / root / fname
-            for root, _, files in self.walk(path_info)
-            for fname in files
-        )
+        existing_files = set(self.walk_files(path_info))
 
         needed_files = {
             path_info / entry[self.PARAM_RELPATH] for entry in dir_info
@@ -853,13 +877,19 @@ class RemoteBASE(object):
             self.safe_remove(path, force)
 
     def checkout(
-        self, path_info, checksum_info, force=False, progress_callback=None
+        self,
+        path_info,
+        checksum_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
     ):
         if path_info.scheme not in ["local", self.scheme]:
             raise NotImplementedError
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
         failed = None
+        skip = False
         if not checksum:
             logger.warning(
                 "No checksum info found for '{}'. "
@@ -868,13 +898,18 @@ class RemoteBASE(object):
             self.safe_remove(path_info, force=force)
             failed = path_info
 
+        elif not relink and not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            logger.debug(msg.format(str(path_info)))
+            skip = True
+
         elif self.changed_cache(checksum):
             msg = "Cache '{}' not found. File '{}' won't be created."
             logger.warning(msg.format(checksum, str(path_info)))
             self.safe_remove(path_info, force=force)
             failed = path_info
 
-        if failed:
+        if failed or skip:
             if progress_callback:
                 progress_callback(
                     str(path_info), self.get_files_number(checksum)
@@ -884,18 +919,23 @@ class RemoteBASE(object):
         msg = "Checking out '{}' with cache '{}'."
         logger.debug(msg.format(str(path_info), checksum))
 
-        self._checkout(path_info, checksum, force, progress_callback)
+        self._checkout(path_info, checksum, force, progress_callback, relink)
         return None
 
     def _checkout(
-        self, path_info, checksum, force=False, progress_callback=None
+        self,
+        path_info,
+        checksum,
+        force=False,
+        progress_callback=None,
+        relink=False,
     ):
         if not self.is_dir_checksum(checksum):
             return self._checkout_file(
                 path_info, checksum, force, progress_callback=progress_callback
             )
         return self._checkout_dir(
-            path_info, checksum, force, progress_callback=progress_callback
+            path_info, checksum, force, progress_callback, relink
         )
 
     def get_files_number(self, checksum):
@@ -908,16 +948,14 @@ class RemoteBASE(object):
         return 1
 
     @staticmethod
-    def unprotect(path_info, allow_copy=True):
+    def unprotect(path_info):
         pass
 
     def _get_unpacked_dir_names(self, checksums):
         return set()
 
-    def extract_used_local_checksums(self, cinfos):
-        from dvc.remote import RemoteLOCAL
-
-        used = {info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]}
+    def extract_used_local_checksums(self, named_cache):
+        used = set(named_cache["local"])
         unpacked = self._get_unpacked_dir_names(used)
         return used | unpacked
 

dvc/remote/config.py

@@ -1,10 +1,11 @@
-import os
 import logging
+import os
 import posixpath
 
+from dvc.config import Config
+from dvc.config import ConfigError
 from dvc.utils import relpath
 from dvc.utils.compat import urlparse
-from dvc.config import Config, ConfigError
 
 
 logger = logging.getLogger(__name__)
@@ -104,7 +105,6 @@ class RemoteConfig(object):
             level=level,
         )
         if default:
-            logger.info("Setting '{}' as a default remote.".format(name))
             self.config.set(
                 Config.SECTION_CORE,
                 Config.SECTION_CORE_REMOTE,
@@ -150,3 +150,8 @@ class RemoteConfig(object):
         self.config.set(
             Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level
         )
+
+    def get_default(self, level=None):
+        return self.config.get(
+            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level
+        )

dvc/remote/gdrive.py

@@ -0,0 +1,306 @@
+from __future__ import unicode_literals
+
+import os
+import posixpath
+import logging
+import threading
+
+from funcy import retry, compose, decorator, wrap_with
+from funcy.py3 import cat
+
+from dvc.progress import Tqdm
+from dvc.scheme import Schemes
+from dvc.path_info import CloudURLInfo
+from dvc.remote.base import RemoteBASE
+from dvc.config import Config
+from dvc.exceptions import DvcException
+from dvc.utils import tmp_fname
+
+logger = logging.getLogger(__name__)
+FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
+
+
+class GDriveRetriableError(DvcException):
+    def __init__(self, msg):
+        super(GDriveRetriableError, self).__init__(msg)
+
+
+@decorator
+def _wrap_pydrive_retriable(call):
+    from apiclient import errors
+    from pydrive.files import ApiRequestError
+
+    try:
+        result = call()
+    except (ApiRequestError, errors.HttpError) as exception:
+        retry_codes = ["403", "500", "502", "503", "504"]
+        if any(
+            "HttpError {}".format(code) in str(exception)
+            for code in retry_codes
+        ):
+            raise GDriveRetriableError(msg="Google API request failed")
+        raise
+    return result
+
+
+gdrive_retry = compose(
+    # 8 tries, start at 0.5s, multiply by golden ratio, cap at 10s
+    retry(
+        8, GDriveRetriableError, timeout=lambda a: min(0.5 * 1.618 ** a, 10)
+    ),
+    _wrap_pydrive_retriable,
+)
+
+
+class RemoteGDrive(RemoteBASE):
+    scheme = Schemes.GDRIVE
+    path_cls = CloudURLInfo
+    REQUIRES = {"pydrive": "pydrive"}
+    GDRIVE_USER_CREDENTIALS_DATA = "GDRIVE_USER_CREDENTIALS_DATA"
+    DEFAULT_USER_CREDENTIALS_FILE = ".dvc/tmp/gdrive-user-credentials.json"
+
+    def __init__(self, repo, config):
+        super(RemoteGDrive, self).__init__(repo, config)
+        self.no_traverse = False
+        self.path_info = self.path_cls(config[Config.SECTION_REMOTE_URL])
+        self.config = config
+        self.init_drive()
+
+    def init_drive(self):
+        self.client_id = self.config.get(Config.SECTION_GDRIVE_CLIENT_ID, None)
+        self.client_secret = self.config.get(
+            Config.SECTION_GDRIVE_CLIENT_SECRET, None
+        )
+        if not self.client_id or not self.client_secret:
+            raise DvcException(
+                "Please specify Google Drive's client id and "
+                "secret in DVC's config. Learn more at "
+                "https://man.dvc.org/remote/add."
+            )
+        self.gdrive_user_credentials_path = (
+            tmp_fname(".dvc/tmp/")
+            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
+            else self.config.get(
+                Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE,
+                self.DEFAULT_USER_CREDENTIALS_FILE,
+            )
+        )
+
+    @gdrive_retry
+    def gdrive_upload_file(
+        self, args, no_progress_bar=True, from_file="", progress_name=""
+    ):
+        item = self.drive.CreateFile(
+            {"title": args["title"], "parents": [{"id": args["parent_id"]}]}
+        )
+
+        with open(from_file, "rb") as fobj:
+            total = os.path.getsize(from_file)
+            with Tqdm.wrapattr(
+                fobj,
+                "read",
+                desc=progress_name,
+                total=total,
+                disable=no_progress_bar,
+            ) as wrapped:
+                # PyDrive doesn't like content property setting for empty files
+                # https://github.com/gsuitedevs/PyDrive/issues/121
+                if total:
+                    item.content = wrapped
+                item.Upload()
+        return item
+
+    @gdrive_retry
+    def gdrive_download_file(
+        self, file_id, to_file, progress_name, no_progress_bar
+    ):
+        gdrive_file = self.drive.CreateFile({"id": file_id})
+        bar_format = (
+            "Donwloading {desc:{ncols_desc}.{ncols_desc}}... "
+            + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
+        )
+        with Tqdm(
+            bar_format=bar_format, desc=progress_name, disable=no_progress_bar
+        ):
+            gdrive_file.GetContentFile(to_file)
+
+    def gdrive_list_item(self, query):
+        file_list = self.drive.ListFile({"q": query, "maxResults": 1000})
+
+        # Isolate and decorate fetching of remote drive items in pages
+        get_list = gdrive_retry(lambda: next(file_list, None))
+
+        # Fetch pages until None is received, lazily flatten the thing
+        return cat(iter(get_list, None))
+
+    def cache_root_dirs(self):
+        cached_dirs = {}
+        cached_ids = {}
+        for dir1 in self.gdrive_list_item(
+            "'{}' in parents and trashed=false".format(self.root_id)
+        ):
+            remote_path = posixpath.join(self.path_info.path, dir1["title"])
+            cached_dirs.setdefault(remote_path, []).append(dir1["id"])
+            cached_ids[dir1["id"]] = dir1["title"]
+        return cached_dirs, cached_ids
+
+    @property
+    def cached_dirs(self):
+        if not hasattr(self, "_cached_dirs"):
+            self.drive
+        return self._cached_dirs
+
+    @property
+    def cached_ids(self):
+        if not hasattr(self, "_cached_ids"):
+            self.drive
+        return self._cached_ids
+
+    @property
+    @wrap_with(threading.RLock())
+    def drive(self):
+        if not hasattr(self, "_gdrive"):
+            from pydrive.auth import GoogleAuth
+            from pydrive.drive import GoogleDrive
+
+            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                with open(
+                    self.gdrive_user_credentials_path, "w"
+                ) as credentials_file:
+                    credentials_file.write(
+                        os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
+                    )
+
+            GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
+            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                "token_uri": "https://oauth2.googleapis.com/token",
+                "revoke_uri": "https://oauth2.googleapis.com/revoke",
+                "redirect_uri": "",
+            }
+            GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
+            GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
+            GoogleAuth.DEFAULT_SETTINGS[
+                "save_credentials_file"
+            ] = self.gdrive_user_credentials_path
+            GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
+            GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
+                "https://www.googleapis.com/auth/drive",
+                "https://www.googleapis.com/auth/drive.appdata",
+            ]
+
+            # Pass non existent settings path to force DEFAULT_SETTINGS loading
+            gauth = GoogleAuth(settings_file="")
+            gauth.CommandLineAuth()
+
+            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                os.remove(self.gdrive_user_credentials_path)
+
+            self._gdrive = GoogleDrive(gauth)
+
+            self.root_id = self.get_remote_id(self.path_info, create=True)
+            self._cached_dirs, self._cached_ids = self.cache_root_dirs()
+
+        return self._gdrive
+
+    @gdrive_retry
+    def create_remote_dir(self, parent_id, title):
+        item = self.drive.CreateFile(
+            {
+                "title": title,
+                "parents": [{"id": parent_id}],
+                "mimeType": FOLDER_MIME_TYPE,
+            }
+        )
+        item.Upload()
+        return item
+
+    @gdrive_retry
+    def get_remote_item(self, name, parents_ids):
+        if not parents_ids:
+            return None
+        query = " or ".join(
+            "'{}' in parents".format(parent_id) for parent_id in parents_ids
+        )
+
+        query += " and trashed=false and title='{}'".format(name)
+
+        # Limit found remote items count to 1 in response
+        item_list = self.drive.ListFile(
+            {"q": query, "maxResults": 1}
+        ).GetList()
+        return next(iter(item_list), None)
+
+    def resolve_remote_item_from_path(self, path_parts, create):
+        parents_ids = ["root"]
+        current_path = ""
+        for path_part in path_parts:
+            current_path = posixpath.join(current_path, path_part)
+            remote_ids = self.get_remote_id_from_cache(current_path)
+            if remote_ids:
+                parents_ids = remote_ids
+                continue
+            item = self.get_remote_item(path_part, parents_ids)
+            if not item and create:
+                item = self.create_remote_dir(parents_ids[0], path_part)
+            elif not item:
+                return None
+            parents_ids = [item["id"]]
+        return item
+
+    def get_remote_id_from_cache(self, remote_path):
+        if hasattr(self, "_cached_dirs"):
+            return self.cached_dirs.get(remote_path, [])
+        return []
+
+    def get_remote_id(self, path_info, create=False):
+        remote_ids = self.get_remote_id_from_cache(path_info.path)
+
+        if remote_ids:
+            return remote_ids[0]
+
+        file1 = self.resolve_remote_item_from_path(
+            path_info.path.split("/"), create
+        )
+        return file1["id"] if file1 else ""
+
+    def exists(self, path_info):
+        return self.get_remote_id(path_info) != ""
+
+    def _upload(self, from_file, to_info, name, no_progress_bar):
+        dirname = to_info.parent
+        if dirname:
+            parent_id = self.get_remote_id(dirname, True)
+        else:
+            parent_id = to_info.bucket
+
+        self.gdrive_upload_file(
+            {"title": to_info.name, "parent_id": parent_id},
+            no_progress_bar,
+            from_file,
+            name,
+        )
+
+    def _download(self, from_info, to_file, name, no_progress_bar):
+        file_id = self.get_remote_id(from_info)
+        self.gdrive_download_file(file_id, to_file, name, no_progress_bar)
+
+    def all(self):
+        if not self.cached_ids:
+            return
+
+        query = " or ".join(
+            "'{}' in parents".format(dir_id) for dir_id in self.cached_ids
+        )
+
+        query += " and trashed=false"
+        for file1 in self.gdrive_list_item(query):
+            parent_id = file1["parents"][0]["id"]
+            path = posixpath.join(self.cached_ids[parent_id], file1["title"])
+            try:
+                yield self.path_to_checksum(path)
+            except ValueError:
+                # We ignore all the non-cache looking files
+                logger.debug('Ignoring path as "non-cache looking"')

dvc/remote/gs.py

@@ -1,19 +1,74 @@
-from __future__ import unicode_literals
+from __future__ import unicode_literals, division
 
 import logging
 from datetime import timedelta
-from funcy import cached_property
-from dvc.utils.compat import FileNotFoundError
+from functools import wraps
+import io
+import os.path
+import threading
+
+from funcy import cached_property, wrap_prop
 
-from dvc.remote.base import RemoteBASE
 from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
+from dvc.utils.compat import FileNotFoundError  # skipcq: PYL-W0622
 
 logger = logging.getLogger(__name__)
 
 
+def dynamic_chunk_size(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        import requests
+        from google.cloud.storage.blob import Blob
+
+        # `ConnectionError` may be due to too large `chunk_size`
+        # (see [#2572]) so try halving on error.
+        # Note: start with 40 * [default: 256K] = 10M.
+        # Note: must be multiple of 256K.
+        #
+        # [#2572]: https://github.com/iterative/dvc/issues/2572
+
+        # skipcq: PYL-W0212
+        multiplier = 40
+        while True:
+            try:
+                # skipcq: PYL-W0212
+                chunk_size = Blob._CHUNK_SIZE_MULTIPLE * multiplier
+                return func(*args, chunk_size=chunk_size, **kwargs)
+            except requests.exceptions.ConnectionError:
+                multiplier //= 2
+                if not multiplier:
+                    raise
+
+    return wrapper
+
+
+@dynamic_chunk_size
+def _upload_to_bucket(
+    bucket,
+    from_file,
+    to_info,
+    chunk_size=None,
+    name=None,
+    no_progress_bar=True,
+):
+    blob = bucket.blob(to_info.path, chunk_size=chunk_size)
+    with io.open(from_file, mode="rb") as fobj:
+        with Tqdm.wrapattr(
+            fobj,
+            "read",
+            desc=name or to_info.path,
+            total=os.path.getsize(from_file),
+            disable=no_progress_bar,
+        ) as wrapped:
+            blob.upload_from_file(wrapped)
+
+
 class RemoteGS(RemoteBASE):
     scheme = Schemes.GS
     path_cls = CloudURLInfo
@@ -23,13 +78,13 @@ class RemoteGS(RemoteBASE):
     def __init__(self, repo, config):
         super(RemoteGS, self).__init__(repo, config)
 
-        storagepath = "gs://" + config.get(Config.SECTION_GCP_STORAGEPATH, "/")
-        url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        url = config.get(Config.SECTION_REMOTE_URL, "gs:///")
         self.path_info = self.path_cls(url)
 
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
         self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def gs(self):
         from google.cloud.storage import Client
@@ -75,26 +130,68 @@ class RemoteGS(RemoteBASE):
 
         blob.delete()
 
-    def _list_paths(self, bucket, prefix):
-        for blob in self.gs.bucket(bucket).list_blobs(prefix=prefix):
+    def _list_paths(self, path_info, max_items=None):
+        for blob in self.gs.bucket(path_info.bucket).list_blobs(
+            prefix=path_info.path, max_results=max_items
+        ):
             yield blob.name
 
     def list_cache_paths(self):
-        return self._list_paths(self.path_info.bucket, self.path_info.path)
+        return self._list_paths(self.path_info)
+
+    def walk_files(self, path_info):
+        for fname in self._list_paths(path_info / ""):
+            # skip nested empty directories
+            if fname.endswith("/"):
+                continue
+            yield path_info.replace(fname)
+
+    def makedirs(self, path_info):
+        self.gs.bucket(path_info.bucket).blob(
+            (path_info / "").path
+        ).upload_from_string("")
+
+    def isdir(self, path_info):
+        dir_path = path_info / ""
+        return bool(list(self._list_paths(dir_path, max_items=1)))
+
+    def isfile(self, path_info):
+        if path_info.path.endswith("/"):
+            return False
+
+        blob = self.gs.bucket(path_info.bucket).blob(path_info.path)
+        return blob.exists()
 
     def exists(self, path_info):
-        paths = set(self._list_paths(path_info.bucket, path_info.path))
-        return any(path_info.path == path for path in paths)
+        """Check if the blob exists. If it does not exist,
+        it could be a part of a directory path.
 
-    def _upload(self, from_file, to_info, **_kwargs):
+        eg: if `data/file.txt` exists, check for `data` should return True
+        """
+        return self.isfile(path_info) or self.isdir(path_info)
+
+    def _upload(self, from_file, to_info, name=None, no_progress_bar=True):
         bucket = self.gs.bucket(to_info.bucket)
-        blob = bucket.blob(to_info.path)
-        blob.upload_from_filename(from_file)
+        _upload_to_bucket(
+            bucket,
+            from_file,
+            to_info,
+            name=name,
+            no_progress_bar=no_progress_bar,
+        )
 
-    def _download(self, from_info, to_file, **_kwargs):
+    def _download(self, from_info, to_file, name=None, no_progress_bar=True):
         bucket = self.gs.bucket(from_info.bucket)
         blob = bucket.get_blob(from_info.path)
-        blob.download_to_filename(to_file)
+        with io.open(to_file, mode="wb") as fobj:
+            with Tqdm.wrapattr(
+                fobj,
+                "write",
+                desc=name or from_info.path,
+                total=blob.size,
+                disable=no_progress_bar,
+            ) as wrapped:
+                blob.download_to_file(wrapped)
 
     def _generate_download_url(self, path_info, expires=3600):
         expiration = timedelta(seconds=int(expires))

dvc/remote/hdfs.py

@@ -1,23 +1,21 @@
 from __future__ import unicode_literals
 
-import os
-import re
+from dvc.utils.compat import FileNotFoundError, urlparse
 import io
-import getpass
-import posixpath
 import logging
-from subprocess import Popen, PIPE
-from contextlib import contextmanager, closing
+import os
+import posixpath
+import re
+from collections import deque
+from contextlib import closing, contextmanager
+import subprocess
 
+from .base import RemoteBASE, RemoteCmdError
+from .pool import get_connection
 from dvc.config import Config
 from dvc.scheme import Schemes
-
-from dvc.utils.compat import urlparse, FileNotFoundError
 from dvc.utils import fix_env, tmp_fname
 
-from .pool import get_connection
-from .base import RemoteBASE, RemoteCmdError
-
 logger = logging.getLogger(__name__)
 
 
@@ -36,11 +34,7 @@ class RemoteHDFS(RemoteBASE):
 
         parsed = urlparse(url)
 
-        user = (
-            parsed.username
-            or config.get(Config.SECTION_REMOTE_USER)
-            or getpass.getuser()
-        )
+        user = parsed.username or config.get(Config.SECTION_REMOTE_USER)
 
         self.path_info = self.path_cls.from_parts(
             scheme=self.scheme,
@@ -70,15 +64,15 @@ class RemoteHDFS(RemoteBASE):
         close_fds = os.name != "nt"
 
         executable = os.getenv("SHELL") if os.name != "nt" else None
-        p = Popen(
+        p = subprocess.Popen(
             cmd,
             shell=True,
             close_fds=close_fds,
             executable=executable,
             env=fix_env(os.environ),
-            stdin=PIPE,
-            stdout=PIPE,
-            stderr=PIPE,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
         )
         out, err = p.communicate()
         if p.returncode != 0:
@@ -106,8 +100,14 @@ class RemoteHDFS(RemoteBASE):
             # NOTE: this is how `hadoop fs -cp` works too: it copies through
             # your local machine.
             with hdfs.open(from_info.path, "rb") as from_fobj:
-                with hdfs.open(to_info.path, "wb") as to_fobj:
-                    to_fobj.upload(from_fobj)
+                tmp_info = to_info.parent / tmp_fname(to_info.name)
+                try:
+                    with hdfs.open(tmp_info.path, "wb") as tmp_fobj:
+                        tmp_fobj.upload(from_fobj)
+                    hdfs.rename(tmp_info.path, to_info.path)
+                except Exception:
+                    self.remove(tmp_info)
+                    raise
 
     def remove(self, path_info):
         if path_info.scheme != "hdfs":
@@ -158,10 +158,9 @@ class RemoteHDFS(RemoteBASE):
 
     def list_cache_paths(self):
         if not self.exists(self.path_info):
-            return []
+            return
 
-        files = []
-        dirs = [self.path_info.path]
+        dirs = deque([self.path_info.path])
 
         with self.hdfs(self.path_info) as hdfs:
             while dirs:
@@ -169,6 +168,4 @@ class RemoteHDFS(RemoteBASE):
                     if entry["kind"] == "directory":
                         dirs.append(urlparse(entry["name"]).path)
                     elif entry["kind"] == "file":
-                        files.append(urlparse(entry["name"]).path)
-
-        return files
+                        yield urlparse(entry["name"]).path

dvc/remote/http.py

@@ -1,20 +1,25 @@
 from __future__ import unicode_literals
 
+import logging
+import threading
+
+from funcy import cached_property, wrap_prop
+
+from dvc.config import Config
+from dvc.config import ConfigError
+from dvc.exceptions import DvcException, HTTPError
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
 from dvc.utils.compat import open
 
-import logging
-
-from dvc.progress import Tqdm
-from dvc.exceptions import DvcException
-from dvc.config import Config, ConfigError
-from dvc.remote.base import RemoteBASE
-
 logger = logging.getLogger(__name__)
 
 
 class RemoteHTTP(RemoteBASE):
     scheme = Schemes.HTTP
+    SESSION_RETRIES = 5
+    SESSION_BACKOFF_FACTOR = 0.1
     REQUEST_TIMEOUT = 10
     CHUNK_SIZE = 2 ** 16
     PARAM_CHECKSUM = "etag"
@@ -32,30 +37,26 @@ class RemoteHTTP(RemoteBASE):
             )
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        request = self._request("GET", from_info.url, stream=True)
+        response = self._request("GET", from_info.url, stream=True)
+        if response.status_code != 200:
+            raise HTTPError(response.status_code, response.reason)
         with Tqdm(
-            total=None if no_progress_bar else self._content_length(from_info),
+            total=None if no_progress_bar else self._content_length(response),
             leave=False,
             bytes=True,
             desc=from_info.url if name is None else name,
             disable=no_progress_bar,
         ) as pbar:
             with open(to_file, "wb") as fd:
-                for chunk in request.iter_content(chunk_size=self.CHUNK_SIZE):
+                for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE):
                     fd.write(chunk)
-                    fd.flush()
                     pbar.update(len(chunk))
 
     def exists(self, path_info):
         return bool(self._request("HEAD", path_info.url))
 
-    def _content_length(self, url_or_request):
-        headers = getattr(
-            url_or_request,
-            "headers",
-            self._request("HEAD", url_or_request).headers,
-        )
-        res = headers.get("Content-Length")
+    def _content_length(self, response):
+        res = response.headers.get("Content-Length")
         return int(res) if res else None
 
     def get_file_checksum(self, path_info):
@@ -77,6 +78,25 @@ class RemoteHTTP(RemoteBASE):
 
         return etag
 
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def _session(self):
+        import requests
+        from requests.adapters import HTTPAdapter
+        from urllib3.util.retry import Retry
+
+        session = requests.Session()
+
+        retries = Retry(
+            total=self.SESSION_RETRIES,
+            backoff_factor=self.SESSION_BACKOFF_FACTOR,
+        )
+
+        session.mount("http://", HTTPAdapter(max_retries=retries))
+        session.mount("https://", HTTPAdapter(max_retries=retries))
+
+        return session
+
     def _request(self, method, url, **kwargs):
         import requests
 
@@ -84,7 +104,7 @@ class RemoteHTTP(RemoteBASE):
         kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
 
         try:
-            return requests.request(method, url, **kwargs)
+            return self._session.request(method, url, **kwargs)
         except requests.exceptions.RequestException:
             raise DvcException("could not perform a {} request".format(method))
 

dvc/remote/https.py

@@ -1,8 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.scheme import Schemes
-
 from .http import RemoteHTTP
+from dvc.scheme import Schemes
 
 
 class RemoteHTTPS(RemoteHTTP):

dvc/remote/local.py

@@ -1,39 +1,38 @@
 from __future__ import unicode_literals
 
-from dvc.scheme import Schemes
-from dvc.utils.compat import str, fspath_py35, open
-
+import errno
+import logging
 import os
 import stat
-import errno
-from shortuuid import uuid
-import logging
+from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 
-from dvc.system import System
-from dvc.remote.base import (
-    RemoteBASE,
-    STATUS_MAP,
-    STATUS_NEW,
-    STATUS_DELETED,
-    STATUS_MISSING,
-)
-from dvc.utils import (
-    remove,
-    move,
-    copyfile,
-    tmp_fname,
-    file_md5,
-    walk_files,
-    relpath,
-    dvc_walk,
-    makedirs,
-)
-from dvc.config import Config
-from dvc.exceptions import DvcException, DownloadError, UploadError
-from dvc.progress import Tqdm, TqdmThreadPoolExecutor
+from shortuuid import uuid
 
+from dvc.config import Config
+from dvc.exceptions import DownloadError
+from dvc.exceptions import DvcException
+from dvc.exceptions import UploadError
 from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.remote.base import STATUS_DELETED
+from dvc.remote.base import STATUS_MAP
+from dvc.remote.base import STATUS_MISSING
+from dvc.remote.base import STATUS_NEW
+from dvc.scheme import Schemes
+from dvc.system import System
+from dvc.utils import copyfile
+from dvc.utils import file_md5
+from dvc.utils import makedirs
+from dvc.utils import relpath
+from dvc.utils import tmp_fname
+from dvc.utils import walk_files
+from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import open
+from dvc.utils.compat import str
+from dvc.utils.fs import move
+from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
@@ -61,17 +60,12 @@ class RemoteLOCAL(RemoteBASE):
             # cache files are set to be read-only for everyone
             self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
 
-        # A clunky way to detect cache dir
-        storagepath = config.get(Config.SECTION_LOCAL_STORAGEPATH, None)
-        cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        cache_dir = config.get(Config.SECTION_REMOTE_URL)
 
         if cache_dir is not None and not os.path.isabs(cache_dir):
             cwd = config[Config.PRIVATE_CWD]
             cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
 
-        if cache_dir is not None and not os.path.exists(cache_dir):
-            os.mkdir(cache_dir)
-
         self.path_info = PathInfo(cache_dir) if cache_dir else None
         self._dir_info = {}
 
@@ -89,17 +83,7 @@ class RemoteLOCAL(RemoteBASE):
 
     def list_cache_paths(self):
         assert self.path_info is not None
-
-        clist = []
-        for entry in os.listdir(fspath_py35(self.path_info)):
-            subdir = self.path_info / entry
-            if not os.path.isdir(fspath_py35(subdir)):
-                continue
-            clist.extend(
-                subdir / cache for cache in os.listdir(fspath_py35(subdir))
-            )
-
-        return clist
+        return walk_files(self.path_info, None)
 
     def get(self, md5):
         if not md5:
@@ -144,15 +128,21 @@ class RemoteLOCAL(RemoteBASE):
     def isdir(path_info):
         return os.path.isdir(fspath_py35(path_info))
 
+    def iscopy(self, path_info):
+        return not (
+            System.is_symlink(path_info) or System.is_hardlink(path_info)
+        )
+
     @staticmethod
     def getsize(path_info):
         return os.path.getsize(fspath_py35(path_info))
 
-    def walk(self, path_info):
-        return dvc_walk(path_info, self.repo.dvcignore)
+    def walk_files(self, path_info):
+        for fname in walk_files(path_info, self.repo.dvcignore):
+            yield PathInfo(fname)
 
     def get_file_checksum(self, path_info):
-        return file_md5(fspath_py35(path_info))[0]
+        return file_md5(path_info)[0]
 
     def remove(self, path_info):
         if path_info.scheme != "local":
@@ -174,9 +164,14 @@ class RemoteLOCAL(RemoteBASE):
 
         move(from_info, to_info, mode=mode)
 
-    @staticmethod
-    def copy(from_info, to_info):
-        System.copy(from_info, to_info)
+    def copy(self, from_info, to_info):
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        try:
+            System.copy(from_info, tmp_info)
+            os.rename(fspath_py35(tmp_info), fspath_py35(to_info))
+        except Exception:
+            self.remove(tmp_info)
+            raise
 
     @staticmethod
     def symlink(from_info, to_info):
@@ -188,7 +183,7 @@ class RemoteLOCAL(RemoteBASE):
         # will get something like: `too many links error`
         #
         # This is because all those empty files will have the same checksum
-        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therfore, they will be
+        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
         # linked to the same file in the cache.
         #
         # From https://en.wikipedia.org/wiki/Hard_link
@@ -227,7 +222,7 @@ class RemoteLOCAL(RemoteBASE):
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        makedirs(fspath_py35(to_info.parent), exist_ok=True)
+        makedirs(to_info.parent, exist_ok=True)
 
         tmp_file = tmp_fname(to_info)
         copyfile(
@@ -239,41 +234,16 @@ class RemoteLOCAL(RemoteBASE):
         self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
     ):
         copyfile(
-            fspath_py35(from_info),
-            to_file,
-            no_progress_bar=no_progress_bar,
-            name=name,
+            from_info, to_file, no_progress_bar=no_progress_bar, name=name
         )
 
     @staticmethod
     def open(path_info, mode="r", encoding=None):
         return open(fspath_py35(path_info), mode=mode, encoding=encoding)
 
-    def _group(self, checksum_infos, show_checksums=False):
-        by_md5 = {}
-
-        for info in checksum_infos:
-            md5 = info[self.PARAM_CHECKSUM]
-
-            if show_checksums:
-                by_md5[md5] = {"name": md5}
-                continue
-
-            name = info["name"]
-            branch = info.get("branch")
-            if branch:
-                name += "({})".format(branch)
-
-            if md5 not in by_md5.keys():
-                by_md5[md5] = {"name": name}
-            else:
-                by_md5[md5]["name"] += " " + name
-
-        return by_md5
-
     def status(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
@@ -282,15 +252,14 @@ class RemoteLOCAL(RemoteBASE):
         logger.debug(
             "Preparing to collect status from {}".format(remote.path_info)
         )
-        ret = self._group(checksum_infos, show_checksums=show_checksums) or {}
-        md5s = list(ret)
+        md5s = list(named_cache[self.scheme])
 
         logger.debug("Collecting information from local cache...")
         local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
 
         # This is a performance optimization. We can safely assume that,
         # if the resources that we want to fetch are already cached,
-        # there's no need to check the remote storage for the existance of
+        # there's no need to check the remote storage for the existence of
         # those files.
         if download and sorted(local_exists) == sorted(md5s):
             remote_exists = local_exists
@@ -302,6 +271,10 @@ class RemoteLOCAL(RemoteBASE):
                 )
             )
 
+        ret = {
+            checksum: {"name": checksum if show_checksums else " ".join(names)}
+            for checksum, names in named_cache[self.scheme].items()
+        }
         self._fill_statuses(ret, local_exists, remote_exists)
 
         self._log_missing_caches(ret)
@@ -341,7 +314,7 @@ class RemoteLOCAL(RemoteBASE):
 
     def _process(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
@@ -369,7 +342,7 @@ class RemoteLOCAL(RemoteBASE):
             jobs = remote.JOBS
 
         status_info = self.status(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
@@ -382,7 +355,7 @@ class RemoteLOCAL(RemoteBASE):
             return 0
 
         if jobs > 1:
-            with TqdmThreadPoolExecutor(max_workers=jobs) as executor:
+            with ThreadPoolExecutor(max_workers=jobs) as executor:
                 fails = sum(executor.map(func, *plans))
         else:
             fails = sum(map(func, *plans))
@@ -394,18 +367,18 @@ class RemoteLOCAL(RemoteBASE):
 
         return len(plans[0])
 
-    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def push(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
             download=False,
         )
 
-    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
@@ -431,22 +404,19 @@ class RemoteLOCAL(RemoteBASE):
             logger.warning(msg)
 
     @staticmethod
-    def _unprotect_file(path, allow_copy=True):
+    def _unprotect_file(path):
         if System.is_symlink(path) or System.is_hardlink(path):
-            if allow_copy:
-                logger.debug("Unprotecting '{}'".format(path))
-                tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
+            logger.debug("Unprotecting '{}'".format(path))
+            tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
 
-                # The operations order is important here - if some application
-                # would access the file during the process of copyfile then it
-                # would get only the part of file. So, at first, the file
-                # should be copied with the temporary name, and then
-                # original file should be replaced by new.
-                copyfile(
-                    path, tmp, name="Unprotecting '{}'".format(relpath(path))
-                )
-                remove(path)
-                os.rename(tmp, path)
+            # The operations order is important here - if some application
+            # would access the file during the process of copyfile then it
+            # would get only the part of file. So, at first, the file should be
+            # copied with the temporary name, and then original file should be
+            # replaced by new.
+            copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
+            remove(path)
+            os.rename(tmp, path)
 
         else:
             logger.debug(
@@ -456,11 +426,11 @@ class RemoteLOCAL(RemoteBASE):
 
         os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
 
-    def _unprotect_dir(self, path, allow_copy=True):
+    def _unprotect_dir(self, path):
         for fname in walk_files(path, self.repo.dvcignore):
-            self._unprotect_file(fname, allow_copy)
+            RemoteLOCAL._unprotect_file(fname)
 
-    def unprotect(self, path_info, allow_copy=True):
+    def unprotect(self, path_info):
         path = path_info.fspath
         if not os.path.exists(path):
             raise DvcException(
@@ -468,9 +438,9 @@ class RemoteLOCAL(RemoteBASE):
             )
 
         if os.path.isdir(path):
-            self._unprotect_dir(path, allow_copy)
+            self._unprotect_dir(path)
         else:
-            self._unprotect_file(path, allow_copy)
+            RemoteLOCAL._unprotect_file(path)
 
     @staticmethod
     def protect(path_info):
@@ -544,36 +514,3 @@ class RemoteLOCAL(RemoteBASE):
             if self.is_dir_checksum(c):
                 unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
         return unpacked
-
-    def _get_cache_type(self, path_info):
-        if self.cache_type_confirmed:
-            return self.cache_types[0]
-
-        workspace_file = path_info.with_name("." + uuid())
-        test_cache_file = self.path_info / ".cache_type_test_file"
-        if not self.exists(test_cache_file):
-            with open(fspath_py35(test_cache_file), "wb") as fobj:
-                fobj.write(bytes(1))
-        try:
-            self.link(test_cache_file, workspace_file)
-        finally:
-            self.remove(workspace_file)
-            self.remove(test_cache_file)
-
-        self.cache_type_confirmed = True
-        return self.cache_types[0]
-
-    def _link_matches(self, path_info):
-        is_hardlink = System.is_hardlink(path_info)
-        is_symlink = System.is_symlink(path_info)
-        is_copy_or_reflink = not is_hardlink and not is_symlink
-
-        cache_type = self._get_cache_type(path_info)
-
-        if cache_type == "symlink":
-            return is_symlink
-
-        if cache_type == "hardlink":
-            return is_hardlink
-
-        return is_copy_or_reflink

dvc/remote/oss.py

@@ -1,15 +1,17 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-import os
 import logging
+import os
+import threading
 
-from dvc.scheme import Schemes
+from funcy import cached_property, wrap_prop
 
 from dvc.config import Config
-from dvc.remote.base import RemoteBASE
-from dvc.progress import Tqdm
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.scheme import Schemes
 
 
 logger = logging.getLogger(__name__)
@@ -62,30 +64,29 @@ class RemoteOSS(RemoteBASE):
             or "defaultSecret"
         )
 
-        self._bucket = None
-
-    @property
+    @wrap_prop(threading.Lock())
+    @cached_property
     def oss_service(self):
         import oss2
 
-        if self._bucket is None:
-            logger.debug("URL {}".format(self.path_info))
-            logger.debug("key id {}".format(self.key_id))
-            logger.debug("key secret {}".format(self.key_secret))
-            auth = oss2.Auth(self.key_id, self.key_secret)
-            self._bucket = oss2.Bucket(
-                auth, self.endpoint, self.path_info.bucket
+        logger.debug("URL {}".format(self.path_info))
+        logger.debug("key id {}".format(self.key_id))
+        logger.debug("key secret {}".format(self.key_secret))
+
+        auth = oss2.Auth(self.key_id, self.key_secret)
+        bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)
+
+        # Ensure bucket exists
+        try:
+            bucket.get_bucket_info()
+        except oss2.exceptions.NoSuchBucket:
+            bucket.create_bucket(
+                oss2.BUCKET_ACL_PUBLIC_READ,
+                oss2.models.BucketCreateConfig(
+                    oss2.BUCKET_STORAGE_CLASS_STANDARD
+                ),
             )
-            try:  # verify that bucket exists
-                self._bucket.get_bucket_info()
-            except oss2.exceptions.NoSuchBucket:
-                self._bucket.create_bucket(
-                    oss2.BUCKET_ACL_PUBLIC_READ,
-                    oss2.models.BucketCreateConfig(
-                        oss2.BUCKET_STORAGE_CLASS_STANDARD
-                    ),
-                )
-        return self._bucket
+        return bucket
 
     def remove(self, path_info):
         if path_info.scheme != self.scheme:

dvc/remote/pool.py

@@ -1,6 +1,8 @@
 from collections import deque
 from contextlib import contextmanager
-from funcy import memoize
+import threading
+
+from funcy import memoize, wrap_with
 
 
 @contextmanager
@@ -16,6 +18,7 @@ def get_connection(conn_func, *args, **kwargs):
         pool.release(conn)
 
 
+@wrap_with(threading.Lock())
 @memoize
 def get_pool(conn_func, *args, **kwargs):
     return Pool(conn_func, *args, **kwargs)

dvc/remote/s3.py

@@ -1,14 +1,18 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import os
 import logging
-from funcy import cached_property
+import os
+import threading
 
-from dvc.progress import Tqdm
-from dvc.config import Config
-from dvc.remote.base import RemoteBASE
-from dvc.exceptions import DvcException, ETagMismatchError
+from funcy import cached_property, wrap_prop
+
+from dvc.config import Config, ConfigError
+from dvc.exceptions import DvcException
+from dvc.exceptions import ETagMismatchError
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
@@ -23,11 +27,7 @@ class RemoteS3(RemoteBASE):
     def __init__(self, repo, config):
         super(RemoteS3, self).__init__(repo, config)
 
-        storagepath = "s3://{}".format(
-            config.get(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
-        )
-
-        url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        url = config.get(Config.SECTION_REMOTE_URL, "s3://")
         self.path_info = self.path_cls(url)
 
         self.region = config.get(Config.SECTION_AWS_REGION)
@@ -36,7 +36,10 @@ class RemoteS3(RemoteBASE):
 
         self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
 
-        self.list_objects = config.get(Config.SECTION_AWS_LIST_OBJECTS)
+        if config.get(Config.SECTION_AWS_LIST_OBJECTS):
+            self.list_objects_api = "list_objects"
+        else:
+            self.list_objects_api = "list_objects_v2"
 
         self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
 
@@ -50,10 +53,13 @@ class RemoteS3(RemoteBASE):
         if self.acl:
             self.extra_args["ACL"] = self.acl
 
+        self._append_aws_grants_to_extra_args(config)
+
         shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
         if shared_creds:
             os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def s3(self):
         import boto3
@@ -139,13 +145,13 @@ class RemoteS3(RemoteBASE):
         # multipart.
         #
         # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
-        # then it was transfered as a single part, which means that the chunk
+        # then it was transferred as a single part, which means that the chunk
         # size used to transfer it was greater or equal to the ContentLength
         # of that object. So to preserve that tag over the next transfer, we
         # could use any value >= ContentLength.
         #
         # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
-        # then it was transfered as a multipart, which means that the chunk
+        # then it was transferred as a multipart, which means that the chunk
         # size used to transfer it was less than ContentLength of that object.
         # Unfortunately, in general, it doesn't mean that the chunk size was
         # the same throughout the transfer, so it means that in order to
@@ -180,27 +186,84 @@ class RemoteS3(RemoteBASE):
         logger.debug("Removing {}".format(path_info))
         self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
 
-    def _list_paths(self, bucket, prefix):
+    def _list_objects(self, path_info, max_items=None):
         """ Read config for list object api, paginate through list objects."""
-        kwargs = {"Bucket": bucket, "Prefix": prefix}
-        if self.list_objects:
-            list_objects_api = "list_objects"
-        else:
-            list_objects_api = "list_objects_v2"
-        paginator = self.s3.get_paginator(list_objects_api)
+        kwargs = {
+            "Bucket": path_info.bucket,
+            "Prefix": path_info.path,
+            "PaginationConfig": {"MaxItems": max_items},
+        }
+        paginator = self.s3.get_paginator(self.list_objects_api)
         for page in paginator.paginate(**kwargs):
             contents = page.get("Contents", None)
             if not contents:
                 continue
             for item in contents:
-                yield item["Key"]
+                yield item
+
+    def _list_paths(self, path_info, max_items=None):
+        return (
+            item["Key"] for item in self._list_objects(path_info, max_items)
+        )
 
     def list_cache_paths(self):
-        return self._list_paths(self.path_info.bucket, self.path_info.path)
+        return self._list_paths(self.path_info)
+
+    def isfile(self, path_info):
+        from botocore.exceptions import ClientError
+
+        if path_info.path.endswith("/"):
+            return False
+
+        try:
+            self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
+        except ClientError as exc:
+            if exc.response["Error"]["Code"] != "404":
+                raise
+            return False
+
+        return True
 
     def exists(self, path_info):
-        paths = self._list_paths(path_info.bucket, path_info.path)
-        return any(path_info.path == path for path in paths)
+        """Check if the blob exists. If it does not exist,
+        it could be a part of a directory path.
+
+        eg: if `data/file.txt` exists, check for `data` should return True
+        """
+        return self.isfile(path_info) or self.isdir(path_info)
+
+    def makedirs(self, path_info):
+        # We need to support creating empty directories, which means
+        # creating an object with an empty body and a trailing slash `/`.
+        #
+        # We are not creating directory objects for every parent prefix,
+        # as it is not required.
+        dir_path = path_info / ""
+        self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
+
+    def isdir(self, path_info):
+        # S3 doesn't have a concept for directories.
+        #
+        # Using `head_object` with a path pointing to a directory
+        # will throw a 404 error.
+        #
+        # A reliable way to know if a given path is a directory is by
+        # checking if there are more files sharing the same prefix
+        # with a `list_objects` call.
+        #
+        # We need to make sure that the path ends with a forward slash,
+        # since we can end with false-positives like the following example:
+        #
+        #       bucket
+        #       └── data
+        #          ├── alice
+        #          └── alpha
+        #
+        # Using `data/al` as prefix will return `[data/alice, data/alpha]`,
+        # While `data/al/` will return nothing.
+        #
+        dir_path = path_info / ""
+        return bool(list(self._list_paths(dir_path, max_items=1)))
 
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
         total = os.path.getsize(from_file)
@@ -234,3 +297,41 @@ class RemoteS3(RemoteBASE):
         return self.s3.generate_presigned_url(
             ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
         )
+
+    def walk_files(self, path_info, max_items=None):
+        for fname in self._list_paths(path_info / "", max_items):
+            if fname.endswith("/"):
+                continue
+
+            yield path_info.replace(path=fname)
+
+    def _append_aws_grants_to_extra_args(self, config):
+        # Keys for extra_args can be one of the following list:
+        # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS
+        """
+          ALLOWED_UPLOAD_ARGS = [
+            'ACL', 'CacheControl', 'ContentDisposition', 'ContentEncoding',
+            'ContentLanguage', 'ContentType', 'Expires', 'GrantFullControl',
+            'GrantRead', 'GrantReadACP', 'GrantWriteACP', 'Metadata',
+            'RequestPayer', 'ServerSideEncryption', 'StorageClass',
+            'SSECustomerAlgorithm', 'SSECustomerKey', 'SSECustomerKeyMD5',
+            'SSEKMSKeyId', 'WebsiteRedirectLocation'
+          ]
+        """
+
+        grants = {
+            Config.SECTION_AWS_GRANT_FULL_CONTROL: "GrantFullControl",
+            Config.SECTION_AWS_GRANT_READ: "GrantRead",
+            Config.SECTION_AWS_GRANT_READ_ACP: "GrantReadACP",
+            Config.SECTION_AWS_GRANT_WRITE_ACP: "GrantWriteACP",
+        }
+
+        for grant_option, extra_args_key in grants.items():
+            if config.get(grant_option, ""):
+                if self.acl:
+                    raise ConfigError(
+                        "`acl` and `grant_*` AWS S3 config options "
+                        "are mutually exclusive"
+                    )
+
+                self.extra_args[extra_args_key] = config.get(grant_option)

dvc/remote/slow_link_detection.py

@@ -1,9 +1,10 @@
 import logging
-import time
 import sys
+import time
 from functools import wraps
 
 import colorama
+
 from dvc.config import Config
 
 
@@ -18,7 +19,7 @@ this.message = (
     "See {blue}https://dvc.org/doc/commands-reference/config#cache{reset} "
     "for more information.\n"
     "To disable this message, run:\n"
-    "'dvc config cache.slow_link_warning false'".format(
+    "`dvc config cache.slow_link_warning false`".format(
         blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
     )
 )

dvc/remote/ssh/__init__.py

@@ -1,29 +1,39 @@
 from __future__ import unicode_literals
 
 import errno
-import itertools
-import io
-import os
 import getpass
+import io
+import itertools
 import logging
+import os
 import threading
 from concurrent.futures import ThreadPoolExecutor
-from contextlib import contextmanager, closing
+from contextlib import closing, contextmanager
+
+from funcy import memoize, wrap_with
 
 import dvc.prompt as prompt
 from dvc.config import Config
-from dvc.utils import to_chunks
-from dvc.utils.compat import urlparse, StringIO
-from dvc.remote.base import RemoteBASE
-from dvc.scheme import Schemes
-from dvc.remote.pool import get_connection
 from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.remote.pool import get_connection
+from dvc.scheme import Schemes
+from dvc.utils import to_chunks
+from dvc.utils.compat import StringIO
+from dvc.utils.compat import urlparse
 
 logger = logging.getLogger(__name__)
 
 
-saved_passwords = {}
-saved_passwords_lock = threading.Lock()
+@wrap_with(threading.Lock())
+@memoize
+def ask_password(host, user, port):
+    return prompt.password(
+        "Enter a private key passphrase or a password for "
+        "host '{host}' port '{port}' user '{user}'".format(
+            host=host, port=port, user=user
+        )
+    )
 
 
 class RemoteSSH(RemoteBASE):
@@ -115,31 +125,25 @@ class RemoteSSH(RemoteBASE):
             return identity_file[0]
         return None
 
-    def ssh(self, path_info):
-        host, user, port = path_info.host, path_info.user, path_info.port
+    def ensure_credentials(self, path_info=None):
+        if path_info is None:
+            path_info = self.path_info
 
         # NOTE: we use the same password regardless of the server :(
         if self.ask_password and self.password is None:
-            with saved_passwords_lock:
-                server_key = (host, user, port)
-                password = saved_passwords.get(server_key)
+            host, user, port = path_info.host, path_info.user, path_info.port
+            self.password = ask_password(host, user, port)
 
-                if password is None:
-                    saved_passwords[server_key] = password = prompt.password(
-                        "Enter a private key passphrase or a password for "
-                        "host '{host}' port '{port}' user '{user}'".format(
-                            host=host, port=port, user=user
-                        )
-                    )
-                self.password = password
+    def ssh(self, path_info):
+        self.ensure_credentials(path_info)
 
         from .connection import SSHConnection
 
         return get_connection(
             SSHConnection,
-            host,
-            username=user,
-            port=port,
+            path_info.host,
+            username=path_info.user,
+            port=path_info.port,
             key_filename=self.keyfile,
             timeout=self.timeout,
             password=self.password,
@@ -174,7 +178,7 @@ class RemoteSSH(RemoteBASE):
             raise NotImplementedError
 
         with self.ssh(from_info) as ssh:
-            ssh.copy(from_info.path, to_info.path)
+            ssh.atomic_copy(from_info.path, to_info.path)
 
     def symlink(self, from_info, to_info):
         if not from_info.scheme == to_info.scheme == self.scheme:
@@ -258,12 +262,14 @@ class RemoteSSH(RemoteBASE):
 
     def list_cache_paths(self):
         with self.ssh(self.path_info) as ssh:
-            return list(ssh.walk_files(self.path_info.path))
+            # If we simply return an iterator then with above closes instantly
+            for path in ssh.walk_files(self.path_info.path):
+                yield path
 
-    def walk(self, path_info):
+    def walk_files(self, path_info):
         with self.ssh(path_info) as ssh:
-            for entry in ssh.walk(path_info.path):
-                yield entry
+            for fname in ssh.walk_files(path_info.path):
+                yield path_info.replace(path=fname)
 
     def makedirs(self, path_info):
         with self.ssh(path_info) as ssh:
@@ -306,6 +312,9 @@ class RemoteSSH(RemoteBASE):
         if not self.no_traverse:
             return list(set(checksums) & set(self.all()))
 
+        # possibly prompt for credentials before "Querying" progress output
+        self.ensure_credentials()
+
         with Tqdm(
             desc="Querying "
             + ("cache in " + name if name else "remote cache"),

dvc/remote/ssh/connection.py

@@ -1,8 +1,9 @@
+import errno
+import logging
 import os
 import posixpath
-import logging
-import errno
 import stat
+
 from funcy import cached_property
 
 try:
@@ -106,7 +107,10 @@ class SSHConnection:
                 # Since paramiko errors are very vague we need to recheck
                 # whether it's because path already exists or something else
                 if e.errno == errno.EACCES or not self.exists(path):
-                    raise
+                    raise DvcException(
+                        "unable to create remote directory '{}'".format(path),
+                        cause=e,
+                    )
 
     def walk(self, directory, topdown=True):
         # NOTE: original os.walk() implementation [1] with default options was
@@ -182,8 +186,26 @@ class SSHConnection:
             self.sftp.get(src, dest, callback=pbar.update_to)
 
     def move(self, src, dst):
+        """Rename src to dst, if it is not possible (in case src and dst are
+        on different filesystems) and actual physical copying of data is
+        happening.
+        """
         self.makedirs(posixpath.dirname(dst))
-        self.sftp.rename(src, dst)
+
+        try:
+            self.sftp.rename(src, dst)
+        except OSError:
+            self.atomic_copy(src, dst)
+            self.remove(src)
+
+    def atomic_copy(self, src, dst):
+        tmp = tmp_fname(dst)
+
+        try:
+            self.copy(src, tmp)
+            self.sftp.rename(tmp, dst)
+        finally:
+            self.remove(tmp)
 
     def upload(self, src, dest, no_progress_bar=False, progress_title=None):
         self.makedirs(posixpath.dirname(dest))

dvc/repo/__init__.py

@@ -1,25 +1,30 @@
 from __future__ import unicode_literals
 
-import os
 import logging
+import os
 from contextlib import contextmanager
+from functools import wraps
 from itertools import chain
 
-from functools import wraps
 from funcy import cached_property
 
+from .graph import check_acyclic
+from .graph import get_pipeline
+from .graph import get_pipelines
+from .graph import get_stages
 from dvc.config import Config
-from dvc.exceptions import (
-    NotDvcRepoError,
-    OutputNotFoundError,
-    TargetNotDirectoryError,
-    OutputFileMissingError,
-)
+from dvc.exceptions import FileMissingError
+from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import OutputNotFoundError
 from dvc.ignore import DvcIgnoreFilter
 from dvc.path_info import PathInfo
 from dvc.remote.base import RemoteActionNotImplemented
-from dvc.utils.compat import open as _open, fspath_py35, FileNotFoundError
 from dvc.utils import relpath
+from dvc.utils.fs import path_isin
+from dvc.utils.compat import FileNotFoundError
+from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import open as _open
+
 
 logger = logging.getLogger(__name__)
 
@@ -27,8 +32,11 @@ logger = logging.getLogger(__name__)
 def locked(f):
     @wraps(f)
     def wrapper(repo, *args, **kwargs):
-        with repo.lock:
-            return f(repo, *args, **kwargs)
+        with repo.lock, repo.state:
+            ret = f(repo, *args, **kwargs)
+            # Our graph cache is no longer valid after we release the repo.lock
+            repo._reset()
+            return ret
 
     return wrapper
 
@@ -61,13 +69,14 @@ class Repo(object):
 
     def __init__(self, root_dir=None):
         from dvc.state import State
-        from dvc.lock import Lock
+        from dvc.lock import make_lock
         from dvc.scm import SCM
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
         from dvc.repo.metrics import Metrics
         from dvc.scm.tree import WorkingTree
         from dvc.repo.tag import Tag
+        from dvc.utils import makedirs
 
         root_dir = self.find_root(root_dir)
 
@@ -80,10 +89,17 @@ class Repo(object):
 
         self.tree = WorkingTree(self.root_dir)
 
-        self.lock = Lock(
+        self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
+        makedirs(self.tmp_dir, exist_ok=True)
+
+        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        self.lock = make_lock(
             os.path.join(self.dvc_dir, "lock"),
             tmp_dir=os.path.join(self.dvc_dir, "tmp"),
+            hardlink_lock=hardlink_lock,
+            friendly=True,
         )
+
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
         self.state = State(self, self.config.config)
@@ -102,6 +118,17 @@ class Repo(object):
 
         self._ignore()
 
+    @property
+    def tree(self):
+        return self._tree
+
+    @tree.setter
+    def tree(self, tree):
+        self._tree = tree
+        # Our graph cache is no longer valid, as it was based on the previous
+        # tree.
+        self._reset()
+
     def __repr__(self):
         return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
 
@@ -143,54 +170,52 @@ class Repo(object):
 
         flist = (
             [self.config.config_local_file, updater.updater_file]
+            + [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir]
             + self.state.files
-            + self.lock.files
-            + updater.lock.files
         )
 
-        if self.cache.local.cache_dir.startswith(self.root_dir):
+        if path_isin(self.cache.local.cache_dir, self.root_dir):
             flist += [self.cache.local.cache_dir]
 
         self.scm.ignore_list(flist)
 
-    def check_dag(self, stages):
+    def check_modified_graph(self, new_stages):
         """Generate graph including the new stage to check for errors"""
-        self.graph(stages=stages)
+        self._collect_graph(self.stages + new_stages)
 
-    @staticmethod
-    def _check_cyclic_graph(graph):
-        import networkx as nx
-        from dvc.exceptions import CyclicGraphError
-
-        cycles = list(nx.simple_cycles(graph))
-
-        if cycles:
-            raise CyclicGraphError(cycles[0])
-
-    def _get_pipeline(self, node):
-        pipelines = [i for i in self.pipelines() if i.has_node(node)]
-        assert len(pipelines) == 1
-        return pipelines[0]
-
-    def collect(self, target, with_deps=False, recursive=False):
+    def collect(self, target, with_deps=False, recursive=False, graph=None):
         import networkx as nx
         from dvc.stage import Stage
 
-        if not target or (recursive and os.path.isdir(target)):
-            return self.stages(target)
+        G = graph or self.graph
+
+        if not target:
+            return get_stages(G)
+
+        target = os.path.abspath(target)
+
+        if recursive and os.path.isdir(target):
+            attrs = nx.get_node_attributes(G, "stage")
+            nodes = [node for node in nx.dfs_postorder_nodes(G)]
+
+            ret = []
+            for node in nodes:
+                stage = attrs[node]
+                if path_isin(stage.path, target):
+                    ret.append(stage)
+            return ret
 
         stage = Stage.load(self, target)
         if not with_deps:
             return [stage]
 
         node = relpath(stage.path, self.root_dir)
-        G = self._get_pipeline(node)
+        pipeline = get_pipeline(get_pipelines(G), node)
 
-        ret = []
-        for n in nx.dfs_postorder_nodes(G, node):
-            ret.append(G.node[n]["stage"])
-
-        return ret
+        return [
+            pipeline.node[n]["stage"]
+            for n in nx.dfs_postorder_nodes(pipeline, node)
+        ]
 
     def used_cache(
         self,
@@ -198,6 +223,7 @@ class Repo(object):
         all_branches=False,
         with_deps=False,
         all_tags=False,
+        all_commits=False,
         remote=None,
         force=False,
         jobs=None,
@@ -216,49 +242,41 @@ class Repo(object):
             A dictionary with Schemes (representing output's location) as keys,
             and a list with the outputs' `dumpd` as values.
         """
+        from dvc.cache import NamedCache
 
-        cache = {}
-        cache["local"] = []
-        cache["s3"] = []
-        cache["gs"] = []
-        cache["hdfs"] = []
-        cache["ssh"] = []
-        cache["azure"] = []
-        cache["repo"] = []
+        cache = NamedCache()
 
         for branch in self.brancher(
-            all_branches=all_branches, all_tags=all_tags
+            all_branches=all_branches,
+            all_tags=all_tags,
+            all_commits=all_commits,
         ):
             if targets:
                 stages = []
                 for target in targets:
-                    if recursive and os.path.isdir(target):
-                        stages.extend(self.stages(target))
-                    else:
-                        stages.extend(
-                            self.collect(target, with_deps=with_deps)
-                        )
+                    collected = self.collect(
+                        target, recursive=recursive, with_deps=with_deps
+                    )
+                    stages.extend(collected)
             else:
-                stages = self.stages()
+                stages = self.stages
 
             for stage in stages:
                 if stage.is_repo_import:
-                    cache["repo"] += stage.deps
+                    dep, = stage.deps
+                    cache.external[dep.repo_pair].add(dep.def_path)
                     continue
 
                 for out in stage.outs:
-                    scheme = out.path_info.scheme
                     used_cache = out.get_used_cache(
                         remote=remote, force=force, jobs=jobs
                     )
-
-                    cache[scheme].extend(
-                        dict(entry, branch=branch) for entry in used_cache
-                    )
+                    suffix = "({})".format(branch) if branch else ""
+                    cache.update(used_cache, suffix=suffix)
 
         return cache
 
-    def graph(self, stages=None, from_directory=None):
+    def _collect_graph(self, stages=None):
         """Generate a graph by using the given stages on the given directory
 
         The nodes of the graph are the stage's path relative to the root.
@@ -288,11 +306,8 @@ class Repo(object):
               (weakly connected components)
 
         Args:
-            stages (list): used to build a graph, if None given, use the ones
-                on the `from_directory`.
-
-            from_directory (str): directory where to look at for stages, if
-                None is given, use the current working directory
+            stages (list): used to build a graph, if None given, collect stages
+                in the repository.
 
         Raises:
             OutputDuplicationError: two outputs with the same path
@@ -308,8 +323,7 @@ class Repo(object):
         )
 
         G = nx.DiGraph()
-        G_active = nx.DiGraph()
-        stages = stages or self.stages(from_directory, check_dag=False)
+        stages = stages or self.collect_stages()
         stages = [stage for stage in stages if stage]
         outs = {}
 
@@ -336,38 +350,29 @@ class Repo(object):
             node = relpath(stage.path, self.root_dir)
 
             G.add_node(node, stage=stage)
-            G_active.add_node(node, stage=stage)
 
             for dep in stage.deps:
                 if dep.path_info is None:
                     continue
 
                 for out in outs:
-                    if (
-                        out == dep.path_info
-                        or dep.path_info.isin(out)
-                        or out.isin(dep.path_info)
-                    ):
+                    if out.overlaps(dep.path_info):
                         dep_stage = outs[out].stage
                         dep_node = relpath(dep_stage.path, self.root_dir)
                         G.add_node(dep_node, stage=dep_stage)
                         G.add_edge(node, dep_node)
-                        if not stage.locked:
-                            G_active.add_node(dep_node, stage=dep_stage)
-                            G_active.add_edge(node, dep_node)
 
-        self._check_cyclic_graph(G)
+        check_acyclic(G)
 
-        return G, G_active
+        return G
 
-    def pipelines(self, from_directory=None):
-        import networkx as nx
+    @cached_property
+    def graph(self):
+        return self._collect_graph()
 
-        G, G_active = self.graph(from_directory=from_directory)
-
-        return [
-            G.subgraph(c).copy() for c in nx.weakly_connected_components(G)
-        ]
+    @cached_property
+    def pipelines(self):
+        return get_pipelines(self.graph)
 
     @staticmethod
     def _filter_out_dirs(dirs, outs, root_dir):
@@ -380,7 +385,7 @@ class Repo(object):
 
         return list(filter(filter_dirs, dirs))
 
-    def stages(self, from_directory=None, check_dag=True):
+    def collect_stages(self):
         """
         Walks down the root directory looking for Dvcfiles,
         skipping the directories that are related with
@@ -392,16 +397,11 @@ class Repo(object):
         """
         from dvc.stage import Stage
 
-        if not from_directory:
-            from_directory = self.root_dir
-        elif not os.path.isdir(from_directory):
-            raise TargetNotDirectoryError(from_directory)
-
         stages = []
         outs = []
 
         for root, dirs, files in self.tree.walk(
-            from_directory, dvcignore=self.dvcignore
+            self.root_dir, dvcignore=self.dvcignore
         ):
             for fname in files:
                 path = os.path.join(root, fname)
@@ -415,18 +415,15 @@ class Repo(object):
 
             dirs[:] = self._filter_out_dirs(dirs, outs, root)
 
-        if check_dag:
-            self.check_dag(stages)
-
         return stages
 
+    @cached_property
+    def stages(self):
+        return get_stages(self.graph)
+
     def find_outs_by_path(self, path, outs=None, recursive=False):
         if not outs:
-            # there is no `from_directory=path` argument because some data
-            # files might be generated to an upper level, and so it is
-            # needed to look at all the files (project root_dir)
-            stages = self.stages()
-            outs = [out for stage in stages for out in stage.outs]
+            outs = [out for stage in self.stages for out in stage.outs]
 
         abs_path = os.path.abspath(path)
         is_dir = self.tree.isdir(abs_path)
@@ -442,7 +439,7 @@ class Repo(object):
 
         matched = list(filter(func, outs))
         if not matched:
-            raise OutputNotFoundError(path)
+            raise OutputNotFoundError(path, self)
 
         return matched
 
@@ -458,14 +455,29 @@ class Repo(object):
     @contextmanager
     def open(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
+        cause = None
         try:
-            with self._open(path, remote, mode, encoding) as fd:
-                yield fd
-        except FileNotFoundError:
-            raise OutputFileMissingError(relpath(path, self.root_dir))
+            out, = self.find_outs_by_path(path)
+        except OutputNotFoundError as e:
+            out = None
+            cause = e
 
-    def _open(self, path, remote=None, mode="r", encoding=None):
-        out, = self.find_outs_by_path(path)
+        if out and out.use_cache:
+            try:
+                with self._open_cached(out, remote, mode, encoding) as fd:
+                    yield fd
+                return
+            except FileNotFoundError as e:
+                raise FileMissingError(relpath(path, self.root_dir), cause=e)
+
+        if self.tree.exists(path):
+            with self.tree.open(path, mode, encoding) as fd:
+                yield fd
+            return
+
+        raise FileMissingError(relpath(path, self.root_dir), cause=cause)
+
+    def _open_cached(self, out, remote=None, mode="r", encoding=None):
         if out.isdir():
             raise ValueError("Can't open a dir")
 
@@ -488,20 +500,11 @@ class Repo(object):
 
     @cached_property
     def dvcignore(self):
-        return DvcIgnoreFilter(self.root_dir)
+        return DvcIgnoreFilter(self.root_dir, self.tree)
 
     def close(self):
         self.scm.close()
 
-    @staticmethod
-    def clone(url, to_path, rev=None):
-        from dvc.scm.git import Git
-
-        git = Git.clone(url, to_path, rev=rev)
-        git.close()
-
-        return Repo(to_path)
-
     @locked
     def checkout(self, *args, **kwargs):
         return self._checkout(*args, **kwargs)
@@ -509,3 +512,9 @@ class Repo(object):
     @locked
     def fetch(self, *args, **kwargs):
         return self._fetch(*args, **kwargs)
+
+    def _reset(self):
+        self.__dict__.pop("graph", None)
+        self.__dict__.pop("stages", None)
+        self.__dict__.pop("pipelines", None)
+        self.__dict__.pop("dvcignore", None)

dvc/repo/add.py

@@ -1,46 +1,67 @@
-import os
-import logging
-import colorama
+from __future__ import unicode_literals
 
-from dvc.repo.scm_context import scm_context
-from dvc.stage import Stage
-from dvc.utils import walk_files, LARGE_DIR_SIZE
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
+import logging
+import os
+
+import colorama
+from six import string_types
 
 from . import locked
-
+from dvc.exceptions import RecursiveAddingWhileUsingFilename
+from dvc.progress import Tqdm
+from dvc.repo.scm_context import scm_context
+from dvc.stage import Stage
+from dvc.utils import LARGE_DIR_SIZE
+from dvc.utils import walk_files
 
 logger = logging.getLogger(__name__)
 
 
 @locked
 @scm_context
-def add(repo, target, recursive=False, no_commit=False, fname=None):
+def add(repo, targets, recursive=False, no_commit=False, fname=None):
     if recursive and fname:
         raise RecursiveAddingWhileUsingFilename()
 
-    targets = _find_all_targets(repo, target, recursive)
+    if isinstance(targets, string_types):
+        targets = [targets]
 
-    if os.path.isdir(target) and len(targets) > LARGE_DIR_SIZE:
-        logger.warning(
-            "You are adding a large directory '{target}' recursively,"
-            " consider tracking it as a whole instead.\n"
-            "{purple}HINT:{nc} Remove the generated DVC-file and then"
-            " run {cyan}dvc add {target}{nc}".format(
-                purple=colorama.Fore.MAGENTA,
-                cyan=colorama.Fore.CYAN,
-                nc=colorama.Style.RESET_ALL,
-                target=target,
-            )
-        )
+    stages_list = []
+    with Tqdm(total=len(targets), desc="Add", unit="file", leave=True) as pbar:
+        for target in targets:
+            sub_targets = _find_all_targets(repo, target, recursive)
+            pbar.total += len(sub_targets) - 1
 
-    stages = _create_stages(repo, targets, fname, no_commit)
+            if os.path.isdir(target) and len(sub_targets) > LARGE_DIR_SIZE:
+                logger.warning(
+                    "You are adding a large directory '{target}' recursively,"
+                    " consider tracking it as a whole instead.\n"
+                    "{purple}HINT:{nc} Remove the generated DVC-file and then"
+                    " run {cyan}dvc add {target}{nc}".format(
+                        purple=colorama.Fore.MAGENTA,
+                        cyan=colorama.Fore.CYAN,
+                        nc=colorama.Style.RESET_ALL,
+                        target=target,
+                    )
+                )
 
-    repo.check_dag(repo.stages() + stages)
+            stages = _create_stages(repo, sub_targets, fname, pbar=pbar)
 
-    for stage in stages:
-        stage.dump()
-    return stages
+            repo.check_modified_graph(stages)
+
+            for stage in stages:
+                stage.save()
+
+                if not no_commit:
+                    stage.commit()
+
+                stage.dump()
+
+            stages_list += stages
+        # remove filled bar bit of progress, leaving stats
+        pbar.bar_format = pbar.BAR_FMT_DEFAULT.replace("|{bar:10}|", " ")
+
+    return stages_list
 
 
 def _find_all_targets(repo, target, recursive):
@@ -56,21 +77,19 @@ def _find_all_targets(repo, target, recursive):
     return [target]
 
 
-def _create_stages(repo, targets, fname, no_commit):
+def _create_stages(repo, targets, fname, pbar=None):
     stages = []
 
-    with repo.state:
-        for out in targets:
-            stage = Stage.create(repo, outs=[out], add=True, fname=fname)
+    for out in targets:
+        stage = Stage.create(repo, outs=[out], add=True, fname=fname)
 
-            if not stage:
-                continue
+        if not stage:
+            if pbar is not None:
+                pbar.total -= 1
+            continue
 
-            stage.save()
-
-            if not no_commit:
-                stage.commit()
-
-            stages.append(stage)
+        stages.append(stage)
+        if pbar is not None:
+            pbar.update_desc(out)
 
     return stages

dvc/repo/brancher.py

@@ -1,8 +1,15 @@
+from funcy import group_by
+
 from dvc.scm.tree import WorkingTree
 
 
 def brancher(  # noqa: E302
-    self, branches=None, all_branches=False, tags=None, all_tags=False
+    self,
+    branches=None,
+    all_branches=False,
+    tags=None,
+    all_tags=False,
+    all_commits=False,
 ):
     """Generator that iterates over specified revisions.
 
@@ -16,9 +23,9 @@ def brancher(  # noqa: E302
         str: the display name for the currently selected tree, it could be:
             - a git revision identifier
             - empty string it there is no branches to iterate over
-            - "Working Tree" if there are uncommited changes in the SCM repo
+            - "Working Tree" if there are uncommitted changes in the SCM repo
     """
-    if not any([branches, all_branches, tags, all_tags]):
+    if not any([branches, all_branches, tags, all_tags, all_commits]):
         yield ""
         return
 
@@ -30,25 +37,28 @@ def brancher(  # noqa: E302
     self.tree = WorkingTree(self.root_dir)
     yield "working tree"
 
-    if all_branches:
-        branches = scm.list_branches()
+    if all_commits:
+        revs = scm.list_all_commits()
+    else:
+        if all_branches:
+            branches = scm.list_branches()
 
-    if all_tags:
-        tags = scm.list_tags()
+        if all_tags:
+            tags = scm.list_tags()
 
-    if branches is not None:
-        revs.extend(branches)
+        if branches is not None:
+            revs.extend(branches)
 
-    if tags is not None:
-        revs.extend(tags)
+        if tags is not None:
+            revs.extend(tags)
 
     # NOTE: it might be a good idea to wrap this loop in try/finally block
     # to don't leave the tree on some unexpected branch after the
     # `brancher()`, but this could cause problems on exception handling
     # code which might expect the tree on which exception was raised to
     # stay in place. This behavior is a subject to change.
-    for rev in revs:
-        self.tree = scm.get_tree(rev)
-        yield rev
+    for sha, names in group_by(scm.resolve_rev, revs).items():
+        self.tree = scm.get_tree(sha)
+        yield ", ".join(names)
 
     self.tree = saved_tree

dvc/repo/checkout.py

@@ -2,21 +2,22 @@ from __future__ import unicode_literals
 
 import logging
 
-from dvc.exceptions import CheckoutErrorSuggestGit, CheckoutError
+from dvc.exceptions import CheckoutError
+from dvc.exceptions import CheckoutErrorSuggestGit
 from dvc.progress import Tqdm
 
 
 logger = logging.getLogger(__name__)
 
 
-def _cleanup_unused_links(self, all_stages):
+def _cleanup_unused_links(repo):
     used = [
         out.fspath
-        for stage in all_stages
+        for stage in repo.stages
         for out in stage.outs
         if out.scheme == "local"
     ]
-    self.state.remove_unused_links(used)
+    repo.state.remove_unused_links(used)
 
 
 def get_all_files_numbers(stages):
@@ -24,41 +25,46 @@ def get_all_files_numbers(stages):
 
 
 def _checkout(
-    self, target=None, with_deps=False, force=False, recursive=False
+    self,
+    targets=None,
+    with_deps=False,
+    force=False,
+    relink=False,
+    recursive=False,
 ):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    all_stages = self.stages()
+    stages = set()
 
-    try:
-        stages = self.collect(target, with_deps=with_deps, recursive=recursive)
-    except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
-        if not target:
-            raise
-        raise CheckoutErrorSuggestGit(target, exc)
+    if not targets:
+        targets = [None]
+        _cleanup_unused_links(self)
 
-    with self.state:
-        _cleanup_unused_links(self, all_stages)
-        total = get_all_files_numbers(stages)
-        if total == 0:
-            logger.info("Nothing to do")
-        failed = []
-        with Tqdm(
-            total=total, unit="file", desc="Checkout", disable=total == 0
-        ) as pbar:
-            for stage in stages:
-                if stage.locked:
-                    logger.warning(
-                        "DVC-file '{path}' is locked. Its dependencies are"
-                        " not going to be checked out.".format(
-                            path=stage.relpath
-                        )
-                    )
+    for target in targets:
+        try:
+            new = self.collect(
+                target, with_deps=with_deps, recursive=recursive
+            )
+            stages.update(new)
+        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+            if not target:
+                raise
+            raise CheckoutErrorSuggestGit(target, exc)
 
-                failed.extend(
-                    stage.checkout(
-                        force=force, progress_callback=pbar.update_desc
-                    )
+    total = get_all_files_numbers(stages)
+    if total == 0:
+        logger.info("Nothing to do")
+    failed = []
+    with Tqdm(
+        total=total, unit="file", desc="Checkout", disable=total == 0
+    ) as pbar:
+        for stage in stages:
+            failed.extend(
+                stage.checkout(
+                    force=force,
+                    progress_callback=pbar.update_desc,
+                    relink=relink,
                 )
-        if failed:
-            raise CheckoutError(failed)
+            )
+    if failed:
+        raise CheckoutError(failed)