dvc: bump to 0.74.0

remote: base: download: don't walk dir twice

.deepsource.toml

@@ -5,7 +5,7 @@ test_patterns = [
 ]
 
 exclude_patterns = [
-  
+  'dvc/utils/compat.py'
 ]
 
 [[analyzers]]

.dvc/.gitignore

@@ -8,3 +8,4 @@
 /cache
 /pkg
 /repos
+/tmp

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,9 +1,8 @@
-* [ ] Have you followed the guidelines in our
-      [Contributing document](https://dvc.org/doc/user-guide/contributing)?
+* [ ] ❗ Have you followed the guidelines in the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) list?
 
-* [ ] Does your PR affect documented changes or does it add new functionality
-      that should be documented? If yes, have you created a PR for
-      [dvc.org](https://github.com/iterative/dvc.org) documenting it or at
-      least opened an issue for it? If so, please add a link to it.
+* [ ] 📖 Check this box if this PR **does not** require [documentation](https://dvc.org/doc) updates, or if it does **and** you have created a separate PR in [dvc.org](https://github.com/iterative/dvc.org) with such updates (or at least opened an issue about it in that repo). Please link below to your PR (or issue) in the [dvc.org](https://github.com/iterative/dvc.org) repo.
+
+* [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect everything to be addressed. Please review them carefully and fix those that actually improve code or fix bugs.
+
+Thank you for the contribution - we'll try to review it as soon as possible. 🙏
 
------

.github/release-drafter.yml

@@ -0,0 +1,5 @@
+# Config for https://github.com/apps/release-drafter
+branches:
+  - master
+template: |
+  $CHANGES

.gitignore

@@ -4,6 +4,7 @@ neatlynx/__pycache__
 cache
 *.pyc
 .env/
+.env2.7/
 
 cache
 .dvc.conf.lock

.pylintrc

@@ -139,7 +139,9 @@ disable=print-statement,
         deprecated-sys-function,
         exception-escape,
         comprehension-escape,
-        useless-object-inheritance
+        useless-object-inheritance,
+        missing-docstring,
+        bad-continuation
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -314,7 +316,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=100
+max-line-length=79
 
 # Maximum number of lines in a module.
 max-module-lines=1000

.restyled.yaml

@@ -0,0 +1,11 @@
+---
+restylers:
+  - name: black
+    image: restyled/restyler-black:v19.3b0
+    command:
+    - black
+    arguments: []
+    include:
+    - "**/*.py"
+    interpreters:
+    - python

.travis.yml

@@ -2,7 +2,6 @@ cache:
   directories:
     - $HOME/.cache/pip
     - $HOME/.pyenv
-sudo: true
 dist: xenial
 env:
   global:
@@ -15,18 +14,20 @@ stages:
   - test
   - build
   - name: deploy
-    if: tag is present
+    if: (branch = master OR tag is present) AND repo = iterative/dvc
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
-# Note that we specify language as ruby-2.4.0 to help us satisfy
+# Note that we specify language as ruby-2.4.3 to help us satisfy
 # fpm dependencies, as travis runs el capitan with an old ruby
 # version, that causes fpm to crash.
+#
+# Each job will use the previous job's `stage` tag if not specified.
 jobs:
   include:
   # patch formatting checks
   - name: "Check patch formatting"
-    stage: "check"
+    stage: check
     os: linux
     language: python
     python: 3.7
@@ -41,35 +42,28 @@ jobs:
     env:
     - PY2=1
   - name: "3.7 on Windows"
-    stage: test
     os: windows           # Windows 10.0.17134 N/A Build 17134
     language: shell       # 'language: python' is an error on Travis CI Windows
   - name: "2.7 on Linux"
-    stage: test
     language: python
     python: 2.7
     env:
     - PY2=1
   - name: "3.5 on Linux"
-    stage: test
     language: python
     python: 3.5
   - name: "3.6 on Linux"
-    stage: test
     language: python
     python: 3.6
   - name: "3.7 on Linux"
-    stage: test
     language: python
     python: 3.7
   - name: "2.7 on Mac"
-    stage: test
     os: osx
     osx_image: xcode9.4
     env:
     - PY2=1
   - name: "3.6 on Mac"
-    stage: test
     os: osx
     osx_image: xcode9.4
   # build jobs
@@ -78,29 +72,46 @@ jobs:
     os: osx
     osx_image: xcode9.4
     language: ruby
-    rvm: 2.4.0
+    rvm: 2.4.3
     install:
     script: ./scripts/build_posix.sh
   - name: "Windows pkg"
-    stage: build
     os: windows
     language: shell
     install:
     script: ./scripts/build_windows.cmd
   - name: "Linux pkgs"
-    stage: build
     language: python
     python: 3.7
     before_install:
     install:
     script: ./scripts/build_posix.sh
   - name: "Pypi pkgs"
-    stage: build
     language: python
     python: 3.7
     script: ./scripts/build_package.sh
+  - name: Snapcraft snap
+    addons:
+     snaps:
+     - name: snapcraft
+       channel: stable
+       confinement: classic
+     - name: lxd
+       channel: stable
+    env:
+    - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}'
+    before_install:
+    - sudo /snap/bin/lxd.migrate -yes
+    - sudo /snap/bin/lxd waitready
+    - sudo /snap/bin/lxd init --auto
+    install:
+    script:
+    - sudo snapcraft --use-lxd
+    after_failure:
+    - sudo journalctl -u snapd
   # deploy jobs
   - name: "Pypi pkgs"
+    if: tag is present
     stage: deploy
     language: python
     python: 3.7
@@ -140,6 +151,7 @@ deploy:
       - dvc*.deb
       - dvc*.pkg
       - dvc*.exe
+      - dvc*.snap
     skip_cleanup: true
     on:
       tags: true

CONTRIBUTING.md

@@ -1 +1 @@
-### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing#contributing).
+### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing/core).

MANIFEST.in

@@ -0,0 +1,2 @@
+include fastentrypoints.py
+include LICENSE

README.rst

@@ -30,25 +30,30 @@
   :target: https://anaconda.org/conda-forge/dvc
   :alt: Conda-forge
 
+.. image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
+  :target: https://snapcraft.io/dvc
+  :alt: Snapcraft
+
 |
 
 **Data Version Control** or **DVC** is an **open-source** tool for data science and machine
 learning projects. Key features:
 
-#. simple **command line** Git-like experience. Does not require installing and maintaining
-   any databases. Does not depend on any proprietary online services;
+#. Simple **command line** Git-like experience. Does not require installing and maintaining
+   any databases. Does not depend on any proprietary online services.
 
-#. it manages and versions **datasets** and **machine learning models**. Data is saved in
-   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS or even local HDD RAID;
+#. Management and versioning of **datasets** and **machine learning models**. Data is saved in
+   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
 
-#. it makes projects **reproducible** and **shareable**, it helps answering question how
-   the model was build;
+#. Makes projects **reproducible** and **shareable**; helping to answer questions about how
+   a model was built.
 
-#. it helps manage experiments with Git tags or branches and **metrics** tracking;
+#. Helps manage experiments with Git tags/branches and **metrics** tracking.
 
-It aims to replace tools like Excel and Docs that are being commonly used as a knowledge repo and
-a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
-data file suffixes and prefixes.
+**DVC** aims to replace spreadsheet and document sharing tools (such as Excel or Google Docs)
+which are being used frequently as both knowledge repositories and team ledgers.
+DVC also replaces both ad-hoc scripts to track, move, and deploy different model versions;
+as well as ad-hoc data file suffixes and prefixes.
 
 .. contents:: **Contents**
   :backlinks: none
@@ -56,34 +61,33 @@ data file suffixes and prefixes.
 How DVC works
 =============
 
-We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ to better understand what DVC
-is and how does it fit your scenarios.
+We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ guide to better understand what DVC
+is and how it can fit your scenarios.
 
-The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-lfs to be precise) + ``makefiles``
+The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-LFS to be precise) & Makefiles
 made right and tailored specifically for ML and Data Science scenarios.
 
-#. ``Git/Git-lfs`` part - DVC helps you storing and sharing data artifacts, models. It connects them with your
-   Git repository.
-#. ``Makefiles`` part - DVC describes how one data or model artifact was build from another data.
+#. ``Git/Git-LFS`` part - DVC helps store and share data artifacts and models, connecting them with a Git repository.
+#. ``Makefile``\ s part - DVC describes how one data or model artifact was built from other data and code.
 
-DVC usually runs along with Git. Git is used as usual to store and version code and DVC meta-files. DVC helps
-to store data and model files seamlessly out of Git while preserving almost the same user experience as if they
-were stored in Git itself. To store and share data files cache DVC supports remotes - any cloud (S3, Azure,
+DVC usually runs along with Git. Git is used as usual to store and version code (including DVC meta-files). DVC helps
+to store data and model files seamlessly out of Git, while preserving almost the same user experience as if they
+were stored in Git itself. To store and share the data cache, DVC supports multiple remotes - any cloud (S3, Azure,
 Google Cloud, etc) or any on-premise network storage (via SSH, for example).
 
 .. image:: https://dvc.org/static/img/flow.gif
    :target: https://dvc.org/static/img/flow.gif
    :alt: how_dvc_works
 
-DVC pipelines (aka computational graph) feature connects code and data together. In a very explicit way you can
-specify, run, and save information that a certain command with certain dependencies needs to be run to produce
-a model. See the quick start section below or check `Get Started <https://dvc.org/doc/get-started>`_ tutorial to
-learn more.
+The DVC pipelines (computational graph) feature connects code and data together. It is possible to explicitly
+specify all steps required to produce a model: input dependencies including data, commands to run,
+and output information to be saved. See the quick start section below or
+the `Get Started <https://dvc.org/doc/get-started>`_ tutorial to learn more.
 
 Quick start
 ===========
 
-Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full version. Common workflow commands include:
+Please read `Get Started <https://dvc.org/doc/get-started>`_ guide for a full version. Common workflow commands include:
 
 +-----------------------------------+-------------------------------------------------------------------+
 | Step                              | Command                                                           |
@@ -108,8 +112,8 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 Installation
 ============
 
-Read this `instruction <https://dvc.org/doc/get-started/install>`_ to get more details. There are four
-options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package:
+There are four options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package.
+Full instructions are `available here <https://dvc.org/doc/get-started/install>`_.
 
 pip (PyPI)
 ----------
@@ -120,8 +124,8 @@ pip (PyPI)
 
 Depending on the remote storage type you plan to use to keep and share your data, you might need to specify
 one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all`` to include them all.
-The command should look like this: ``pip install dvc[s3]`` - it installs the ``boto3`` library along with
-DVC to support the AWS S3 storage.
+The command should look like this: ``pip install dvc[s3]`` (in this case AWS S3 dependencies such as ``boto3``
+will be installed automatically).
 
 To install the development version, run:
 
@@ -134,14 +138,9 @@ Homebrew
 
 .. code-block:: bash
 
-   brew install iterative/homebrew-dvc/dvc
+   brew install dvc
 
-or:
 
-.. code-block:: bash
-
-   brew cask install iterative/homebrew-dvc/dvc
-   
 Conda (Anaconda)
 ----------------
 
@@ -149,13 +148,28 @@ Conda (Anaconda)
 
    conda install -c conda-forge dvc
 
-Currently, it supports only python version 2.7, 3.6 and 3.7.
+Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
+
+Snap (Snapcraft)
+----------------
+
+Download the latest ``dvc_*.snap`` from the
+GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+
+.. code-block:: bash
+
+   snap install dvc_*.snap --dangerous --classic
+
+Once ``dvc`` is released on the snap store
+(`pending request <https://forum.snapcraft.io/t/classic-confinement-request-for-dvc/14124>`_)
+there will be no need to download ``dvc_*.snap`` or use ``--dangerous``
+(``snap install dvc --classic`` would suffice).
 
 Package
 -------
 
-Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
-GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+Self-contained packages for Linux, Windows, and Mac are available. The latest version of the packages
+can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
 
 Ubuntu / Debian (deb)
 ^^^^^^^^^^^^^^^^^^^^^
@@ -173,36 +187,27 @@ Fedora / CentOS (rpm)
    sudo yum update
    sudo yum install dvc
 
-Arch Linux (AUR)
-^^^^^^^^^^^^^^^^
-*Unofficial package*, any inquiries regarding the AUR package,
-`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
+Comparison to related technologies
+==================================
 
-.. code-block:: bash
-
-   yay -S dvc
-
-Related technologies
-====================
-
-#. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (that you
-   don't want to see in your Git repository) in a local key-value store and uses file hardlinks/symlinks instead of the
-   copying actual files.
+#. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (which should
+   not be in a Git repository) in a local key-value store, and uses file hardlinks/symlinks instead of
+   copying/duplicating files.
 
 #. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with any remote storage (S3, Google Cloud, Azure, SSH,
-   etc). DVC utilizes reflinks or hardlinks to avoid copy operation on checkouts which makes much more efficient for
-   large data files.
+   etc). DVC also uses reflinks or hardlinks to avoid copy operations on checkouts; thus handling large data files
+   much more efficiently.
 
-#. *Makefile* (and its analogues). DVC tracks dependencies (DAG).
+#. *Makefile* (and analogues including ad-hoc scripts) - DVC tracks dependencies (in a directed acyclic graph).
 
-#. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_. DVC is a workflow
+#. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_ - DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-#. `DAGsHub <https://dagshub.com/>`_ Is a Github equivalent for DVC - pushing your Git+DVC based repo to DAGsHub will give you a high level dashboard of your project, including DVC pipeline and metrics visualizations, as well as links to DVC managed files if they are in cloud storage.
+#. `DAGsHub <https://dagshub.com/>`_ - This is a Github equivalent for DVC. Pushing Git+DVC based repositories to DAGsHub will produce in a high level project dashboard; including DVC pipelines and metrics visualizations, as well as links to any DVC-managed files present in cloud storage.
 
 Contributing
 ============
-Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/>`_ for more
+Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
@@ -247,5 +252,5 @@ Copyright
 
 This project is distributed under the Apache license version 2.0 (see the LICENSE file in the project root).
 
-By submitting a pull request for this project, you agree to license your contribution under the Apache license version
+By submitting a pull request to this project, you agree to license your contribution under the Apache license version
 2.0 to this project.

dvc/__init__.py

@@ -3,12 +3,12 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-
 from __future__ import unicode_literals
 
-from dvc.version import __version__  # noqa: F401
 import warnings
+
 import dvc.logger
+from dvc.version import __version__  # noqa: F401
 
 
 dvc.logger.setup()

dvc/__main__.py

@@ -1,5 +1,4 @@
 """Main entry point for dvc command line tool."""
-
 from __future__ import unicode_literals
 
 import sys
@@ -7,4 +6,4 @@ import sys
 from dvc.main import main
 
 if __name__ == "__main__":
-    main(sys.argv[1:])
+    sys.exit(main(sys.argv[1:]))

dvc/analytics.py

@@ -1,15 +1,14 @@
 """Collect and send usage analytics"""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str
-
-import os
-import json
 import errno
+import json
 import logging
+import os
 
 from dvc import __version__
+from dvc.utils import env2bool
+from dvc.utils.compat import str
 
 
 logger = logging.getLogger(__name__)
@@ -66,7 +65,7 @@ class Analytics(object):
                 raise
 
         self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
-        self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + ".lock")
+        self.user_id_file_lock = Lock(self.user_id_file + ".lock")
 
     @staticmethod
     def load(path):
@@ -113,7 +112,7 @@ class Analytics(object):
                 return user_id
         except LockError:
             msg = "Failed to acquire '{}'"
-            logger.debug(msg.format(self.user_id_file_lock.lock_file))
+            logger.debug(msg.format(self.user_id_file_lock.lockfile))
 
     def _collect_windows(self):
         import sys
@@ -205,45 +204,19 @@ class Analytics(object):
             json.dump(self.info, fobj)
             return fobj.name
 
-    @staticmethod
-    def _is_enabled_config(config):
-        from dvc.config import Config
-
-        core = config.config.get(Config.SECTION_CORE, {})
-        return core.get(Config.SECTION_CORE_ANALYTICS, True)
-
-    @staticmethod
-    def _get_current_config():
-        from dvc.config import Config
-        from dvc.repo import Repo
-        from dvc.exceptions import NotDvcRepoError
-
-        try:
-            dvc_dir = Repo.find_dvc_dir()
-            config = Config(dvc_dir)
-        except NotDvcRepoError:
-            config = Config(validate=False)
-        return config
-
     @staticmethod
     def is_enabled(cmd=None):
+        from dvc.config import Config, to_bool
         from dvc.command.daemon import CmdDaemonBase
 
-        if os.getenv("DVC_TEST"):
+        if env2bool("DVC_TEST"):
             return False
 
         if isinstance(cmd, CmdDaemonBase):
             return False
 
-        config = (
-            Analytics._get_current_config()
-            if cmd is None or not hasattr(cmd, "config")
-            else cmd.config
-        )
-
-        assert config is not None
-
-        enabled = Analytics._is_enabled_config(config)
+        core = Config(validate=False).config.get(Config.SECTION_CORE, {})
+        enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true"))
         logger.debug(
             "Analytics is {}.".format("enabled" if enabled else "disabled")
         )

dvc/api.py

@@ -1,5 +1,5 @@
-from contextlib import contextmanager
 import os
+from contextlib import contextmanager
 
 try:
     from contextlib import _GeneratorContextManager as GCM

dvc/cache.py

@@ -1,10 +1,13 @@
 """Manages cache of a dvc repo."""
-
 from __future__ import unicode_literals
 
 import os
+from collections import defaultdict
+
+from funcy import cached_property
 
 from dvc.config import Config
+from dvc.utils.compat import builtin_str
 
 
 class CacheConfig(object):
@@ -21,6 +24,42 @@ class CacheConfig(object):
         )
 
 
+def _make_remote_property(name):
+    """
+    The config file is stored in a way that allows you to have a
+    cache for each remote.
+
+    This is needed when specifying external outputs
+    (as they require you to have an external cache location).
+
+    Imagine a config file like the following:
+
+            ['remote "dvc-storage"']
+            url = ssh://localhost/tmp
+            ask_password = true
+
+            [cache]
+            ssh = dvc-storage
+
+    This method creates a cached property, containing cache named `name`:
+
+        self.config == {'ssh': 'dvc-storage'}
+        self.ssh  # a RemoteSSH instance
+    """
+
+    def getter(self):
+        from dvc.remote import Remote
+
+        remote = self.config.get(name)
+        if not remote:
+            return None
+
+        return Remote(self.repo, name=remote)
+
+    getter.__name__ = builtin_str(name)
+    return cached_property(getter)
+
+
 class Cache(object):
     """Class that manages cache locations of a dvc repo.
 
@@ -35,7 +74,7 @@ class Cache(object):
 
         self.repo = repo
 
-        config = repo.config.config[Config.SECTION_CACHE]
+        self.config = config = repo.config.config[Config.SECTION_CACHE]
         local = config.get(Config.SECTION_CACHE_LOCAL)
 
         if local:
@@ -59,48 +98,42 @@ class Cache(object):
             }
 
         self.local = Remote(repo, **settings)
-        self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
-        self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
-        self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
-        self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
-        self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
 
-    def _get_remote(self, config, name):
-        """
-        The config file is stored in a way that allows you to have a
-        cache for each remote.
+    s3 = _make_remote_property(Config.SECTION_CACHE_S3)
+    gs = _make_remote_property(Config.SECTION_CACHE_GS)
+    ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
+    hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
+    azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
 
-        This is needed when specifying external outputs
-        (as they require you to have an external cache location).
 
-        Imagine a config file like the following:
+class NamedCache(object):
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(set))
+        self.external = defaultdict(set)
 
-                ['remote "dvc-storage"']
-                url = ssh://localhost/tmp
-                ask_password = true
+    @classmethod
+    def make(cls, scheme, checksum, name):
+        cache = cls()
+        cache.add(scheme, checksum, name)
+        return cache
 
-                [cache]
-                ssh = dvc-storage
+    def __getitem__(self, key):
+        return self._items[key]
 
-        This method resolves the name under the cache section into the
-        correct Remote instance.
+    def add(self, scheme, checksum, name):
+        self._items[scheme][checksum].add(name)
 
-        Args:
-            config (dict): The cache section on the config file
-            name (str): Name of the section we are interested in to retrieve
+    def add_external(self, url, rev, path):
+        self.external[url, rev].add(path)
 
-        Returns:
-            remote (dvc.Remote): Remote instance that the section is referring.
-                None when there's no remote with that name.
+    def update(self, cache, suffix=""):
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme]
+            for checksum, names in src.items():
+                if suffix:
+                    dst[checksum].update(n + suffix for n in names)
+                else:
+                    dst[checksum].update(names)
 
-        Example:
-            >>> _get_remote(config={'ssh': 'dvc-storage'}, name='ssh')
-        """
-        from dvc.remote import Remote
-
-        remote = config.get(name)
-
-        if not remote:
-            return None
-
-        return Remote(self.repo, name=remote)
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)

dvc/cli.py

@@ -1,42 +1,41 @@
 """Command line interface for dvc."""
-
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import sys
 import argparse
 import logging
+import sys
 
-from dvc.command.base import fix_subparsers
-import dvc.command.init as init
+import dvc.command.add as add
+import dvc.command.cache as cache
+import dvc.command.checkout as checkout
+import dvc.command.commit as commit
+import dvc.command.config as config
+import dvc.command.daemon as daemon
+import dvc.command.data_sync as data_sync
+import dvc.command.destroy as destroy
+import dvc.command.diff as diff
+import dvc.command.gc as gc
 import dvc.command.get as get
 import dvc.command.get_url as get_url
-import dvc.command.destroy as destroy
-import dvc.command.remove as remove
-import dvc.command.move as move
-import dvc.command.unprotect as unprotect
-import dvc.command.run as run
-import dvc.command.repro as repro
-import dvc.command.data_sync as data_sync
-import dvc.command.gc as gc
-import dvc.command.add as add
 import dvc.command.imp as imp
 import dvc.command.imp_url as imp_url
-import dvc.command.config as config
-import dvc.command.checkout as checkout
-import dvc.command.remote as remote
-import dvc.command.cache as cache
-import dvc.command.metrics as metrics
+import dvc.command.init as init
 import dvc.command.install as install
-import dvc.command.root as root
 import dvc.command.lock as lock
+import dvc.command.metrics as metrics
+import dvc.command.move as move
 import dvc.command.pipeline as pipeline
-import dvc.command.daemon as daemon
-import dvc.command.commit as commit
+import dvc.command.remote as remote
+import dvc.command.remove as remove
+import dvc.command.repro as repro
+import dvc.command.root as root
+import dvc.command.run as run
 import dvc.command.tag as tag
-import dvc.command.diff as diff
-import dvc.command.version as version
+import dvc.command.unprotect as unprotect
 import dvc.command.update as update
+import dvc.command.version as version
+from dvc.command.base import fix_subparsers
 from dvc.exceptions import DvcParserError
 
 
@@ -106,7 +105,7 @@ def get_parent_parser():
     """Create instances of a parser containing common arguments shared among
     all the commands.
 
-    When overwritting `-q` or `-v`, you need to instantiate a new object
+    When overwriting `-q` or `-v`, you need to instantiate a new object
     in order to prevent some weird behavior.
     """
     parent_parser = argparse.ArgumentParser(add_help=False)
@@ -144,6 +143,19 @@ def parse_args(argv=None):
         description=desc,
         parents=[parent_parser],
         formatter_class=argparse.RawTextHelpFormatter,
+        add_help=False,
+    )
+
+    # NOTE: We are doing this to capitalize help message.
+    # Unfortunately, there is no easier and clearer way to do it,
+    # as adding this argument in get_parent_parser() either in
+    # log_level_group or on parent_parser itself will cause unexpected error.
+    parser.add_argument(
+        "-h",
+        "--help",
+        action="help",
+        default=argparse.SUPPRESS,
+        help="Show this help message and exit.",
     )
 
     # NOTE: On some python versions action='version' prints to stderr

dvc/command/add.py

@@ -3,8 +3,10 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import RecursiveAddingWhileUsingFilename
 
 
 logger = logging.getLogger(__name__)
@@ -14,49 +16,50 @@ class CmdAdd(CmdBase):
     def run(self):
         try:
             if len(self.args.targets) > 1 and self.args.file:
-                raise DvcException("can't use '--file' with multiple targets")
+                raise RecursiveAddingWhileUsingFilename()
 
-            for target in self.args.targets:
-                self.repo.add(
-                    target,
-                    recursive=self.args.recursive,
-                    no_commit=self.args.no_commit,
-                    fname=self.args.file,
-                )
+            self.repo.add(
+                self.args.targets,
+                recursive=self.args.recursive,
+                no_commit=self.args.no_commit,
+                fname=self.args.file,
+            )
 
         except DvcException:
-            logger.exception("failed to add file")
+            logger.exception("")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    ADD_HELP = "Take data files or directories under DVC control."
+    ADD_HELP = "Track data files or directories with DVC."
 
-    add_parser = subparsers.add_parser(
+    parser = subparsers.add_parser(
         "add",
         parents=[parent_parser],
         description=append_doc_link(ADD_HELP, "add"),
         help=ADD_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    add_parser.add_argument(
+    parser.add_argument(
         "-R",
         "--recursive",
         action="store_true",
         default=False,
-        help="Recursively add each file under the directory.",
+        help="Recursively add files under directory targets.",
     )
-    add_parser.add_argument(
+    parser.add_argument(
         "--no-commit",
         action="store_true",
         default=False,
         help="Don't put files/directories into cache.",
     )
-    add_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC-file it generates."
+    parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
     )
-    add_parser.add_argument(
+    parser.add_argument(
         "targets", nargs="+", help="Input files/directories to add."
     )
-    add_parser.set_defaults(func=CmdAdd)
+    parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -1,8 +1,9 @@
 from __future__ import unicode_literals
 
-import colorama
 import logging
 
+import colorama
+
 
 logger = logging.getLogger(__name__)
 
@@ -26,7 +27,7 @@ def append_doc_link(help_message, path):
     if not path:
         return help_message
     doc_base = "https://man.dvc.org/"
-    return "{message}\ndocumentation: {blue}{base}{path}{nc}".format(
+    return "{message}\nDocumentation: <{blue}{base}{path}{nc}>".format(
         message=help_message,
         base=doc_base,
         path=path,
@@ -55,24 +56,11 @@ class CmdBase(object):
         logger.warning(msg)
         return [Stage.STAGE_FILE]
 
-    def run_cmd(self):
-        from dvc.lock import LockError
-
-        try:
-            with self.repo.lock:
-                return self.run()
-        except LockError:
-            logger.exception("failed to lock before running a command")
-            return 1
-
     # Abstract methods that have to be implemented by any inheritance class
     def run(self):
-        pass
+        raise NotImplementedError
 
 
 class CmdBaseNoRepo(CmdBase):
     def __init__(self, args):
         self.args = args
-
-    def run_cmd(self):
-        return self.run()

dvc/command/cache.py

@@ -2,9 +2,10 @@ from __future__ import unicode_literals
 
 import argparse
 
-from dvc.command.base import fix_subparsers, append_doc_link
-from dvc.command.config import CmdConfig
 from dvc.cache import CacheConfig
+from dvc.command.base import append_doc_link
+from dvc.command.base import fix_subparsers
+from dvc.command.config import CmdConfig
 
 
 class CmdCacheDir(CmdConfig):

dvc/command/checkout.py

@@ -2,21 +2,18 @@ from __future__ import unicode_literals
 
 import argparse
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 
 
 class CmdCheckout(CmdBase):
     def run(self):
-        if not self.args.targets:
-            self.repo.checkout(force=self.args.force)
-        else:
-            for target in self.args.targets:
-                self.repo.checkout(
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    force=self.args.force,
-                    recursive=self.args.recursive,
-                )
+        self.repo.checkout(
+            targets=self.args.targets,
+            with_deps=self.args.with_deps,
+            force=self.args.force,
+            recursive=self.args.recursive,
+        )
         return 0
 
 

dvc/command/commit.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/config.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 from dvc.config import Config
 
 

dvc/command/daemon.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.command.base import CmdBaseNoRepo
+from dvc.command.base import fix_subparsers
 
 
 class CmdDaemonBase(CmdBaseNoRepo):

dvc/command/data_sync.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 

dvc/command/destroy.py

@@ -4,7 +4,8 @@ import argparse
 import logging
 
 import dvc.prompt as prompt
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 
@@ -12,7 +13,7 @@ logger = logging.getLogger(__name__)
 
 
 class CmdDestroy(CmdBase):
-    def run_cmd(self):
+    def run(self):
         try:
             statement = (
                 "This will destroy all information about your pipelines,"
@@ -24,7 +25,7 @@ class CmdDestroy(CmdBase):
             if not self.args.force and not prompt.confirm(statement):
                 raise DvcException(
                     "cannot destroy without a confirmation from the user."
-                    " Use '-f' to force."
+                    " Use `-f` to force."
                 )
 
             self.repo.destroy()

dvc/command/diff.py

@@ -1,15 +1,15 @@
 from __future__ import unicode_literals
 
 import argparse
+import logging
 
 import humanize
 import inflect
-import logging
 from funcy import compact
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
-import dvc.repo.diff as diff
 
 
 logger = logging.getLogger(__name__)
@@ -37,6 +37,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_dir_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         engine = inflect.engine()
         changes_msg = (
             "{} {} untouched, {} {} modified, {} {} added, "
@@ -57,6 +59,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_file_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         if (
             dct.get(diff.DIFF_OLD_FILE)
             and dct.get(diff.DIFF_NEW_FILE)
@@ -79,6 +83,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_royal_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
             if dct.get("is_dir"):
                 return cls._get_dir_changes(dct)
@@ -88,6 +94,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _show(cls, diff_dct):
+        import dvc.repo.diff as diff
+
         msg = "dvc diff from {} to {}".format(
             diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
         )

dvc/command/gc.py

@@ -1,11 +1,12 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
-import dvc.prompt as prompt
 import logging
+import os
 
-from dvc.command.base import CmdBase, append_doc_link
+import dvc.prompt as prompt
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 
 
 logger = logging.getLogger(__name__)
@@ -13,17 +14,19 @@ logger = logging.getLogger(__name__)
 
 class CmdGC(CmdBase):
     def run(self):
-        msg = "this will remove all cache except the cache that is used in "
-        if not self.args.all_branches and not self.args.all_tags:
-            msg += "the current git branch"
-        elif self.args.all_branches and not self.args.all_tags:
-            msg += "all git branches"
-        elif not self.args.all_branches and self.args.all_tags:
-            msg += "all git tags"
-        else:
-            msg += "all git branches and all git tags"
+        msg = "This will remove all cache except items used in "
 
-        if self.args.repos is not None and len(self.args.repos) > 0:
+        msg += "the working tree"
+        if self.args.all_commits:
+            msg += " and all git commits"
+        elif self.args.all_branches and self.args.all_tags:
+            msg += " and all git branches and tags"
+        elif self.args.all_branches:
+            msg += " and all git branches"
+        elif self.args.all_tags:
+            msg += " and all git tags"
+
+        if self.args.repos:
             msg += " of the current and the following repos:"
 
             for repo_path in self.args.repos:
@@ -40,6 +43,7 @@ class CmdGC(CmdBase):
         self.repo.gc(
             all_branches=self.args.all_branches,
             all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
             cloud=self.args.cloud,
             remote=self.args.remote,
             force=self.args.force,
@@ -76,6 +80,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Keep data files for all git tags.",
     )
+    gc_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     gc_parser.add_argument(
         "-c",
         "--cloud",

dvc/command/get.py

@@ -3,9 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
+from .base import append_doc_link
+from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
-from .base import CmdBaseNoRepo, append_doc_link
 
 
 logger = logging.getLogger(__name__)
@@ -13,6 +13,8 @@ logger = logging.getLogger(__name__)
 
 class CmdGet(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         try:
             Repo.get(
                 self.args.url,

dvc/command/get_url.py

@@ -3,9 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
+from .base import append_doc_link
+from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
-from .base import CmdBaseNoRepo, append_doc_link
 
 
 logger = logging.getLogger(__name__)
@@ -13,6 +13,8 @@ logger = logging.getLogger(__name__)
 
 class CmdGetUrl(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         try:
             Repo.get_url(self.args.url, out=self.args.out)
             return 0

dvc/command/imp.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/imp_url.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)
@@ -19,7 +20,7 @@ class CmdImportUrl(CmdBase):
         except DvcException:
             logger.exception(
                 "failed to import {}. You could also try downloading "
-                "it manually and adding it with `dvc add` command.".format(
+                "it manually, and adding it with `dvc add`.".format(
                     self.args.url
                 )
             )

dvc/command/init.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 
 
 logger = logging.getLogger(__name__)
@@ -53,7 +54,7 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
         help=(
-            "Overwrite existing '.dvc' directory. "
+            "Overwrite existing '.dvc/' directory. "
             "This operation removes local cache."
         ),
     )

dvc/command/install.py

@@ -3,14 +3,15 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 
 
 logger = logging.getLogger(__name__)
 
 
 class CmdInstall(CmdBase):
-    def run_cmd(self):
+    def run(self):
         try:
             self.repo.install()
         except Exception:

dvc/command/lock.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/metrics.py

@@ -3,8 +3,11 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.exceptions import DvcException, BadMetricError
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
+from dvc.exceptions import BadMetricError
+from dvc.exceptions import DvcException
 
 
 logger = logging.getLogger(__name__)

dvc/command/move.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/pipeline.py

@@ -1,13 +1,14 @@
 from __future__ import unicode_literals
 
-from dvc.utils import relpath
-from dvc.utils.compat import str
-
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.utils import relpath
+from dvc.utils.compat import str
 
 
 logger = logging.getLogger(__name__)
@@ -19,7 +20,7 @@ class CmdPipelineShow(CmdBase):
         from dvc.stage import Stage
 
         stage = Stage.load(self.repo, target)
-        G = self.repo.graph()[0]
+        G = self.repo.graph
         stages = networkx.get_node_attributes(G, "stage")
         node = relpath(stage.path, self.repo.root_dir)
         nodes = networkx.dfs_postorder_nodes(G, node)
@@ -41,20 +42,16 @@ class CmdPipelineShow(CmdBase):
     def __build_graph(self, target, commands, outs):
         import networkx
         from dvc.stage import Stage
+        from dvc.repo.graph import get_pipeline
 
         stage = Stage.load(self.repo, target)
         node = relpath(stage.path, self.repo.root_dir)
 
-        pipelines = list(
-            filter(lambda g: node in g.nodes(), self.repo.pipelines())
-        )
-
-        assert len(pipelines) == 1
-        G = pipelines[0]
+        G = get_pipeline(self.repo.pipelines, node)
         stages = networkx.get_node_attributes(G, "stage")
 
         nodes = []
-        for n in G.nodes():
+        for n in G:
             stage = stages[n]
             if commands:
                 if stage.cmd is None:
@@ -168,7 +165,7 @@ class CmdPipelineList(CmdBase):
     def run(self):
         import networkx
 
-        pipelines = self.repo.pipelines()
+        pipelines = self.repo.pipelines
         for p in pipelines:
             stages = networkx.get_node_attributes(p, "stage")
             for stage in stages:

dvc/command/remote.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link, fix_subparsers
+from dvc.command.base import append_doc_link
+from dvc.command.base import fix_subparsers
 from dvc.command.config import CmdConfig
 from dvc.remote.config import RemoteConfig
 
@@ -47,9 +48,13 @@ class CmdRemoteModify(CmdRemoteConfig):
 
 class CmdRemoteDefault(CmdRemoteConfig):
     def run(self):
-        self.remote_config.set_default(
-            self.args.name, unset=self.args.unset, level=self.args.level
-        )
+        if self.args.name is None and not self.args.unset:
+            name = self.remote_config.get_default(level=self.args.level)
+            print(name)
+        else:
+            self.remote_config.set_default(
+                self.args.name, unset=self.args.unset, level=self.args.level
+            )
         return 0
 
 

dvc/command/remove.py

@@ -4,8 +4,9 @@ import argparse
 import logging
 
 import dvc.prompt as prompt
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)
@@ -28,7 +29,7 @@ class CmdRemove(CmdBase):
 
         raise DvcException(
             "Cannot purge without a confirmation from the user."
-            " Use '-f' to force."
+            " Use `-f` to force."
         )
 
     def run(self):

dvc/command/repro.py

@@ -1,10 +1,11 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
 import logging
+import os
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
 from dvc.exceptions import DvcException

dvc/command/root.py

@@ -3,8 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
@@ -12,6 +12,8 @@ logger = logging.getLogger(__name__)
 
 class CmdRoot(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         logger.info(relpath(Repo.find_root()))
         return 0
 

dvc/command/run.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 
@@ -27,9 +28,9 @@ class CmdRun(CmdBase):
             ]
         ):  # pragma: no cover
             logger.error(
-                "too few arguments. Specify at least one: '-d', '-o', '-O',"
-                " '-m', '-M', '--outs-persist', '--outs-persist-no-cache',"
-                " 'command'."
+                "too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
+                "`-m`, `-M`, `--outs-persist`, `--outs-persist-no-cache`, "
+                "`command`."
             )
             return 1
 

dvc/command/tag.py

@@ -1,8 +1,10 @@
 import logging
 
-from dvc.utils import to_yaml_string
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.utils import to_yaml_string
 
 
 logger = logging.getLogger(__name__)

dvc/command/unprotect.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/update.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/version.py

@@ -1,9 +1,10 @@
 from __future__ import unicode_literals
 
+import argparse
+import itertools
+import logging
 import os
 import platform
-import argparse
-import logging
 import uuid
 
 try:
@@ -11,44 +12,59 @@ try:
 except ImportError:
     psutil = None
 
-from dvc.utils import is_binary
+from dvc.utils import is_binary, relpath
 from dvc.utils.compat import pathlib
-from dvc.repo import Repo
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.system import System
-
+from dvc.utils.pkg import PKG
 
 logger = logging.getLogger(__name__)
 
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         dvc_version = __version__
         python_version = platform.python_version()
         platform_type = platform.platform()
         binary = is_binary()
-
         info = (
             "DVC version: {dvc_version}\n"
             "Python version: {python_version}\n"
             "Platform: {platform_type}\n"
             "Binary: {binary}\n"
+            "Package: {package}\n"
         ).format(
             dvc_version=dvc_version,
             python_version=python_version,
             platform_type=platform_type,
             binary=binary,
+            package=PKG,
         )
 
         try:
             repo = Repo()
             root_directory = repo.root_dir
 
-            info += "Cache: {cache}\n".format(
-                cache=self.get_linktype_support_info(repo)
-            )
+            # cache_dir might not exist yet (e.g. after `dvc init`), and we
+            # can't auto-create it, as it might cause issues if the user
+            # later decides to enable shared cache mode with
+            # `dvc config cache.shared group`.
+            if os.path.exists(repo.cache.local.cache_dir):
+                info += "Cache: {cache}\n".format(
+                    cache=self.get_linktype_support_info(repo)
+                )
+            else:
+                logger.warning(
+                    "Unable to detect supported link types, as cache "
+                    "directory '{}' doesn't exist. It is usually auto-created "
+                    "by commands such as `dvc add/fetch/pull/run/import`, "
+                    "but you could create it manually to enable this "
+                    "check.".format(relpath(repo.cache.local.cache_dir))
+                )
 
             if psutil:
                 info += (
@@ -69,12 +85,15 @@ class CmdVersion(CmdBaseNoRepo):
     def get_fs_type(path):
         partition = {
             pathlib.Path(part.mountpoint): (part.fstype, part.device)
-            for part in psutil.disk_partitions()
+            for part in psutil.disk_partitions(all=True)
         }
-        for parent in pathlib.Path(path).parents:
+
+        path = pathlib.Path(path)
+
+        for parent in itertools.chain([path], path.parents):
             if parent in partition:
                 return partition[parent]
-        return ("unkown", "none")
+        return ("unknown", "none")
 
     @staticmethod
     def get_linktype_support_info(repo):

dvc/config.py

@@ -1,18 +1,20 @@
 """DVC config objects."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, open
-
-import os
-import re
 import copy
 import errno
-import configobj
 import logging
+import os
+import re
 
-from schema import Schema, Optional, And, Use, Regex, SchemaError
-from dvc.exceptions import DvcException, NotDvcRepoError
+import configobj
+from voluptuous import Schema, Required, Optional, Invalid
+from voluptuous import All, Any, Lower, Range, Coerce, Match
+
+from dvc.exceptions import DvcException
+from dvc.exceptions import NotDvcRepoError
+from dvc.utils.compat import open
+from dvc.utils.compat import str
 
 logger = logging.getLogger(__name__)
 
@@ -31,83 +33,56 @@ class ConfigError(DvcException):
         )
 
 
+class NoRemoteError(ConfigError):
+    def __init__(self, command, cause=None):
+        msg = (
+            "no remote specified. Setup default remote with\n"
+            "    dvc config core.remote <name>\n"
+            "or use:\n"
+            "    dvc {} -r <name>\n".format(command)
+        )
+
+        super(NoRemoteError, self).__init__(msg, cause=cause)
+
+
 def supported_cache_type(types):
-    """Checks if link type config option has a valid value.
+    """Checks if link type config option consists only of valid values.
 
     Args:
         types (list/string): type(s) of links that dvc should try out.
     """
+    if types is None:
+        return None
     if isinstance(types, str):
         types = [typ.strip() for typ in types.split(",")]
-    for typ in types:
-        if typ not in ["reflink", "hardlink", "symlink", "copy"]:
-            return False
-    return True
+
+    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
+    if unsupported:
+        raise Invalid(
+            "Unsupported cache type(s): {}".format(", ".join(unsupported))
+        )
+
+    return types
 
 
-def is_bool(val):
-    """Checks that value is a boolean.
-
-    Args:
-        val (str): string value verify.
-
-    Returns:
-        bool: True if value stands for boolean, False otherwise.
-    """
-    return val.lower() in ["true", "false"]
+# Checks that value is either true or false and converts it to bool
+Bool = All(
+    Lower,
+    Any("true", "false"),
+    lambda v: v == "true",
+    msg="expected true or false",
+)
+to_bool = Schema(Bool)
 
 
-def to_bool(val):
-    """Converts value to boolean.
-
-    Args:
-        val (str): string to convert to boolean.
-
-    Returns:
-        bool: True if value.lower() == 'true', False otherwise.
-    """
-    return val.lower() == "true"
-
-
-def is_whole(val):
-    """Checks that value is a whole integer.
-
-    Args:
-        val (str): number string to verify.
-
-    Returns:
-        bool: True if val is a whole number, False otherwise.
-    """
-    return int(val) >= 0
-
-
-def is_percent(val):
-    """Checks that value is a percent.
-
-    Args:
-        val (str): number string to verify.
-
-    Returns:
-        bool: True if 0<=value<=100, False otherwise.
-    """
-    return int(val) >= 0 and int(val) <= 100
-
-
-class Choices(object):
+def Choices(*choices):
     """Checks that value belongs to the specified set of values
 
     Args:
         *choices: pass allowed values as arguments, or pass a list or
             tuple as a single argument
     """
-
-    def __init__(self, *choices):
-        if len(choices) == 1 and isinstance(choices[0], (list, tuple)):
-            choices = choices[0]
-        self.choices = choices
-
-    def __call__(self, value):
-        return value in self.choices
+    return Any(*choices, msg="expected one of {}".format(",".join(choices)))
 
 
 class Config(object):  # pylint: disable=too-many-instance-attributes
@@ -135,33 +110,29 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
+    CREDENTIALPATH = "credentialpath"
+
     LEVEL_LOCAL = 0
     LEVEL_REPO = 1
     LEVEL_GLOBAL = 2
     LEVEL_SYSTEM = 3
 
-    BOOL_SCHEMA = And(str, is_bool, Use(to_bool))
-
     SECTION_CORE = "core"
     SECTION_CORE_LOGLEVEL = "loglevel"
-    SECTION_CORE_LOGLEVEL_SCHEMA = And(
-        Use(str.lower), Choices("info", "debug", "warning", "error")
+    SECTION_CORE_LOGLEVEL_SCHEMA = All(
+        Lower, Choices("info", "debug", "warning", "error")
     )
     SECTION_CORE_REMOTE = "remote"
-    SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA
     SECTION_CORE_INTERACTIVE = "interactive"
     SECTION_CORE_ANALYTICS = "analytics"
-    SECTION_CORE_ANALYTICS_SCHEMA = BOOL_SCHEMA
     SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_CORE_CHECKSUM_JOBS_SCHEMA = And(Use(int), lambda x: x > 0)
 
     SECTION_CACHE = "cache"
     SECTION_CACHE_DIR = "dir"
     SECTION_CACHE_TYPE = "type"
-    SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
     SECTION_CACHE_PROTECTED = "protected"
     SECTION_CACHE_SHARED = "shared"
-    SECTION_CACHE_SHARED_SCHEMA = And(Use(str.lower), Choices("group"))
+    SECTION_CACHE_SHARED_SCHEMA = All(Lower, Choices("group"))
     SECTION_CACHE_LOCAL = "local"
     SECTION_CACHE_S3 = "s3"
     SECTION_CACHE_GS = "gs"
@@ -170,40 +141,30 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_CACHE_AZURE = "azure"
     SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
     SECTION_CACHE_SCHEMA = {
-        Optional(SECTION_CACHE_LOCAL): str,
-        Optional(SECTION_CACHE_S3): str,
-        Optional(SECTION_CACHE_GS): str,
-        Optional(SECTION_CACHE_HDFS): str,
-        Optional(SECTION_CACHE_SSH): str,
-        Optional(SECTION_CACHE_AZURE): str,
-        Optional(SECTION_CACHE_DIR): str,
-        Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
-        Optional(SECTION_CACHE_SHARED): SECTION_CACHE_SHARED_SCHEMA,
-        Optional(PRIVATE_CWD): str,
-        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): BOOL_SCHEMA,
+        SECTION_CACHE_LOCAL: str,
+        SECTION_CACHE_S3: str,
+        SECTION_CACHE_GS: str,
+        SECTION_CACHE_HDFS: str,
+        SECTION_CACHE_SSH: str,
+        SECTION_CACHE_AZURE: str,
+        SECTION_CACHE_DIR: str,
+        SECTION_CACHE_TYPE: supported_cache_type,
+        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
+        SECTION_CACHE_SHARED: SECTION_CACHE_SHARED_SCHEMA,
+        PRIVATE_CWD: str,
+        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): Bool,
     }
 
     SECTION_CORE_SCHEMA = {
-        Optional(SECTION_CORE_LOGLEVEL): And(
-            str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA
-        ),
-        Optional(SECTION_CORE_REMOTE, default=""): And(str, Use(str.lower)),
-        Optional(
-            SECTION_CORE_INTERACTIVE, default=False
-        ): SECTION_CORE_INTERACTIVE_SCHEMA,
-        Optional(
-            SECTION_CORE_ANALYTICS, default=True
-        ): SECTION_CORE_ANALYTICS_SCHEMA,
-        Optional(
-            SECTION_CORE_CHECKSUM_JOBS, default=None
-        ): SECTION_CORE_CHECKSUM_JOBS_SCHEMA,
+        SECTION_CORE_LOGLEVEL: SECTION_CORE_LOGLEVEL_SCHEMA,
+        SECTION_CORE_REMOTE: Lower,
+        Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
+        Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
+        SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
     }
 
-    # backward compatibility
-    SECTION_AWS = "aws"
-    SECTION_AWS_STORAGEPATH = "storagepath"
-    SECTION_AWS_CREDENTIALPATH = "credentialpath"
+    # aws specific options
+    SECTION_AWS_CREDENTIALPATH = CREDENTIALPATH
     SECTION_AWS_ENDPOINT_URL = "endpointurl"
     SECTION_AWS_LIST_OBJECTS = "listobjects"
     SECTION_AWS_REGION = "region"
@@ -211,39 +172,24 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_AWS_USE_SSL = "use_ssl"
     SECTION_AWS_SSE = "sse"
     SECTION_AWS_ACL = "acl"
-    SECTION_AWS_SCHEMA = {
-        SECTION_AWS_STORAGEPATH: str,
-        Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE): str,
-        Optional(SECTION_AWS_CREDENTIALPATH): str,
-        Optional(SECTION_AWS_ENDPOINT_URL): str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
-        Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
-        Optional(SECTION_AWS_SSE): str,
-        Optional(SECTION_AWS_ACL): str,
-    }
 
-    # backward compatibility
-    SECTION_GCP = "gcp"
-    SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
-    SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH
+    # gcp specific options
+    SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH
     SECTION_GCP_PROJECTNAME = "projectname"
-    SECTION_GCP_SCHEMA = {
-        SECTION_GCP_STORAGEPATH: str,
-        Optional(SECTION_GCP_PROJECTNAME): str,
-    }
-
-    # backward compatibility
-    SECTION_LOCAL = "local"
-    SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
-    SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str}
 
+    # azure specific option
     SECTION_AZURE_CONNECTION_STRING = "connection_string"
+
     # Alibabacloud oss options
     SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
     SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
     SECTION_OSS_ENDPOINT = "oss_endpoint"
 
+    # GDrive options
+    SECTION_GDRIVE_CLIENT_ID = "gdrive_client_id"
+    SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret"
+    SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file"
+
     SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
     SECTION_REMOTE_FMT = 'remote "{}"'
     SECTION_REMOTE_URL = "url"
@@ -256,51 +202,51 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_REMOTE_GSS_AUTH = "gss_auth"
     SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
     SECTION_REMOTE_SCHEMA = {
-        SECTION_REMOTE_URL: str,
-        Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE): str,
-        Optional(SECTION_AWS_CREDENTIALPATH): str,
-        Optional(SECTION_AWS_ENDPOINT_URL): str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
-        Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
-        Optional(SECTION_AWS_SSE): str,
-        Optional(SECTION_AWS_ACL): str,
-        Optional(SECTION_GCP_PROJECTNAME): str,
-        Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
-        Optional(SECTION_REMOTE_USER): str,
-        Optional(SECTION_REMOTE_PORT): Use(int),
-        Optional(SECTION_REMOTE_KEY_FILE): str,
-        Optional(SECTION_REMOTE_TIMEOUT): Use(int),
-        Optional(SECTION_REMOTE_PASSWORD): str,
-        Optional(SECTION_REMOTE_ASK_PASSWORD): BOOL_SCHEMA,
-        Optional(SECTION_REMOTE_GSS_AUTH): BOOL_SCHEMA,
-        Optional(SECTION_AZURE_CONNECTION_STRING): str,
-        Optional(SECTION_OSS_ACCESS_KEY_ID): str,
-        Optional(SECTION_OSS_ACCESS_KEY_SECRET): str,
-        Optional(SECTION_OSS_ENDPOINT): str,
-        Optional(PRIVATE_CWD): str,
-        Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): BOOL_SCHEMA,
+        Required(SECTION_REMOTE_URL): str,
+        SECTION_AWS_REGION: str,
+        SECTION_AWS_PROFILE: str,
+        SECTION_AWS_CREDENTIALPATH: str,
+        SECTION_AWS_ENDPOINT_URL: str,
+        Optional(SECTION_AWS_LIST_OBJECTS, default=False): Bool,
+        Optional(SECTION_AWS_USE_SSL, default=True): Bool,
+        SECTION_AWS_SSE: str,
+        SECTION_AWS_ACL: str,
+        SECTION_GCP_PROJECTNAME: str,
+        SECTION_CACHE_TYPE: supported_cache_type,
+        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
+        SECTION_REMOTE_USER: str,
+        SECTION_REMOTE_PORT: Coerce(int),
+        SECTION_REMOTE_KEY_FILE: str,
+        SECTION_REMOTE_TIMEOUT: Coerce(int),
+        SECTION_REMOTE_PASSWORD: str,
+        SECTION_REMOTE_ASK_PASSWORD: Bool,
+        SECTION_REMOTE_GSS_AUTH: Bool,
+        SECTION_AZURE_CONNECTION_STRING: str,
+        SECTION_OSS_ACCESS_KEY_ID: str,
+        SECTION_OSS_ACCESS_KEY_SECRET: str,
+        SECTION_OSS_ENDPOINT: str,
+        SECTION_GDRIVE_CLIENT_ID: str,
+        SECTION_GDRIVE_CLIENT_SECRET: str,
+        SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
+        PRIVATE_CWD: str,
+        Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): Bool,
     }
 
     SECTION_STATE = "state"
     SECTION_STATE_ROW_LIMIT = "row_limit"
     SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
     SECTION_STATE_SCHEMA = {
-        Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
-        Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
+        SECTION_STATE_ROW_LIMIT: All(Coerce(int), Range(1)),
+        SECTION_STATE_ROW_CLEANUP_QUOTA: All(Coerce(int), Range(0, 100)),
     }
 
     SCHEMA = {
         Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
-        Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
+        Match(SECTION_REMOTE_REGEX): SECTION_REMOTE_SCHEMA,
         Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
         Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
-        # backward compatibility
-        Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
-        Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
-        Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
     }
+    COMPILED_SCHEMA = Schema(SCHEMA)
 
     def __init__(self, dvc_dir=None, validate=True):
         self.dvc_dir = dvc_dir
@@ -439,9 +385,9 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
 
         d = self.config.dict()
         try:
-            d = Schema(self.SCHEMA).validate(d)
-        except SchemaError as exc:
-            raise ConfigError("config format error", cause=exc)
+            d = self.COMPILED_SCHEMA(d)
+        except Invalid as exc:
+            raise ConfigError(str(exc), cause=exc)
         self.config = configobj.ConfigObj(d, write_empty_values=True)
 
     def save(self, config=None):

dvc/daemon.py

@@ -1,15 +1,15 @@
 """Launch `dvc daemon` command in a separate detached process."""
-
 from __future__ import unicode_literals
 
-import os
-import sys
 import inspect
 import logging
+import os
+import sys
 from subprocess import Popen
 
 from dvc.env import DVC_DAEMON
-from dvc.utils import is_binary, fix_env
+from dvc.utils import fix_env
+from dvc.utils import is_binary
 from dvc.utils.compat import cast_bytes_py2
 
 
@@ -39,7 +39,7 @@ def _spawn_windows(cmd, env):
 
 def _spawn_posix(cmd, env):
     # NOTE: using os._exit instead of sys.exit, because dvc built
-    # with PyInstaller has trouble with SystemExit exeption and throws
+    # with PyInstaller has trouble with SystemExit exception and throws
     # errors such as "[26338] Failed to execute script __main__"
     try:
         pid = os.fork()
@@ -94,7 +94,7 @@ def daemon(args):
 
     cmd = [sys.executable]
     if not is_binary():
-        cmd += ["-m", "dvc"]
+        cmd += [sys.argv[0]]
     cmd += ["daemon", "-q"] + args
 
     env = fix_env()

dvc/dagascii.py

@@ -1,14 +1,16 @@
 """Draws DAG in ASCII."""
-
-from __future__ import unicode_literals
 from __future__ import print_function
+from __future__ import unicode_literals
 
-import sys
 import math
+import sys
 
-from grandalf.graphs import Vertex, Edge, Graph
+from grandalf.graphs import Edge
+from grandalf.graphs import Graph
+from grandalf.graphs import Vertex
 from grandalf.layouts import SugiyamaLayout
-from grandalf.routing import route_with_lines, EdgeViewer
+from grandalf.routing import EdgeViewer
+from grandalf.routing import route_with_lines
 
 
 class VertexViewer(object):

dvc/data_cloud.py

@@ -1,19 +1,11 @@
-"""Manages dvc remotes that user can use with push/pull/status comamnds."""
-
+"""Manages dvc remotes that user can use with push/pull/status commands."""
 from __future__ import unicode_literals
 
 import logging
 
-from dvc.config import Config, ConfigError
+from dvc.config import Config
+from dvc.config import NoRemoteError
 from dvc.remote import Remote
-from dvc.remote.s3 import RemoteS3
-from dvc.remote.gs import RemoteGS
-from dvc.remote.azure import RemoteAZURE
-from dvc.remote.oss import RemoteOSS
-from dvc.remote.ssh import RemoteSSH
-from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.http import RemoteHTTP
 
 
 logger = logging.getLogger(__name__)
@@ -30,18 +22,6 @@ class DataCloud(object):
         config.ConfigError: thrown when config has invalid format.
     """
 
-    CLOUD_MAP = {
-        "aws": RemoteS3,
-        "gcp": RemoteGS,
-        "azure": RemoteAZURE,
-        "oss": RemoteOSS,
-        "ssh": RemoteSSH,
-        "hdfs": RemoteHDFS,
-        "local": RemoteLOCAL,
-        "http": RemoteHTTP,
-        "https": RemoteHTTP,
-    }
-
     def __init__(self, repo):
         self.repo = repo
 
@@ -60,21 +40,16 @@ class DataCloud(object):
         if remote:
             return self._init_remote(remote)
 
-        raise ConfigError(
-            "No remote repository specified. Setup default repository with\n"
-            "    dvc config core.remote <name>\n"
-            "or use:\n"
-            "    dvc {} -r <name>\n".format(command)
-        )
+        raise NoRemoteError(command)
 
     def _init_remote(self, remote):
         return Remote(self.repo, name=remote)
 
-    def push(self, targets, jobs=None, remote=None, show_checksums=False):
+    def push(self, cache, jobs=None, remote=None, show_checksums=False):
         """Push data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to push to the cloud.
+            cache (NamedCache): named checksums to push to the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to push to.
                 By default remote from core.remote config option is used.
@@ -82,17 +57,17 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.push(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "push"),
             show_checksums=show_checksums,
         )
 
-    def pull(self, targets, jobs=None, remote=None, show_checksums=False):
+    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
         """Pull data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to pull from the cloud.
+            cache (NamedCache): named checksums to pull from the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
                 By default remote from core.remote config option is used.
@@ -100,25 +75,25 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.pull(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "pull"),
             show_checksums=show_checksums,
         )
 
-    def status(self, targets, jobs=None, remote=None, show_checksums=False):
+    def status(self, cache, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to check status for.
+            cache (NamedCache): named checksums to check status for.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to compare
-                targets to. By default remote from core.remote config option
+                cache to. By default remote from core.remote config option
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
         remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            targets, jobs=jobs, remote=remote, show_checksums=show_checksums
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )

dvc/dependency/__init__.py

@@ -1,21 +1,17 @@
 from __future__ import unicode_literals
 
-import schema
-
-from dvc.scheme import Schemes
-
 import dvc.output as output
-from dvc.output.base import OutputBase
-from dvc.dependency.s3 import DependencyS3
 from dvc.dependency.gs import DependencyGS
-from dvc.dependency.local import DependencyLOCAL
 from dvc.dependency.hdfs import DependencyHDFS
-from dvc.dependency.ssh import DependencySSH
 from dvc.dependency.http import DependencyHTTP
 from dvc.dependency.https import DependencyHTTPS
-from .repo import DependencyREPO
-
+from dvc.dependency.local import DependencyLOCAL
+from dvc.dependency.s3 import DependencyS3
+from dvc.dependency.ssh import DependencySSH
+from dvc.output.base import OutputBase
 from dvc.remote import Remote
+from dvc.scheme import Schemes
+from .repo import DependencyREPO
 
 
 DEPS = [
@@ -44,9 +40,9 @@ DEP_MAP = {
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
 # output is a metric file and how to parse it, see `-M` flag for `dvc run`).
 SCHEMA = output.SCHEMA.copy()
-del SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)]
-del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)]
-SCHEMA[schema.Optional(DependencyREPO.PARAM_REPO)] = DependencyREPO.REPO_SCHEMA
+del SCHEMA[OutputBase.PARAM_CACHE]
+del SCHEMA[OutputBase.PARAM_METRIC]
+SCHEMA[DependencyREPO.PARAM_REPO] = DependencyREPO.REPO_SCHEMA
 
 
 def _get(stage, p, info):

dvc/dependency/base.py

@@ -15,11 +15,19 @@ class DependencyIsNotFileOrDirError(DvcException):
         super(DependencyIsNotFileOrDirError, self).__init__(msg)
 
 
+class DependencyIsStageFileError(DvcException):
+    def __init__(self, path):
+        super(DependencyIsStageFileError, self).__init__(
+            "Stage file '{}' cannot be a dependency.".format(path)
+        )
+
+
 class DependencyBase(object):
     IS_DEPENDENCY = True
 
     DoesNotExistError = DependencyDoesNotExistError
     IsNotFileOrDirError = DependencyIsNotFileOrDirError
+    IsStageFileError = DependencyIsStageFileError
 
     def update(self):
         pass

dvc/dependency/gs.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.gs import OutputGS
 from dvc.dependency.base import DependencyBase
+from dvc.output.gs import OutputGS
 
 
 class DependencyGS(DependencyBase, OutputGS):

dvc/dependency/hdfs.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.hdfs import OutputHDFS
 from dvc.dependency.base import DependencyBase
+from dvc.output.hdfs import OutputHDFS
 
 
 class DependencyHDFS(DependencyBase, OutputHDFS):

dvc/dependency/http.py

@@ -1,8 +1,8 @@
 from __future__ import unicode_literals
 
+from dvc.dependency.base import DependencyBase
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP
-from dvc.dependency.base import DependencyBase
 
 
 class DependencyHTTP(DependencyBase, OutputBase):

dvc/dependency/https.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.remote.https import RemoteHTTPS
 from .http import DependencyHTTP
+from dvc.remote.https import RemoteHTTPS
 
 
 class DependencyHTTPS(DependencyHTTP):

dvc/dependency/repo.py

@@ -1,15 +1,13 @@
 from __future__ import unicode_literals
 
 import copy
-
-from funcy import merge
-from schema import Optional
 from contextlib import contextmanager
 
-from dvc.external_repo import external_repo
-from dvc.utils.compat import str
+from funcy import merge
 
 from .local import DependencyLOCAL
+from dvc.external_repo import external_repo
+from dvc.utils.compat import str
 
 
 class DependencyREPO(DependencyLOCAL):
@@ -18,11 +16,7 @@ class DependencyREPO(DependencyLOCAL):
     PARAM_REV = "rev"
     PARAM_REV_LOCK = "rev_lock"
 
-    REPO_SCHEMA = {
-        Optional(PARAM_URL): str,
-        Optional(PARAM_REV): str,
-        Optional(PARAM_REV_LOCK): str,
-    }
+    REPO_SCHEMA = {PARAM_URL: str, PARAM_REV: str, PARAM_REV_LOCK: str}
 
     def __init__(self, def_repo, stage, *args, **kwargs):
         self.def_repo = def_repo
@@ -35,6 +29,11 @@ class DependencyREPO(DependencyLOCAL):
     def is_in_repo(self):
         return False
 
+    @property
+    def repo_pair(self):
+        d = self.def_repo
+        return d[self.PARAM_URL], d[self.PARAM_REV_LOCK] or d[self.PARAM_REV]
+
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 
@@ -61,7 +60,7 @@ class DependencyREPO(DependencyLOCAL):
     def dumpd(self):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
-    def download(self, to, resume=False):
+    def fetch(self):
         with self._make_repo(
             cache_dir=self.repo.cache.local.cache_dir
         ) as repo:
@@ -70,8 +69,13 @@ class DependencyREPO(DependencyLOCAL):
             out = repo.find_out_by_relpath(self.def_path)
             with repo.state:
                 repo.cloud.pull(out.get_used_cache())
-            to.info = copy.copy(out.info)
-            to.checkout()
+
+        return out
+
+    def download(self, to):
+        out = self.fetch()
+        to.info = copy.copy(out.info)
+        to.checkout()
 
     def update(self):
         with self._make_repo(rev_lock=None) as repo:

dvc/dependency/s3.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.s3 import OutputS3
 from dvc.dependency.base import DependencyBase
+from dvc.output.s3 import OutputS3
 
 
 class DependencyS3(DependencyBase, OutputS3):

dvc/dependency/ssh.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.ssh import OutputSSH
 from dvc.dependency.base import DependencyBase
+from dvc.output.ssh import OutputSSH
 
 
 class DependencySSH(DependencyBase, OutputSSH):

dvc/exceptions.py

@@ -1,12 +1,11 @@
 """Exceptions raised by the dvc."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, builtin_str
-
 import traceback
 
 from dvc.utils import relpath
+from dvc.utils.compat import builtin_str
+from dvc.utils.compat import str
 
 
 class DvcException(Exception):
@@ -47,7 +46,7 @@ class OutputDuplicationError(DvcException):
             for stage in stages
         )
         msg = (
-            "file/directory '{}' is specified as an output in more than one"
+            "file/directory '{}' is specified as an output in more than one "
             "stage: {}"
         ).format(output, "\n    ".join(stages))
         super(OutputDuplicationError, self).__init__(msg)
@@ -60,10 +59,12 @@ class OutputNotFoundError(DvcException):
         output (unicode): path to the file/directory.
     """
 
-    def __init__(self, output):
+    def __init__(self, output, repo=None):
+        self.output = output
+        self.repo = repo
         super(OutputNotFoundError, self).__init__(
             "unable to find DVC-file with output '{path}'".format(
-                path=relpath(output)
+                path=relpath(self.output)
             )
         )
 
@@ -82,8 +83,8 @@ class StagePathAsOutputError(DvcException):
         assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
         assert isinstance(fname, str) or isinstance(fname, builtin_str)
         msg = (
-            "current working directory '{cwd}' is specified as an output in"
-            " '{fname}'. Use another CWD to prevent any data removal.".format(
+            "current working directory '{cwd}' is specified as an output in "
+            "'{fname}'. Use another CWD to prevent any data removal.".format(
                 cwd=wdir, fname=fname
             )
         )
@@ -167,8 +168,8 @@ class CyclicGraphError(DvcException):
         assert isinstance(stages, list)
         stages = "\n".join("\t- {}".format(stage) for stage in stages)
         msg = (
-            "you've introduced a cycle in your pipeline that involves"
-            " the following stages:"
+            "you've introduced a cycle in your pipeline that involves "
+            "the following stages:"
             "\n"
             "{stages}".format(stages=stages)
         )
@@ -179,7 +180,7 @@ class ConfirmRemoveError(DvcException):
     def __init__(self, path):
         super(ConfirmRemoveError, self).__init__(
             "unable to remove '{}' without a confirmation from the user. Use "
-            "'-f' to force.".format(path)
+            "`-f` to force.".format(path)
         )
 
 
@@ -209,7 +210,7 @@ class NoMetricsError(DvcException):
     def __init__(self):
         super(NoMetricsError, self).__init__(
             "no metric files in this repository. "
-            "Use 'dvc metrics add' to add a metric file to track."
+            "Use `dvc metrics add` to add a metric file to track."
         )
 
 
@@ -226,7 +227,7 @@ class StageFileCorruptedError(DvcException):
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
         super(RecursiveAddingWhileUsingFilename, self).__init__(
-            "using fname with recursive is not allowed."
+            "cannot use `fname` with multiple targets or `-R|--recursive`"
         )
 
 
@@ -244,17 +245,10 @@ class OverlappingOutputPathsError(DvcException):
         )
 
 
-class TargetNotDirectoryError(DvcException):
-    def __init__(self, path):
-        super(TargetNotDirectoryError, self).__init__(
-            "Target: {} is not a directory".format(path)
-        )
-
-
 class CheckoutErrorSuggestGit(DvcException):
     def __init__(self, target, cause):
         super(CheckoutErrorSuggestGit, self).__init__(
-            "Did you mean 'git checkout {}'?".format(target), cause=cause
+            "Did you mean `git checkout {}`?".format(target), cause=cause
         )
 
 
@@ -266,10 +260,11 @@ class ETagMismatchError(DvcException):
         )
 
 
-class OutputFileMissingError(DvcException):
-    def __init__(self, path):
-        super(OutputFileMissingError, self).__init__(
-            "Can't find {} neither locally nor on remote".format(path)
+class FileMissingError(DvcException):
+    def __init__(self, path, cause=None):
+        super(FileMissingError, self).__init__(
+            "Can't find '{}' neither locally nor on remote".format(path),
+            cause=cause,
         )
 
 
@@ -294,3 +289,65 @@ class GetDVCFileError(DvcException):
             "the given path is a DVC-file, you must specify a data file "
             "or a directory"
         )
+
+
+class GitHookAlreadyExistsError(DvcException):
+    def __init__(self, hook_name):
+        super(GitHookAlreadyExistsError, self).__init__(
+            "Hook '{}' already exists. Please refer to "
+            "https://man.dvc.org/install "
+            "for more info.".format(hook_name)
+        )
+
+
+class DownloadError(DvcException):
+    def __init__(self, amount):
+        self.amount = amount
+
+        super(DownloadError, self).__init__(
+            "{amount} files failed to download".format(amount=amount)
+        )
+
+
+class UploadError(DvcException):
+    def __init__(self, amount):
+        self.amount = amount
+
+        super(UploadError, self).__init__(
+            "{amount} files failed to upload".format(amount=amount)
+        )
+
+
+class CheckoutError(DvcException):
+    def __init__(self, target_infos):
+        targets = [str(t) for t in target_infos]
+        m = (
+            "Checkout failed for following targets:\n {}\nDid you "
+            "forget to fetch?".format("\n".join(targets))
+        )
+        super(CheckoutError, self).__init__(m)
+
+
+class CollectCacheError(DvcException):
+    pass
+
+
+class NoRemoteInExternalRepoError(DvcException):
+    def __init__(self, url):
+        super(NoRemoteInExternalRepoError, self).__init__(
+            "No DVC remote is specified in target repository '{}'.".format(url)
+        )
+
+
+class NoOutputInExternalRepoError(DvcException):
+    def __init__(self, path, external_repo_path, external_repo_url):
+        super(NoOutputInExternalRepoError, self).__init__(
+            "Output '{}' not found in target repository '{}'".format(
+                relpath(path, external_repo_path), external_repo_url
+            )
+        )
+
+
+class HTTPError(DvcException):
+    def __init__(self, code, reason):
+        super(HTTPError, self).__init__("'{} {}'".format(code, reason))

dvc/external_repo.py

@@ -1,50 +1,115 @@
 from __future__ import unicode_literals
 
 import os
-import logging
 import tempfile
+from contextlib import contextmanager
+from distutils.dir_util import copy_tree
 
 from funcy import retry
-from contextlib import contextmanager
 
-from dvc.config import Config
-from dvc.cache import CacheConfig
-from dvc.utils import remove
+from dvc.config import NoRemoteError
+from dvc.exceptions import NoRemoteInExternalRepoError
+from dvc.exceptions import NoOutputInExternalRepoError
+from dvc.exceptions import OutputNotFoundError
+from dvc.utils.fs import remove
 
 
-logger = logging.getLogger(__name__)
-
-
-def _clone(cache_dir=None, url=None, rev=None, rev_lock=None):
-    from dvc.repo import Repo
-
-    _path = tempfile.mkdtemp("dvc-repo")
-
-    repo = Repo.clone(url, _path, rev=(rev_lock or rev))
-
-    try:
-        if cache_dir:
-            cache_config = CacheConfig(repo.config)
-            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
-    finally:
-        repo.close()
-
-    return Repo(_path)
-
-
-def _remove(repo):
-    repo.close()
-
-    if os.name == "nt":
-        # git.exe may hang for a while not permitting to remove temp dir
-        os_retry = retry(5, errors=OSError, timeout=0.1)
-        os_retry(remove)(repo.root_dir)
-    else:
-        remove(repo.root_dir)
+REPO_CACHE = {}
 
 
 @contextmanager
-def external_repo(**kwargs):
-    repo = _clone(**kwargs)
-    yield repo
-    _remove(repo)
+def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
+    from dvc.repo import Repo
+
+    path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
+    repo = Repo(path)
+    try:
+        yield repo
+    except NoRemoteError:
+        raise NoRemoteInExternalRepoError(url)
+    except OutputNotFoundError as exc:
+        if exc.repo is repo:
+            raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url)
+        raise
+    repo.close()
+
+
+def _external_repo(url=None, rev=None, cache_dir=None):
+    from dvc.config import Config
+    from dvc.cache import CacheConfig
+    from dvc.repo import Repo
+
+    key = (url, rev, cache_dir)
+    if key in REPO_CACHE:
+        return REPO_CACHE[key]
+
+    new_path = tempfile.mkdtemp("dvc-erepo")
+
+    # Copy and adjust existing clone
+    if (url, None, None) in REPO_CACHE:
+        old_path = REPO_CACHE[url, None, None]
+
+        # This one unlike shutil.copytree() works with an existing dir
+        copy_tree(old_path, new_path)
+    else:
+        # Create a new clone
+        _clone_repo(url, new_path)
+
+        # Save clean clone dir so that we will have access to a default branch
+        clean_clone_path = tempfile.mkdtemp("dvc-erepo")
+        copy_tree(new_path, clean_clone_path)
+        REPO_CACHE[url, None, None] = clean_clone_path
+
+    # Adjust new clone/copy to fit rev and cache_dir
+
+    # Checkout needs to be done first because current branch might not be
+    # DVC repository
+    if rev is not None:
+        _git_checkout(new_path, rev)
+
+    repo = Repo(new_path)
+    try:
+        if cache_dir is not None:
+            cache_config = CacheConfig(repo.config)
+            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
+    finally:
+        # Need to close/reopen repo to force config reread
+        repo.close()
+
+    REPO_CACHE[key] = new_path
+    return new_path
+
+
+def _git_checkout(repo_path, revision):
+    from dvc.scm import Git
+
+    git = Git(repo_path)
+    try:
+        git.checkout(revision)
+    finally:
+        git.close()
+
+
+def clean_repos():
+    # Outside code should not see cache while we are removing
+    repo_paths = list(REPO_CACHE.values())
+    REPO_CACHE.clear()
+
+    for path in repo_paths:
+        _remove(path)
+
+
+def _remove(path):
+    if os.name == "nt":
+        # git.exe may hang for a while not permitting to remove temp dir
+        os_retry = retry(5, errors=OSError, timeout=0.1)
+        os_retry(remove)(path)
+    else:
+        remove(path)
+
+
+def _clone_repo(url, path):
+    from dvc.scm.git import Git
+
+    git = Git.clone(url, path)
+    git.close()

dvc/ignore.py

@@ -2,13 +2,12 @@ from __future__ import unicode_literals
 
 import logging
 import os
+
 from pathspec import PathSpec
 from pathspec.patterns import GitWildMatchPattern
 
 from dvc.utils import relpath
 
-from dvc.utils.compat import open
-
 logger = logging.getLogger(__name__)
 
 
@@ -20,13 +19,13 @@ class DvcIgnore(object):
 
 
 class DvcIgnorePatterns(DvcIgnore):
-    def __init__(self, ignore_file_path):
+    def __init__(self, ignore_file_path, tree):
         assert os.path.isabs(ignore_file_path)
 
         self.ignore_file_path = ignore_file_path
         self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
 
-        with open(ignore_file_path, encoding="utf-8") as fobj:
+        with tree.open(ignore_file_path, encoding="utf-8") as fobj:
             self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
 
     def __call__(self, root, dirs, files):
@@ -47,6 +46,9 @@ class DvcIgnorePatterns(DvcIgnore):
         return hash(self.ignore_file_path)
 
     def __eq__(self, other):
+        if not isinstance(other, DvcIgnorePatterns):
+            return NotImplemented
+
         return self.ignore_file_path == other.ignore_file_path
 
 
@@ -59,19 +61,29 @@ class DvcIgnoreDirs(DvcIgnore):
 
         return dirs, files
 
+    def __hash__(self):
+        return hash(tuple(self.basenames))
+
+    def __eq__(self, other):
+        if not isinstance(other, DvcIgnoreDirs):
+            return NotImplemented
+
+        return self.basenames == other.basenames
+
 
 class DvcIgnoreFilter(object):
-    def __init__(self, root_dir):
+    def __init__(self, root_dir, tree):
+        self.tree = tree
         self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
         self._update(root_dir)
-        for root, dirs, _ in os.walk(root_dir):
+        for root, dirs, _ in self.tree.walk(root_dir, dvcignore=self):
             for d in dirs:
                 self._update(os.path.join(root, d))
 
     def _update(self, dirname):
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if os.path.exists(ignore_file_path):
-            self.ignores.add(DvcIgnorePatterns(ignore_file_path))
+        if self.tree.exists(ignore_file_path):
+            self.ignores.add(DvcIgnorePatterns(ignore_file_path, self.tree))
 
     def __call__(self, root, dirs, files):
         for ignore in self.ignores:

dvc/istextfile.py

@@ -1,8 +1,8 @@
 """Use heuristics to guess if it is a text file or a binary file."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import is_py3, open
+from dvc.utils.compat import is_py3
+from dvc.utils.compat import open
 
 # Based on https://eli.thegreenplace.net/2011/10/19/
 # perls-guess-if-file-is-text-or-binary-implemented-in-python

dvc/lock.py

@@ -1,60 +1,144 @@
 """Manages dvc lock file."""
-
 from __future__ import unicode_literals
 
+import hashlib
 import os
 import time
-import zc.lockfile
+from datetime import timedelta
+
+from funcy.py3 import lkeep
 
 from dvc.exceptions import DvcException
+from dvc.utils import makedirs
+from dvc.utils.compat import is_py3
+
+
+DEFAULT_TIMEOUT = 5
+
+FAILED_TO_LOCK_MESSAGE = (
+    "cannot perform the command because another DVC process seems to be "
+    "running on this project. If that is not the case, manually remove "
+    "`.dvc/lock` and try again."
+)
 
 
 class LockError(DvcException):
     """Thrown when unable to acquire the lock for dvc repo."""
 
 
-class Lock(object):
-    """Class for dvc repo lock.
+if is_py3:
+    import flufl.lock
 
-    Args:
-        dvc_dir (str): path to the directory that the lock should be created
-            in.
-        name (str): name of the lock file.
-    """
+    class Lock(flufl.lock.Lock):
+        """Class for dvc repo lock.
 
-    LOCK_FILE = "lock"
-    TIMEOUT = 5
+        Args:
+            lockfile (str): the lock filename
+                in.
+            tmp_dir (str): a directory to store claim files.
+        """
 
-    def __init__(self, dvc_dir, name=LOCK_FILE):
-        self.lock_file = os.path.join(dvc_dir, name)
-        self._lock = None
+        def __init__(self, lockfile, tmp_dir=None):
+            import socket
 
-    def _do_lock(self):
-        try:
-            self._lock = zc.lockfile.LockFile(self.lock_file)
-        except zc.lockfile.LockError:
-            raise LockError(
-                "cannot perform the cmd since DVC is busy and "
-                "locked. Please retry the cmd later."
-            )
+            self._tmp_dir = tmp_dir
+            if self._tmp_dir is not None:
+                makedirs(self._tmp_dir, exist_ok=True)
+
+            # NOTE: this is basically Lock.__init__ copy-paste, except that
+            # instead of using `socket.getfqdn()` we use `socket.gethostname()`
+            # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
+            # anything, which is way too slow. `gethostname()` is actually a
+            # fallback for `getfqdn()` when it is not able to resolve a
+            # canonical hostname through network. The claimfile that uses
+            # `self._hostname` is still usable, as it uses `pid` and random
+            # number to generate the resulting lock file name, which is unique
+            # enough for our application.
+            #
+            # [1] https://github.com/iterative/dvc/issues/2582
+            self._hostname = socket.gethostname()
+
+            self._lockfile = lockfile
+            self._lifetime = timedelta(days=365)  # Lock for good by default
+            self._separator = flufl.lock.SEP
+            self._set_claimfile()
+            self._owned = True
+            self._retry_errnos = []
+
+        @property
+        def lockfile(self):
+            return self._lockfile
+
+        @property
+        def files(self):
+            return lkeep([self._lockfile, self._tmp_dir])
+
+        def lock(self):
+            try:
+                super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
+            except flufl.lock.TimeOutError:
+                raise LockError(FAILED_TO_LOCK_MESSAGE)
+
+        def _set_claimfile(self, pid=None):
+            super(Lock, self)._set_claimfile(pid)
+
+            if self._tmp_dir is not None:
+                # Under Windows file path length is limited so we hash it
+                filename = hashlib.md5(self._claimfile.encode()).hexdigest()
+                self._claimfile = os.path.join(
+                    self._tmp_dir, filename + ".lock"
+                )
+
+        # Fix for __del__ bug in flufl.lock [1] which is causing errors on
+        # Python shutdown [2].
+        # [1] https://gitlab.com/warsaw/flufl.lock/issues/7
+        # [2] https://github.com/iterative/dvc/issues/2573
+        def __del__(self):
+            try:
+                if self._owned:
+                    self.finalize()
+            except ImportError:
+                pass
+
+
+else:
+    import zc.lockfile
+
+    class Lock(object):
+        """Class for dvc repo lock.
+
+        Uses zc.lockfile as backend.
+        """
+
+        def __init__(self, lockfile, tmp_dir=None):
+            self.lockfile = lockfile
+            self._lock = None
+
+        @property
+        def files(self):
+            return [self.lockfile]
+
+        def _do_lock(self):
+            try:
+                self._lock = zc.lockfile.LockFile(self.lockfile)
+            except zc.lockfile.LockError:
+                raise LockError(FAILED_TO_LOCK_MESSAGE)
+
+        def lock(self):
+            try:
+                self._do_lock()
+                return
+            except LockError:
+                time.sleep(DEFAULT_TIMEOUT)
 
-    def lock(self):
-        """Acquire lock for dvc repo."""
-        try:
             self._do_lock()
-            return
-        except LockError:
-            time.sleep(self.TIMEOUT)
 
-        self._do_lock()
+        def unlock(self):
+            self._lock.close()
+            self._lock = None
 
-    def unlock(self):
-        """Release lock for dvc repo."""
-        self._lock.close()
-        self._lock = None
+        def __enter__(self):
+            self.lock()
 
-    def __enter__(self):
-        self.lock()
-
-    def __exit__(self, typ, value, tbck):
-        self.unlock()
+        def __exit__(self, typ, value, tbck):
+            self.unlock()

dvc/logger.py

@@ -1,15 +1,27 @@
 """Manages logging configuration for dvc repo."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, StringIO
-from dvc.progress import Tqdm
-
-import logging
-import logging.handlers
 import logging.config
+import logging.handlers
+
 import colorama
 
+from dvc.progress import Tqdm
+from dvc.utils.compat import RecursionError
+from dvc.utils.compat import str
+from dvc.utils.compat import StringIO
+
+
+FOOTER = (
+    "\n{yellow}Having any troubles?{nc}"
+    " Hit us up at {blue}https://dvc.org/support{nc},"
+    " we are always happy to help!"
+).format(
+    blue=colorama.Fore.BLUE,
+    nc=colorama.Fore.RESET,
+    yellow=colorama.Fore.YELLOW,
+)
+
 
 class LoggingException(Exception):
     def __init__(self, record):
@@ -19,7 +31,12 @@ class LoggingException(Exception):
 
 class ExcludeErrorsFilter(logging.Filter):
     def filter(self, record):
-        return record.levelno < logging.ERROR
+        return record.levelno < logging.WARNING
+
+
+class ExcludeInfoFilter(logging.Filter):
+    def filter(self, record):
+        return record.levelno < logging.INFO
 
 
 class ColorFormatter(logging.Formatter):
@@ -32,7 +49,7 @@ class ColorFormatter(logging.Formatter):
     __ https://pypi.python.org/pypi/colorama
 
     For records containing `exc_info`, it will use a custom `_walk_exc` to
-    retrieve the whole tracebak.
+    retrieve the whole traceback.
     """
 
     color_code = {
@@ -43,16 +60,6 @@ class ColorFormatter(logging.Formatter):
         "CRITICAL": colorama.Fore.RED,
     }
 
-    footer = (
-        "{yellow}Having any troubles?{nc}."
-        " Hit us up at {blue}https://dvc.org/support{nc},"
-        " we are always happy to help!"
-    ).format(
-        blue=colorama.Fore.BLUE,
-        nc=colorama.Fore.RESET,
-        yellow=colorama.Fore.YELLOW,
-    )
-
     def format(self, record):
         if record.levelname == "INFO":
             return record.msg
@@ -61,10 +68,7 @@ class ColorFormatter(logging.Formatter):
             exception, stack_trace = self._parse_exc(record.exc_info)
 
             return (
-                "{color}{levelname}{nc}: {description}"
-                "{stack_trace}\n"
-                "\n"
-                "{footer}"
+                "{color}{levelname}{nc}: {description}" "{stack_trace}\n"
             ).format(
                 color=self.color_code.get(record.levelname, ""),
                 nc=colorama.Fore.RESET,
@@ -72,7 +76,6 @@ class ColorFormatter(logging.Formatter):
                 description=self._description(record.msg, exception),
                 msg=record.msg,
                 stack_trace=stack_trace,
-                footer=self.footer,
             )
 
         return "{color}{levelname}{nc}: {msg}".format(
@@ -151,7 +154,7 @@ class LoggerHandler(logging.StreamHandler):
         raise LoggingException(record)
 
     def emit(self, record):
-        """Write to Tqdm's stream so as to not break progressbars"""
+        """Write to Tqdm's stream so as to not break progress-bars"""
         try:
             msg = self.format(record)
             Tqdm.write(
@@ -170,19 +173,29 @@ def setup(level=logging.INFO):
     logging.config.dictConfig(
         {
             "version": 1,
-            "filters": {"exclude_errors": {"()": ExcludeErrorsFilter}},
+            "filters": {
+                "exclude_errors": {"()": ExcludeErrorsFilter},
+                "exclude_info": {"()": ExcludeInfoFilter},
+            },
             "formatters": {"color": {"()": ColorFormatter}},
             "handlers": {
-                "console": {
+                "console_info": {
                     "class": "dvc.logger.LoggerHandler",
-                    "level": "DEBUG",
+                    "level": "INFO",
                     "formatter": "color",
                     "stream": "ext://sys.stdout",
                     "filters": ["exclude_errors"],
                 },
+                "console_debug": {
+                    "class": "dvc.logger.LoggerHandler",
+                    "level": "DEBUG",
+                    "formatter": "color",
+                    "stream": "ext://sys.stdout",
+                    "filters": ["exclude_info"],
+                },
                 "console_errors": {
                     "class": "dvc.logger.LoggerHandler",
-                    "level": "ERROR",
+                    "level": "WARNING",
                     "formatter": "color",
                     "stream": "ext://sys.stderr",
                 },
@@ -190,11 +203,27 @@ def setup(level=logging.INFO):
             "loggers": {
                 "dvc": {
                     "level": level,
-                    "handlers": ["console", "console_errors"],
+                    "handlers": [
+                        "console_info",
+                        "console_debug",
+                        "console_errors",
+                    ],
                 },
                 "paramiko": {
                     "level": logging.CRITICAL,
-                    "handlers": ["console", "console_errors"],
+                    "handlers": [
+                        "console_info",
+                        "console_debug",
+                        "console_errors",
+                    ],
+                },
+                "flufl.lock": {
+                    "level": logging.CRITICAL,
+                    "handlers": [
+                        "console_info",
+                        "console_debug",
+                        "console_errors",
+                    ],
                 },
             },
         }

dvc/main.py

@@ -1,14 +1,23 @@
 """Main entry point for dvc CLI."""
-
 from __future__ import unicode_literals
 
 import logging
 
+from dvc.analytics import Analytics
 from dvc.cli import parse_args
 from dvc.config import ConfigError
-from dvc.analytics import Analytics
-from dvc.exceptions import NotDvcRepoError, DvcParserError
+from dvc.exceptions import DvcParserError
+from dvc.exceptions import NotDvcRepoError
+from dvc.external_repo import clean_repos
+from dvc.logger import FOOTER
 from dvc.remote.pool import close_pools
+from dvc.utils.compat import is_py2
+
+
+# Workaround for CPython bug. See [1] and [2] for more info.
+# [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
+# [2] https://bugs.python.org/issue29288
+"".encode("idna")
 
 
 logger = logging.getLogger("dvc")
@@ -37,7 +46,7 @@ def main(argv=None):
             logger.setLevel(logging.DEBUG)
 
         cmd = args.func(args)
-        ret = cmd.run_cmd()
+        ret = cmd.run()
     except ConfigError:
         logger.exception("configuration error")
         ret = 251
@@ -49,15 +58,29 @@ def main(argv=None):
         ret = 253
     except DvcParserError:
         ret = 254
-    except Exception:  # pylint: disable=broad-except
-        logger.exception("unexpected error")
+    except Exception as exc:  # pylint: disable=broad-except
+        if isinstance(exc, UnicodeError) and is_py2:
+            logger.exception(
+                "unicode is not supported in DVC for Python 2 "
+                "(end-of-life January 1, 2020), please upgrade to Python 3"
+            )
+        else:
+            logger.exception("unexpected error")
         ret = 255
     finally:
         logger.setLevel(outerLogLevel)
+
         # Python 2 fails to close these clean occasionally and users see
         # weird error messages, so we do it manually
         close_pools()
 
+        # Remove cached repos in the end of the call, these are anonymous
+        # so won't be reused by any other subsequent run anyway.
+        clean_repos()
+
+    if ret != 0:
+        logger.info(FOOTER)
+
     Analytics().send_cmd(cmd, args, ret)
 
     return ret

dvc/output/__init__.py

@@ -1,21 +1,20 @@
 from __future__ import unicode_literals
 
-import schema
-
-from dvc.scheme import Schemes
-from dvc.utils.compat import urlparse, str
+from voluptuous import Any, Required
 
 from dvc.output.base import OutputBase
-from dvc.output.s3 import OutputS3
 from dvc.output.gs import OutputGS
-from dvc.output.local import OutputLOCAL
 from dvc.output.hdfs import OutputHDFS
+from dvc.output.local import OutputLOCAL
+from dvc.output.s3 import OutputS3
 from dvc.output.ssh import OutputSSH
-
 from dvc.remote import Remote
-from dvc.remote.s3 import RemoteS3
 from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.local import RemoteLOCAL
+from dvc.remote.s3 import RemoteS3
+from dvc.scheme import Schemes
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
 
 OUTS = [
     OutputHDFS,
@@ -42,19 +41,19 @@ OUTS_MAP = {
 # so when a few types of outputs share the same name, we only need
 # specify it once.
 CHECKSUM_SCHEMA = {
-    schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None),
-    schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None),
-    schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
+    RemoteLOCAL.PARAM_CHECKSUM: Any(str, None),
+    RemoteS3.PARAM_CHECKSUM: Any(str, None),
+    RemoteHDFS.PARAM_CHECKSUM: Any(str, None),
 }
 
-TAGS_SCHEMA = {schema.Optional(str): CHECKSUM_SCHEMA}
+TAGS_SCHEMA = {str: CHECKSUM_SCHEMA}
 
 SCHEMA = CHECKSUM_SCHEMA.copy()
-SCHEMA[OutputBase.PARAM_PATH] = str
-SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)] = bool
-SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)] = OutputBase.METRIC_SCHEMA
-SCHEMA[schema.Optional(OutputBase.PARAM_TAGS)] = TAGS_SCHEMA
-SCHEMA[schema.Optional(OutputBase.PARAM_PERSIST)] = bool
+SCHEMA[Required(OutputBase.PARAM_PATH)] = str
+SCHEMA[OutputBase.PARAM_CACHE] = bool
+SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA
+SCHEMA[OutputBase.PARAM_TAGS] = TAGS_SCHEMA
+SCHEMA[OutputBase.PARAM_PERSIST] = bool
 
 
 def _get(stage, p, info, cache, metric, persist=False, tags=None):

dvc/output/base.py

@@ -3,12 +3,15 @@ from __future__ import unicode_literals
 import logging
 from copy import copy
 
-from schema import Or, Optional
+from voluptuous import Any
 
 import dvc.prompt as prompt
+from dvc.cache import NamedCache
+from dvc.exceptions import CollectCacheError
 from dvc.exceptions import DvcException
-from dvc.utils.compat import str, urlparse
 from dvc.remote.base import RemoteBASE
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -32,6 +35,13 @@ class OutputAlreadyTrackedError(DvcException):
         super(OutputAlreadyTrackedError, self).__init__(msg)
 
 
+class OutputIsStageFileError(DvcException):
+    def __init__(self, path):
+        super(OutputIsStageFileError, self).__init__(
+            "Stage file '{}' cannot be an output.".format(path)
+        )
+
+
 class OutputBase(object):
     IS_DEPENDENCY = False
 
@@ -44,12 +54,12 @@ class OutputBase(object):
     PARAM_METRIC_XPATH = "xpath"
     PARAM_PERSIST = "persist"
 
-    METRIC_SCHEMA = Or(
+    METRIC_SCHEMA = Any(
         None,
         bool,
         {
-            Optional(PARAM_METRIC_TYPE): Or(str, None),
-            Optional(PARAM_METRIC_XPATH): Or(str, None),
+            PARAM_METRIC_TYPE: Any(str, None),
+            PARAM_METRIC_XPATH: Any(str, None),
         },
     )
 
@@ -57,6 +67,7 @@ class OutputBase(object):
 
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
+    IsStageFileError = OutputIsStageFileError
 
     sep = "/"
 
@@ -71,6 +82,7 @@ class OutputBase(object):
         persist=False,
         tags=None,
     ):
+        self._validate_output_path(path)
         # This output (and dependency) objects have too many paths/urls
         # here is a list and comments:
         #
@@ -158,12 +170,7 @@ class OutputBase(object):
         return self.remote.exists(self.path_info)
 
     def changed_checksum(self):
-        return (
-            self.checksum
-            != self.remote.save_info(self.path_info)[
-                self.remote.PARAM_CHECKSUM
-            ]
-        )
+        return self.checksum != self.remote.get_checksum(self.path_info)
 
     def changed_cache(self):
         if not self.use_cache or not self.checksum:
@@ -277,15 +284,16 @@ class OutputBase(object):
 
     def checkout(self, force=False, progress_callback=None, tag=None):
         if not self.use_cache:
-            progress_callback(str(self.path_info), self.get_files_number())
-            return
+            if progress_callback:
+                progress_callback(str(self.path_info), self.get_files_number())
+            return None
 
         if tag:
             info = self.tags[tag]
         else:
             info = self.info
 
-        self.cache.checkout(
+        return self.cache.checkout(
             self.path_info,
             info,
             force=force,
@@ -324,7 +332,7 @@ class OutputBase(object):
             self.remote.unprotect(self.path_info)
 
     def _collect_used_dir_cache(self, remote=None, force=False, jobs=None):
-        """Get a list of `info`s retaled to the given directory.
+        """Get a list of `info`s related to the given directory.
 
         - Pull the directory entry from the remote cache if it was changed.
 
@@ -336,25 +344,19 @@ class OutputBase(object):
             $ echo "bar" > directory/bar
             $ dvc add directory
 
-            It will return something similar to the following list:
+            It will return a NamedCache like:
 
-            [
-                { 'path': 'directory/foo', 'md5': 'c157a79031e1', ... },
-                { 'path': 'directory/bar', 'md5': 'd3b07384d113', ... },
-            ]
+            nc = NamedCache()
+            nc.add(self.scheme, 'c157a79031e1', 'directory/foo')
+            nc.add(self.scheme, 'd3b07384d113', 'directory/bar')
         """
 
-        ret = []
+        cache = NamedCache()
 
         if self.cache.changed_cache_file(self.checksum):
             try:
                 self.repo.cloud.pull(
-                    [
-                        {
-                            self.remote.PARAM_CHECKSUM: self.checksum,
-                            "name": str(self),
-                        }
-                    ],
+                    NamedCache.make("local", self.checksum, str(self)),
                     jobs=jobs,
                     remote=remote,
                     show_checksums=False,
@@ -369,20 +371,19 @@ class OutputBase(object):
                 "Would you like to continue? Use '-f' to force."
             )
             if not force and not prompt.confirm(msg.format(self.path_info)):
-                raise DvcException(
+                raise CollectCacheError(
                     "unable to fully collect used cache"
                     " without cache for directory '{}'".format(self)
                 )
             else:
-                return ret
+                return cache
 
         for entry in self.dir_cache:
-            info = copy(entry)
+            checksum = entry[self.remote.PARAM_CHECKSUM]
             path_info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            info["name"] = str(path_info)
-            ret.append(info)
+            cache.add(self.scheme, checksum, str(path_info))
 
-        return ret
+        return cache
 
     def get_used_cache(self, **kwargs):
         """Get a dumpd of the given `out`, with an entry including the branch.
@@ -393,11 +394,8 @@ class OutputBase(object):
         include the `info` of its files.
         """
 
-        if self.stage.is_repo_import:
-            return []
-
         if not self.use_cache:
-            return []
+            return NamedCache()
 
         if not self.info:
             logger.warning(
@@ -405,13 +403,20 @@ class OutputBase(object):
                 "not be collected. Use dvc repro to get your pipeline up to "
                 "date.".format(self, self.stage)
             )
-            return []
+            return NamedCache()
 
-        ret = [{self.remote.PARAM_CHECKSUM: self.checksum, "name": str(self)}]
+        ret = NamedCache.make(self.scheme, self.checksum, str(self))
 
         if not self.is_dir_checksum:
             return ret
 
-        ret.extend(self._collect_used_dir_cache(**kwargs))
+        ret.update(self._collect_used_dir_cache(**kwargs))
 
         return ret
+
+    @classmethod
+    def _validate_output_path(cls, path):
+        from dvc.stage import Stage
+
+        if Stage.is_valid_filename(path):
+            raise cls.IsStageFileError(path)

dvc/output/gs.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.remote.gs import RemoteGS
 from dvc.output.s3 import OutputS3
+from dvc.remote.gs import RemoteGS
 
 
 class OutputGS(OutputS3):

dvc/output/local.py

@@ -1,13 +1,17 @@
 from __future__ import unicode_literals
 
-import os
 import logging
+import os
 
-from dvc.utils.compat import urlparse, fspath_py35, str
-from dvc.istextfile import istextfile
 from dvc.exceptions import DvcException
-from dvc.remote.local import RemoteLOCAL
+from dvc.istextfile import istextfile
 from dvc.output.base import OutputBase
+from dvc.remote.local import RemoteLOCAL
+from dvc.utils import relpath
+from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
+from dvc.utils.fs import path_isin
 
 
 logger = logging.getLogger(__name__)
@@ -26,7 +30,7 @@ class OutputLOCAL(OutputBase):
             # so we should expect both posix and windows style paths.
             # PathInfo accepts both, i.e. / works everywhere, \ only on win.
             #
-            # FIXME: if we have Windows path containig / or posix one with \
+            # FIXME: if we have Windows path containing / or posix one with \
             # then we have #2059 bug and can't really handle that.
             p = self.REMOTE.path_cls(path)
             if not p.is_absolute():
@@ -36,7 +40,14 @@ class OutputLOCAL(OutputBase):
         return self.REMOTE.path_cls(abs_p)
 
     def __str__(self):
-        return str(self.path_info)
+        if not self.is_in_repo:
+            return str(self.def_path)
+
+        cur_dir = os.getcwd()
+        if path_isin(cur_dir, self.repo.root_dir):
+            return relpath(self.path_info, cur_dir)
+
+        return relpath(self.path_info, self.repo.root_dir)
 
     @property
     def fspath(self):

dvc/output/s3.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.remote.s3 import RemoteS3
 from dvc.output.base import OutputBase
+from dvc.remote.s3 import RemoteS3
 
 
 class OutputS3(OutputBase):

dvc/path_info.py

@@ -1,20 +1,24 @@
-from __future__ import unicode_literals, absolute_import
+from __future__ import absolute_import
+from __future__ import unicode_literals
 
-import sys
 import os
 import posixpath
+import sys
 
 from funcy import cached_property
 
-from dvc.utils.compat import str, builtin_str, basestring, is_py2
-from dvc.utils.compat import pathlib, urlparse
+from dvc.utils import relpath
+from dvc.utils.compat import basestring
+from dvc.utils.compat import builtin_str
+from dvc.utils.compat import is_py2
+from dvc.utils.compat import pathlib
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
 
 
 # On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
 # which is lossy, thus we can't use that,
 # see https://github.com/mcmtroffaes/pathlib2/issues/56.
-from dvc.utils import relpath
-
 if is_py2:
     fs_encoding = "utf-8"
 
@@ -36,6 +40,15 @@ class PathInfo(pathlib.PurePath):
     def from_posix(cls, s):
         return cls(PosixPathInfo(s))
 
+    def as_posix(self):
+        f = self._flavour
+        # Unlike original implementation [1] that uses `str()` we actually need
+        # to use `fspath`, because we've overridden `__str__` method to return
+        # relative paths, which will break original `as_posix`.
+        #
+        # [1] https://github.com/python/cpython/blob/v3.7.0/Lib/pathlib.py#L692
+        return self.fspath.replace(f.sep, "/")
+
     def __str__(self):
         path = self.__fspath__()
         return relpath(path)
@@ -94,10 +107,6 @@ class PathInfo(pathlib.PurePath):
         def with_name(self, name):
             return pathlib.PurePath.with_name(self, name.encode(fs_encoding))
 
-        def as_posix(self):
-            f = self._flavour
-            return str(self).replace(f.sep, "/")
-
 
 class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
     pass
@@ -107,52 +116,80 @@ class PosixPathInfo(PathInfo, pathlib.PurePosixPath):
     pass
 
 
+class _URLPathInfo(PosixPathInfo):
+    def __str__(self):
+        return self.__fspath__()
+
+    __unicode__ = __str__
+
+
 class _URLPathParents(object):
-    def __init__(self, pathcls, scheme, netloc, path):
-        self._scheme = scheme
-        self._netloc = netloc
-        self._parents = path.parents
-        self._pathcls = pathcls
+    def __init__(self, src):
+        self.src = src
+        self._parents = self.src._path.parents
 
     def __len__(self):
         return len(self._parents)
 
     def __getitem__(self, idx):
-        return self._pathcls.from_parts(
-            scheme=self._scheme,
-            netloc=self._netloc,
-            path=self._parents[idx].fspath,
-        )
+        return self.src.replace(path=self._parents[idx])
 
     def __repr__(self):
-        return "<{}.parents>".format(self._pathcls.__name__)
+        return "<{}.parents>".format(self.src)
 
 
 class URLInfo(object):
     DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22, "hdfs": 0}
 
     def __init__(self, url):
-        self.parsed = urlparse(url)
-        assert self.parsed.scheme != "remote"
+        p = urlparse(url)
+        assert not p.query and not p.params and not p.fragment
+        assert p.password is None
+
+        self.fill_parts(p.scheme, p.hostname, p.username, p.port, p.path)
 
     @classmethod
     def from_parts(
-        cls, scheme=None, netloc=None, host=None, user=None, port=None, path=""
+        cls, scheme=None, host=None, user=None, port=None, path="", netloc=None
     ):
-        assert scheme and (bool(host) ^ bool(netloc))
+        assert bool(host) ^ bool(netloc)
 
-        if netloc is None:
-            netloc = host
-            if user:
-                netloc = user + "@" + host
-            if port:
-                netloc += ":" + str(port)
-        return cls("{}://{}{}".format(scheme, netloc, path))
+        if netloc is not None:
+            return cls("{}://{}{}".format(scheme, netloc, path))
+
+        obj = cls.__new__(cls)
+        obj.fill_parts(scheme, host, user, port, path)
+        return obj
+
+    def fill_parts(self, scheme, host, user, port, path):
+        assert scheme != "remote"
+        assert isinstance(path, (basestring, _URLPathInfo))
+
+        self.scheme, self.host, self.user = scheme, host, user
+        self.port = int(port) if port else self.DEFAULT_PORTS.get(self.scheme)
+
+        if isinstance(path, _URLPathInfo):
+            self._spath = builtin_str(path)
+            self._path = path
+        else:
+            if path and path[0] != "/":
+                path = "/" + path
+            self._spath = path
+
+    @property
+    def _base_parts(self):
+        return (self.scheme, self.host, self.user, self.port)
+
+    @property
+    def parts(self):
+        return self._base_parts + self._path.parts
+
+    def replace(self, path=None):
+        return self.from_parts(*self._base_parts, path=path)
 
     @cached_property
     def url(self):
-        p = self.parsed
-        return "{}://{}{}".format(p.scheme, self.netloc, p.path)
+        return "{}://{}{}".format(self.scheme, self.netloc, self._spath)
 
     def __str__(self):
         return self.url
@@ -165,92 +202,60 @@ class URLInfo(object):
             other = self.__class__(other)
         return (
             self.__class__ == other.__class__
-            and self.scheme == other.scheme
-            and self.netloc == other.netloc
+            and self._base_parts == other._base_parts
             and self._path == other._path
         )
 
     def __hash__(self):
-        return hash(self.url)
+        return hash(self.parts)
 
     def __div__(self, other):
-        p = self.parsed
-        new_path = posixpath.join(p.path, str(other))
-        if not new_path.startswith("/"):
-            new_path = "/" + new_path
-        new_url = "{}://{}{}".format(p.scheme, p.netloc, new_path)
-        return self.__class__(new_url)
+        return self.replace(path=posixpath.join(self._spath, other))
 
     __truediv__ = __div__
 
-    def __getattr__(self, name):
-        # When deepcopy is called, it creates and object without __init__,
-        # self.parsed is not initialized and it causes infinite recursion.
-        # More on this special casing here:
-        # https://stackoverflow.com/a/47300262/298182
-        if name.startswith("__"):
-            raise AttributeError(name)
-        return getattr(self.parsed, name)
-
-    @cached_property
-    def netloc(self):
-        p = self.parsed
-        netloc = p.hostname
-        if p.username:
-            netloc = p.username + "@" + netloc
-        if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme):
-            netloc += ":" + str(p.port)
-        return netloc
-
     @property
-    def port(self):
-        return self.parsed.port or self.DEFAULT_PORTS.get(self.parsed.scheme)
-
-    @property
-    def host(self):
-        return self.parsed.hostname
-
-    @property
-    def user(self):
-        return self.parsed.username
+    def path(self):
+        return self._spath
 
     @cached_property
     def _path(self):
-        return PosixPathInfo(self.parsed.path)
+        return _URLPathInfo(self._spath)
 
     @property
     def name(self):
         return self._path.name
 
-    @property
-    def parts(self):
-        return (self.scheme, self.netloc) + self._path.parts
+    @cached_property
+    def netloc(self):
+        netloc = self.host
+        if self.user:
+            netloc = self.user + "@" + netloc
+        if self.port and int(self.port) != self.DEFAULT_PORTS.get(self.scheme):
+            netloc += ":" + str(self.port)
+        return netloc
 
     @property
     def bucket(self):
-        return self.parsed.netloc
+        return self.netloc
 
     @property
     def parent(self):
-        return self.from_parts(
-            scheme=self.scheme,
-            netloc=self.parsed.netloc,
-            path=self._path.parent.fspath,
-        )
+        return self.replace(path=self._path.parent)
 
     @property
     def parents(self):
-        return _URLPathParents(
-            type(self), self.scheme, self.parsed.netloc, self._path
-        )
+        return _URLPathParents(self)
 
     def relative_to(self, other):
-        if isinstance(other, str):
-            other = URLInfo(other)
-        if self.scheme != other.scheme or self.netloc != other.netloc:
-            raise ValueError(
-                "'{}' does not start with '{}'".format(self, other)
-            )
+        if isinstance(other, basestring):
+            other = self.__class__(other)
+        if self.__class__ != other.__class__:
+            msg = "'{}' has incompatible class with '{}'".format(self, other)
+            raise ValueError(msg)
+        if self._base_parts != other._base_parts:
+            msg = "'{}' does not start with '{}'".format(self, other)
+            raise ValueError(msg)
         return self._path.relative_to(other._path)
 
     def isin(self, other):
@@ -258,14 +263,12 @@ class URLInfo(object):
             other = self.__class__(other)
         elif self.__class__ != other.__class__:
             return False
-        return (
-            self.scheme == other.scheme
-            and self.netloc == other.netloc
-            and self._path.isin(other._path)
+        return self._base_parts == other._base_parts and self._path.isin(
+            other._path
         )
 
 
 class CloudURLInfo(URLInfo):
     @property
     def path(self):
-        return self.parsed.path.lstrip("/")
+        return self._spath.lstrip("/")

dvc/progress.py

@@ -1,33 +1,17 @@
 """Manages progress bars for dvc repo."""
 from __future__ import print_function
+
 import logging
-from tqdm import tqdm
-from concurrent.futures import ThreadPoolExecutor
+import sys
+from threading import RLock
+
 from funcy import merge
+from tqdm import tqdm
+
+from dvc.utils import env2bool
 
 logger = logging.getLogger(__name__)
-
-
-class TqdmThreadPoolExecutor(ThreadPoolExecutor):
-    """
-    Ensure worker progressbars are cleared away properly.
-    """
-
-    def __enter__(self):
-        """
-        Creates a blank initial dummy progress bar if needed so that workers
-        are forced to create "nested" bars.
-        """
-        blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False)
-        if blank_bar.pos > 0:
-            # already nested - don't need a placeholder bar
-            blank_bar.close()
-        self.bar = blank_bar
-        return super(TqdmThreadPoolExecutor, self).__enter__()
-
-    def __exit__(self, *a, **k):
-        super(TqdmThreadPoolExecutor, self).__exit__(*a, **k)
-        self.bar.close()
+tqdm.set_lock(RLock())
 
 
 class Tqdm(tqdm):
@@ -36,12 +20,18 @@ class Tqdm(tqdm):
     """
 
     BAR_FMT_DEFAULT = (
-        "{percentage:3.0f}%|{bar:10}|"
-        "{desc:{ncols_desc}.{ncols_desc}}{n}/{total}"
+        "{percentage:3.0f}% {desc}|{bar}|"
+        "{n_fmt}/{total_fmt}"
+        " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
+    )
+    # nested bars should have fixed bar widths to align nicely
+    BAR_FMT_DEFAULT_NESTED = (
+        "{percentage:3.0f}%|{bar:10}|{desc:{ncols_desc}.{ncols_desc}}"
+        "{n_fmt}/{total_fmt}"
         " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
     )
     BAR_FMT_NOTOTAL = (
-        "{desc:{ncols_desc}.{ncols_desc}}{n}"
+        "{desc:{ncols_desc}.{ncols_desc}}{n_fmt}"
         " [{elapsed}<??:??, {rate_fmt:>11}{postfix}]"
     )
 
@@ -54,6 +44,7 @@ class Tqdm(tqdm):
         leave=False,
         bar_format=None,
         bytes=False,  # pylint: disable=W0622
+        file=None,
         **kwargs
     ):
         """
@@ -62,6 +53,9 @@ class Tqdm(tqdm):
         desc  : persists after `close()`
         level  : effective logging level for determining `disable`;
             used only if `disable` is unspecified
+        disable  : If (default: None), will be determined by logging level.
+            May be overridden to `True` due to non-TTY status.
+            Skip override by specifying env var `DVC_IGNORE_ISATTY`.
         kwargs  : anything accepted by `tqdm.tqdm()`
         """
         kwargs = kwargs.copy()
@@ -71,20 +65,35 @@ class Tqdm(tqdm):
                 unit="B", unit_scale=True, unit_divisor=1024, miniters=1
             )
             kwargs = merge(bytes_defaults, kwargs)
+        if file is None:
+            file = sys.stderr
         self.desc_persist = desc
+        # auto-disable based on `logger.level`
         if disable is None:
             disable = logger.getEffectiveLevel() > level
+        # auto-disable based on TTY
+        if (
+            not disable
+            and not env2bool("DVC_IGNORE_ISATTY")
+            and hasattr(file, "isatty")
+        ):
+            disable = not file.isatty()
         super(Tqdm, self).__init__(
             iterable=iterable,
             disable=disable,
             leave=leave,
             desc=desc,
             bar_format="!",
+            lock_args=(False,),
             **kwargs
         )
         if bar_format is None:
             if self.__len__():
-                self.bar_format = self.BAR_FMT_DEFAULT
+                self.bar_format = (
+                    self.BAR_FMT_DEFAULT_NESTED
+                    if self.pos
+                    else self.BAR_FMT_DEFAULT
+                )
             else:
                 self.bar_format = self.BAR_FMT_NOTOTAL
         else:
@@ -114,5 +123,11 @@ class Tqdm(tqdm):
         d = super(Tqdm, self).format_dict
         ncols = d["ncols"] or 80
         ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1
-        d["ncols_desc"] = max(ncols_desc, 0)
+        ncols_desc = max(ncols_desc, 0)
+        if ncols_desc:
+            d["ncols_desc"] = ncols_desc
+        else:
+            # work-around for zero-width description
+            d["ncols_desc"] = 1
+            d["prefix"] = ""
         return d

dvc/prompt.py

@@ -1,10 +1,9 @@
 """Manages user prompts."""
-
-from __future__ import unicode_literals
 from __future__ import print_function
+from __future__ import unicode_literals
 
-import sys
 import logging
+import sys
 from getpass import getpass
 
 from dvc.utils.compat import input

dvc/remote/__init__.py

@@ -1,20 +1,21 @@
 from __future__ import unicode_literals
 
+from .config import RemoteConfig
 from dvc.remote.azure import RemoteAZURE
+from dvc.remote.gdrive import RemoteGDrive
 from dvc.remote.gs import RemoteGS
 from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
-from dvc.remote.ssh import RemoteSSH
 from dvc.remote.http import RemoteHTTP
 from dvc.remote.https import RemoteHTTPS
+from dvc.remote.local import RemoteLOCAL
 from dvc.remote.oss import RemoteOSS
-
-from .config import RemoteConfig
+from dvc.remote.s3 import RemoteS3
+from dvc.remote.ssh import RemoteSSH
 
 
 REMOTES = [
     RemoteAZURE,
+    RemoteGDrive,
     RemoteGS,
     RemoteHDFS,
     RemoteHTTP,

dvc/remote/azure.py

@@ -1,26 +1,20 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+import logging
 import os
 import re
-import logging
 from datetime import datetime, timedelta
+import threading
 
-from funcy import cached_property
+from funcy import cached_property, wrap_prop
 
-from dvc.scheme import Schemes
-
-try:
-    from azure.storage.blob import BlockBlobService, BlobPermissions
-    from azure.common import AzureMissingResourceHttpError
-except ImportError:
-    BlockBlobService = None
-
-from dvc.utils.compat import urlparse
-from dvc.progress import Tqdm
 from dvc.config import Config
-from dvc.remote.base import RemoteBASE
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.scheme import Schemes
+from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -35,7 +29,7 @@ class RemoteAZURE(RemoteBASE):
         r"(ContainerName=(?P<container_name>[^;]+);?)?"
         r"(?P<connection_string>.+)?)?)$"
     )
-    REQUIRES = {"azure-storage-blob": BlockBlobService}
+    REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
 
@@ -70,8 +64,12 @@ class RemoteAZURE(RemoteBASE):
             else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
         )
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def blob_service(self):
+        from azure.storage.blob import BlockBlobService
+        from azure.common import AzureMissingResourceHttpError
+
         logger.debug("URL {}".format(self.path_info))
         logger.debug("Connection string {}".format(self.connection_string))
         blob_service = BlockBlobService(
@@ -139,6 +137,8 @@ class RemoteAZURE(RemoteBASE):
         return any(path_info.path == path for path in paths)
 
     def _generate_download_url(self, path_info, expires=3600):
+        from azure.storage.blob import BlobPermissions
+
         expires_at = datetime.utcnow() + timedelta(seconds=expires)
 
         sas_token = self.blob_service.generate_blob_shared_access_signature(

dvc/remote/base.py

@@ -1,41 +1,45 @@
 from __future__ import unicode_literals
 
-from dvc.ignore import DvcIgnore
-from dvc.utils.compat import str, basestring, urlparse
-
-import os
+import itertools
 import json
 import logging
 import tempfile
-import itertools
-from operator import itemgetter
-from multiprocessing import cpu_count
-from functools import partial
 from concurrent.futures import ThreadPoolExecutor
+from copy import copy
+from functools import partial
+from multiprocessing import cpu_count
+from operator import itemgetter
+
+from shortuuid import uuid
 
 import dvc.prompt as prompt
 from dvc.config import Config
-from dvc.exceptions import (
-    DvcException,
-    ConfirmRemoveError,
-    DvcIgnoreInCollectedDirError,
-)
-from dvc.progress import Tqdm, TqdmThreadPoolExecutor
-from dvc.utils import LARGE_DIR_SIZE, tmp_fname, move, relpath, makedirs
+from dvc.exceptions import ConfirmRemoveError
+from dvc.exceptions import DvcException
+from dvc.exceptions import DvcIgnoreInCollectedDirError
+from dvc.ignore import DvcIgnore
+from dvc.path_info import PathInfo
+from dvc.path_info import URLInfo
+from dvc.progress import Tqdm
+from dvc.remote.slow_link_detection import slow_link_guard
 from dvc.state import StateNoop
-from dvc.path_info import PathInfo, URLInfo
+from dvc.utils import makedirs
+from dvc.utils import relpath
+from dvc.utils import tmp_fname
+from dvc.utils.compat import basestring
+from dvc.utils.compat import FileNotFoundError
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
+from dvc.utils.fs import move
 from dvc.utils.http import open_url
 
-
 logger = logging.getLogger(__name__)
 
-
 STATUS_OK = 1
 STATUS_MISSING = 2
 STATUS_NEW = 3
 STATUS_DELETED = 4
 
-
 STATUS_MAP = {
     # (local_exists, remote_exists)
     (True, True): STATUS_OK,
@@ -70,6 +74,14 @@ class RemoteMissingDepsError(DvcException):
     pass
 
 
+class DirCacheError(DvcException):
+    def __init__(self, checksum, cause=None):
+        super(DirCacheError, self).__init__(
+            "Failed to load dir cache for checksum: '{}'.".format(checksum),
+            cause=cause,
+        )
+
+
 class RemoteBASE(object):
     scheme = "base"
     path_cls = URLInfo
@@ -79,36 +91,14 @@ class RemoteBASE(object):
     PARAM_RELPATH = "relpath"
     CHECKSUM_DIR_SUFFIX = ".dir"
     CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
+    DEFAULT_CACHE_TYPES = ["copy"]
 
     state = StateNoop()
 
     def __init__(self, repo, config):
         self.repo = repo
-        deps_ok = all(self.REQUIRES.values())
-        if not deps_ok:
-            missing = [k for k, v in self.REQUIRES.items() if v is None]
-            url = config.get(
-                Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
-            )
-            msg = (
-                "URL '{}' is supported but requires these missing "
-                "dependencies: {}. If you have installed dvc using pip, "
-                "choose one of these options to proceed: \n"
-                "\n"
-                "    1) Install specific missing dependencies:\n"
-                "        pip install {}\n"
-                "    2) Install dvc package that includes those missing "
-                "dependencies: \n"
-                "        pip install 'dvc[{}]'\n"
-                "    3) Install dvc package with all possible "
-                "dependencies included: \n"
-                "        pip install 'dvc[all]'\n"
-                "\n"
-                "If you have installed dvc from a binary package and you "
-                "are still seeing this message, please report it to us "
-                "using https://github.com/iterative/dvc/issues. Thank you!"
-            ).format(url, missing, " ".join(missing), self.scheme)
-            raise RemoteMissingDepsError(msg)
+
+        self._check_requires(config)
 
         core = config.get(Config.SECTION_CORE, {})
         self.checksum_jobs = core.get(
@@ -119,6 +109,52 @@ class RemoteBASE(object):
         self.no_traverse = config.get(Config.SECTION_REMOTE_NO_TRAVERSE, True)
         self._dir_info = {}
 
+        types = config.get(Config.SECTION_CACHE_TYPE, None)
+        if types:
+            if isinstance(types, str):
+                types = [t.strip() for t in types.split(",")]
+            self.cache_types = types
+        else:
+            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
+        self.cache_type_confirmed = False
+
+    def _check_requires(self, config):
+        import importlib
+
+        missing = []
+
+        for package, module in self.REQUIRES.items():
+            try:
+                importlib.import_module(module)
+            except ImportError:
+                missing.append(package)
+
+        if not missing:
+            return
+
+        url = config.get(
+            Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
+        )
+        msg = (
+            "URL '{}' is supported but requires these missing "
+            "dependencies: {}. If you have installed dvc using pip, "
+            "choose one of these options to proceed: \n"
+            "\n"
+            "    1) Install specific missing dependencies:\n"
+            "        pip install {}\n"
+            "    2) Install dvc package that includes those missing "
+            "dependencies: \n"
+            "        pip install 'dvc[{}]'\n"
+            "    3) Install dvc package with all possible "
+            "dependencies included: \n"
+            "        pip install 'dvc[all]'\n"
+            "\n"
+            "If you have installed dvc from a binary package and you "
+            "are still seeing this message, please report it to us "
+            "using https://github.com/iterative/dvc/issues. Thank you!"
+        ).format(url, missing, " ".join(missing), self.scheme)
+        raise RemoteMissingDepsError(msg)
+
     def __repr__(self):
         return "{class_name}: '{path_info}'".format(
             class_name=type(self).__name__,
@@ -145,31 +181,25 @@ class RemoteBASE(object):
 
     def _calculate_checksums(self, file_infos):
         file_infos = list(file_infos)
-        with TqdmThreadPoolExecutor(
-            max_workers=self.checksum_jobs
-        ) as executor:
+        with ThreadPoolExecutor(max_workers=self.checksum_jobs) as executor:
             tasks = executor.map(self.get_file_checksum, file_infos)
-
-            if len(file_infos) > LARGE_DIR_SIZE:
-                logger.info(
-                    (
-                        "Computing md5 for a large number of files. "
-                        "This is only done once."
-                    )
-                )
-                tasks = Tqdm(tasks, total=len(file_infos), unit="md5")
-            checksums = dict(zip(file_infos, tasks))
+            with Tqdm(
+                tasks,
+                total=len(file_infos),
+                unit="md5",
+                desc="Computing hashes (only done once)",
+            ) as tasks:
+                checksums = dict(zip(file_infos, tasks))
         return checksums
 
     def _collect_dir(self, path_info):
-
         file_infos = set()
-        for root, _dirs, files in self.walk(path_info):
 
-            if DvcIgnore.DVCIGNORE_FILE in files:
-                raise DvcIgnoreInCollectedDirError(root)
+        for fname in self.walk_files(path_info):
+            if DvcIgnore.DVCIGNORE_FILE == fname.name:
+                raise DvcIgnoreInCollectedDirError(fname.parent)
 
-            file_infos.update(path_info / root / fname for fname in files)
+            file_infos.add(fname)
 
         checksums = {fi: self.state.get(fi) for fi in file_infos}
         not_in_state = {
@@ -231,30 +261,26 @@ class RemoteBASE(object):
         if dir_info:
             return dir_info
 
-        dir_info = self.load_dir_cache(checksum)
+        try:
+            dir_info = self.load_dir_cache(checksum)
+        except DirCacheError:
+            dir_info = []
+
         self._dir_info[checksum] = dir_info
         return dir_info
 
     def load_dir_cache(self, checksum):
         path_info = self.checksum_to_path_info(checksum)
 
-        fobj = tempfile.NamedTemporaryFile(delete=False)
-        path = fobj.name
-        to_info = PathInfo(path)
-        self.cache.download(path_info, to_info, no_progress_bar=True)
-
         try:
-            with open(path, "r") as fobj:
+            with self.cache.open(path_info, "r") as fobj:
                 d = json.load(fobj)
-        except ValueError:
-            logger.exception("Failed to load dir cache '{}'".format(path_info))
-            return []
-        finally:
-            os.unlink(path)
+        except (ValueError, FileNotFoundError) as exc:
+            raise DirCacheError(checksum, cause=exc)
 
         if not isinstance(d, list):
             msg = "dir cache file format error '{}' [skipping the file]"
-            logger.error(msg.format(relpath(path)))
+            logger.error(msg.format(relpath(path_info)))
             return []
 
         for info in d:
@@ -269,6 +295,8 @@ class RemoteBASE(object):
         return checksum.endswith(cls.CHECKSUM_DIR_SUFFIX)
 
     def get_checksum(self, path_info):
+        assert path_info.scheme == self.scheme
+
         if not self.exists(path_info):
             return None
 
@@ -298,7 +326,6 @@ class RemoteBASE(object):
         return checksum
 
     def save_info(self, path_info):
-        assert path_info.scheme == self.scheme
         return {self.PARAM_CHECKSUM: self.get_checksum(path_info)}
 
     def changed(self, path_info, checksum_info):
@@ -339,7 +366,7 @@ class RemoteBASE(object):
             )
             return True
 
-        actual = self.save_info(path_info)[self.PARAM_CHECKSUM]
+        actual = self.get_checksum(path_info)
         if checksum != actual:
             logger.debug(
                 "checksum '{}'(actual '{}') for '{}' has changed.".format(
@@ -352,7 +379,47 @@ class RemoteBASE(object):
         return False
 
     def link(self, from_info, to_info):
-        self.copy(from_info, to_info)
+        self._link(from_info, to_info, self.cache_types)
+
+    def _link(self, from_info, to_info, link_types):
+        assert self.isfile(from_info)
+
+        self.makedirs(to_info.parent)
+
+        self._try_links(from_info, to_info, link_types)
+
+    @slow_link_guard
+    def _try_links(self, from_info, to_info, link_types):
+        while link_types:
+            link_method = getattr(self, link_types[0])
+            try:
+                self._do_link(from_info, to_info, link_method)
+                self.cache_type_confirmed = True
+                return
+
+            except DvcException as exc:
+                msg = "Cache type '{}' is not supported: {}"
+                logger.debug(msg.format(link_types[0], str(exc)))
+                del link_types[0]
+
+        raise DvcException("no possible cache types left to try out.")
+
+    def _do_link(self, from_info, to_info, link_method):
+        if self.exists(to_info):
+            raise DvcException("Link '{}' already exists!".format(to_info))
+
+        link_method(from_info, to_info)
+
+        if self.protected:
+            self.protect(to_info)
+
+        msg = "Created {}'{}': {} -> {}".format(
+            "protected " if self.protected else "",
+            self.cache_types[0],
+            from_info,
+            to_info,
+        )
+        logger.debug(msg)
 
     def _save_file(self, path_info, checksum, save_link=True):
         assert checksum
@@ -360,10 +427,16 @@ class RemoteBASE(object):
         cache_info = self.checksum_to_path_info(checksum)
         if self.changed_cache(checksum):
             self.move(path_info, cache_info)
+            self.link(cache_info, path_info)
+        elif self.iscopy(path_info) and self._cache_is_copy(path_info):
+            # Default relink procedure involves unneeded copy
+            if self.protected:
+                self.protect(path_info)
+            else:
+                self.unprotect(path_info)
         else:
             self.remove(path_info)
-
-        self.link(cache_info, path_info)
+            self.link(cache_info, path_info)
 
         if save_link:
             self.state.save_link(path_info)
@@ -374,6 +447,28 @@ class RemoteBASE(object):
         self.state.save(path_info, checksum)
         self.state.save(cache_info, checksum)
 
+    def _cache_is_copy(self, path_info):
+        """Checks whether cache uses copies."""
+        if self.cache_type_confirmed:
+            return self.cache_types[0] == "copy"
+
+        if set(self.cache_types) <= {"copy"}:
+            return True
+
+        workspace_file = path_info.with_name("." + uuid())
+        test_cache_file = self.path_info / ".cache_type_test_file"
+        if not self.exists(test_cache_file):
+            with self.open(test_cache_file, "wb") as fobj:
+                fobj.write(bytes(1))
+        try:
+            self.link(test_cache_file, workspace_file)
+        finally:
+            self.remove(workspace_file)
+            self.remove(test_cache_file)
+
+        self.cache_type_confirmed = True
+        return self.cache_types[0] == "copy"
+
     def _save_dir(self, path_info, checksum):
         cache_info = self.checksum_to_path_info(checksum)
         dir_info = self.get_dir_cache(checksum)
@@ -391,12 +486,23 @@ class RemoteBASE(object):
         return False
 
     def isfile(self, path_info):
-        raise NotImplementedError
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return True
 
     def isdir(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
         return False
 
-    def walk(self, path_info):
+    def iscopy(self, path_info):
+        """Check if this file is an independent copy."""
+        return False  # We can't be sure by default
+
+    def walk_files(self, path_info):
+        """Return a generator with `PathInfo`s to all the files"""
         raise NotImplementedError
 
     @staticmethod
@@ -411,15 +517,11 @@ class RemoteBASE(object):
             )
 
         checksum = checksum_info[self.PARAM_CHECKSUM]
-        if not self.changed_cache(checksum):
-            self._checkout(path_info, checksum)
-            return
-
         self._save(path_info, checksum)
 
     def _save(self, path_info, checksum):
         to_info = self.checksum_to_path_info(checksum)
-        logger.info("Saving '{}' to '{}'.".format(path_info, to_info))
+        logger.debug("Saving '{}' to '{}'.".format(path_info, to_info))
         if self.isdir(path_info):
             self._save_dir(path_info, checksum)
             return
@@ -475,11 +577,48 @@ class RemoteBASE(object):
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
+        if self.isdir(from_info):
+            return self._download_dir(
+                from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+            )
+        return self._download_file(
+            from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+        )
 
+    def _download_dir(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        from_infos = list(self.walk_files(from_info))
+        to_infos = (
+            to_info / info.relative_to(from_info) for info in from_infos
+        )
+
+        with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
+            download_files = partial(
+                self._download_file,
+                name=name,
+                no_progress_bar=True,
+                file_mode=file_mode,
+                dir_mode=dir_mode,
+            )
+            futures = executor.map(download_files, from_infos, to_infos)
+            with Tqdm(
+                futures,
+                total=len(from_infos),
+                desc="Downloading directory",
+                unit="Files",
+                disable=no_progress_bar,
+            ) as futures:
+                return sum(futures)
+
+    def _download_file(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
+
+        logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
         name = name or to_info.name
 
-        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
         tmp_file = tmp_fname(to_info)
 
         try:
@@ -512,6 +651,15 @@ class RemoteBASE(object):
     def copy(self, from_info, to_info):
         raise RemoteActionNotImplemented("copy", self.scheme)
 
+    def symlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("symlink", self.scheme)
+
+    def hardlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("hardlink", self.scheme)
+
+    def reflink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("reflink", self.scheme)
+
     def exists(self, path_info):
         raise NotImplementedError
 
@@ -541,14 +689,11 @@ class RemoteBASE(object):
                 # We ignore all the non-cache looking files
                 pass
 
-    def gc(self, cinfos):
-        used = self.extract_used_local_checksums(cinfos)
+    def gc(self, named_cache):
+        used = self.extract_used_local_checksums(named_cache)
 
         if self.scheme != "":
-            used |= {
-                info[self.PARAM_CHECKSUM]
-                for info in cinfos.get(self.scheme, [])
-            }
+            used.update(named_cache[self.scheme])
 
         removed = False
         for checksum in self.all():
@@ -680,47 +825,26 @@ class RemoteBASE(object):
         self.remove(path_info)
 
     def _checkout_file(
-        self,
-        path_info,
-        checksum,
-        force,
-        progress_callback=None,
-        save_link=True,
+        self, path_info, checksum, force, progress_callback=None
     ):
-        # NOTE: In case if path_info is already cached and path_info's
-        # link type matches cache link type, we would like to avoid
-        # relinking.
-        if self.changed(
-            path_info, {self.PARAM_CHECKSUM: checksum}
-        ) or not self._link_matches(path_info):
+        """The file is changed we need to checkout a new copy"""
+        cache_info = self.checksum_to_path_info(checksum)
+        if self.exists(path_info):
+            msg = "data '{}' exists. Removing before checkout."
+            logger.warning(msg.format(str(path_info)))
             self.safe_remove(path_info, force=force)
 
-            cache_info = self.checksum_to_path_info(checksum)
-            self.link(cache_info, path_info)
-
-            if save_link:
-                self.state.save_link(path_info)
-
-            self.state.save(path_info, checksum)
-        else:
-            # NOTE: performing (un)protection costs us +/- the same as checking
-            # if path_info is protected. Instead of implementing logic,
-            # just (un)protect according to self.protected.
-            if self.protected:
-                self.protect(path_info)
-            else:
-                # NOTE dont allow copy, because we checked before that link
-                # type matches cache, and we don't want data duplication
-                self.unprotect(path_info, allow_copy=False)
-
+        self.link(cache_info, path_info)
+        self.state.save_link(path_info)
+        self.state.save(path_info, checksum)
         if progress_callback:
             progress_callback(str(path_info))
 
-    def _link_matches(self, path_info):
-        return True
-
     def makedirs(self, path_info):
-        raise NotImplementedError
+        """Optional: Implement only if the remote needs to create
+        directories before copying/linking/moving data
+        """
+        pass
 
     def _checkout_dir(
         self, path_info, checksum, force, progress_callback=None
@@ -737,14 +861,17 @@ class RemoteBASE(object):
         for entry in dir_info:
             relative_path = entry[self.PARAM_RELPATH]
             entry_checksum = entry[self.PARAM_CHECKSUM]
+            entry_cache_info = self.checksum_to_path_info(entry_checksum)
             entry_info = path_info / relative_path
-            self._checkout_file(
-                entry_info,
-                entry_checksum,
-                force,
-                progress_callback,
-                save_link=False,
-            )
+
+            entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
+            if self.changed(entry_info, entry_checksum_info):
+                if self.exists(entry_info):
+                    self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+                self.state.save(entry_info, entry_checksum)
+            if progress_callback:
+                progress_callback(str(entry_info))
 
         self._remove_redundant_files(path_info, dir_info, force)
 
@@ -752,11 +879,7 @@ class RemoteBASE(object):
         self.state.save(path_info, checksum)
 
     def _remove_redundant_files(self, path_info, dir_info, force):
-        existing_files = set(
-            path_info / root / fname
-            for root, _, files in self.walk(path_info)
-            for fname in files
-        )
+        existing_files = set(self.walk_files(path_info))
 
         needed_files = {
             path_info / entry[self.PARAM_RELPATH] for entry in dir_info
@@ -772,6 +895,7 @@ class RemoteBASE(object):
             raise NotImplementedError
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
+        failed = None
         skip = False
         if not checksum:
             logger.warning(
@@ -779,7 +903,7 @@ class RemoteBASE(object):
                 "It won't be created.".format(str(path_info))
             )
             self.safe_remove(path_info, force=force)
-            skip = True
+            failed = path_info
 
         elif not self.changed(path_info, checksum_info):
             msg = "Data '{}' didn't change."
@@ -790,19 +914,20 @@ class RemoteBASE(object):
             msg = "Cache '{}' not found. File '{}' won't be created."
             logger.warning(msg.format(checksum, str(path_info)))
             self.safe_remove(path_info, force=force)
-            skip = True
+            failed = path_info
 
-        if skip:
+        if failed or skip:
             if progress_callback:
                 progress_callback(
                     str(path_info), self.get_files_number(checksum)
                 )
-            return
+            return failed
 
         msg = "Checking out '{}' with cache '{}'."
         logger.debug(msg.format(str(path_info), checksum))
 
         self._checkout(path_info, checksum, force, progress_callback)
+        return None
 
     def _checkout(
         self, path_info, checksum, force=False, progress_callback=None
@@ -825,16 +950,14 @@ class RemoteBASE(object):
         return 1
 
     @staticmethod
-    def unprotect(path_info, allow_copy=True):
+    def unprotect(path_info):
         pass
 
     def _get_unpacked_dir_names(self, checksums):
         return set()
 
-    def extract_used_local_checksums(self, cinfos):
-        from dvc.remote import RemoteLOCAL
-
-        used = {info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]}
+    def extract_used_local_checksums(self, named_cache):
+        used = set(named_cache["local"])
         unpacked = self._get_unpacked_dir_names(used)
         return used | unpacked
 

dvc/remote/config.py

@@ -1,10 +1,11 @@
-import os
 import logging
+import os
 import posixpath
 
+from dvc.config import Config
+from dvc.config import ConfigError
 from dvc.utils import relpath
 from dvc.utils.compat import urlparse
-from dvc.config import Config, ConfigError
 
 
 logger = logging.getLogger(__name__)
@@ -150,3 +151,8 @@ class RemoteConfig(object):
         self.config.set(
             Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level
         )
+
+    def get_default(self, level=None):
+        return self.config.get(
+            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level
+        )

dvc/remote/gdrive.py

@@ -0,0 +1,306 @@
+from __future__ import unicode_literals
+
+import os
+import posixpath
+import logging
+import threading
+
+from funcy import retry, compose, decorator, wrap_with
+from funcy.py3 import cat
+
+from dvc.progress import Tqdm
+from dvc.scheme import Schemes
+from dvc.path_info import CloudURLInfo
+from dvc.remote.base import RemoteBASE
+from dvc.config import Config
+from dvc.exceptions import DvcException
+from dvc.utils import tmp_fname
+
+logger = logging.getLogger(__name__)
+FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
+
+
+class GDriveRetriableError(DvcException):
+    def __init__(self, msg):
+        super(GDriveRetriableError, self).__init__(msg)
+
+
+@decorator
+def _wrap_pydrive_retriable(call):
+    from apiclient import errors
+    from pydrive.files import ApiRequestError
+
+    try:
+        result = call()
+    except (ApiRequestError, errors.HttpError) as exception:
+        retry_codes = ["403", "500", "502", "503", "504"]
+        if any(
+            "HttpError {}".format(code) in str(exception)
+            for code in retry_codes
+        ):
+            raise GDriveRetriableError(msg="Google API request failed")
+        raise
+    return result
+
+
+gdrive_retry = compose(
+    # 8 tries, start at 0.5s, multiply by golden ratio, cap at 10s
+    retry(
+        8, GDriveRetriableError, timeout=lambda a: min(0.5 * 1.618 ** a, 10)
+    ),
+    _wrap_pydrive_retriable,
+)
+
+
+class RemoteGDrive(RemoteBASE):
+    scheme = Schemes.GDRIVE
+    path_cls = CloudURLInfo
+    REQUIRES = {"pydrive": "pydrive"}
+    GDRIVE_USER_CREDENTIALS_DATA = "GDRIVE_USER_CREDENTIALS_DATA"
+    DEFAULT_USER_CREDENTIALS_FILE = ".dvc/tmp/gdrive-user-credentials.json"
+
+    def __init__(self, repo, config):
+        super(RemoteGDrive, self).__init__(repo, config)
+        self.no_traverse = False
+        self.path_info = self.path_cls(config[Config.SECTION_REMOTE_URL])
+        self.config = config
+        self.init_drive()
+
+    def init_drive(self):
+        self.client_id = self.config.get(Config.SECTION_GDRIVE_CLIENT_ID, None)
+        self.client_secret = self.config.get(
+            Config.SECTION_GDRIVE_CLIENT_SECRET, None
+        )
+        if not self.client_id or not self.client_secret:
+            raise DvcException(
+                "Please specify Google Drive's client id and "
+                "secret in DVC's config. Learn more at "
+                "https://man.dvc.org/remote/add."
+            )
+        self.gdrive_user_credentials_path = (
+            tmp_fname(".dvc/tmp/")
+            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
+            else self.config.get(
+                Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE,
+                self.DEFAULT_USER_CREDENTIALS_FILE,
+            )
+        )
+
+    @gdrive_retry
+    def gdrive_upload_file(
+        self, args, no_progress_bar=True, from_file="", progress_name=""
+    ):
+        item = self.drive.CreateFile(
+            {"title": args["title"], "parents": [{"id": args["parent_id"]}]}
+        )
+
+        with open(from_file, "rb") as fobj:
+            total = os.path.getsize(from_file)
+            with Tqdm.wrapattr(
+                fobj,
+                "read",
+                desc=progress_name,
+                total=total,
+                disable=no_progress_bar,
+            ) as wrapped:
+                # PyDrive doesn't like content property setting for empty files
+                # https://github.com/gsuitedevs/PyDrive/issues/121
+                if total:
+                    item.content = wrapped
+                item.Upload()
+        return item
+
+    @gdrive_retry
+    def gdrive_download_file(
+        self, file_id, to_file, progress_name, no_progress_bar
+    ):
+        gdrive_file = self.drive.CreateFile({"id": file_id})
+        bar_format = (
+            "Donwloading {desc:{ncols_desc}.{ncols_desc}}... "
+            + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
+        )
+        with Tqdm(
+            bar_format=bar_format, desc=progress_name, disable=no_progress_bar
+        ):
+            gdrive_file.GetContentFile(to_file)
+
+    def gdrive_list_item(self, query):
+        file_list = self.drive.ListFile({"q": query, "maxResults": 1000})
+
+        # Isolate and decorate fetching of remote drive items in pages
+        get_list = gdrive_retry(lambda: next(file_list, None))
+
+        # Fetch pages until None is received, lazily flatten the thing
+        return cat(iter(get_list, None))
+
+    def cache_root_dirs(self):
+        cached_dirs = {}
+        cached_ids = {}
+        for dir1 in self.gdrive_list_item(
+            "'{}' in parents and trashed=false".format(self.root_id)
+        ):
+            remote_path = posixpath.join(self.path_info.path, dir1["title"])
+            cached_dirs.setdefault(remote_path, []).append(dir1["id"])
+            cached_ids[dir1["id"]] = dir1["title"]
+        return cached_dirs, cached_ids
+
+    @property
+    def cached_dirs(self):
+        if not hasattr(self, "_cached_dirs"):
+            self.drive
+        return self._cached_dirs
+
+    @property
+    def cached_ids(self):
+        if not hasattr(self, "_cached_ids"):
+            self.drive
+        return self._cached_ids
+
+    @property
+    @wrap_with(threading.RLock())
+    def drive(self):
+        if not hasattr(self, "_gdrive"):
+            from pydrive.auth import GoogleAuth
+            from pydrive.drive import GoogleDrive
+
+            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                with open(
+                    self.gdrive_user_credentials_path, "w"
+                ) as credentials_file:
+                    credentials_file.write(
+                        os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
+                    )
+
+            GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
+            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                "token_uri": "https://oauth2.googleapis.com/token",
+                "revoke_uri": "https://oauth2.googleapis.com/revoke",
+                "redirect_uri": "",
+            }
+            GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
+            GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
+            GoogleAuth.DEFAULT_SETTINGS[
+                "save_credentials_file"
+            ] = self.gdrive_user_credentials_path
+            GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
+            GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
+                "https://www.googleapis.com/auth/drive",
+                "https://www.googleapis.com/auth/drive.appdata",
+            ]
+
+            # Pass non existent settings path to force DEFAULT_SETTINGS loading
+            gauth = GoogleAuth(settings_file="")
+            gauth.CommandLineAuth()
+
+            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                os.remove(self.gdrive_user_credentials_path)
+
+            self._gdrive = GoogleDrive(gauth)
+
+            self.root_id = self.get_remote_id(self.path_info, create=True)
+            self._cached_dirs, self._cached_ids = self.cache_root_dirs()
+
+        return self._gdrive
+
+    @gdrive_retry
+    def create_remote_dir(self, parent_id, title):
+        item = self.drive.CreateFile(
+            {
+                "title": title,
+                "parents": [{"id": parent_id}],
+                "mimeType": FOLDER_MIME_TYPE,
+            }
+        )
+        item.Upload()
+        return item
+
+    @gdrive_retry
+    def get_remote_item(self, name, parents_ids):
+        if not parents_ids:
+            return None
+        query = " or ".join(
+            "'{}' in parents".format(parent_id) for parent_id in parents_ids
+        )
+
+        query += " and trashed=false and title='{}'".format(name)
+
+        # Limit found remote items count to 1 in response
+        item_list = self.drive.ListFile(
+            {"q": query, "maxResults": 1}
+        ).GetList()
+        return next(iter(item_list), None)
+
+    def resolve_remote_item_from_path(self, path_parts, create):
+        parents_ids = ["root"]
+        current_path = ""
+        for path_part in path_parts:
+            current_path = posixpath.join(current_path, path_part)
+            remote_ids = self.get_remote_id_from_cache(current_path)
+            if remote_ids:
+                parents_ids = remote_ids
+                continue
+            item = self.get_remote_item(path_part, parents_ids)
+            if not item and create:
+                item = self.create_remote_dir(parents_ids[0], path_part)
+            elif not item:
+                return None
+            parents_ids = [item["id"]]
+        return item
+
+    def get_remote_id_from_cache(self, remote_path):
+        if hasattr(self, "_cached_dirs"):
+            return self.cached_dirs.get(remote_path, [])
+        return []
+
+    def get_remote_id(self, path_info, create=False):
+        remote_ids = self.get_remote_id_from_cache(path_info.path)
+
+        if remote_ids:
+            return remote_ids[0]
+
+        file1 = self.resolve_remote_item_from_path(
+            path_info.path.split("/"), create
+        )
+        return file1["id"] if file1 else ""
+
+    def exists(self, path_info):
+        return self.get_remote_id(path_info) != ""
+
+    def _upload(self, from_file, to_info, name, no_progress_bar):
+        dirname = to_info.parent
+        if dirname:
+            parent_id = self.get_remote_id(dirname, True)
+        else:
+            parent_id = to_info.bucket
+
+        self.gdrive_upload_file(
+            {"title": to_info.name, "parent_id": parent_id},
+            no_progress_bar,
+            from_file,
+            name,
+        )
+
+    def _download(self, from_info, to_file, name, no_progress_bar):
+        file_id = self.get_remote_id(from_info)
+        self.gdrive_download_file(file_id, to_file, name, no_progress_bar)
+
+    def all(self):
+        if not self.cached_ids:
+            return
+
+        query = " or ".join(
+            "'{}' in parents".format(dir_id) for dir_id in self.cached_ids
+        )
+
+        query += " and trashed=false"
+        for file1 in self.gdrive_list_item(query):
+            parent_id = file1["parents"][0]["id"]
+            path = posixpath.join(self.cached_ids[parent_id], file1["title"])
+            try:
+                yield self.path_to_checksum(path)
+            except ValueError:
+                # We ignore all the non-cache looking files
+                logger.debug('Ignoring path as "non-cache looking"')

dvc/remote/gs.py

@@ -1,46 +1,98 @@
-from __future__ import unicode_literals
+from __future__ import unicode_literals, division
 
 import logging
 from datetime import timedelta
-from funcy import cached_property
-from dvc.utils.compat import FileNotFoundError
+from functools import wraps
+import io
+import os.path
+import threading
 
-try:
-    from google.cloud import storage
-except ImportError:
-    storage = None
+from funcy import cached_property, wrap_prop
 
-from dvc.remote.base import RemoteBASE
 from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
+from dvc.utils.compat import FileNotFoundError  # skipcq: PYL-W0622
 
 logger = logging.getLogger(__name__)
 
 
+def dynamic_chunk_size(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        import requests
+        from google.cloud.storage.blob import Blob
+
+        # `ConnectionError` may be due to too large `chunk_size`
+        # (see [#2572]) so try halving on error.
+        # Note: start with 40 * [default: 256K] = 10M.
+        # Note: must be multiple of 256K.
+        #
+        # [#2572]: https://github.com/iterative/dvc/issues/2572
+
+        # skipcq: PYL-W0212
+        multiplier = 40
+        while True:
+            try:
+                # skipcq: PYL-W0212
+                chunk_size = Blob._CHUNK_SIZE_MULTIPLE * multiplier
+                return func(*args, chunk_size=chunk_size, **kwargs)
+            except requests.exceptions.ConnectionError:
+                multiplier //= 2
+                if not multiplier:
+                    raise
+
+    return wrapper
+
+
+@dynamic_chunk_size
+def _upload_to_bucket(
+    bucket,
+    from_file,
+    to_info,
+    chunk_size=None,
+    name=None,
+    no_progress_bar=True,
+):
+    blob = bucket.blob(to_info.path, chunk_size=chunk_size)
+    with io.open(from_file, mode="rb") as fobj:
+        with Tqdm.wrapattr(
+            fobj,
+            "read",
+            desc=name or to_info.path,
+            total=os.path.getsize(from_file),
+            disable=no_progress_bar,
+        ) as wrapped:
+            blob.upload_from_file(wrapped)
+
+
 class RemoteGS(RemoteBASE):
     scheme = Schemes.GS
     path_cls = CloudURLInfo
-    REQUIRES = {"google.cloud.storage": storage}
+    REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
     PARAM_CHECKSUM = "md5"
 
     def __init__(self, repo, config):
         super(RemoteGS, self).__init__(repo, config)
 
-        storagepath = "gs://" + config.get(Config.SECTION_GCP_STORAGEPATH, "/")
-        url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        url = config.get(Config.SECTION_REMOTE_URL, "gs:///")
         self.path_info = self.path_cls(url)
 
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
         self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def gs(self):
+        from google.cloud.storage import Client
+
         return (
-            storage.Client.from_service_account_json(self.credentialpath)
+            Client.from_service_account_json(self.credentialpath)
             if self.credentialpath
-            else storage.Client(self.projectname)
+            else Client(self.projectname)
         )
 
     def get_file_checksum(self, path_info):
@@ -78,26 +130,68 @@ class RemoteGS(RemoteBASE):
 
         blob.delete()
 
-    def _list_paths(self, bucket, prefix):
-        for blob in self.gs.bucket(bucket).list_blobs(prefix=prefix):
+    def _list_paths(self, path_info, max_items=None):
+        for blob in self.gs.bucket(path_info.bucket).list_blobs(
+            prefix=path_info.path, max_results=max_items
+        ):
             yield blob.name
 
     def list_cache_paths(self):
-        return self._list_paths(self.path_info.bucket, self.path_info.path)
+        return self._list_paths(self.path_info)
+
+    def walk_files(self, path_info):
+        for fname in self._list_paths(path_info / ""):
+            # skip nested empty directories
+            if fname.endswith("/"):
+                continue
+            yield path_info.replace(fname)
+
+    def makedirs(self, path_info):
+        self.gs.bucket(path_info.bucket).blob(
+            (path_info / "").path
+        ).upload_from_string("")
+
+    def isdir(self, path_info):
+        dir_path = path_info / ""
+        return bool(list(self._list_paths(dir_path, max_items=1)))
+
+    def isfile(self, path_info):
+        if path_info.path.endswith("/"):
+            return False
+
+        blob = self.gs.bucket(path_info.bucket).blob(path_info.path)
+        return blob.exists()
 
     def exists(self, path_info):
-        paths = set(self._list_paths(path_info.bucket, path_info.path))
-        return any(path_info.path == path for path in paths)
+        """Check if the blob exists. If it does not exist,
+        it could be a part of a directory path.
 
-    def _upload(self, from_file, to_info, **_kwargs):
+        eg: if `data/file.txt` exists, check for `data` should return True
+        """
+        return self.isfile(path_info) or self.isdir(path_info)
+
+    def _upload(self, from_file, to_info, name=None, no_progress_bar=True):
         bucket = self.gs.bucket(to_info.bucket)
-        blob = bucket.blob(to_info.path)
-        blob.upload_from_filename(from_file)
+        _upload_to_bucket(
+            bucket,
+            from_file,
+            to_info,
+            name=name,
+            no_progress_bar=no_progress_bar,
+        )
 
-    def _download(self, from_info, to_file, **_kwargs):
+    def _download(self, from_info, to_file, name=None, no_progress_bar=True):
         bucket = self.gs.bucket(from_info.bucket)
         blob = bucket.get_blob(from_info.path)
-        blob.download_to_filename(to_file)
+        with io.open(to_file, mode="wb") as fobj:
+            with Tqdm.wrapattr(
+                fobj,
+                "write",
+                desc=name or from_info.path,
+                total=blob.size,
+                disable=no_progress_bar,
+            ) as wrapped:
+                blob.download_to_file(wrapped)
 
     def _generate_download_url(self, path_info, expires=3600):
         expiration = timedelta(seconds=int(expires))

dvc/remote/hdfs.py

@@ -1,28 +1,21 @@
 from __future__ import unicode_literals
 
-import os
-import re
+from dvc.utils.compat import FileNotFoundError, urlparse
 import io
-import getpass
-import posixpath
 import logging
-from subprocess import Popen, PIPE
-from contextlib import contextmanager, closing
-
-try:
-    import pyarrow
-except ImportError:
-    pyarrow = None
+import os
+import posixpath
+import re
+from collections import deque
+from contextlib import closing, contextmanager
+import subprocess
 
+from .base import RemoteBASE, RemoteCmdError
+from .pool import get_connection
 from dvc.config import Config
 from dvc.scheme import Schemes
-
-from dvc.utils.compat import urlparse, FileNotFoundError
 from dvc.utils import fix_env, tmp_fname
 
-from .pool import get_connection
-from .base import RemoteBASE, RemoteCmdError
-
 logger = logging.getLogger(__name__)
 
 
@@ -30,7 +23,7 @@ class RemoteHDFS(RemoteBASE):
     scheme = Schemes.HDFS
     REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
     PARAM_CHECKSUM = "checksum"
-    REQUIRES = {"pyarrow": pyarrow}
+    REQUIRES = {"pyarrow": "pyarrow"}
 
     def __init__(self, repo, config):
         super(RemoteHDFS, self).__init__(repo, config)
@@ -41,11 +34,7 @@ class RemoteHDFS(RemoteBASE):
 
         parsed = urlparse(url)
 
-        user = (
-            parsed.username
-            or config.get(Config.SECTION_REMOTE_USER)
-            or getpass.getuser()
-        )
+        user = parsed.username or config.get(Config.SECTION_REMOTE_USER)
 
         self.path_info = self.path_cls.from_parts(
             scheme=self.scheme,
@@ -55,8 +44,9 @@ class RemoteHDFS(RemoteBASE):
             path=parsed.path,
         )
 
-    @staticmethod
-    def hdfs(path_info):
+    def hdfs(self, path_info):
+        import pyarrow
+
         return get_connection(
             pyarrow.hdfs.connect,
             path_info.host,
@@ -74,15 +64,15 @@ class RemoteHDFS(RemoteBASE):
         close_fds = os.name != "nt"
 
         executable = os.getenv("SHELL") if os.name != "nt" else None
-        p = Popen(
+        p = subprocess.Popen(
             cmd,
             shell=True,
             close_fds=close_fds,
             executable=executable,
             env=fix_env(os.environ),
-            stdin=PIPE,
-            stdout=PIPE,
-            stderr=PIPE,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
         )
         out, err = p.communicate()
         if p.returncode != 0:
@@ -162,10 +152,9 @@ class RemoteHDFS(RemoteBASE):
 
     def list_cache_paths(self):
         if not self.exists(self.path_info):
-            return []
+            return
 
-        files = []
-        dirs = [self.path_info.path]
+        dirs = deque([self.path_info.path])
 
         with self.hdfs(self.path_info) as hdfs:
             while dirs:
@@ -173,6 +162,4 @@ class RemoteHDFS(RemoteBASE):
                     if entry["kind"] == "directory":
                         dirs.append(urlparse(entry["name"]).path)
                     elif entry["kind"] == "file":
-                        files.append(urlparse(entry["name"]).path)
-
-        return files
+                        yield urlparse(entry["name"]).path

dvc/remote/http.py

@@ -1,21 +1,25 @@
 from __future__ import unicode_literals
 
+import logging
+import threading
+
+from funcy import cached_property, wrap_prop
+
+from dvc.config import Config
+from dvc.config import ConfigError
+from dvc.exceptions import DvcException, HTTPError
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
 from dvc.utils.compat import open
 
-import requests
-import logging
-
-from dvc.progress import Tqdm
-from dvc.exceptions import DvcException
-from dvc.config import Config, ConfigError
-from dvc.remote.base import RemoteBASE
-
 logger = logging.getLogger(__name__)
 
 
 class RemoteHTTP(RemoteBASE):
     scheme = Schemes.HTTP
+    SESSION_RETRIES = 5
+    SESSION_BACKOFF_FACTOR = 0.1
     REQUEST_TIMEOUT = 10
     CHUNK_SIZE = 2 ** 16
     PARAM_CHECKSUM = "etag"
@@ -33,30 +37,26 @@ class RemoteHTTP(RemoteBASE):
             )
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        request = self._request("GET", from_info.url, stream=True)
+        response = self._request("GET", from_info.url, stream=True)
+        if response.status_code != 200:
+            raise HTTPError(response.status_code, response.reason)
         with Tqdm(
-            total=None if no_progress_bar else self._content_length(from_info),
+            total=None if no_progress_bar else self._content_length(response),
             leave=False,
             bytes=True,
             desc=from_info.url if name is None else name,
             disable=no_progress_bar,
         ) as pbar:
             with open(to_file, "wb") as fd:
-                for chunk in request.iter_content(chunk_size=self.CHUNK_SIZE):
+                for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE):
                     fd.write(chunk)
-                    fd.flush()
                     pbar.update(len(chunk))
 
     def exists(self, path_info):
         return bool(self._request("HEAD", path_info.url))
 
-    def _content_length(self, url_or_request):
-        headers = getattr(
-            url_or_request,
-            "headers",
-            self._request("HEAD", url_or_request).headers,
-        )
-        res = headers.get("Content-Length")
+    def _content_length(self, response):
+        res = response.headers.get("Content-Length")
         return int(res) if res else None
 
     def get_file_checksum(self, path_info):
@@ -78,12 +78,33 @@ class RemoteHTTP(RemoteBASE):
 
         return etag
 
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def _session(self):
+        import requests
+        from requests.adapters import HTTPAdapter
+        from urllib3.util.retry import Retry
+
+        session = requests.Session()
+
+        retries = Retry(
+            total=self.SESSION_RETRIES,
+            backoff_factor=self.SESSION_BACKOFF_FACTOR,
+        )
+
+        session.mount("http://", HTTPAdapter(max_retries=retries))
+        session.mount("https://", HTTPAdapter(max_retries=retries))
+
+        return session
+
     def _request(self, method, url, **kwargs):
+        import requests
+
         kwargs.setdefault("allow_redirects", True)
         kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
 
         try:
-            return requests.request(method, url, **kwargs)
+            return self._session.request(method, url, **kwargs)
         except requests.exceptions.RequestException:
             raise DvcException("could not perform a {} request".format(method))
 

dvc/remote/https.py

@@ -1,8 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.scheme import Schemes
-
 from .http import RemoteHTTP
+from dvc.scheme import Schemes
 
 
 class RemoteHTTPS(RemoteHTTP):

dvc/remote/local.py

@@ -1,43 +1,38 @@
 from __future__ import unicode_literals
 
-from copy import copy
-
-from dvc.scheme import Schemes
-from dvc.remote.local.slow_link_detection import slow_link_guard
-from dvc.utils.compat import str, fspath_py35, open
-
+import errno
+import logging
 import os
 import stat
-import errno
-from shortuuid import uuid
-import shutil
-import logging
+from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 
-from dvc.system import System
-from dvc.remote.base import (
-    RemoteBASE,
-    STATUS_MAP,
-    STATUS_NEW,
-    STATUS_DELETED,
-    STATUS_MISSING,
-)
-from dvc.utils import (
-    remove,
-    move,
-    copyfile,
-    tmp_fname,
-    file_md5,
-    walk_files,
-    relpath,
-    dvc_walk,
-    makedirs,
-)
-from dvc.config import Config
-from dvc.exceptions import DvcException
-from dvc.progress import Tqdm, TqdmThreadPoolExecutor
+from shortuuid import uuid
 
+from dvc.config import Config
+from dvc.exceptions import DownloadError
+from dvc.exceptions import DvcException
+from dvc.exceptions import UploadError
 from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.remote.base import STATUS_DELETED
+from dvc.remote.base import STATUS_MAP
+from dvc.remote.base import STATUS_MISSING
+from dvc.remote.base import STATUS_NEW
+from dvc.scheme import Schemes
+from dvc.system import System
+from dvc.utils import copyfile
+from dvc.utils import file_md5
+from dvc.utils import makedirs
+from dvc.utils import relpath
+from dvc.utils import tmp_fname
+from dvc.utils import walk_files
+from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import open
+from dvc.utils.compat import str
+from dvc.utils.fs import move
+from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
@@ -51,12 +46,6 @@ class RemoteLOCAL(RemoteBASE):
     UNPACKED_DIR_SUFFIX = ".unpacked"
 
     DEFAULT_CACHE_TYPES = ["reflink", "copy"]
-    CACHE_TYPE_MAP = {
-        "copy": shutil.copyfile,
-        "symlink": System.symlink,
-        "hardlink": System.hardlink,
-        "reflink": System.reflink,
-    }
 
     SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
 
@@ -71,26 +60,12 @@ class RemoteLOCAL(RemoteBASE):
             # cache files are set to be read-only for everyone
             self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
 
-        types = config.get(Config.SECTION_CACHE_TYPE, None)
-        if types:
-            if isinstance(types, str):
-                types = [t.strip() for t in types.split(",")]
-            self.cache_types = types
-        else:
-            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
-        self.cache_type_confirmed = False
-
-        # A clunky way to detect cache dir
-        storagepath = config.get(Config.SECTION_LOCAL_STORAGEPATH, None)
-        cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        cache_dir = config.get(Config.SECTION_REMOTE_URL)
 
         if cache_dir is not None and not os.path.isabs(cache_dir):
             cwd = config[Config.PRIVATE_CWD]
             cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
 
-        if cache_dir is not None and not os.path.exists(cache_dir):
-            os.mkdir(cache_dir)
-
         self.path_info = PathInfo(cache_dir) if cache_dir else None
         self._dir_info = {}
 
@@ -108,17 +83,7 @@ class RemoteLOCAL(RemoteBASE):
 
     def list_cache_paths(self):
         assert self.path_info is not None
-
-        clist = []
-        for entry in os.listdir(fspath_py35(self.path_info)):
-            subdir = self.path_info / entry
-            if not os.path.isdir(fspath_py35(subdir)):
-                continue
-            clist.extend(
-                subdir / cache for cache in os.listdir(fspath_py35(subdir))
-            )
-
-        return clist
+        return walk_files(self.path_info, None)
 
     def get(self, md5):
         if not md5:
@@ -126,80 +91,14 @@ class RemoteLOCAL(RemoteBASE):
 
         return self.checksum_to_path_info(md5).url
 
-    def exists(self, path_info):
+    @staticmethod
+    def exists(path_info):
         assert path_info.scheme == "local"
         return os.path.lexists(fspath_py35(path_info))
 
     def makedirs(self, path_info):
         makedirs(path_info, exist_ok=True, mode=self._dir_mode)
 
-    def link(self, from_info, to_info):
-        self._link(from_info, to_info, self.cache_types)
-
-    def _link(self, from_info, to_info, link_types):
-        from_path = from_info.fspath
-        to_path = to_info.fspath
-
-        assert os.path.isfile(from_path)
-
-        dname = os.path.dirname(to_path)
-        if not os.path.exists(dname):
-            os.makedirs(dname)
-
-        # NOTE: just create an empty file for an empty cache
-        if os.path.getsize(from_path) == 0:
-            open(to_path, "w+").close()
-
-            msg = "Created empty file: {} -> {}".format(from_path, to_path)
-            logger.debug(msg)
-            return
-
-        self._try_links(from_info, to_info, link_types)
-
-    @classmethod
-    def _get_link_method(cls, link_type):
-        try:
-            return cls.CACHE_TYPE_MAP[link_type]
-        except KeyError:
-            raise DvcException(
-                "Cache type: '{}' not supported!".format(link_type)
-            )
-
-    def _do_link(self, from_info, to_info, link_method):
-        if self.exists(to_info):
-            raise DvcException("Link '{}' already exists!".format(to_info))
-        else:
-            link_method(from_info.fspath, to_info.fspath)
-
-        if self.protected:
-            self.protect(to_info)
-
-        msg = "Created {}'{}': {} -> {}".format(
-            "protected " if self.protected else "",
-            self.cache_types[0],
-            from_info,
-            to_info,
-        )
-        logger.debug(msg)
-
-    @slow_link_guard
-    def _try_links(self, from_info, to_info, link_types):
-        i = len(link_types)
-        while i > 0:
-            link_method = self._get_link_method(link_types[0])
-            try:
-                self._do_link(from_info, to_info, link_method)
-                self.cache_type_confirmed = True
-                return
-
-            except DvcException as exc:
-                msg = "Cache type '{}' is not supported: {}"
-                logger.debug(msg.format(link_types[0], str(exc)))
-                del link_types[0]
-                i -= 1
-
-        raise DvcException("no possible cache types left to try out.")
-
     def already_cached(self, path_info):
         assert path_info.scheme in ["", "local"]
 
@@ -221,17 +120,29 @@ class RemoteLOCAL(RemoteBASE):
 
         return False
 
-    def isfile(self, path_info):
+    @staticmethod
+    def isfile(path_info):
         return os.path.isfile(fspath_py35(path_info))
 
-    def isdir(self, path_info):
+    @staticmethod
+    def isdir(path_info):
         return os.path.isdir(fspath_py35(path_info))
 
-    def walk(self, path_info):
-        return dvc_walk(path_info, self.repo.dvcignore)
+    def iscopy(self, path_info):
+        return not (
+            System.is_symlink(path_info) or System.is_hardlink(path_info)
+        )
+
+    @staticmethod
+    def getsize(path_info):
+        return os.path.getsize(fspath_py35(path_info))
+
+    def walk_files(self, path_info):
+        for fname in walk_files(path_info, self.repo.dvcignore):
+            yield PathInfo(fname)
 
     def get_file_checksum(self, path_info):
-        return file_md5(fspath_py35(path_info))[0]
+        return file_md5(path_info)[0]
 
     def remove(self, path_info):
         if path_info.scheme != "local":
@@ -253,6 +164,44 @@ class RemoteLOCAL(RemoteBASE):
 
         move(from_info, to_info, mode=mode)
 
+    @staticmethod
+    def copy(from_info, to_info):
+        System.copy(from_info, to_info)
+
+    @staticmethod
+    def symlink(from_info, to_info):
+        System.symlink(from_info, to_info)
+
+    def hardlink(self, from_info, to_info):
+        # If there are a lot of empty files (which happens a lot in datasets),
+        # and the cache type is `hardlink`, we might reach link limits and
+        # will get something like: `too many links error`
+        #
+        # This is because all those empty files will have the same checksum
+        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
+        # linked to the same file in the cache.
+        #
+        # From https://en.wikipedia.org/wiki/Hard_link
+        #   * ext4 limits the number of hard links on a file to 65,000
+        #   * Windows with NTFS has a limit of 1024 hard links on a file
+        #
+        # That's why we simply create an empty file rather than a link.
+        if self.getsize(from_info) == 0:
+            self.open(to_info, "w").close()
+
+            logger.debug(
+                "Created empty file: {src} -> {dest}".format(
+                    src=str(from_info), dest=str(to_info)
+                )
+            )
+            return
+
+        System.hardlink(from_info, to_info)
+
+    @staticmethod
+    def reflink(from_info, to_info):
+        System.reflink(from_info, to_info)
+
     def cache_exists(self, checksums, jobs=None, name=None):
         return [
             checksum
@@ -268,7 +217,7 @@ class RemoteLOCAL(RemoteBASE):
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        makedirs(fspath_py35(to_info.parent), exist_ok=True)
+        makedirs(to_info.parent, exist_ok=True)
 
         tmp_file = tmp_fname(to_info)
         copyfile(
@@ -280,41 +229,16 @@ class RemoteLOCAL(RemoteBASE):
         self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
     ):
         copyfile(
-            fspath_py35(from_info),
-            to_file,
-            no_progress_bar=no_progress_bar,
-            name=name,
+            from_info, to_file, no_progress_bar=no_progress_bar, name=name
         )
 
-    def open(self, path_info, mode="r", encoding=None):
-        assert mode in {"r", "rt", "rb"}
+    @staticmethod
+    def open(path_info, mode="r", encoding=None):
         return open(fspath_py35(path_info), mode=mode, encoding=encoding)
 
-    def _group(self, checksum_infos, show_checksums=False):
-        by_md5 = {}
-
-        for info in checksum_infos:
-            md5 = info[self.PARAM_CHECKSUM]
-
-            if show_checksums:
-                by_md5[md5] = {"name": md5}
-                continue
-
-            name = info["name"]
-            branch = info.get("branch")
-            if branch:
-                name += "({})".format(branch)
-
-            if md5 not in by_md5.keys():
-                by_md5[md5] = {"name": name}
-            else:
-                by_md5[md5]["name"] += " " + name
-
-        return by_md5
-
     def status(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
@@ -323,15 +247,14 @@ class RemoteLOCAL(RemoteBASE):
         logger.debug(
             "Preparing to collect status from {}".format(remote.path_info)
         )
-        ret = self._group(checksum_infos, show_checksums=show_checksums) or {}
-        md5s = list(ret)
+        md5s = list(named_cache[self.scheme])
 
         logger.debug("Collecting information from local cache...")
         local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
 
         # This is a performance optimization. We can safely assume that,
         # if the resources that we want to fetch are already cached,
-        # there's no need to check the remote storage for the existance of
+        # there's no need to check the remote storage for the existence of
         # those files.
         if download and sorted(local_exists) == sorted(md5s):
             remote_exists = local_exists
@@ -343,6 +266,10 @@ class RemoteLOCAL(RemoteBASE):
                 )
             )
 
+        ret = {
+            checksum: {"name": checksum if show_checksums else " ".join(names)}
+            for checksum, names in named_cache[self.scheme].items()
+        }
         self._fill_statuses(ret, local_exists, remote_exists)
 
         self._log_missing_caches(ret)
@@ -382,7 +309,7 @@ class RemoteLOCAL(RemoteBASE):
 
     def _process(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
@@ -410,7 +337,7 @@ class RemoteLOCAL(RemoteBASE):
             jobs = remote.JOBS
 
         status_info = self.status(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
@@ -423,31 +350,30 @@ class RemoteLOCAL(RemoteBASE):
             return 0
 
         if jobs > 1:
-            with TqdmThreadPoolExecutor(max_workers=jobs) as executor:
+            with ThreadPoolExecutor(max_workers=jobs) as executor:
                 fails = sum(executor.map(func, *plans))
         else:
             fails = sum(map(func, *plans))
 
         if fails:
-            msg = "{} files failed to {}"
-            raise DvcException(
-                msg.format(fails, "download" if download else "upload")
-            )
+            if download:
+                raise DownloadError(fails)
+            raise UploadError(fails)
 
         return len(plans[0])
 
-    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def push(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
             download=False,
         )
 
-    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
@@ -473,22 +399,19 @@ class RemoteLOCAL(RemoteBASE):
             logger.warning(msg)
 
     @staticmethod
-    def _unprotect_file(path, allow_copy=True):
+    def _unprotect_file(path):
         if System.is_symlink(path) or System.is_hardlink(path):
-            if allow_copy:
-                logger.debug("Unprotecting '{}'".format(path))
-                tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
+            logger.debug("Unprotecting '{}'".format(path))
+            tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
 
-                # The operations order is important here - if some application
-                # would access the file during the process of copyfile then it
-                # would get only the part of file. So, at first, the file
-                # should be copied with the temporary name, and then
-                # original file should be replaced by new.
-                copyfile(
-                    path, tmp, name="Unprotecting '{}'".format(relpath(path))
-                )
-                remove(path)
-                os.rename(tmp, path)
+            # The operations order is important here - if some application
+            # would access the file during the process of copyfile then it
+            # would get only the part of file. So, at first, the file should be
+            # copied with the temporary name, and then original file should be
+            # replaced by new.
+            copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
+            remove(path)
+            os.rename(tmp, path)
 
         else:
             logger.debug(
@@ -498,11 +421,11 @@ class RemoteLOCAL(RemoteBASE):
 
         os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
 
-    def _unprotect_dir(self, path, allow_copy=True):
+    def _unprotect_dir(self, path):
         for fname in walk_files(path, self.repo.dvcignore):
-            self._unprotect_file(fname, allow_copy)
+            RemoteLOCAL._unprotect_file(fname)
 
-    def unprotect(self, path_info, allow_copy=True):
+    def unprotect(self, path_info):
         path = path_info.fspath
         if not os.path.exists(path):
             raise DvcException(
@@ -510,9 +433,9 @@ class RemoteLOCAL(RemoteBASE):
             )
 
         if os.path.isdir(path):
-            self._unprotect_dir(path, allow_copy)
+            self._unprotect_dir(path)
         else:
-            self._unprotect_file(path, allow_copy)
+            RemoteLOCAL._unprotect_file(path)
 
     @staticmethod
     def protect(path_info):
@@ -586,36 +509,3 @@ class RemoteLOCAL(RemoteBASE):
             if self.is_dir_checksum(c):
                 unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
         return unpacked
-
-    def _get_cache_type(self, path_info):
-        if self.cache_type_confirmed:
-            return self.cache_types[0]
-
-        workspace_file = path_info.with_name("." + uuid())
-        test_cache_file = self.path_info / ".cache_type_test_file"
-        if not self.exists(test_cache_file):
-            with open(fspath_py35(test_cache_file), "wb") as fobj:
-                fobj.write(bytes(1))
-        try:
-            self.link(test_cache_file, workspace_file)
-        finally:
-            self.remove(workspace_file)
-            self.remove(test_cache_file)
-
-        self.cache_type_confirmed = True
-        return self.cache_types[0]
-
-    def _link_matches(self, path_info):
-        is_hardlink = System.is_hardlink(path_info)
-        is_symlink = System.is_symlink(path_info)
-        is_copy_or_reflink = not is_hardlink and not is_symlink
-
-        cache_type = self._get_cache_type(path_info)
-
-        if cache_type == "symlink":
-            return is_symlink
-
-        if cache_type == "hardlink":
-            return is_hardlink
-
-        return is_copy_or_reflink

dvc/remote/local/slow_link_detection.py

@@ -1,62 +0,0 @@
-import logging
-import time
-
-import colorama
-from dvc.config import Config
-
-logger = logging.getLogger(__name__)
-
-
-class SlowLinkDetectorDecorator(object):
-    LINKING_TIMEOUT_SECONDS = 10.0
-    was_displayed = False
-
-    @classmethod
-    def should_display(cls):
-        if not cls.was_displayed:
-            cls.was_displayed = True
-            return True
-        return False
-
-    def __init__(self, method):
-        self.method = method
-
-    def __call__(self, *args, **kwargs):
-        start = time.time()
-        result = self.method(*args, **kwargs)
-        execution_time_seconds = time.time() - start
-
-        if (
-            execution_time_seconds >= self.LINKING_TIMEOUT_SECONDS
-            and self.should_display()
-        ):
-            msg = (
-                "You can cut execution time considerably by using a different"
-                " `cache.type` configuration.\nSee "
-                "{blue}https://dvc.org/doc/commands-reference/config#cache{"
-                "reset} for more information.\n"
-                "To disable this message, run:\n"
-                "'dvc config cache.slow_link_warning false'".format(
-                    blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
-                )
-            )
-            logger.warning(msg)
-
-        return result
-
-
-def slow_link_guard(method):
-    def call(remote_local, *args, **kwargs):
-        cache_config = remote_local.repo.config.config.get(
-            Config.SECTION_CACHE, {}
-        )
-        should_warn = cache_config.get(
-            Config.SECTION_CACHE_SLOW_LINK_WARNING, True
-        ) and not cache_config.get(Config.SECTION_CACHE_TYPE, None)
-
-        if should_warn:
-            decorated = SlowLinkDetectorDecorator(method)
-            return decorated(remote_local, *args, **kwargs)
-        return method(remote_local, *args, **kwargs)
-
-    return call

dvc/remote/oss.py

@@ -1,20 +1,17 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-import os
 import logging
+import os
+import threading
 
-from dvc.scheme import Schemes
-
-try:
-    import oss2
-except ImportError:
-    oss2 = None
+from funcy import cached_property, wrap_prop
 
 from dvc.config import Config
-from dvc.remote.base import RemoteBASE
-from dvc.progress import Tqdm
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.scheme import Schemes
 
 
 logger = logging.getLogger(__name__)
@@ -41,7 +38,7 @@ class RemoteOSS(RemoteBASE):
 
     scheme = Schemes.OSS
     path_cls = CloudURLInfo
-    REQUIRES = {"oss2": oss2}
+    REQUIRES = {"oss2": "oss2"}
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
 
@@ -67,28 +64,29 @@ class RemoteOSS(RemoteBASE):
             or "defaultSecret"
         )
 
-        self._bucket = None
-
-    @property
+    @wrap_prop(threading.Lock())
+    @cached_property
     def oss_service(self):
-        if self._bucket is None:
-            logger.debug("URL {}".format(self.path_info))
-            logger.debug("key id {}".format(self.key_id))
-            logger.debug("key secret {}".format(self.key_secret))
-            auth = oss2.Auth(self.key_id, self.key_secret)
-            self._bucket = oss2.Bucket(
-                auth, self.endpoint, self.path_info.bucket
+        import oss2
+
+        logger.debug("URL {}".format(self.path_info))
+        logger.debug("key id {}".format(self.key_id))
+        logger.debug("key secret {}".format(self.key_secret))
+
+        auth = oss2.Auth(self.key_id, self.key_secret)
+        bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)
+
+        # Ensure bucket exists
+        try:
+            bucket.get_bucket_info()
+        except oss2.exceptions.NoSuchBucket:
+            bucket.create_bucket(
+                oss2.BUCKET_ACL_PUBLIC_READ,
+                oss2.models.BucketCreateConfig(
+                    oss2.BUCKET_STORAGE_CLASS_STANDARD
+                ),
             )
-            try:  # verify that bucket exists
-                self._bucket.get_bucket_info()
-            except oss2.exceptions.NoSuchBucket:
-                self._bucket.create_bucket(
-                    oss2.BUCKET_ACL_PUBLIC_READ,
-                    oss2.models.BucketCreateConfig(
-                        oss2.BUCKET_STORAGE_CLASS_STANDARD
-                    ),
-                )
-        return self._bucket
+        return bucket
 
     def remove(self, path_info):
         if path_info.scheme != self.scheme:
@@ -98,6 +96,8 @@ class RemoteOSS(RemoteBASE):
         self.oss_service.delete_object(path_info.path)
 
     def _list_paths(self, prefix):
+        import oss2
+
         for blob in oss2.ObjectIterator(self.oss_service, prefix=prefix):
             yield blob.key
 

dvc/remote/pool.py

@@ -1,6 +1,8 @@
 from collections import deque
 from contextlib import contextmanager
-from funcy import memoize
+import threading
+
+from funcy import memoize, wrap_with
 
 
 @contextmanager
@@ -16,6 +18,7 @@ def get_connection(conn_func, *args, **kwargs):
         pool.release(conn)
 
 
+@wrap_with(threading.Lock())
 @memoize
 def get_pool(conn_func, *args, **kwargs):
     return Pool(conn_func, *args, **kwargs)

dvc/remote/s3.py

@@ -1,19 +1,19 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import os
 import logging
-from funcy import cached_property
+import os
+import threading
 
-try:
-    import boto3
-except ImportError:
-    boto3 = None
+from botocore.exceptions import ClientError
+from funcy import cached_property, wrap_prop
 
-from dvc.progress import Tqdm
 from dvc.config import Config
-from dvc.remote.base import RemoteBASE
-from dvc.exceptions import DvcException, ETagMismatchError
+from dvc.exceptions import DvcException
+from dvc.exceptions import ETagMismatchError
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
@@ -22,17 +22,13 @@ logger = logging.getLogger(__name__)
 class RemoteS3(RemoteBASE):
     scheme = Schemes.S3
     path_cls = CloudURLInfo
-    REQUIRES = {"boto3": boto3}
+    REQUIRES = {"boto3": "boto3"}
     PARAM_CHECKSUM = "etag"
 
     def __init__(self, repo, config):
         super(RemoteS3, self).__init__(repo, config)
 
-        storagepath = "s3://{}".format(
-            config.get(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
-        )
-
-        url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        url = config.get(Config.SECTION_REMOTE_URL, "s3://")
         self.path_info = self.path_cls(url)
 
         self.region = config.get(Config.SECTION_AWS_REGION)
@@ -41,7 +37,10 @@ class RemoteS3(RemoteBASE):
 
         self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
 
-        self.list_objects = config.get(Config.SECTION_AWS_LIST_OBJECTS)
+        if config.get(Config.SECTION_AWS_LIST_OBJECTS):
+            self.list_objects_api = "list_objects"
+        else:
+            self.list_objects_api = "list_objects_v2"
 
         self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
 
@@ -59,8 +58,11 @@ class RemoteS3(RemoteBASE):
         if shared_creds:
             os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def s3(self):
+        import boto3
+
         session = boto3.session.Session(
             profile_name=self.profile, region_name=self.region
         )
@@ -142,13 +144,13 @@ class RemoteS3(RemoteBASE):
         # multipart.
         #
         # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
-        # then it was transfered as a single part, which means that the chunk
+        # then it was transferred as a single part, which means that the chunk
         # size used to transfer it was greater or equal to the ContentLength
         # of that object. So to preserve that tag over the next transfer, we
         # could use any value >= ContentLength.
         #
         # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
-        # then it was transfered as a multipart, which means that the chunk
+        # then it was transferred as a multipart, which means that the chunk
         # size used to transfer it was less than ContentLength of that object.
         # Unfortunately, in general, it doesn't mean that the chunk size was
         # the same throughout the transfer, so it means that in order to
@@ -183,27 +185,82 @@ class RemoteS3(RemoteBASE):
         logger.debug("Removing {}".format(path_info))
         self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
 
-    def _list_paths(self, bucket, prefix):
+    def _list_objects(self, path_info, max_items=None):
         """ Read config for list object api, paginate through list objects."""
-        kwargs = {"Bucket": bucket, "Prefix": prefix}
-        if self.list_objects:
-            list_objects_api = "list_objects"
-        else:
-            list_objects_api = "list_objects_v2"
-        paginator = self.s3.get_paginator(list_objects_api)
+        kwargs = {
+            "Bucket": path_info.bucket,
+            "Prefix": path_info.path,
+            "PaginationConfig": {"MaxItems": max_items},
+        }
+        paginator = self.s3.get_paginator(self.list_objects_api)
         for page in paginator.paginate(**kwargs):
             contents = page.get("Contents", None)
             if not contents:
                 continue
             for item in contents:
-                yield item["Key"]
+                yield item
+
+    def _list_paths(self, path_info, max_items=None):
+        return (
+            item["Key"] for item in self._list_objects(path_info, max_items)
+        )
 
     def list_cache_paths(self):
-        return self._list_paths(self.path_info.bucket, self.path_info.path)
+        return self._list_paths(self.path_info)
+
+    def isfile(self, path_info):
+        if path_info.path.endswith("/"):
+            return False
+
+        try:
+            self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
+        except ClientError as exc:
+            if exc.response["Error"]["Code"] != "404":
+                raise
+            return False
+
+        return True
 
     def exists(self, path_info):
-        paths = self._list_paths(path_info.bucket, path_info.path)
-        return any(path_info.path == path for path in paths)
+        """Check if the blob exists. If it does not exist,
+        it could be a part of a directory path.
+
+        eg: if `data/file.txt` exists, check for `data` should return True
+        """
+        return self.isfile(path_info) or self.isdir(path_info)
+
+    def makedirs(self, path_info):
+        # We need to support creating empty directories, which means
+        # creating an object with an empty body and a trailing slash `/`.
+        #
+        # We are not creating directory objects for every parent prefix,
+        # as it is not required.
+        dir_path = path_info / ""
+        self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
+
+    def isdir(self, path_info):
+        # S3 doesn't have a concept for directories.
+        #
+        # Using `head_object` with a path pointing to a directory
+        # will throw a 404 error.
+        #
+        # A reliable way to know if a given path is a directory is by
+        # checking if there are more files sharing the same prefix
+        # with a `list_objects` call.
+        #
+        # We need to make sure that the path ends with a forward slash,
+        # since we can end with false-positives like the following example:
+        #
+        #       bucket
+        #       └── data
+        #          ├── alice
+        #          └── alpha
+        #
+        # Using `data/al` as prefix will return `[data/alice, data/alpha]`,
+        # While `data/al/` will return nothing.
+        #
+        dir_path = path_info / ""
+        return bool(list(self._list_paths(dir_path, max_items=1)))
 
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
         total = os.path.getsize(from_file)
@@ -237,3 +294,10 @@ class RemoteS3(RemoteBASE):
         return self.s3.generate_presigned_url(
             ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
         )
+
+    def walk_files(self, path_info, max_items=None):
+        for fname in self._list_paths(path_info / "", max_items):
+            if fname.endswith("/"):
+                continue
+
+            yield path_info.replace(path=fname)

dvc/remote/slow_link_detection.py

@@ -0,0 +1,51 @@
+import logging
+import sys
+import time
+from functools import wraps
+
+import colorama
+
+from dvc.config import Config
+
+
+logger = logging.getLogger(__name__)
+this = sys.modules[__name__]
+
+this.timeout_seconds = 10.0
+this.already_displayed = False
+this.message = (
+    "You can cut execution time considerably by using a different "
+    "`cache.type` configuration.\n"
+    "See {blue}https://dvc.org/doc/commands-reference/config#cache{reset} "
+    "for more information.\n"
+    "To disable this message, run:\n"
+    "`dvc config cache.slow_link_warning false`".format(
+        blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
+    )
+)
+
+
+def slow_link_guard(f):
+    @wraps(f)
+    def wrapper(remote, *args, **kwargs):
+        if this.already_displayed:
+            return f(remote, *args, **kwargs)
+
+        config = remote.repo.config.config.get(Config.SECTION_CACHE, {})
+        cache_type = config.get(Config.SECTION_CACHE_TYPE)
+        should_warn = config.get(Config.SECTION_CACHE_SLOW_LINK_WARNING, True)
+
+        if not should_warn or cache_type:
+            return f(remote, *args, **kwargs)
+
+        start = time.time()
+        result = f(remote, *args, **kwargs)
+        delta = time.time() - start
+
+        if delta >= this.timeout_seconds:
+            logger.warning(this.message)
+            this.already_displayed = True
+
+        return result
+
+    return wrapper

dvc/remote/ssh/__init__.py

@@ -1,42 +1,44 @@
 from __future__ import unicode_literals
 
 import errno
-import itertools
-import io
-import os
 import getpass
+import io
+import itertools
 import logging
+import os
 import threading
 from concurrent.futures import ThreadPoolExecutor
-from contextlib import contextmanager, closing
+from contextlib import closing, contextmanager
 
-try:
-    import paramiko
-except ImportError:
-    paramiko = None
+from funcy import memoize, wrap_with
 
 import dvc.prompt as prompt
 from dvc.config import Config
-from dvc.utils import to_chunks
-from dvc.utils.compat import urlparse, StringIO
-from dvc.remote.base import RemoteBASE
-from dvc.scheme import Schemes
-from dvc.remote.pool import get_connection
 from dvc.progress import Tqdm
-
-from .connection import SSHConnection
-
+from dvc.remote.base import RemoteBASE
+from dvc.remote.pool import get_connection
+from dvc.scheme import Schemes
+from dvc.utils import to_chunks
+from dvc.utils.compat import StringIO
+from dvc.utils.compat import urlparse
 
 logger = logging.getLogger(__name__)
 
 
-saved_passwords = {}
-saved_passwords_lock = threading.Lock()
+@wrap_with(threading.Lock())
+@memoize
+def ask_password(host, user, port):
+    return prompt.password(
+        "Enter a private key passphrase or a password for "
+        "host '{host}' port '{port}' user '{user}'".format(
+            host=host, port=port, user=user
+        )
+    )
 
 
 class RemoteSSH(RemoteBASE):
     scheme = Schemes.SSH
-    REQUIRES = {"paramiko": paramiko}
+    REQUIRES = {"paramiko": "paramiko"}
 
     JOBS = 4
     PARAM_CHECKSUM = "md5"
@@ -47,9 +49,10 @@ class RemoteSSH(RemoteBASE):
     # We use conservative setting of 4 instead to not exhaust max sessions.
     CHECKSUM_JOBS = 4
 
+    DEFAULT_CACHE_TYPES = ["copy"]
+
     def __init__(self, repo, config):
         super(RemoteSSH, self).__init__(repo, config)
-
         url = config.get(Config.SECTION_REMOTE_URL)
         if url:
             parsed = urlparse(url)
@@ -95,6 +98,8 @@ class RemoteSSH(RemoteBASE):
 
     @staticmethod
     def _load_user_ssh_config(hostname):
+        import paramiko
+
         user_config_file = RemoteSSH.ssh_config_filename()
         user_ssh_config = {}
         if hostname and os.path.exists(user_config_file):
@@ -120,29 +125,25 @@ class RemoteSSH(RemoteBASE):
             return identity_file[0]
         return None
 
-    def ssh(self, path_info):
-        host, user, port = path_info.host, path_info.user, path_info.port
+    def ensure_credentials(self, path_info=None):
+        if path_info is None:
+            path_info = self.path_info
 
         # NOTE: we use the same password regardless of the server :(
         if self.ask_password and self.password is None:
-            with saved_passwords_lock:
-                server_key = (host, user, port)
-                password = saved_passwords.get(server_key)
+            host, user, port = path_info.host, path_info.user, path_info.port
+            self.password = ask_password(host, user, port)
 
-                if password is None:
-                    saved_passwords[server_key] = password = prompt.password(
-                        "Enter a private key passphrase or a password for "
-                        "host '{host}' port '{port}' user '{user}'".format(
-                            host=host, port=port, user=user
-                        )
-                    )
-                self.password = password
+    def ssh(self, path_info):
+        self.ensure_credentials(path_info)
+
+        from .connection import SSHConnection
 
         return get_connection(
             SSHConnection,
-            host,
-            username=user,
-            port=port,
+            path_info.host,
+            username=path_info.user,
+            port=path_info.port,
             key_filename=self.keyfile,
             timeout=self.timeout,
             password=self.password,
@@ -164,12 +165,54 @@ class RemoteSSH(RemoteBASE):
         with self.ssh(path_info) as ssh:
             return ssh.isdir(path_info.path)
 
+    def isfile(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.isfile(path_info.path)
+
+    def getsize(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.getsize(path_info.path)
+
     def copy(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.copy(from_info.path, to_info.path)
+
+    def symlink(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.symlink(from_info.path, to_info.path)
+
+    def hardlink(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        # See dvc/remote/local/__init__.py - hardlink()
+        if self.getsize(from_info) == 0:
+
+            with self.ssh(to_info) as ssh:
+                ssh.sftp.open(to_info.path, "w").close()
+
+            logger.debug(
+                "Created empty file: {src} -> {dest}".format(
+                    src=str(from_info), dest=str(to_info)
+                )
+            )
+            return
+
+        with self.ssh(from_info) as ssh:
+            ssh.hardlink(from_info.path, to_info.path)
+
+    def reflink(self, from_info, to_info):
         if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
             raise NotImplementedError
 
         with self.ssh(from_info) as ssh:
-            ssh.cp(from_info.path, to_info.path)
+            ssh.reflink(from_info.path, to_info.path)
 
     def remove(self, path_info):
         if path_info.scheme != self.scheme:
@@ -207,24 +250,26 @@ class RemoteSSH(RemoteBASE):
 
     @contextmanager
     def open(self, path_info, mode="r", encoding=None):
-        assert mode in {"r", "rt", "rb"}
+        assert mode in {"r", "rt", "rb", "wb"}
 
         with self.ssh(path_info) as ssh, closing(
-            ssh.sftp.file(path_info.path, mode="r")
+            ssh.sftp.open(path_info.path, mode)
         ) as fd:
-            if mode == "rb":
+            if "b" in mode:
                 yield fd
             else:
                 yield io.TextIOWrapper(fd, encoding=encoding)
 
     def list_cache_paths(self):
         with self.ssh(self.path_info) as ssh:
-            return list(ssh.walk_files(self.path_info.path))
+            # If we simply return an iterator then with above closes instantly
+            for path in ssh.walk_files(self.path_info.path):
+                yield path
 
-    def walk(self, path_info):
+    def walk_files(self, path_info):
         with self.ssh(path_info) as ssh:
-            for entry in ssh.walk(path_info.path):
-                yield entry
+            for fname in ssh.walk_files(path_info.path):
+                yield path_info.replace(path=fname)
 
     def makedirs(self, path_info):
         with self.ssh(path_info) as ssh:
@@ -267,6 +312,9 @@ class RemoteSSH(RemoteBASE):
         if not self.no_traverse:
             return list(set(checksums) & set(self.all()))
 
+        # possibly prompt for credentials before "Querying" progress output
+        self.ensure_credentials()
+
         with Tqdm(
             desc="Querying "
             + ("cache in " + name if name else "remote cache"),

dvc/remote/ssh/connection.py

@@ -1,8 +1,9 @@
+import errno
+import logging
 import os
 import posixpath
-import logging
-import errno
 import stat
+
 from funcy import cached_property
 
 try:
@@ -60,7 +61,13 @@ class SSHConnection:
 
     def st_mode(self, path):
         with ignore_file_not_found():
-            return self.sftp.stat(path).st_mode
+            return self.sftp.lstat(path).st_mode
+
+        return 0
+
+    def getsize(self, path):
+        with ignore_file_not_found():
+            return self.sftp.lstat(path).st_size
 
         return 0
 
@@ -100,7 +107,10 @@ class SSHConnection:
                 # Since paramiko errors are very vague we need to recheck
                 # whether it's because path already exists or something else
                 if e.errno == errno.EACCES or not self.exists(path):
-                    raise
+                    raise DvcException(
+                        "unable to create remote directory '{}'".format(path),
+                        cause=e,
+                    )
 
     def walk(self, directory, topdown=True):
         # NOTE: original os.walk() implementation [1] with default options was
@@ -176,8 +186,26 @@ class SSHConnection:
             self.sftp.get(src, dest, callback=pbar.update_to)
 
     def move(self, src, dst):
+        """Rename src to dst, if it is not possible (in case src and dst are
+        on different filesystems) and actual physical copying of data is
+        happening.
+        """
         self.makedirs(posixpath.dirname(dst))
-        self.sftp.rename(src, dst)
+
+        try:
+            self.sftp.rename(src, dst)
+        except OSError:
+            self._atomic_copy(src, dst)
+            self.remove(src)
+
+    def _atomic_copy(self, src, dst):
+        tmp = tmp_fname(dst)
+
+        try:
+            self.copy(src, tmp)
+            self.sftp.rename(tmp, dst)
+        finally:
+            self.remove(tmp)
 
     def upload(self, src, dest, no_progress_bar=False, progress_title=None):
         self.makedirs(posixpath.dirname(dest))
@@ -267,8 +295,7 @@ class SSHConnection:
         assert len(md5) == 32
         return md5
 
-    def cp(self, src, dest):
-        self.makedirs(posixpath.dirname(dest))
+    def copy(self, src, dest):
         self.execute("cp {} {}".format(src, dest))
 
     def open_max_sftp_channels(self):
@@ -282,3 +309,23 @@ class SSHConnection:
                         raise
                     break
         return self._sftp_channels
+
+    def open(self, *args, **kwargs):
+        return self.sftp.open(*args, **kwargs)
+
+    def symlink(self, src, dest):
+        self.sftp.symlink(src, dest)
+
+    def reflink(self, src, dest):
+        if self.uname == "Linux":
+            return self.execute("cp --reflink {} {}".format(src, dest))
+
+        if self.uname == "Darwin":
+            return self.execute("cp -c {} {}".format(src, dest))
+
+        raise DvcException(
+            "'{}' is not supported as a SSH remote".format(self.uname)
+        )
+
+    def hardlink(self, src, dest):
+        self.execute("ln {} {}".format(src, dest))

dvc/repo/__init__.py

@@ -1,28 +1,46 @@
 from __future__ import unicode_literals
 
-import os
 import logging
+import os
 from contextlib import contextmanager
+from functools import wraps
 from itertools import chain
 
 from funcy import cached_property
 
+from .graph import check_acyclic
+from .graph import get_pipeline
+from .graph import get_pipelines
+from .graph import get_stages
 from dvc.config import Config
-from dvc.exceptions import (
-    NotDvcRepoError,
-    OutputNotFoundError,
-    TargetNotDirectoryError,
-    OutputFileMissingError,
-)
+from dvc.exceptions import FileMissingError
+from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import OutputNotFoundError
 from dvc.ignore import DvcIgnoreFilter
 from dvc.path_info import PathInfo
 from dvc.remote.base import RemoteActionNotImplemented
-from dvc.utils.compat import open as _open, fspath_py35, FileNotFoundError
 from dvc.utils import relpath
+from dvc.utils.fs import path_isin
+from dvc.utils.compat import FileNotFoundError
+from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import open as _open
+
 
 logger = logging.getLogger(__name__)
 
 
+def locked(f):
+    @wraps(f)
+    def wrapper(repo, *args, **kwargs):
+        with repo.lock, repo.state:
+            ret = f(repo, *args, **kwargs)
+            # Our graph cache is no longer valid after we release the repo.lock
+            repo._reset()
+            return ret
+
+    return wrapper
+
+
 class Repo(object):
     DVC_DIR = ".dvc"
 
@@ -36,9 +54,9 @@ class Repo(object):
     from dvc.repo.imp import imp
     from dvc.repo.imp_url import imp_url
     from dvc.repo.reproduce import reproduce
-    from dvc.repo.checkout import checkout
+    from dvc.repo.checkout import _checkout
     from dvc.repo.push import push
-    from dvc.repo.fetch import fetch
+    from dvc.repo.fetch import _fetch
     from dvc.repo.pull import pull
     from dvc.repo.status import status
     from dvc.repo.gc import gc
@@ -70,7 +88,10 @@ class Repo(object):
 
         self.tree = WorkingTree(self.root_dir)
 
-        self.lock = Lock(self.dvc_dir)
+        self.lock = Lock(
+            os.path.join(self.dvc_dir, "lock"),
+            tmp_dir=os.path.join(self.dvc_dir, "tmp"),
+        )
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
         self.state = State(self, self.config.config)
@@ -89,6 +110,17 @@ class Repo(object):
 
         self._ignore()
 
+    @property
+    def tree(self):
+        return self._tree
+
+    @tree.setter
+    def tree(self, tree):
+        self._tree = tree
+        # Our graph cache is no longer valid, as it was based on the previous
+        # tree.
+        self._reset()
+
     def __repr__(self):
         return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
 
@@ -128,56 +160,55 @@ class Repo(object):
 
         updater = Updater(self.dvc_dir)
 
-        flist = [
-            self.lock.lock_file,
-            self.config.config_local_file,
-            updater.updater_file,
-            updater.lock.lock_file,
-        ] + self.state.files
+        flist = (
+            [self.config.config_local_file, updater.updater_file]
+            + self.state.files
+            + self.lock.files
+            + updater.lock.files
+        )
 
-        if self.cache.local.cache_dir.startswith(self.root_dir):
+        if path_isin(self.cache.local.cache_dir, self.root_dir):
             flist += [self.cache.local.cache_dir]
 
         self.scm.ignore_list(flist)
 
-    def check_dag(self, stages):
+    def check_modified_graph(self, new_stages):
         """Generate graph including the new stage to check for errors"""
-        self.graph(stages=stages)
+        self._collect_graph(self.stages + new_stages)
 
-    @staticmethod
-    def _check_cyclic_graph(graph):
-        import networkx as nx
-        from dvc.exceptions import CyclicGraphError
-
-        cycles = list(nx.simple_cycles(graph))
-
-        if cycles:
-            raise CyclicGraphError(cycles[0])
-
-    def _get_pipeline(self, node):
-        pipelines = [i for i in self.pipelines() if i.has_node(node)]
-        assert len(pipelines) == 1
-        return pipelines[0]
-
-    def collect(self, target, with_deps=False, recursive=False):
+    def collect(self, target, with_deps=False, recursive=False, graph=None):
         import networkx as nx
         from dvc.stage import Stage
 
-        if not target or (recursive and os.path.isdir(target)):
-            return self.stages(target)
+        G = graph or self.graph
+
+        if not target:
+            return get_stages(G)
+
+        target = os.path.abspath(target)
+
+        if recursive and os.path.isdir(target):
+            attrs = nx.get_node_attributes(G, "stage")
+            nodes = [node for node in nx.dfs_postorder_nodes(G)]
+
+            ret = []
+            for node in nodes:
+                stage = attrs[node]
+                if path_isin(stage.path, target):
+                    ret.append(stage)
+            return ret
 
         stage = Stage.load(self, target)
         if not with_deps:
             return [stage]
 
         node = relpath(stage.path, self.root_dir)
-        G = self._get_pipeline(node)
+        pipeline = get_pipeline(get_pipelines(G), node)
 
-        ret = []
-        for n in nx.dfs_postorder_nodes(G, node):
-            ret.append(G.node[n]["stage"])
-
-        return ret
+        return [
+            pipeline.node[n]["stage"]
+            for n in nx.dfs_postorder_nodes(pipeline, node)
+        ]
 
     def used_cache(
         self,
@@ -185,6 +216,7 @@ class Repo(object):
         all_branches=False,
         with_deps=False,
         all_tags=False,
+        all_commits=False,
         remote=None,
         force=False,
         jobs=None,
@@ -203,44 +235,41 @@ class Repo(object):
             A dictionary with Schemes (representing output's location) as keys,
             and a list with the outputs' `dumpd` as values.
         """
+        from dvc.cache import NamedCache
 
-        cache = {}
-        cache["local"] = []
-        cache["s3"] = []
-        cache["gs"] = []
-        cache["hdfs"] = []
-        cache["ssh"] = []
-        cache["azure"] = []
+        cache = NamedCache()
 
         for branch in self.brancher(
-            all_branches=all_branches, all_tags=all_tags
+            all_branches=all_branches,
+            all_tags=all_tags,
+            all_commits=all_commits,
         ):
             if targets:
                 stages = []
                 for target in targets:
-                    if recursive and os.path.isdir(target):
-                        stages.extend(self.stages(target))
-                    else:
-                        stages.extend(
-                            self.collect(target, with_deps=with_deps)
-                        )
+                    collected = self.collect(
+                        target, recursive=recursive, with_deps=with_deps
+                    )
+                    stages.extend(collected)
             else:
-                stages = self.stages()
+                stages = self.stages
 
             for stage in stages:
+                if stage.is_repo_import:
+                    dep, = stage.deps
+                    cache.external[dep.repo_pair].add(dep.def_path)
+                    continue
+
                 for out in stage.outs:
-                    scheme = out.path_info.scheme
                     used_cache = out.get_used_cache(
                         remote=remote, force=force, jobs=jobs
                     )
-
-                    cache[scheme].extend(
-                        dict(entry, branch=branch) for entry in used_cache
-                    )
+                    suffix = "({})".format(branch) if branch else ""
+                    cache.update(used_cache, suffix=suffix)
 
         return cache
 
-    def graph(self, stages=None, from_directory=None):
+    def _collect_graph(self, stages=None):
         """Generate a graph by using the given stages on the given directory
 
         The nodes of the graph are the stage's path relative to the root.
@@ -270,11 +299,8 @@ class Repo(object):
               (weakly connected components)
 
         Args:
-            stages (list): used to build a graph, if None given, use the ones
-                on the `from_directory`.
-
-            from_directory (str): directory where to look at for stages, if
-                None is given, use the current working directory
+            stages (list): used to build a graph, if None given, collect stages
+                in the repository.
 
         Raises:
             OutputDuplicationError: two outputs with the same path
@@ -290,8 +316,7 @@ class Repo(object):
         )
 
         G = nx.DiGraph()
-        G_active = nx.DiGraph()
-        stages = stages or self.stages(from_directory, check_dag=False)
+        stages = stages or self.collect_stages()
         stages = [stage for stage in stages if stage]
         outs = {}
 
@@ -318,7 +343,6 @@ class Repo(object):
             node = relpath(stage.path, self.root_dir)
 
             G.add_node(node, stage=stage)
-            G_active.add_node(node, stage=stage)
 
             for dep in stage.deps:
                 if dep.path_info is None:
@@ -334,22 +358,18 @@ class Repo(object):
                         dep_node = relpath(dep_stage.path, self.root_dir)
                         G.add_node(dep_node, stage=dep_stage)
                         G.add_edge(node, dep_node)
-                        if not stage.locked:
-                            G_active.add_node(dep_node, stage=dep_stage)
-                            G_active.add_edge(node, dep_node)
 
-        self._check_cyclic_graph(G)
+        check_acyclic(G)
 
-        return G, G_active
+        return G
 
-    def pipelines(self, from_directory=None):
-        import networkx as nx
+    @cached_property
+    def graph(self):
+        return self._collect_graph()
 
-        G, G_active = self.graph(from_directory=from_directory)
-
-        return [
-            G.subgraph(c).copy() for c in nx.weakly_connected_components(G)
-        ]
+    @cached_property
+    def pipelines(self):
+        return get_pipelines(self.graph)
 
     @staticmethod
     def _filter_out_dirs(dirs, outs, root_dir):
@@ -362,7 +382,7 @@ class Repo(object):
 
         return list(filter(filter_dirs, dirs))
 
-    def stages(self, from_directory=None, check_dag=True):
+    def collect_stages(self):
         """
         Walks down the root directory looking for Dvcfiles,
         skipping the directories that are related with
@@ -374,16 +394,11 @@ class Repo(object):
         """
         from dvc.stage import Stage
 
-        if not from_directory:
-            from_directory = self.root_dir
-        elif not os.path.isdir(from_directory):
-            raise TargetNotDirectoryError(from_directory)
-
         stages = []
         outs = []
 
         for root, dirs, files in self.tree.walk(
-            from_directory, dvcignore=self.dvcignore
+            self.root_dir, dvcignore=self.dvcignore
         ):
             for fname in files:
                 path = os.path.join(root, fname)
@@ -397,18 +412,15 @@ class Repo(object):
 
             dirs[:] = self._filter_out_dirs(dirs, outs, root)
 
-        if check_dag:
-            self.check_dag(stages)
-
         return stages
 
+    @cached_property
+    def stages(self):
+        return get_stages(self.graph)
+
     def find_outs_by_path(self, path, outs=None, recursive=False):
         if not outs:
-            # there is no `from_directory=path` argument because some data
-            # files might be generated to an upper level, and so it is
-            # needed to look at all the files (project root_dir)
-            stages = self.stages()
-            outs = [out for stage in stages for out in stage.outs]
+            outs = [out for stage in self.stages for out in stage.outs]
 
         abs_path = os.path.abspath(path)
         is_dir = self.tree.isdir(abs_path)
@@ -424,7 +436,7 @@ class Repo(object):
 
         matched = list(filter(func, outs))
         if not matched:
-            raise OutputNotFoundError(path)
+            raise OutputNotFoundError(path, self)
 
         return matched
 
@@ -440,14 +452,29 @@ class Repo(object):
     @contextmanager
     def open(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
+        cause = None
         try:
-            with self._open(path, remote, mode, encoding) as fd:
-                yield fd
-        except FileNotFoundError:
-            raise OutputFileMissingError(relpath(path, self.root_dir))
+            out, = self.find_outs_by_path(path)
+        except OutputNotFoundError as e:
+            out = None
+            cause = e
 
-    def _open(self, path, remote=None, mode="r", encoding=None):
-        out, = self.find_outs_by_path(path)
+        if out and out.use_cache:
+            try:
+                with self._open_cached(out, remote, mode, encoding) as fd:
+                    yield fd
+                return
+            except FileNotFoundError as e:
+                raise FileMissingError(relpath(path, self.root_dir), cause=e)
+
+        if self.tree.exists(path):
+            with self.tree.open(path, mode, encoding) as fd:
+                yield fd
+            return
+
+        raise FileMissingError(relpath(path, self.root_dir), cause=cause)
+
+    def _open_cached(self, out, remote=None, mode="r", encoding=None):
         if out.isdir():
             raise ValueError("Can't open a dir")
 
@@ -470,16 +497,21 @@ class Repo(object):
 
     @cached_property
     def dvcignore(self):
-        return DvcIgnoreFilter(self.root_dir)
+        return DvcIgnoreFilter(self.root_dir, self.tree)
 
     def close(self):
         self.scm.close()
 
-    @staticmethod
-    def clone(url, to_path, rev=None):
-        from dvc.scm.git import Git
+    @locked
+    def checkout(self, *args, **kwargs):
+        return self._checkout(*args, **kwargs)
 
-        git = Git.clone(url, to_path, rev=rev)
-        git.close()
+    @locked
+    def fetch(self, *args, **kwargs):
+        return self._fetch(*args, **kwargs)
 
-        return Repo(to_path)
+    def _reset(self):
+        self.__dict__.pop("graph", None)
+        self.__dict__.pop("stages", None)
+        self.__dict__.pop("pipelines", None)
+        self.__dict__.pop("dvcignore", None)

dvc/repo/add.py

@@ -1,43 +1,67 @@
-import os
-import logging
-import colorama
+from __future__ import unicode_literals
 
+import logging
+import os
+
+import colorama
+from six import string_types
+
+from . import locked
+from dvc.exceptions import RecursiveAddingWhileUsingFilename
+from dvc.progress import Tqdm
 from dvc.repo.scm_context import scm_context
 from dvc.stage import Stage
-from dvc.utils import walk_files, LARGE_DIR_SIZE
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
-
+from dvc.utils import LARGE_DIR_SIZE
+from dvc.utils import walk_files
 
 logger = logging.getLogger(__name__)
 
 
+@locked
 @scm_context
-def add(repo, target, recursive=False, no_commit=False, fname=None):
+def add(repo, targets, recursive=False, no_commit=False, fname=None):
     if recursive and fname:
         raise RecursiveAddingWhileUsingFilename()
 
-    targets = _find_all_targets(repo, target, recursive)
+    if isinstance(targets, string_types):
+        targets = [targets]
 
-    if os.path.isdir(target) and len(targets) > LARGE_DIR_SIZE:
-        logger.warning(
-            "You are adding a large directory '{target}' recursively,"
-            " consider tracking it as a whole instead.\n"
-            "{purple}HINT:{nc} Remove the generated DVC-file and then"
-            " run {cyan}dvc add {target}{nc}".format(
-                purple=colorama.Fore.MAGENTA,
-                cyan=colorama.Fore.CYAN,
-                nc=colorama.Style.RESET_ALL,
-                target=target,
-            )
-        )
+    stages_list = []
+    with Tqdm(total=len(targets), desc="Add", unit="file", leave=True) as pbar:
+        for target in targets:
+            sub_targets = _find_all_targets(repo, target, recursive)
+            pbar.total += len(sub_targets) - 1
 
-    stages = _create_stages(repo, targets, fname, no_commit)
+            if os.path.isdir(target) and len(sub_targets) > LARGE_DIR_SIZE:
+                logger.warning(
+                    "You are adding a large directory '{target}' recursively,"
+                    " consider tracking it as a whole instead.\n"
+                    "{purple}HINT:{nc} Remove the generated DVC-file and then"
+                    " run {cyan}dvc add {target}{nc}".format(
+                        purple=colorama.Fore.MAGENTA,
+                        cyan=colorama.Fore.CYAN,
+                        nc=colorama.Style.RESET_ALL,
+                        target=target,
+                    )
+                )
 
-    repo.check_dag(repo.stages() + stages)
+            stages = _create_stages(repo, sub_targets, fname, pbar=pbar)
 
-    for stage in stages:
-        stage.dump()
-    return stages
+            repo.check_modified_graph(stages)
+
+            for stage in stages:
+                stage.save()
+
+                if not no_commit:
+                    stage.commit()
+
+                stage.dump()
+
+            stages_list += stages
+        # remove filled bar bit of progress, leaving stats
+        pbar.bar_format = pbar.BAR_FMT_DEFAULT.replace("|{bar:10}|", " ")
+
+    return stages_list
 
 
 def _find_all_targets(repo, target, recursive):
@@ -53,21 +77,19 @@ def _find_all_targets(repo, target, recursive):
     return [target]
 
 
-def _create_stages(repo, targets, fname, no_commit):
+def _create_stages(repo, targets, fname, pbar=None):
     stages = []
 
-    with repo.state:
-        for out in targets:
-            stage = Stage.create(repo, outs=[out], add=True, fname=fname)
+    for out in targets:
+        stage = Stage.create(repo, outs=[out], add=True, fname=fname)
 
-            if not stage:
-                continue
+        if not stage:
+            if pbar is not None:
+                pbar.total -= 1
+            continue
 
-            stage.save()
-
-            if not no_commit:
-                stage.commit()
-
-            stages.append(stage)
+        stages.append(stage)
+        if pbar is not None:
+            pbar.update_desc(out)
 
     return stages

dvc/repo/brancher.py

@@ -1,8 +1,15 @@
+from funcy import group_by
+
 from dvc.scm.tree import WorkingTree
 
 
 def brancher(  # noqa: E302
-    self, branches=None, all_branches=False, tags=None, all_tags=False
+    self,
+    branches=None,
+    all_branches=False,
+    tags=None,
+    all_tags=False,
+    all_commits=False,
 ):
     """Generator that iterates over specified revisions.
 
@@ -16,9 +23,9 @@ def brancher(  # noqa: E302
         str: the display name for the currently selected tree, it could be:
             - a git revision identifier
             - empty string it there is no branches to iterate over
-            - "Working Tree" if there are uncommited changes in the SCM repo
+            - "Working Tree" if there are uncommitted changes in the SCM repo
     """
-    if not any([branches, all_branches, tags, all_tags]):
+    if not any([branches, all_branches, tags, all_tags, all_commits]):
         yield ""
         return
 
@@ -30,25 +37,28 @@ def brancher(  # noqa: E302
     self.tree = WorkingTree(self.root_dir)
     yield "working tree"
 
-    if all_branches:
-        branches = scm.list_branches()
+    if all_commits:
+        revs = scm.list_all_commits()
+    else:
+        if all_branches:
+            branches = scm.list_branches()
 
-    if all_tags:
-        tags = scm.list_tags()
+        if all_tags:
+            tags = scm.list_tags()
 
-    if branches is not None:
-        revs.extend(branches)
+        if branches is not None:
+            revs.extend(branches)
 
-    if tags is not None:
-        revs.extend(tags)
+        if tags is not None:
+            revs.extend(tags)
 
     # NOTE: it might be a good idea to wrap this loop in try/finally block
     # to don't leave the tree on some unexpected branch after the
     # `brancher()`, but this could cause problems on exception handling
     # code which might expect the tree on which exception was raised to
     # stay in place. This behavior is a subject to change.
-    for rev in revs:
-        self.tree = scm.get_tree(rev)
-        yield rev
+    for sha, names in group_by(scm.resolve_rev, revs).items():
+        self.tree = scm.get_tree(sha)
+        yield ", ".join(names)
 
     self.tree = saved_tree

dvc/repo/checkout.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import logging
 
+from dvc.exceptions import CheckoutError
 from dvc.exceptions import CheckoutErrorSuggestGit
 from dvc.progress import Tqdm
 
@@ -9,47 +10,52 @@ from dvc.progress import Tqdm
 logger = logging.getLogger(__name__)
 
 
-def _cleanup_unused_links(self, all_stages):
+def _cleanup_unused_links(repo):
     used = [
         out.fspath
-        for stage in all_stages
+        for stage in repo.stages
         for out in stage.outs
         if out.scheme == "local"
     ]
-    self.state.remove_unused_links(used)
+    repo.state.remove_unused_links(used)
 
 
 def get_all_files_numbers(stages):
     return sum(stage.get_all_files_number() for stage in stages)
 
 
-def checkout(self, target=None, with_deps=False, force=False, recursive=False):
+def _checkout(
+    self, targets=None, with_deps=False, force=False, recursive=False
+):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    all_stages = self.stages()
+    stages = set()
 
-    try:
-        stages = self.collect(target, with_deps=with_deps, recursive=recursive)
-    except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
-        if not target:
-            raise
-        raise CheckoutErrorSuggestGit(target, exc)
+    if not targets:
+        targets = [None]
+        _cleanup_unused_links(self)
 
-    with self.state:
-        _cleanup_unused_links(self, all_stages)
-        total = get_all_files_numbers(stages)
-        if total == 0:
-            logger.info("Nothing to do")
-        with Tqdm(
-            total=total, unit="file", desc="Checkout", disable=total == 0
-        ) as pbar:
-            for stage in stages:
-                if stage.locked:
-                    logger.warning(
-                        "DVC-file '{path}' is locked. Its dependencies are"
-                        " not going to be checked out.".format(
-                            path=stage.relpath
-                        )
-                    )
+    for target in targets:
+        try:
+            new = self.collect(
+                target, with_deps=with_deps, recursive=recursive
+            )
+            stages.update(new)
+        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+            if not target:
+                raise
+            raise CheckoutErrorSuggestGit(target, exc)
 
+    total = get_all_files_numbers(stages)
+    if total == 0:
+        logger.info("Nothing to do")
+    failed = []
+    with Tqdm(
+        total=total, unit="file", desc="Checkout", disable=total == 0
+    ) as pbar:
+        for stage in stages:
+            failed.extend(
                 stage.checkout(force=force, progress_callback=pbar.update_desc)
+            )
+    if failed:
+        raise CheckoutError(failed)

dvc/repo/commit.py

@@ -1,7 +1,10 @@
+from . import locked
+
+
+@locked
 def commit(self, target, with_deps=False, recursive=False, force=False):
     stages = self.collect(target, with_deps=with_deps, recursive=recursive)
-    with self.state:
-        for stage in stages:
-            stage.check_can_commit(force=force)
-            stage.commit()
-            stage.dump()
+    for stage in stages:
+        stage.check_can_commit(force=force)
+        stage.commit()
+        stage.dump()

dvc/repo/destroy.py

@@ -1,8 +1,8 @@
-from dvc.utils import remove
+from dvc.utils.fs import remove
 
 
 def destroy(self):
-    for stage in self.stages():
+    for stage in self.stages:
         stage.remove(remove_outs=False)
 
     remove(self.dvc_dir)

dvc/repo/diff.py

@@ -3,16 +3,14 @@ from __future__ import unicode_literals
 import os
 from errno import ENOENT
 
-
 import dvc.logger as logger
+from . import locked
 from dvc.scm.base import FileNotInCommitError
-from dvc.scm.git import (
-    DIFF_A_TREE,
-    DIFF_B_TREE,
-    DIFF_A_REF,
-    DIFF_B_REF,
-    DIFF_EQUAL,
-)
+from dvc.scm.git import DIFF_A_REF
+from dvc.scm.git import DIFF_A_TREE
+from dvc.scm.git import DIFF_B_REF
+from dvc.scm.git import DIFF_B_TREE
+from dvc.scm.git import DIFF_EQUAL
 
 
 DIFF_TARGET = "target"
@@ -140,9 +138,9 @@ def _is_dir(path, a_outs, b_outs):
 
 def _get_diff_outs(self, diff_dct):
     self.tree = diff_dct[DIFF_A_TREE]
-    a_outs = {str(out): out for st in self.stages() for out in st.outs}
+    a_outs = {str(out): out for st in self.stages for out in st.outs}
     self.tree = diff_dct[DIFF_B_TREE]
-    b_outs = {str(out): out for st in self.stages() for out in st.outs}
+    b_outs = {str(out): out for st in self.stages for out in st.outs}
     outs_paths = set(a_outs.keys())
     outs_paths.update(b_outs.keys())
     results = {}
@@ -220,8 +218,9 @@ def _diff_royal(self, target, diff_dct):
     return _diff_file(self, target, diff_dct)
 
 
+@locked
 def diff(self, a_ref, target=None, b_ref=None):
-    """Gerenates diff message string output
+    """Generates diff message string output
 
     Args:
         target(str) - file/directory to check diff of

dvc/repo/fetch.py

@@ -1,7 +1,19 @@
 from __future__ import unicode_literals
 
+import logging
 
-def fetch(
+from dvc.cache import NamedCache
+from dvc.config import NoRemoteError
+from dvc.exceptions import DownloadError
+from dvc.exceptions import OutputNotFoundError
+from dvc.external_repo import external_repo
+from dvc.scm.base import CloneError
+
+
+logger = logging.getLogger(__name__)
+
+
+def _fetch(
     self,
     targets=None,
     jobs=None,
@@ -12,17 +24,81 @@ def fetch(
     all_tags=False,
     recursive=False,
 ):
-    with self.state:
-        used = self.used_cache(
-            targets,
-            all_branches=all_branches,
-            all_tags=all_tags,
-            with_deps=with_deps,
-            force=True,
-            remote=remote,
-            jobs=jobs,
-            recursive=recursive,
-        )["local"]
-        return self.cloud.pull(
+    """Download data items from a cloud and imported repositories
+
+    Returns:
+        int: number of successfully downloaded files
+
+    Raises:
+        DownloadError: thrown when there are failed downloads, either
+            during `cloud.pull` or trying to fetch imported files
+
+        config.NoRemoteError: thrown when downloading only local files and no
+            remote is configured
+    """
+    used = self.used_cache(
+        targets,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        with_deps=with_deps,
+        force=True,
+        remote=remote,
+        jobs=jobs,
+        recursive=recursive,
+    )
+
+    downloaded = 0
+    failed = 0
+
+    try:
+        downloaded += self.cloud.pull(
             used, jobs, remote=remote, show_checksums=show_checksums
         )
+    except NoRemoteError:
+        if not used.external and used["local"]:
+            raise
+    except DownloadError as exc:
+        failed += exc.amount
+
+    for (repo_url, repo_rev), files in used.external.items():
+        d, f = _fetch_external(self, repo_url, repo_rev, files)
+        downloaded += d
+        failed += f
+
+    if failed:
+        raise DownloadError(failed)
+
+    return downloaded
+
+
+def _fetch_external(self, repo_url, repo_rev, files):
+    failed = 0
+
+    cache_dir = self.cache.local.cache_dir
+    try:
+        with external_repo(repo_url, repo_rev, cache_dir=cache_dir) as repo:
+            with repo.state:
+                cache = NamedCache()
+                for name in files:
+                    try:
+                        out = repo.find_out_by_relpath(name)
+                    except OutputNotFoundError:
+                        failed += 1
+                        logger.exception(
+                            "failed to fetch data for '{}'".format(name)
+                        )
+                        continue
+                    else:
+                        cache.update(out.get_used_cache())
+
+                try:
+                    return repo.cloud.pull(cache), failed
+                except DownloadError as exc:
+                    failed += exc.amount
+    except CloneError:
+        failed += 1
+        logger.exception(
+            "failed to fetch data for '{}'".format(", ".join(files))
+        )
+
+    return 0, failed