dvc: bump to 0.76.0

dvc: make flufl.lock opt-in and use zc.lockfile

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,9 +1,8 @@
-* [ ] Have you followed the guidelines in our
-      [Contributing document](https://dvc.org/doc/user-guide/contributing/core)?
+* [ ] ❗ Have you followed the guidelines in the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) list?
 
-* [ ] Does your PR affect documented changes or does it add new functionality
-      that should be documented? If yes, have you created a PR for
-      [dvc.org](https://github.com/iterative/dvc.org) documenting it or at
-      least opened an issue for it? If so, please add a link to it.
+* [ ] 📖 Check this box if this PR **does not** require [documentation](https://dvc.org/doc) updates, or if it does **and** you have created a separate PR in [dvc.org](https://github.com/iterative/dvc.org) with such updates (or at least opened an issue about it in that repo). Please link below to your PR (or issue) in the [dvc.org](https://github.com/iterative/dvc.org) repo.
+
+* [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect everything to be addressed. Please review them carefully and fix those that actually improve code or fix bugs.
+
+Thank you for the contribution - we'll try to review it as soon as possible. 🙏
 
------

.github/release-drafter.yml

@@ -0,0 +1,5 @@
+# Config for https://github.com/apps/release-drafter
+branches:
+  - master
+template: |
+  $CHANGES

.gitignore

@@ -4,6 +4,7 @@ neatlynx/__pycache__
 cache
 *.pyc
 .env/
+.env2.7/
 
 cache
 .dvc.conf.lock

.restyled.yaml

@@ -0,0 +1,11 @@
+---
+restylers:
+  - name: black
+    image: restyled/restyler-black:v19.3b0
+    command:
+    - black
+    arguments: []
+    include:
+    - "**/*.py"
+    interpreters:
+    - python

.travis.yml

@@ -2,7 +2,6 @@ cache:
   directories:
     - $HOME/.cache/pip
     - $HOME/.pyenv
-sudo: true
 dist: xenial
 env:
   global:
@@ -15,18 +14,20 @@ stages:
   - test
   - build
   - name: deploy
-    if: tag is present
+    if: (branch = master OR tag is present) AND repo = iterative/dvc
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
 # Note that we specify language as ruby-2.4.3 to help us satisfy
 # fpm dependencies, as travis runs el capitan with an old ruby
 # version, that causes fpm to crash.
+#
+# Each job will use the previous job's `stage` tag if not specified.
 jobs:
   include:
   # patch formatting checks
   - name: "Check patch formatting"
-    stage: "check"
+    stage: check
     os: linux
     language: python
     python: 3.7
@@ -41,35 +42,28 @@ jobs:
     env:
     - PY2=1
   - name: "3.7 on Windows"
-    stage: test
     os: windows           # Windows 10.0.17134 N/A Build 17134
     language: shell       # 'language: python' is an error on Travis CI Windows
   - name: "2.7 on Linux"
-    stage: test
     language: python
     python: 2.7
     env:
     - PY2=1
   - name: "3.5 on Linux"
-    stage: test
     language: python
     python: 3.5
   - name: "3.6 on Linux"
-    stage: test
     language: python
     python: 3.6
   - name: "3.7 on Linux"
-    stage: test
     language: python
     python: 3.7
   - name: "2.7 on Mac"
-    stage: test
     os: osx
     osx_image: xcode9.4
     env:
     - PY2=1
   - name: "3.6 on Mac"
-    stage: test
     os: osx
     osx_image: xcode9.4
   # build jobs
@@ -82,25 +76,42 @@ jobs:
     install:
     script: ./scripts/build_posix.sh
   - name: "Windows pkg"
-    stage: build
     os: windows
     language: shell
     install:
     script: ./scripts/build_windows.cmd
   - name: "Linux pkgs"
-    stage: build
     language: python
     python: 3.7
     before_install:
     install:
     script: ./scripts/build_posix.sh
   - name: "Pypi pkgs"
-    stage: build
     language: python
     python: 3.7
     script: ./scripts/build_package.sh
+  - name: Snapcraft snap
+    addons:
+     snaps:
+     - name: snapcraft
+       channel: stable
+       confinement: classic
+     - name: lxd
+       channel: stable
+    env:
+    - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}'
+    before_install:
+    - sudo /snap/bin/lxd.migrate -yes
+    - sudo /snap/bin/lxd waitready
+    - sudo /snap/bin/lxd init --auto
+    install:
+    script:
+    - sudo snapcraft --use-lxd
+    after_failure:
+    - sudo journalctl -u snapd
   # deploy jobs
   - name: "Pypi pkgs"
+    if: tag is present
     stage: deploy
     language: python
     python: 3.7
@@ -140,6 +151,7 @@ deploy:
       - dvc*.deb
       - dvc*.pkg
       - dvc*.exe
+      - dvc*.snap
     skip_cleanup: true
     on:
       tags: true

README.rst

@@ -30,25 +30,30 @@
   :target: https://anaconda.org/conda-forge/dvc
   :alt: Conda-forge
 
+.. image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
+  :target: https://snapcraft.io/dvc
+  :alt: Snapcraft
+
 |
 
 **Data Version Control** or **DVC** is an **open-source** tool for data science and machine
 learning projects. Key features:
 
-#. simple **command line** Git-like experience. Does not require installing and maintaining
-   any databases. Does not depend on any proprietary online services;
+#. Simple **command line** Git-like experience. Does not require installing and maintaining
+   any databases. Does not depend on any proprietary online services.
 
-#. it manages and versions **datasets** and **machine learning models**. Data is saved in
-   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS or even local HDD RAID;
+#. Management and versioning of **datasets** and **machine learning models**. Data is saved in
+   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
 
-#. it makes projects **reproducible** and **shareable**, it helps answering question how
-   the model was build;
+#. Makes projects **reproducible** and **shareable**; helping to answer questions about how
+   a model was built.
 
-#. it helps manage experiments with Git tags or branches and **metrics** tracking;
+#. Helps manage experiments with Git tags/branches and **metrics** tracking.
 
-**DVC** aims to replace tools like Excel and Google Docs that are being commonly used as a knowledge repo and
-a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
-data file suffixes and prefixes.
+**DVC** aims to replace spreadsheet and document sharing tools (such as Excel or Google Docs)
+which are being used frequently as both knowledge repositories and team ledgers.
+DVC also replaces both ad-hoc scripts to track, move, and deploy different model versions;
+as well as ad-hoc data file suffixes and prefixes.
 
 .. contents:: **Contents**
   :backlinks: none
@@ -56,34 +61,33 @@ data file suffixes and prefixes.
 How DVC works
 =============
 
-We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ to better understand what DVC
-is and how does it fit your scenarios.
+We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ guide to better understand what DVC
+is and how it can fit your scenarios.
 
-The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-lfs to be precise) + ``makefiles``
+The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-LFS to be precise) & Makefiles
 made right and tailored specifically for ML and Data Science scenarios.
 
-#. ``Git/Git-lfs`` part - DVC helps you storing and sharing data artifacts, models. It connects them with your
-   Git repository.
-#. ``Makefiles`` part - DVC describes how one data or model artifact was build from another data.
+#. ``Git/Git-LFS`` part - DVC helps store and share data artifacts and models, connecting them with a Git repository.
+#. ``Makefile``\ s part - DVC describes how one data or model artifact was built from other data and code.
 
-DVC usually runs along with Git. Git is used as usual to store and version code and DVC meta-files. DVC helps
-to store data and model files seamlessly out of Git while preserving almost the same user experience as if they
-were stored in Git itself. To store and share data files cache DVC supports remotes - any cloud (S3, Azure,
+DVC usually runs along with Git. Git is used as usual to store and version code (including DVC meta-files). DVC helps
+to store data and model files seamlessly out of Git, while preserving almost the same user experience as if they
+were stored in Git itself. To store and share the data cache, DVC supports multiple remotes - any cloud (S3, Azure,
 Google Cloud, etc) or any on-premise network storage (via SSH, for example).
 
 .. image:: https://dvc.org/static/img/flow.gif
    :target: https://dvc.org/static/img/flow.gif
    :alt: how_dvc_works
 
-DVC pipelines (aka computational graph) feature connects code and data together. In a very explicit way you can
-specify, run, and save information that a certain command with certain dependencies needs to be run to produce
-a model. See the quick start section below or check `Get Started <https://dvc.org/doc/get-started>`_ tutorial to
-learn more.
+The DVC pipelines (computational graph) feature connects code and data together. It is possible to explicitly
+specify all steps required to produce a model: input dependencies including data, commands to run,
+and output information to be saved. See the quick start section below or
+the `Get Started <https://dvc.org/doc/get-started>`_ tutorial to learn more.
 
 Quick start
 ===========
 
-Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full version. Common workflow commands include:
+Please read `Get Started <https://dvc.org/doc/get-started>`_ guide for a full version. Common workflow commands include:
 
 +-----------------------------------+-------------------------------------------------------------------+
 | Step                              | Command                                                           |
@@ -108,8 +112,8 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 Installation
 ============
 
-Read this `instruction <https://dvc.org/doc/get-started/install>`_ to get more details. There are four
-options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package:
+There are four options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package.
+Full instructions are `available here <https://dvc.org/doc/get-started/install>`_.
 
 pip (PyPI)
 ----------
@@ -120,8 +124,8 @@ pip (PyPI)
 
 Depending on the remote storage type you plan to use to keep and share your data, you might need to specify
 one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all`` to include them all.
-The command should look like this: ``pip install dvc[s3]`` - it installs the ``boto3`` library along with
-DVC to support the AWS S3 storage.
+The command should look like this: ``pip install dvc[s3]`` (in this case AWS S3 dependencies such as ``boto3``
+will be installed automatically).
 
 To install the development version, run:
 
@@ -144,13 +148,28 @@ Conda (Anaconda)
 
    conda install -c conda-forge dvc
 
-Currently, it supports only python version 2.7, 3.6 and 3.7.
+Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
+
+Snap (Snapcraft)
+----------------
+
+Download the latest ``dvc_*.snap`` from the
+GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+
+.. code-block:: bash
+
+   snap install dvc_*.snap --dangerous --classic
+
+Once ``dvc`` is released on the snap store
+(`pending request <https://forum.snapcraft.io/t/classic-confinement-request-for-dvc/14124>`_)
+there will be no need to download ``dvc_*.snap`` or use ``--dangerous``
+(``snap install dvc --classic`` would suffice).
 
 Package
 -------
 
-Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
-GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+Self-contained packages for Linux, Windows, and Mac are available. The latest version of the packages
+can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
 
 Ubuntu / Debian (deb)
 ^^^^^^^^^^^^^^^^^^^^^
@@ -168,32 +187,23 @@ Fedora / CentOS (rpm)
    sudo yum update
    sudo yum install dvc
 
-Arch Linux (AUR)
-^^^^^^^^^^^^^^^^
-*Unofficial package*, any inquiries regarding the AUR package,
-`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
+Comparison to related technologies
+==================================
 
-.. code-block:: bash
-
-   yay -S dvc
-
-Related technologies
-====================
-
-#. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (that you
-   don't want to see in your Git repository) in a local key-value store and uses file hardlinks/symlinks instead of the
-   copying actual files.
+#. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (which should
+   not be in a Git repository) in a local key-value store, and uses file hardlinks/symlinks instead of
+   copying/duplicating files.
 
 #. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with any remote storage (S3, Google Cloud, Azure, SSH,
-   etc). DVC utilizes reflinks or hardlinks to avoid copy operation on checkouts which makes much more efficient for
-   large data files.
+   etc). DVC also uses reflinks or hardlinks to avoid copy operations on checkouts; thus handling large data files
+   much more efficiently.
 
-#. *Makefile* (and its analogues). DVC tracks dependencies (DAG).
+#. *Makefile* (and analogues including ad-hoc scripts) - DVC tracks dependencies (in a directed acyclic graph).
 
-#. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_. DVC is a workflow
+#. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_ - DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-#. `DAGsHub <https://dagshub.com/>`_ Is a Github equivalent for DVC - pushing your Git+DVC based repo to DAGsHub will give you a high level dashboard of your project, including DVC pipeline and metrics visualizations, as well as links to DVC managed files if they are in cloud storage.
+#. `DAGsHub <https://dagshub.com/>`_ - This is a Github equivalent for DVC. Pushing Git+DVC based repositories to DAGsHub will produce in a high level project dashboard; including DVC pipelines and metrics visualizations, as well as links to any DVC-managed files present in cloud storage.
 
 Contributing
 ============
@@ -242,5 +252,5 @@ Copyright
 
 This project is distributed under the Apache license version 2.0 (see the LICENSE file in the project root).
 
-By submitting a pull request for this project, you agree to license your contribution under the Apache license version
+By submitting a pull request to this project, you agree to license your contribution under the Apache license version
 2.0 to this project.

debian/changelog

@@ -0,0 +1,5 @@
+dvc (0.75.b1) experimental; urgency=low
+
+  * First native Debian package version.
+
+ -- Dmitry Petrov <[email protected]>  Sat, 07 Dec 2019 09:49:28 +0300

debian/compat

@@ -0,0 +1 @@
+10

debian/control

@@ -0,0 +1,33 @@
+Source: dvc
+Maintainer: Dmitry Petrov <[email protected]>
+Section: python
+Priority: optional
+Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
+Standards-Version: 4.0.0
+Homepage: http://dvc.org
+Vcs-Git: https://github.com/iterative/dvc.git
+Vcs-browser: https://github.com/iterative/dvc
+
+Package: dvc
+Architecture: all
+Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
+Recommends: python3-arrow (=0.14.0),
+            python3-paramiko (>=2.5.0),
+            python3-gssapi (>= 2.5.0),
+            python3-boto3 (>=1.9.86),
+            python3-pydrive (=1.3.1),
+            python3-azure-storage-blob (=12.1.0),
+            python3-oss2 (=2.6.1),
+            python3-google-cloud-storage (=1.19.0)
+Description: Git for data scientists - manage your code and data together
+ Data Version Control or DVC is an open-source tool for data science and machine learning projects.
+ .
+ Key features:
+ .
+ 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
+ .
+ 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
+ .
+ 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
+ .
+ 4. Helps manage experiments with Git tags/branches and metrics tracking.

debian/copyright

@@ -0,0 +1,11 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: DVC
+Upstream-Contact: Dmitry Petrov <[email protected]>
+Source: https://github.com/iterative/dvc
+
+Files: *
+Copyright: Dmitry Petrov <[email protected]>
+License: Apache 2.0
+ On Debian systems, the full text of the Apache 2.0
+ License can be found in the file
+ `/usr/share/common-licenses/Apache-2.0`.

debian/rules

@@ -0,0 +1,6 @@
+#!/usr/bin/make -f
+
+export PYBUILD_NAME=dvc
+%:
+	dh $@ --with python3 --buildsystem=pybuild
+

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

debian/watch

@@ -0,0 +1,4 @@
+# please also check http://pypi.debian.net/dvc/watch
+version=3
+opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
+http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

dvc/__init__.py

@@ -3,13 +3,11 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-
 from __future__ import unicode_literals
 
 import warnings
 
 import dvc.logger
-
 from dvc.version import __version__  # noqa: F401
 
 

dvc/__main__.py

@@ -1,5 +1,4 @@
 """Main entry point for dvc command line tool."""
-
 from __future__ import unicode_literals
 
 import sys
@@ -7,4 +6,4 @@ import sys
 from dvc.main import main
 
 if __name__ == "__main__":
-    main(sys.argv[1:])
+    sys.exit(main(sys.argv[1:]))

dvc/analytics.py

@@ -1,284 +1,167 @@
-"""Collect and send usage analytics"""
-
-from __future__ import unicode_literals
-
-from dvc.utils.compat import str
-
-import os
 import json
-import errno
 import logging
+import os
+import platform
+import requests
+import sys
+import tempfile
+import uuid
+
+import distro
 
 from dvc import __version__
-from dvc.utils import env2bool
+from dvc.config import Config, to_bool
+from dvc.daemon import daemon
+from dvc.exceptions import NotDvcRepoError
+from dvc.lock import Lock, LockError
+from dvc.repo import Repo
+from dvc.scm import SCM
+from dvc.utils import env2bool, is_binary, makedirs
+from dvc.utils.compat import str, FileNotFoundError
 
 
 logger = logging.getLogger(__name__)
 
 
-class Analytics(object):
-    """Class for collecting and sending usage analytics.
-
-    Args:
-        info (dict): optional existing analytics report.
+def collect_and_send_report(args=None, return_code=None):
     """
+    Collect information from the runtime/environment and the command
+    being executed into a report and send it over the network.
 
-    URL = "https://analytics.dvc.org"
-    TIMEOUT_POST = 5
+    To prevent analytics from blocking the execution of the main thread,
+    sending the report is done in a separate process.
 
-    USER_ID_FILE = "user_id"
+    The inter-process communication happens through a file containing the
+    report as a JSON, where the _collector_ generates it and the _sender_
+    removes it after sending it.
+    """
+    report = _runtime_info()
 
-    PARAM_DVC_VERSION = "dvc_version"
-    PARAM_USER_ID = "user_id"
-    PARAM_SYSTEM_INFO = "system_info"
+    # Include command execution information on the report only when available.
+    if args and hasattr(args, "func"):
+        report.update({"cmd_class": args.func.__name__})
 
-    PARAM_OS = "os"
+    if return_code is not None:
+        report.update({"cmd_return_code": return_code})
 
-    PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major"
-    PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor"
-    PARAM_WINDOWS_VERSION_BUILD = "windows_version_build"
-    PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack"
+    with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
+        json.dump(report, fobj)
+        daemon(["analytics", fobj.name])
 
-    PARAM_MAC_VERSION = "mac_version"
 
-    PARAM_LINUX_DISTRO = "linux_distro"
-    PARAM_LINUX_DISTRO_VERSION = "linux_distro_version"
-    PARAM_LINUX_DISTRO_LIKE = "linux_distro_like"
+def is_enabled():
+    if env2bool("DVC_TEST"):
+        return False
 
-    PARAM_SCM_CLASS = "scm_class"
-    PARAM_IS_BINARY = "is_binary"
-    PARAM_CMD_CLASS = "cmd_class"
-    PARAM_CMD_RETURN_CODE = "cmd_return_code"
+    enabled = to_bool(
+        Config(validate=False)
+        .config.get(Config.SECTION_CORE, {})
+        .get(Config.SECTION_CORE_ANALYTICS, "true")
+    )
 
-    def __init__(self, info=None):
-        from dvc.config import Config
-        from dvc.lock import Lock
+    logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
 
-        if info is None:
-            info = {}
+    return enabled
 
-        self.info = info
 
-        cdir = Config.get_global_config_dir()
-        try:
-            os.makedirs(cdir)
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                raise
+def send(report):
+    """
+    Side effect: Removes the report after sending it.
 
-        self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
-        self.user_id_file_lock = Lock(self.user_id_file + ".lock")
+    The report is generated and stored in a temporary file, see:
+    `collect_and_send_report`. Sending happens on another process,
+    thus, the need of removing such file afterwards.
+    """
+    url = "https://analytics.dvc.org"
+    headers = {"content-type": "application/json"}
 
-    @staticmethod
-    def load(path):
-        """Loads analytics report from json file specified by path.
+    with open(report, "rb") as fobj:
+        requests.post(url, data=fobj, headers=headers, timeout=5)
 
-        Args:
-            path (str): path to json file with analytics report.
-        """
-        with open(path, "r") as fobj:
-            analytics = Analytics(info=json.load(fobj))
-        os.unlink(path)
-        return analytics
+    os.remove(report)
 
-    def _write_user_id(self):
-        import uuid
 
-        with open(self.user_id_file, "w+") as fobj:
-            user_id = str(uuid.uuid4())
-            info = {self.PARAM_USER_ID: user_id}
-            json.dump(info, fobj)
+def _scm_in_use():
+    try:
+        scm = SCM(root_dir=Repo.find_root())
+        return type(scm).__name__
+    except NotDvcRepoError:
+        pass
+
+
+def _runtime_info():
+    """
+    Gather information from the environment where DVC runs to fill a report.
+    """
+    return {
+        "dvc_version": __version__,
+        "is_binary": is_binary(),
+        "scm_class": _scm_in_use(),
+        "system_info": _system_info(),
+        "user_id": _find_or_create_user_id(),
+    }
+
+
+def _system_info():
+    system = platform.system()
+
+    if system == "Windows":
+        version = sys.getwindowsversion()
+
+        return {
+            "os": "windows",
+            "windows_version_build": version.build,
+            "windows_version_major": version.major,
+            "windows_version_minor": version.minor,
+            "windows_version_service_pack": version.service_pack,
+        }
+
+    if system == "Darwin":
+        return {"os": "mac", "mac_version": platform.mac_ver()[0]}
+
+    if system == "Linux":
+        return {
+            "os": "linux",
+            "linux_distro": distro.id(),
+            "linux_distro_like": distro.like(),
+            "linux_distro_version": distro.version(),
+        }
+
+    # We don't collect data for any other system.
+    raise NotImplementedError
+
+
+def _find_or_create_user_id():
+    """
+    The user's ID is stored on a file under the global config directory.
+
+    The file should contain a JSON with a "user_id" key:
+
+        {"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"}
+
+    IDs are generated randomly with UUID.
+    """
+    config_dir = Config.get_global_config_dir()
+    fname = os.path.join(config_dir, "user_id")
+    lockfile = os.path.join(config_dir, "user_id.lock")
+
+    # Since the `fname` and `lockfile` are under the global config,
+    # we need to make sure such directory exist already.
+    makedirs(config_dir, exist_ok=True)
+
+    try:
+        with Lock(lockfile):
+            try:
+                with open(fname, "r") as fobj:
+                    user_id = json.load(fobj)["user_id"]
+
+            except (FileNotFoundError, ValueError, KeyError):
+                user_id = str(uuid.uuid4())
+
+                with open(fname, "w") as fobj:
+                    json.dump({"user_id": user_id}, fobj)
+
             return user_id
 
-    def _read_user_id(self):
-        if not os.path.exists(self.user_id_file):
-            return None
-
-        with open(self.user_id_file, "r") as fobj:
-            try:
-                info = json.load(fobj)
-            except ValueError as exc:
-                logger.debug("Failed to load user_id: {}".format(exc))
-                return None
-
-            return info[self.PARAM_USER_ID]
-
-    def _get_user_id(self):
-        from dvc.lock import LockError
-
-        try:
-            with self.user_id_file_lock:
-                user_id = self._read_user_id()
-                if user_id is None:
-                    user_id = self._write_user_id()
-                return user_id
-        except LockError:
-            msg = "Failed to acquire '{}'"
-            logger.debug(msg.format(self.user_id_file_lock.lockfile))
-
-    def _collect_windows(self):
-        import sys
-
-        version = sys.getwindowsversion()  # pylint: disable=no-member
-        info = {}
-        info[self.PARAM_OS] = "windows"
-        info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
-        info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
-        info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
-        info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack
-        return info
-
-    def _collect_darwin(self):
-        import platform
-
-        info = {}
-        info[self.PARAM_OS] = "mac"
-        info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
-        return info
-
-    def _collect_linux(self):
-        import distro
-
-        info = {}
-        info[self.PARAM_OS] = "linux"
-        info[self.PARAM_LINUX_DISTRO] = distro.id()
-        info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
-        info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
-        return info
-
-    def _collect_system_info(self):
-        import platform
-
-        system = platform.system()
-
-        if system == "Windows":
-            return self._collect_windows()
-
-        if system == "Darwin":
-            return self._collect_darwin()
-
-        if system == "Linux":
-            return self._collect_linux()
-
-        raise NotImplementedError
-
-    def collect(self):
-        """Collect analytics report."""
-        from dvc.scm import SCM
-        from dvc.utils import is_binary
-        from dvc.repo import Repo
-        from dvc.exceptions import NotDvcRepoError
-
-        self.info[self.PARAM_DVC_VERSION] = __version__
-        self.info[self.PARAM_IS_BINARY] = is_binary()
-        self.info[self.PARAM_USER_ID] = self._get_user_id()
-
-        self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
-
-        try:
-            scm = SCM(root_dir=Repo.find_root())
-            self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
-        except NotDvcRepoError:
-            pass
-
-    def collect_cmd(self, args, ret):
-        """Collect analytics info from a CLI command."""
-        from dvc.command.daemon import CmdDaemonAnalytics
-
-        assert isinstance(ret, int) or ret is None
-
-        if ret is not None:
-            self.info[self.PARAM_CMD_RETURN_CODE] = ret
-
-        if args is not None and hasattr(args, "func"):
-            assert args.func != CmdDaemonAnalytics
-            self.info[self.PARAM_CMD_CLASS] = args.func.__name__
-
-    def dump(self):
-        """Save analytics report to a temporary file.
-
-        Returns:
-            str: path to the temporary file that contains the analytics report.
-        """
-        import tempfile
-
-        with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
-            json.dump(self.info, fobj)
-            return fobj.name
-
-    @staticmethod
-    def _is_enabled_config(config):
-        from dvc.config import Config
-
-        core = config.config.get(Config.SECTION_CORE, {})
-        return core.get(Config.SECTION_CORE_ANALYTICS, True)
-
-    @staticmethod
-    def _get_current_config():
-        from dvc.config import Config
-        from dvc.repo import Repo
-        from dvc.exceptions import NotDvcRepoError
-
-        try:
-            dvc_dir = Repo.find_dvc_dir()
-            config = Config(dvc_dir)
-        except NotDvcRepoError:
-            config = Config(validate=False)
-        return config
-
-    @staticmethod
-    def is_enabled(cmd=None):
-        from dvc.command.daemon import CmdDaemonBase
-
-        if env2bool("DVC_TEST"):
-            return False
-
-        if isinstance(cmd, CmdDaemonBase):
-            return False
-
-        config = (
-            Analytics._get_current_config()
-            if cmd is None or not hasattr(cmd, "config")
-            else cmd.config
-        )
-
-        assert config is not None
-
-        enabled = Analytics._is_enabled_config(config)
-        logger.debug(
-            "Analytics is {}.".format("enabled" if enabled else "disabled")
-        )
-        return enabled
-
-    @staticmethod
-    def send_cmd(cmd, args, ret):
-        """Collect and send analytics for CLI command.
-
-        Args:
-            args (list): parsed args for the CLI command.
-            ret (int): return value of the CLI command.
-        """
-        from dvc.daemon import daemon
-
-        if not Analytics.is_enabled(cmd):
-            return
-
-        analytics = Analytics()
-        analytics.collect_cmd(args, ret)
-        daemon(["analytics", analytics.dump()])
-
-    def send(self):
-        """Collect and send analytics."""
-        import requests
-
-        if not self.is_enabled():
-            return
-
-        self.collect()
-
-        logger.debug("Sending analytics: {}".format(self.info))
-
-        try:
-            requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
-        except requests.exceptions.RequestException as exc:
-            logger.debug("Failed to send analytics: {}".format(str(exc)))
+    except LockError:
+        logger.debug("Failed to acquire {lockfile}".format(lockfile=lockfile))

dvc/api.py

@@ -1,5 +1,5 @@
-from contextlib import contextmanager
 import os
+from contextlib import contextmanager
 
 try:
     from contextlib import _GeneratorContextManager as GCM

dvc/cache.py

@@ -1,13 +1,13 @@
 """Manages cache of a dvc repo."""
-
 from __future__ import unicode_literals
 
 import os
 from collections import defaultdict
+
 from funcy import cached_property
 
-from dvc.utils.compat import builtin_str
 from dvc.config import Config
+from dvc.utils.compat import builtin_str
 
 
 class CacheConfig(object):

dvc/cli.py

@@ -1,42 +1,41 @@
 """Command line interface for dvc."""
-
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import sys
 import argparse
 import logging
+import sys
 
-from dvc.command.base import fix_subparsers
-import dvc.command.init as init
+import dvc.command.add as add
+import dvc.command.cache as cache
+import dvc.command.checkout as checkout
+import dvc.command.commit as commit
+import dvc.command.config as config
+import dvc.command.daemon as daemon
+import dvc.command.data_sync as data_sync
+import dvc.command.destroy as destroy
+import dvc.command.diff as diff
+import dvc.command.gc as gc
 import dvc.command.get as get
 import dvc.command.get_url as get_url
-import dvc.command.destroy as destroy
-import dvc.command.remove as remove
-import dvc.command.move as move
-import dvc.command.unprotect as unprotect
-import dvc.command.run as run
-import dvc.command.repro as repro
-import dvc.command.data_sync as data_sync
-import dvc.command.gc as gc
-import dvc.command.add as add
 import dvc.command.imp as imp
 import dvc.command.imp_url as imp_url
-import dvc.command.config as config
-import dvc.command.checkout as checkout
-import dvc.command.remote as remote
-import dvc.command.cache as cache
-import dvc.command.metrics as metrics
+import dvc.command.init as init
 import dvc.command.install as install
-import dvc.command.root as root
 import dvc.command.lock as lock
+import dvc.command.metrics as metrics
+import dvc.command.move as move
 import dvc.command.pipeline as pipeline
-import dvc.command.daemon as daemon
-import dvc.command.commit as commit
+import dvc.command.remote as remote
+import dvc.command.remove as remove
+import dvc.command.repro as repro
+import dvc.command.root as root
+import dvc.command.run as run
 import dvc.command.tag as tag
-import dvc.command.diff as diff
-import dvc.command.version as version
+import dvc.command.unprotect as unprotect
 import dvc.command.update as update
+import dvc.command.version as version
+from dvc.command.base import fix_subparsers
 from dvc.exceptions import DvcParserError
 
 
@@ -106,7 +105,7 @@ def get_parent_parser():
     """Create instances of a parser containing common arguments shared among
     all the commands.
 
-    When overwritting `-q` or `-v`, you need to instantiate a new object
+    When overwriting `-q` or `-v`, you need to instantiate a new object
     in order to prevent some weird behavior.
     """
     parent_parser = argparse.ArgumentParser(add_help=False)

dvc/command/add.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import RecursiveAddingWhileUsingFilename
 
 

dvc/command/base.py

@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import colorama
 import logging
 
 
@@ -23,15 +22,13 @@ def fix_subparsers(subparsers):
 
 
 def append_doc_link(help_message, path):
+    from dvc.utils import format_link
+
     if not path:
         return help_message
     doc_base = "https://man.dvc.org/"
-    return "{message}\nDocumentation: <{blue}{base}{path}{nc}>".format(
-        message=help_message,
-        base=doc_base,
-        path=path,
-        blue=colorama.Fore.CYAN,
-        nc=colorama.Fore.RESET,
+    return "{message}\nDocumentation: {link}".format(
+        message=help_message, link=format_link(doc_base + path)
     )
 
 
@@ -43,7 +40,8 @@ class CmdBase(object):
         self.repo = Repo()
         self.config = self.repo.config
         self.args = args
-        updater = Updater(self.repo.dvc_dir)
+        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
         updater.check()
 
     @property

dvc/command/cache.py

@@ -2,9 +2,10 @@ from __future__ import unicode_literals
 
 import argparse
 
-from dvc.command.base import fix_subparsers, append_doc_link
-from dvc.command.config import CmdConfig
 from dvc.cache import CacheConfig
+from dvc.command.base import append_doc_link
+from dvc.command.base import fix_subparsers
+from dvc.command.config import CmdConfig
 
 
 class CmdCacheDir(CmdConfig):

dvc/command/checkout.py

@@ -2,7 +2,8 @@ from __future__ import unicode_literals
 
 import argparse
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 
 
 class CmdCheckout(CmdBase):

dvc/command/commit.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/config.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 from dvc.config import Config
 
 

dvc/command/daemon.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.command.base import CmdBaseNoRepo
+from dvc.command.base import fix_subparsers
 
 
 class CmdDaemonBase(CmdBaseNoRepo):
@@ -11,11 +12,16 @@ class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
         from dvc.repo import Repo
+        from dvc.config import Config
         from dvc.updater import Updater
 
         root_dir = Repo.find_root()
         dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
-        updater = Updater(dvc_dir)
+        config = Config(dvc_dir, verify=False)
+        hardlink_lock = config.config.get("core", {}).get(
+            "hardlink_lock", False
+        )
+        updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
         updater.fetch(detach=False)
 
         return 0
@@ -23,10 +29,9 @@ class CmdDaemonUpdater(CmdDaemonBase):
 
 class CmdDaemonAnalytics(CmdDaemonBase):
     def run(self):
-        from dvc.analytics import Analytics
+        from dvc import analytics
 
-        analytics = Analytics.load(self.args.target)
-        analytics.send()
+        analytics.send(self.args.target)
 
         return 0
 

dvc/command/data_sync.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 

dvc/command/destroy.py

@@ -4,7 +4,8 @@ import argparse
 import logging
 
 import dvc.prompt as prompt
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 

dvc/command/diff.py

@@ -1,14 +1,15 @@
 from __future__ import unicode_literals
 
 import argparse
+import logging
 
 import humanize
 import inflect
-import logging
 from funcy import compact
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/gc.py

@@ -1,11 +1,12 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
-import dvc.prompt as prompt
 import logging
+import os
 
-from dvc.command.base import CmdBase, append_doc_link
+import dvc.prompt as prompt
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 
 
 logger = logging.getLogger(__name__)

dvc/command/get.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from .base import append_doc_link
+from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
-from .base import CmdBaseNoRepo, append_doc_link
 
 
 logger = logging.getLogger(__name__)
@@ -32,7 +33,7 @@ class CmdGet(CmdBaseNoRepo):
 
 
 def add_parser(subparsers, parent_parser):
-    GET_HELP = "Download data from DVC repository."
+    GET_HELP = "Download/copy files or directories from DVC repository."
     get_parser = subparsers.add_parser(
         "get",
         parents=[parent_parser],
@@ -43,9 +44,14 @@ def add_parser(subparsers, parent_parser):
     get_parser.add_argument(
         "url", help="URL of Git repository with DVC project to download from."
     )
-    get_parser.add_argument("path", help="Path to data within DVC repository.")
     get_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to put data to."
+        "path", help="Path to a file or directory within a DVC repository."
+    )
+    get_parser.add_argument(
+        "-o",
+        "--out",
+        nargs="?",
+        help="Destination path to copy/download files to.",
     )
     get_parser.add_argument(
         "--rev", nargs="?", help="DVC repository git revision."

dvc/command/get_url.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from .base import append_doc_link
+from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
-from .base import CmdBaseNoRepo, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/imp.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/imp_url.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)
@@ -43,12 +44,17 @@ def add_parser(subparsers, parent_parser):
         "url",
         help="Supported urls:\n"
         "/path/to/file\n"
+        "/path/to/directory\n"
         "C:\\\\path\\to\\file\n"
+        "C:\\\\path\\to\\directory\n"
         "https://example.com/path/to/file\n"
         "s3://bucket/path/to/file\n"
+        "s3://bucket/path/to/directory\n"
         "gs://bucket/path/to/file\n"
+        "gs://bucket/path/to/directory\n"
         "hdfs://example.com/path/to/file\n"
         "ssh://example.com:/path/to/file\n"
+        "ssh://example.com:/path/to/directory\n"
         "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(

dvc/command/init.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 
 
 logger = logging.getLogger(__name__)

dvc/command/install.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 
 
 logger = logging.getLogger(__name__)

dvc/command/lock.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/metrics.py

@@ -3,8 +3,11 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.exceptions import DvcException, BadMetricError
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
+from dvc.exceptions import BadMetricError
+from dvc.exceptions import DvcException
 
 
 logger = logging.getLogger(__name__)

dvc/command/move.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/pipeline.py

@@ -1,13 +1,14 @@
 from __future__ import unicode_literals
 
-from dvc.utils import relpath
-from dvc.utils.compat import str
-
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.utils import relpath
+from dvc.utils.compat import str
 
 
 logger = logging.getLogger(__name__)

dvc/command/remote.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link, fix_subparsers
+from dvc.command.base import append_doc_link
+from dvc.command.base import fix_subparsers
 from dvc.command.config import CmdConfig
 from dvc.remote.config import RemoteConfig
 
@@ -18,6 +19,10 @@ class CmdRemoteConfig(CmdConfig):
 
 class CmdRemoteAdd(CmdRemoteConfig):
     def run(self):
+        if self.args.default:
+            logger.info(
+                "Setting '{}' as a default remote.".format(self.args.name)
+            )
         self.remote_config.add(
             self.args.name,
             self.args.url,
@@ -47,9 +52,13 @@ class CmdRemoteModify(CmdRemoteConfig):
 
 class CmdRemoteDefault(CmdRemoteConfig):
     def run(self):
-        self.remote_config.set_default(
-            self.args.name, unset=self.args.unset, level=self.args.level
-        )
+        if self.args.name is None and not self.args.unset:
+            name = self.remote_config.get_default(level=self.args.level)
+            print(name)
+        else:
+            self.remote_config.set_default(
+                self.args.name, unset=self.args.unset, level=self.args.level
+            )
         return 0
 
 

dvc/command/remove.py

@@ -4,8 +4,9 @@ import argparse
 import logging
 
 import dvc.prompt as prompt
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/repro.py

@@ -1,10 +1,11 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
 import logging
+import os
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
 from dvc.exceptions import DvcException

dvc/command/root.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)

dvc/command/run.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 

dvc/command/tag.py

@@ -1,8 +1,10 @@
 import logging
 
-from dvc.utils import to_yaml_string
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.utils import to_yaml_string
 
 
 logger = logging.getLogger(__name__)

dvc/command/unprotect.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/update.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/version.py

@@ -1,11 +1,11 @@
 from __future__ import unicode_literals
 
+import argparse
+import itertools
+import logging
 import os
 import platform
-import argparse
-import logging
 import uuid
-import itertools
 
 try:
     import psutil
@@ -18,7 +18,7 @@ from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.system import System
-
+from dvc.utils.pkg import PKG
 
 logger = logging.getLogger(__name__)
 
@@ -31,17 +31,18 @@ class CmdVersion(CmdBaseNoRepo):
         python_version = platform.python_version()
         platform_type = platform.platform()
         binary = is_binary()
-
         info = (
             "DVC version: {dvc_version}\n"
             "Python version: {python_version}\n"
             "Platform: {platform_type}\n"
             "Binary: {binary}\n"
+            "Package: {package}\n"
         ).format(
             dvc_version=dvc_version,
             python_version=python_version,
             platform_type=platform_type,
             binary=binary,
+            package=PKG,
         )
 
         try:
@@ -92,7 +93,7 @@ class CmdVersion(CmdBaseNoRepo):
         for parent in itertools.chain([path], path.parents):
             if parent in partition:
                 return partition[parent]
-        return ("unkown", "none")
+        return ("unknown", "none")
 
     @staticmethod
     def get_linktype_support_info(repo):

dvc/config.py

@@ -1,18 +1,19 @@
 """DVC config objects."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, open
-
-import os
-import re
 import copy
 import errno
-import configobj
 import logging
+import os
+import re
 
-from schema import Schema, Optional, And, Use, Regex, SchemaError
-from dvc.exceptions import DvcException, NotDvcRepoError
+import configobj
+from voluptuous import Schema, Required, Optional, Invalid
+from voluptuous import All, Any, Lower, Range, Coerce, Match
+
+from dvc.exceptions import DvcException
+from dvc.exceptions import NotDvcRepoError
+from dvc.utils.compat import open, str
 
 logger = logging.getLogger(__name__)
 
@@ -44,82 +45,43 @@ class NoRemoteError(ConfigError):
 
 
 def supported_cache_type(types):
-    """Checks if link type config option has a valid value.
+    """Checks if link type config option consists only of valid values.
 
     Args:
         types (list/string): type(s) of links that dvc should try out.
     """
+    if types is None:
+        return None
     if isinstance(types, str):
         types = [typ.strip() for typ in types.split(",")]
-    for typ in types:
-        if typ not in ["reflink", "hardlink", "symlink", "copy"]:
-            return False
-    return True
+
+    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
+    if unsupported:
+        raise Invalid(
+            "Unsupported cache type(s): {}".format(", ".join(unsupported))
+        )
+
+    return types
 
 
-def is_bool(val):
-    """Checks that value is a boolean.
-
-    Args:
-        val (str): string value verify.
-
-    Returns:
-        bool: True if value stands for boolean, False otherwise.
-    """
-    return val.lower() in ["true", "false"]
+# Checks that value is either true or false and converts it to bool
+Bool = All(
+    Lower,
+    Any("true", "false"),
+    lambda v: v == "true",
+    msg="expected true or false",
+)
+to_bool = Schema(Bool)
 
 
-def to_bool(val):
-    """Converts value to boolean.
-
-    Args:
-        val (str): string to convert to boolean.
-
-    Returns:
-        bool: True if value.lower() == 'true', False otherwise.
-    """
-    return val.lower() == "true"
-
-
-def is_whole(val):
-    """Checks that value is a whole integer.
-
-    Args:
-        val (str): number string to verify.
-
-    Returns:
-        bool: True if val is a whole number, False otherwise.
-    """
-    return int(val) >= 0
-
-
-def is_percent(val):
-    """Checks that value is a percent.
-
-    Args:
-        val (str): number string to verify.
-
-    Returns:
-        bool: True if 0<=value<=100, False otherwise.
-    """
-    return int(val) >= 0 and int(val) <= 100
-
-
-class Choices(object):
+def Choices(*choices):
     """Checks that value belongs to the specified set of values
 
     Args:
         *choices: pass allowed values as arguments, or pass a list or
             tuple as a single argument
     """
-
-    def __init__(self, *choices):
-        if len(choices) == 1 and isinstance(choices[0], (list, tuple)):
-            choices = choices[0]
-        self.choices = choices
-
-    def __call__(self, value):
-        return value in self.choices
+    return Any(*choices, msg="expected one of {}".format(",".join(choices)))
 
 
 class Config(object):  # pylint: disable=too-many-instance-attributes
@@ -147,33 +109,30 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
+    CREDENTIALPATH = "credentialpath"
+
     LEVEL_LOCAL = 0
     LEVEL_REPO = 1
     LEVEL_GLOBAL = 2
     LEVEL_SYSTEM = 3
 
-    BOOL_SCHEMA = And(str, is_bool, Use(to_bool))
-
     SECTION_CORE = "core"
     SECTION_CORE_LOGLEVEL = "loglevel"
-    SECTION_CORE_LOGLEVEL_SCHEMA = And(
-        Use(str.lower), Choices("info", "debug", "warning", "error")
+    SECTION_CORE_LOGLEVEL_SCHEMA = All(
+        Lower, Choices("info", "debug", "warning", "error")
     )
     SECTION_CORE_REMOTE = "remote"
-    SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA
     SECTION_CORE_INTERACTIVE = "interactive"
     SECTION_CORE_ANALYTICS = "analytics"
-    SECTION_CORE_ANALYTICS_SCHEMA = BOOL_SCHEMA
     SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_CORE_CHECKSUM_JOBS_SCHEMA = And(Use(int), lambda x: x > 0)
+    SECTION_CORE_HARDLINK_LOCK = "hardlink_lock"
 
     SECTION_CACHE = "cache"
     SECTION_CACHE_DIR = "dir"
     SECTION_CACHE_TYPE = "type"
-    SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
     SECTION_CACHE_PROTECTED = "protected"
     SECTION_CACHE_SHARED = "shared"
-    SECTION_CACHE_SHARED_SCHEMA = And(Use(str.lower), Choices("group"))
+    SECTION_CACHE_SHARED_SCHEMA = All(Lower, Choices("group"))
     SECTION_CACHE_LOCAL = "local"
     SECTION_CACHE_S3 = "s3"
     SECTION_CACHE_GS = "gs"
@@ -182,40 +141,31 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_CACHE_AZURE = "azure"
     SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
     SECTION_CACHE_SCHEMA = {
-        Optional(SECTION_CACHE_LOCAL): str,
-        Optional(SECTION_CACHE_S3): str,
-        Optional(SECTION_CACHE_GS): str,
-        Optional(SECTION_CACHE_HDFS): str,
-        Optional(SECTION_CACHE_SSH): str,
-        Optional(SECTION_CACHE_AZURE): str,
-        Optional(SECTION_CACHE_DIR): str,
-        Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
-        Optional(SECTION_CACHE_SHARED): SECTION_CACHE_SHARED_SCHEMA,
-        Optional(PRIVATE_CWD): str,
-        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): BOOL_SCHEMA,
+        SECTION_CACHE_LOCAL: str,
+        SECTION_CACHE_S3: str,
+        SECTION_CACHE_GS: str,
+        SECTION_CACHE_HDFS: str,
+        SECTION_CACHE_SSH: str,
+        SECTION_CACHE_AZURE: str,
+        SECTION_CACHE_DIR: str,
+        SECTION_CACHE_TYPE: supported_cache_type,
+        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
+        SECTION_CACHE_SHARED: SECTION_CACHE_SHARED_SCHEMA,
+        PRIVATE_CWD: str,
+        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): Bool,
     }
 
     SECTION_CORE_SCHEMA = {
-        Optional(SECTION_CORE_LOGLEVEL): And(
-            str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA
-        ),
-        Optional(SECTION_CORE_REMOTE, default=""): And(str, Use(str.lower)),
-        Optional(
-            SECTION_CORE_INTERACTIVE, default=False
-        ): SECTION_CORE_INTERACTIVE_SCHEMA,
-        Optional(
-            SECTION_CORE_ANALYTICS, default=True
-        ): SECTION_CORE_ANALYTICS_SCHEMA,
-        Optional(
-            SECTION_CORE_CHECKSUM_JOBS, default=None
-        ): SECTION_CORE_CHECKSUM_JOBS_SCHEMA,
+        SECTION_CORE_LOGLEVEL: SECTION_CORE_LOGLEVEL_SCHEMA,
+        SECTION_CORE_REMOTE: Lower,
+        Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
+        Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
+        SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
+        Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool,
     }
 
-    # backward compatibility
-    SECTION_AWS = "aws"
-    SECTION_AWS_STORAGEPATH = "storagepath"
-    SECTION_AWS_CREDENTIALPATH = "credentialpath"
+    # aws specific options
+    SECTION_AWS_CREDENTIALPATH = CREDENTIALPATH
     SECTION_AWS_ENDPOINT_URL = "endpointurl"
     SECTION_AWS_LIST_OBJECTS = "listobjects"
     SECTION_AWS_REGION = "region"
@@ -223,39 +173,24 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_AWS_USE_SSL = "use_ssl"
     SECTION_AWS_SSE = "sse"
     SECTION_AWS_ACL = "acl"
-    SECTION_AWS_SCHEMA = {
-        SECTION_AWS_STORAGEPATH: str,
-        Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE): str,
-        Optional(SECTION_AWS_CREDENTIALPATH): str,
-        Optional(SECTION_AWS_ENDPOINT_URL): str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
-        Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
-        Optional(SECTION_AWS_SSE): str,
-        Optional(SECTION_AWS_ACL): str,
-    }
 
-    # backward compatibility
-    SECTION_GCP = "gcp"
-    SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
-    SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH
+    # gcp specific options
+    SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH
     SECTION_GCP_PROJECTNAME = "projectname"
-    SECTION_GCP_SCHEMA = {
-        SECTION_GCP_STORAGEPATH: str,
-        Optional(SECTION_GCP_PROJECTNAME): str,
-    }
-
-    # backward compatibility
-    SECTION_LOCAL = "local"
-    SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
-    SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str}
 
+    # azure specific option
     SECTION_AZURE_CONNECTION_STRING = "connection_string"
+
     # Alibabacloud oss options
     SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
     SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
     SECTION_OSS_ENDPOINT = "oss_endpoint"
 
+    # GDrive options
+    SECTION_GDRIVE_CLIENT_ID = "gdrive_client_id"
+    SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret"
+    SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file"
+
     SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
     SECTION_REMOTE_FMT = 'remote "{}"'
     SECTION_REMOTE_URL = "url"
@@ -268,51 +203,51 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_REMOTE_GSS_AUTH = "gss_auth"
     SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
     SECTION_REMOTE_SCHEMA = {
-        SECTION_REMOTE_URL: str,
-        Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE): str,
-        Optional(SECTION_AWS_CREDENTIALPATH): str,
-        Optional(SECTION_AWS_ENDPOINT_URL): str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
-        Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
-        Optional(SECTION_AWS_SSE): str,
-        Optional(SECTION_AWS_ACL): str,
-        Optional(SECTION_GCP_PROJECTNAME): str,
-        Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
-        Optional(SECTION_REMOTE_USER): str,
-        Optional(SECTION_REMOTE_PORT): Use(int),
-        Optional(SECTION_REMOTE_KEY_FILE): str,
-        Optional(SECTION_REMOTE_TIMEOUT): Use(int),
-        Optional(SECTION_REMOTE_PASSWORD): str,
-        Optional(SECTION_REMOTE_ASK_PASSWORD): BOOL_SCHEMA,
-        Optional(SECTION_REMOTE_GSS_AUTH): BOOL_SCHEMA,
-        Optional(SECTION_AZURE_CONNECTION_STRING): str,
-        Optional(SECTION_OSS_ACCESS_KEY_ID): str,
-        Optional(SECTION_OSS_ACCESS_KEY_SECRET): str,
-        Optional(SECTION_OSS_ENDPOINT): str,
-        Optional(PRIVATE_CWD): str,
-        Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): BOOL_SCHEMA,
+        Required(SECTION_REMOTE_URL): str,
+        SECTION_AWS_REGION: str,
+        SECTION_AWS_PROFILE: str,
+        SECTION_AWS_CREDENTIALPATH: str,
+        SECTION_AWS_ENDPOINT_URL: str,
+        Optional(SECTION_AWS_LIST_OBJECTS, default=False): Bool,
+        Optional(SECTION_AWS_USE_SSL, default=True): Bool,
+        SECTION_AWS_SSE: str,
+        SECTION_AWS_ACL: str,
+        SECTION_GCP_PROJECTNAME: str,
+        SECTION_CACHE_TYPE: supported_cache_type,
+        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
+        SECTION_REMOTE_USER: str,
+        SECTION_REMOTE_PORT: Coerce(int),
+        SECTION_REMOTE_KEY_FILE: str,
+        SECTION_REMOTE_TIMEOUT: Coerce(int),
+        SECTION_REMOTE_PASSWORD: str,
+        SECTION_REMOTE_ASK_PASSWORD: Bool,
+        SECTION_REMOTE_GSS_AUTH: Bool,
+        SECTION_AZURE_CONNECTION_STRING: str,
+        SECTION_OSS_ACCESS_KEY_ID: str,
+        SECTION_OSS_ACCESS_KEY_SECRET: str,
+        SECTION_OSS_ENDPOINT: str,
+        SECTION_GDRIVE_CLIENT_ID: str,
+        SECTION_GDRIVE_CLIENT_SECRET: str,
+        SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
+        PRIVATE_CWD: str,
+        Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): Bool,
     }
 
     SECTION_STATE = "state"
     SECTION_STATE_ROW_LIMIT = "row_limit"
     SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
     SECTION_STATE_SCHEMA = {
-        Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
-        Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
+        SECTION_STATE_ROW_LIMIT: All(Coerce(int), Range(1)),
+        SECTION_STATE_ROW_CLEANUP_QUOTA: All(Coerce(int), Range(0, 100)),
     }
 
     SCHEMA = {
         Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
-        Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
+        Match(SECTION_REMOTE_REGEX): SECTION_REMOTE_SCHEMA,
         Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
         Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
-        # backward compatibility
-        Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
-        Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
-        Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
     }
+    COMPILED_SCHEMA = Schema(SCHEMA)
 
     def __init__(self, dvc_dir=None, validate=True):
         self.dvc_dir = dvc_dir
@@ -451,9 +386,9 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
 
         d = self.config.dict()
         try:
-            d = Schema(self.SCHEMA).validate(d)
-        except SchemaError as exc:
-            raise ConfigError("config format error", cause=exc)
+            d = self.COMPILED_SCHEMA(d)
+        except Invalid as exc:
+            raise ConfigError(str(exc), cause=exc)
         self.config = configobj.ConfigObj(d, write_empty_values=True)
 
     def save(self, config=None):

dvc/daemon.py

@@ -1,15 +1,15 @@
 """Launch `dvc daemon` command in a separate detached process."""
-
 from __future__ import unicode_literals
 
-import os
-import sys
 import inspect
 import logging
+import os
+import sys
 from subprocess import Popen
 
 from dvc.env import DVC_DAEMON
-from dvc.utils import is_binary, fix_env
+from dvc.utils import fix_env
+from dvc.utils import is_binary
 from dvc.utils.compat import cast_bytes_py2
 
 
@@ -39,7 +39,7 @@ def _spawn_windows(cmd, env):
 
 def _spawn_posix(cmd, env):
     # NOTE: using os._exit instead of sys.exit, because dvc built
-    # with PyInstaller has trouble with SystemExit exeption and throws
+    # with PyInstaller has trouble with SystemExit exception and throws
     # errors such as "[26338] Failed to execute script __main__"
     try:
         pid = os.fork()

dvc/dagascii.py

@@ -1,14 +1,55 @@
 """Draws DAG in ASCII."""
-
-from __future__ import unicode_literals
 from __future__ import print_function
+from __future__ import unicode_literals
 
-import sys
+import logging
 import math
+import os
+import pydoc
+import sys
 
-from grandalf.graphs import Vertex, Edge, Graph
+from grandalf.graphs import Edge
+from grandalf.graphs import Graph
+from grandalf.graphs import Vertex
 from grandalf.layouts import SugiyamaLayout
-from grandalf.routing import route_with_lines, EdgeViewer
+from grandalf.routing import EdgeViewer
+from grandalf.routing import route_with_lines
+
+from dvc.env import DVC_PAGER
+
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_PAGER = "less"
+DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
+    DEFAULT_PAGER
+)
+
+
+def make_pager(cmd):
+    def pager(text):
+        return pydoc.tempfilepager(pydoc.plain(text), cmd)
+
+    return pager
+
+
+def find_pager():
+    if not sys.stdout.isatty():
+        return pydoc.plainpager
+
+    env_pager = os.getenv(DVC_PAGER)
+    if env_pager:
+        return make_pager(env_pager)
+
+    if os.system("({}) 2>{}".format(DEFAULT_PAGER, os.devnull)) == 0:
+        return make_pager(DEFAULT_PAGER_FORMATTED)
+
+    logger.warning(
+        "Unable to find `less` in the PATH. Check out "
+        "man.dvc.org/doc/command-reference/pipeline/show for more info."
+    )
+    return pydoc.plainpager
 
 
 class VertexViewer(object):
@@ -58,99 +99,10 @@ class AsciiCanvas(object):
 
     def draw(self):
         """Draws ASCII canvas on the screen."""
-        if sys.stdout.isatty():  # pragma: no cover
-            from asciimatics.screen import Screen
-
-            Screen.wrapper(self._do_draw)
-        else:
-            for line in self.canvas:
-                print("".join(line))
-
-    def _do_draw(self, screen):  # pragma: no cover
-        # pylint: disable=too-many-locals
-        # pylint: disable=too-many-branches, too-many-statements
-        from dvc.system import System
-        from asciimatics.event import KeyboardEvent
-
-        offset_x = 0
-        offset_y = 0
-        smaxrow, smaxcol = screen.dimensions
-        assert smaxrow > 1
-        assert smaxcol > 1
-        smaxrow -= 1
-        smaxcol -= 1
-
-        if self.lines + 1 > smaxrow:
-            max_y = self.lines + 1 - smaxrow
-        else:
-            max_y = 0
-
-        if self.cols + 1 > smaxcol:
-            max_x = self.cols + 1 - smaxcol
-        else:
-            max_x = 0
-
-        while True:
-            for y in range(smaxrow + 1):
-                y_index = offset_y + y
-                line = []
-                for x in range(smaxcol + 1):
-                    x_index = offset_x + x
-                    if (
-                        len(self.canvas) > y_index
-                        and len(self.canvas[y_index]) > x_index
-                    ):
-                        line.append(self.canvas[y_index][x_index])
-                    else:
-                        line.append(" ")
-                assert len(line) == (smaxcol + 1)
-                screen.print_at("".join(line), 0, y)
-
-            screen.refresh()
-
-            # NOTE: get_event() doesn't block by itself,
-            # so we have to do the blocking ourselves.
-            #
-            # NOTE: using this workaround while waiting for PR [1]
-            # to get merged and released. After that need to adjust
-            # asciimatics version requirements.
-            #
-            # [1] https://github.com/peterbrittain/asciimatics/pull/188
-            System.wait_for_input(self.TIMEOUT)
-
-            event = screen.get_event()
-            if not isinstance(event, KeyboardEvent):
-                continue
-
-            k = event.key_code
-            if k == screen.KEY_DOWN or k == ord("s"):
-                offset_y += 1
-            elif k == screen.KEY_PAGE_DOWN or k == ord("S"):
-                offset_y += smaxrow
-            elif k == screen.KEY_UP or k == ord("w"):
-                offset_y -= 1
-            elif k == screen.KEY_PAGE_UP or k == ord("W"):
-                offset_y -= smaxrow
-            elif k == screen.KEY_RIGHT or k == ord("d"):
-                offset_x += 1
-            elif k == ord("D"):
-                offset_x += smaxcol
-            elif k == screen.KEY_LEFT or k == ord("a"):
-                offset_x -= 1
-            elif k == ord("A"):
-                offset_x -= smaxcol
-            elif k == ord("q") or k == ord("Q"):
-                break
-
-            if offset_y > max_y:
-                offset_y = max_y
-            elif offset_y < 0:
-                offset_y = 0
-
-            if offset_x > max_x:
-                offset_x = max_x
-            elif offset_x < 0:
-                offset_x = 0
+        pager = find_pager()
+        lines = map("".join, self.canvas)
+        joined_lines = os.linesep.join(lines)
+        pager(joined_lines)
 
     def point(self, x, y, char):
         """Create a point on ASCII canvas.

dvc/data_cloud.py

@@ -1,10 +1,10 @@
-"""Manages dvc remotes that user can use with push/pull/status comamnds."""
-
+"""Manages dvc remotes that user can use with push/pull/status commands."""
 from __future__ import unicode_literals
 
 import logging
 
-from dvc.config import Config, NoRemoteError
+from dvc.config import Config
+from dvc.config import NoRemoteError
 from dvc.remote import Remote
 
 

dvc/dependency/__init__.py

@@ -1,21 +1,17 @@
 from __future__ import unicode_literals
 
-import schema
-
-from dvc.scheme import Schemes
-
 import dvc.output as output
-from dvc.output.base import OutputBase
-from dvc.dependency.s3 import DependencyS3
 from dvc.dependency.gs import DependencyGS
-from dvc.dependency.local import DependencyLOCAL
 from dvc.dependency.hdfs import DependencyHDFS
-from dvc.dependency.ssh import DependencySSH
 from dvc.dependency.http import DependencyHTTP
 from dvc.dependency.https import DependencyHTTPS
-from .repo import DependencyREPO
-
+from dvc.dependency.local import DependencyLOCAL
+from dvc.dependency.s3 import DependencyS3
+from dvc.dependency.ssh import DependencySSH
+from dvc.output.base import OutputBase
 from dvc.remote import Remote
+from dvc.scheme import Schemes
+from .repo import DependencyREPO
 
 
 DEPS = [
@@ -44,9 +40,9 @@ DEP_MAP = {
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
 # output is a metric file and how to parse it, see `-M` flag for `dvc run`).
 SCHEMA = output.SCHEMA.copy()
-del SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)]
-del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)]
-SCHEMA[schema.Optional(DependencyREPO.PARAM_REPO)] = DependencyREPO.REPO_SCHEMA
+del SCHEMA[OutputBase.PARAM_CACHE]
+del SCHEMA[OutputBase.PARAM_METRIC]
+SCHEMA[DependencyREPO.PARAM_REPO] = DependencyREPO.REPO_SCHEMA
 
 
 def _get(stage, p, info):

dvc/dependency/gs.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.gs import OutputGS
 from dvc.dependency.base import DependencyBase
+from dvc.output.gs import OutputGS
 
 
 class DependencyGS(DependencyBase, OutputGS):

dvc/dependency/hdfs.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.hdfs import OutputHDFS
 from dvc.dependency.base import DependencyBase
+from dvc.output.hdfs import OutputHDFS
 
 
 class DependencyHDFS(DependencyBase, OutputHDFS):

dvc/dependency/http.py

@@ -1,8 +1,8 @@
 from __future__ import unicode_literals
 
+from dvc.dependency.base import DependencyBase
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP
-from dvc.dependency.base import DependencyBase
 
 
 class DependencyHTTP(DependencyBase, OutputBase):

dvc/dependency/https.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.remote.https import RemoteHTTPS
 from .http import DependencyHTTP
+from dvc.remote.https import RemoteHTTPS
 
 
 class DependencyHTTPS(DependencyHTTP):

dvc/dependency/repo.py

@@ -1,15 +1,13 @@
 from __future__ import unicode_literals
 
 import copy
-
-from funcy import merge
-from schema import Optional
 from contextlib import contextmanager
 
-from dvc.external_repo import external_repo
-from dvc.utils.compat import str
+from funcy import merge
 
 from .local import DependencyLOCAL
+from dvc.external_repo import external_repo
+from dvc.utils.compat import str
 
 
 class DependencyREPO(DependencyLOCAL):
@@ -18,11 +16,7 @@ class DependencyREPO(DependencyLOCAL):
     PARAM_REV = "rev"
     PARAM_REV_LOCK = "rev_lock"
 
-    REPO_SCHEMA = {
-        Optional(PARAM_URL): str,
-        Optional(PARAM_REV): str,
-        Optional(PARAM_REV_LOCK): str,
-    }
+    REPO_SCHEMA = {PARAM_URL: str, PARAM_REV: str, PARAM_REV_LOCK: str}
 
     def __init__(self, def_repo, stage, *args, **kwargs):
         self.def_repo = def_repo

dvc/dependency/s3.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.s3 import OutputS3
 from dvc.dependency.base import DependencyBase
+from dvc.output.s3 import OutputS3
 
 
 class DependencyS3(DependencyBase, OutputS3):

dvc/dependency/ssh.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.ssh import OutputSSH
 from dvc.dependency.base import DependencyBase
+from dvc.output.ssh import OutputSSH
 
 
 class DependencySSH(DependencyBase, OutputSSH):

dvc/env.py

@@ -1 +1,2 @@
 DVC_DAEMON = "DVC_DAEMON"
+DVC_PAGER = "DVC_PAGER"

dvc/exceptions.py

@@ -1,12 +1,11 @@
 """Exceptions raised by the dvc."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, builtin_str
-
 import traceback
 
 from dvc.utils import relpath
+from dvc.utils.compat import builtin_str
+from dvc.utils.compat import str
 
 
 class DvcException(Exception):
@@ -60,10 +59,12 @@ class OutputNotFoundError(DvcException):
         output (unicode): path to the file/directory.
     """
 
-    def __init__(self, output):
+    def __init__(self, output, repo=None):
+        self.output = output
+        self.repo = repo
         super(OutputNotFoundError, self).__init__(
             "unable to find DVC-file with output '{path}'".format(
-                path=relpath(output)
+                path=relpath(self.output)
             )
         )
 
@@ -282,14 +283,6 @@ class UrlNotDvcRepoError(DvcException):
         )
 
 
-class GetDVCFileError(DvcException):
-    def __init__(self):
-        super(GetDVCFileError, self).__init__(
-            "the given path is a DVC-file, you must specify a data file "
-            "or a directory"
-        )
-
-
 class GitHookAlreadyExistsError(DvcException):
     def __init__(self, hook_name):
         super(GitHookAlreadyExistsError, self).__init__(
@@ -329,3 +322,24 @@ class CheckoutError(DvcException):
 
 class CollectCacheError(DvcException):
     pass
+
+
+class NoRemoteInExternalRepoError(DvcException):
+    def __init__(self, url):
+        super(NoRemoteInExternalRepoError, self).__init__(
+            "No DVC remote is specified in target repository '{}'.".format(url)
+        )
+
+
+class NoOutputInExternalRepoError(DvcException):
+    def __init__(self, path, external_repo_path, external_repo_url):
+        super(NoOutputInExternalRepoError, self).__init__(
+            "Output '{}' not found in target repository '{}'".format(
+                relpath(path, external_repo_path), external_repo_url
+            )
+        )
+
+
+class HTTPError(DvcException):
+    def __init__(self, code, reason):
+        super(HTTPError, self).__init__("'{} {}'".format(code, reason))

dvc/external_repo.py

@@ -2,12 +2,17 @@ from __future__ import unicode_literals
 
 import os
 import tempfile
+from contextlib import contextmanager
 from distutils.dir_util import copy_tree
 
-from contextlib import contextmanager
 from funcy import retry
 
-from dvc.utils import remove
+from dvc.config import NoRemoteError, ConfigError
+from dvc.exceptions import NoRemoteInExternalRepoError
+from dvc.remote import RemoteConfig
+from dvc.exceptions import NoOutputInExternalRepoError
+from dvc.exceptions import OutputNotFoundError
+from dvc.utils.fs import remove
 
 
 REPO_CACHE = {}
@@ -19,7 +24,14 @@ def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
 
     path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
     repo = Repo(path)
-    yield repo
+    try:
+        yield repo
+    except NoRemoteError:
+        raise NoRemoteInExternalRepoError(url)
+    except OutputNotFoundError as exc:
+        if exc.repo is repo:
+            raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url)
+        raise
     repo.close()
 
 
@@ -50,10 +62,29 @@ def _external_repo(url=None, rev=None, cache_dir=None):
         REPO_CACHE[url, None, None] = clean_clone_path
 
     # Adjust new clone/copy to fit rev and cache_dir
+
+    # Checkout needs to be done first because current branch might not be
+    # DVC repository
+    if rev is not None:
+        _git_checkout(new_path, rev)
+
     repo = Repo(new_path)
     try:
-        if rev is not None:
-            repo.scm.checkout(rev)
+        # check if the URL is local and no default remote is present
+        # add default remote pointing to the original repo's cache location
+        if os.path.isdir(url):
+            rconfig = RemoteConfig(repo.config)
+            if not _default_remote_set(rconfig):
+                original_repo = Repo(url)
+                try:
+                    rconfig.add(
+                        "auto-generated-upstream",
+                        original_repo.cache.local.cache_dir,
+                        default=True,
+                        level=Config.LEVEL_LOCAL,
+                    )
+                finally:
+                    original_repo.close()
 
         if cache_dir is not None:
             cache_config = CacheConfig(repo.config)
@@ -66,6 +97,16 @@ def _external_repo(url=None, rev=None, cache_dir=None):
     return new_path
 
 
+def _git_checkout(repo_path, revision):
+    from dvc.scm import Git
+
+    git = Git(repo_path)
+    try:
+        git.checkout(revision)
+    finally:
+        git.close()
+
+
 def clean_repos():
     # Outside code should not see cache while we are removing
     repo_paths = list(REPO_CACHE.values())
@@ -89,3 +130,19 @@ def _clone_repo(url, path):
 
     git = Git.clone(url, path)
     git.close()
+
+
+def _default_remote_set(rconfig):
+    """
+    Checks if default remote config is present.
+    Args:
+        rconfig: a remote config
+
+    Returns:
+        True if the default remote config is set, else False
+    """
+    try:
+        rconfig.get_default()
+        return True
+    except ConfigError:
+        return False

dvc/ignore.py

@@ -2,13 +2,12 @@ from __future__ import unicode_literals
 
 import logging
 import os
+
 from pathspec import PathSpec
 from pathspec.patterns import GitWildMatchPattern
 
 from dvc.utils import relpath
 
-from dvc.utils.compat import open
-
 logger = logging.getLogger(__name__)
 
 
@@ -20,13 +19,13 @@ class DvcIgnore(object):
 
 
 class DvcIgnorePatterns(DvcIgnore):
-    def __init__(self, ignore_file_path):
+    def __init__(self, ignore_file_path, tree):
         assert os.path.isabs(ignore_file_path)
 
         self.ignore_file_path = ignore_file_path
         self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
 
-        with open(ignore_file_path, encoding="utf-8") as fobj:
+        with tree.open(ignore_file_path, encoding="utf-8") as fobj:
             self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
 
     def __call__(self, root, dirs, files):
@@ -47,6 +46,9 @@ class DvcIgnorePatterns(DvcIgnore):
         return hash(self.ignore_file_path)
 
     def __eq__(self, other):
+        if not isinstance(other, DvcIgnorePatterns):
+            return NotImplemented
+
         return self.ignore_file_path == other.ignore_file_path
 
 
@@ -59,19 +61,29 @@ class DvcIgnoreDirs(DvcIgnore):
 
         return dirs, files
 
+    def __hash__(self):
+        return hash(tuple(self.basenames))
+
+    def __eq__(self, other):
+        if not isinstance(other, DvcIgnoreDirs):
+            return NotImplemented
+
+        return self.basenames == other.basenames
+
 
 class DvcIgnoreFilter(object):
-    def __init__(self, root_dir):
+    def __init__(self, root_dir, tree):
+        self.tree = tree
         self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
         self._update(root_dir)
-        for root, dirs, _ in os.walk(root_dir):
+        for root, dirs, _ in self.tree.walk(root_dir, dvcignore=self):
             for d in dirs:
                 self._update(os.path.join(root, d))
 
     def _update(self, dirname):
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if os.path.exists(ignore_file_path):
-            self.ignores.add(DvcIgnorePatterns(ignore_file_path))
+        if self.tree.exists(ignore_file_path):
+            self.ignores.add(DvcIgnorePatterns(ignore_file_path, self.tree))
 
     def __call__(self, root, dirs, files):
         for ignore in self.ignores:

dvc/istextfile.py

@@ -1,8 +1,8 @@
 """Use heuristics to guess if it is a text file or a binary file."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import is_py3, open
+from dvc.utils.compat import is_py3
+from dvc.utils.compat import open
 
 # Based on https://eli.thegreenplace.net/2011/10/19/
 # perls-guess-if-file-is-text-or-binary-implemented-in-python

dvc/lock.py

@@ -1,30 +1,86 @@
 """Manages dvc lock file."""
-
 from __future__ import unicode_literals
 
-import os
 import hashlib
+import os
 import time
 from datetime import timedelta
 
+import zc.lockfile
 from funcy.py3 import lkeep
 
 from dvc.exceptions import DvcException
-from dvc.utils import makedirs
-from dvc.utils.compat import is_py3
-
+from dvc.utils import makedirs, format_link
+from dvc.utils.compat import is_py3, is_py2
+from dvc.progress import Tqdm
 
 DEFAULT_TIMEOUT = 5
 
+FAILED_TO_LOCK_MESSAGE = (
+    "cannot perform the command because another DVC process seems to be "
+    "running on this project. If that is not the case, manually remove "
+    "`.dvc/lock` and try again."
+)
+
 
 class LockError(DvcException):
     """Thrown when unable to acquire the lock for dvc repo."""
 
 
+class Lock(object):
+    """Class for dvc repo lock.
+
+    Uses zc.lockfile as backend.
+    """
+
+    def __init__(self, lockfile, friendly=False, **kwargs):
+        self._friendly = friendly
+        self.lockfile = lockfile
+        self._lock = None
+
+    @property
+    def files(self):
+        return [self.lockfile]
+
+    def _do_lock(self):
+        try:
+            with Tqdm(
+                bar_format="{desc}",
+                disable=not self._friendly,
+                desc=(
+                    "If DVC froze, see `hardlink_lock` in {}".format(
+                        format_link("man.dvc.org/config#core")
+                    )
+                ),
+            ):
+                self._lock = zc.lockfile.LockFile(self.lockfile)
+        except zc.lockfile.LockError:
+            raise LockError(FAILED_TO_LOCK_MESSAGE)
+
+    def lock(self):
+        try:
+            self._do_lock()
+            return
+        except LockError:
+            time.sleep(DEFAULT_TIMEOUT)
+
+        self._do_lock()
+
+    def unlock(self):
+        self._lock.close()
+        self._lock = None
+
+    def __enter__(self):
+        self.lock()
+
+    def __exit__(self, typ, value, tbck):
+        self.unlock()
+
+
 if is_py3:
     import flufl.lock
 
-    class Lock(flufl.lock.Lock):
+    class HardlinkLock(flufl.lock.Lock):
         """Class for dvc repo lock.
 
         Args:
@@ -33,7 +89,7 @@ if is_py3:
             tmp_dir (str): a directory to store claim files.
         """
 
-        def __init__(self, lockfile, tmp_dir=None):
+        def __init__(self, lockfile, tmp_dir=None, **kwargs):
             import socket
 
             self._tmp_dir = tmp_dir
@@ -72,10 +128,7 @@ if is_py3:
             try:
                 super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
             except flufl.lock.TimeOutError:
-                raise LockError(
-                    "cannot perform the cmd since DVC is busy and "
-                    "locked. Please retry the cmd later."
-                )
+                raise LockError(FAILED_TO_LOCK_MESSAGE)
 
         def _set_claimfile(self, pid=None):
             super(Lock, self)._set_claimfile(pid)
@@ -99,47 +152,14 @@ if is_py3:
                 pass
 
 
-else:
-    import zc.lockfile
+def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False):
+    if hardlink_lock and is_py2:
+        raise DvcException(
+            "Hardlink locks are not supported on Python <3.5. "
+            "See `hardlink_lock` in {}".format(
+                format_link("man.dvc.org/config#core")
+            )
+        )
 
-    class Lock(object):
-        """Class for dvc repo lock.
-
-        Uses zc.lockfile as backend.
-        """
-
-        def __init__(self, lockfile, tmp_dir=None):
-            self.lockfile = lockfile
-            self._lock = None
-
-        @property
-        def files(self):
-            return [self.lockfile]
-
-        def _do_lock(self):
-            try:
-                self._lock = zc.lockfile.LockFile(self.lockfile)
-            except zc.lockfile.LockError:
-                raise LockError(
-                    "cannot perform the cmd since DVC is busy and "
-                    "locked. Please retry the cmd later."
-                )
-
-        def lock(self):
-            try:
-                self._do_lock()
-                return
-            except LockError:
-                time.sleep(DEFAULT_TIMEOUT)
-
-            self._do_lock()
-
-        def unlock(self):
-            self._lock.close()
-            self._lock = None
-
-        def __enter__(self):
-            self.lock()
-
-        def __exit__(self, typ, value, tbck):
-            self.unlock()
+    cls = HardlinkLock if hardlink_lock else Lock
+    return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly)

dvc/logger.py

@@ -1,15 +1,27 @@
 """Manages logging configuration for dvc repo."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, StringIO, RecursionError
-from dvc.progress import Tqdm
-
-import logging
-import logging.handlers
 import logging.config
+import logging.handlers
+
 import colorama
 
+from dvc.progress import Tqdm
+from dvc.utils.compat import RecursionError
+from dvc.utils.compat import str
+from dvc.utils.compat import StringIO
+
+
+FOOTER = (
+    "\n{yellow}Having any troubles?{nc}"
+    " Hit us up at {blue}https://dvc.org/support{nc},"
+    " we are always happy to help!"
+).format(
+    blue=colorama.Fore.BLUE,
+    nc=colorama.Fore.RESET,
+    yellow=colorama.Fore.YELLOW,
+)
+
 
 class LoggingException(Exception):
     def __init__(self, record):
@@ -37,7 +49,7 @@ class ColorFormatter(logging.Formatter):
     __ https://pypi.python.org/pypi/colorama
 
     For records containing `exc_info`, it will use a custom `_walk_exc` to
-    retrieve the whole tracebak.
+    retrieve the whole traceback.
     """
 
     color_code = {
@@ -48,16 +60,6 @@ class ColorFormatter(logging.Formatter):
         "CRITICAL": colorama.Fore.RED,
     }
 
-    footer = (
-        "{yellow}Having any troubles?{nc}"
-        " Hit us up at {blue}https://dvc.org/support{nc},"
-        " we are always happy to help!"
-    ).format(
-        blue=colorama.Fore.BLUE,
-        nc=colorama.Fore.RESET,
-        yellow=colorama.Fore.YELLOW,
-    )
-
     def format(self, record):
         if record.levelname == "INFO":
             return record.msg
@@ -66,10 +68,7 @@ class ColorFormatter(logging.Formatter):
             exception, stack_trace = self._parse_exc(record.exc_info)
 
             return (
-                "{color}{levelname}{nc}: {description}"
-                "{stack_trace}\n"
-                "\n"
-                "{footer}"
+                "{color}{levelname}{nc}: {description}" "{stack_trace}\n"
             ).format(
                 color=self.color_code.get(record.levelname, ""),
                 nc=colorama.Fore.RESET,
@@ -77,7 +76,6 @@ class ColorFormatter(logging.Formatter):
                 description=self._description(record.msg, exception),
                 msg=record.msg,
                 stack_trace=stack_trace,
-                footer=self.footer,
             )
 
         return "{color}{levelname}{nc}: {msg}".format(
@@ -156,7 +154,7 @@ class LoggerHandler(logging.StreamHandler):
         raise LoggingException(record)
 
     def emit(self, record):
-        """Write to Tqdm's stream so as to not break progressbars"""
+        """Write to Tqdm's stream so as to not break progress-bars"""
         try:
             msg = self.format(record)
             Tqdm.write(

dvc/main.py

@@ -1,19 +1,25 @@
 """Main entry point for dvc CLI."""
-
 from __future__ import unicode_literals
 
 import logging
 
+from dvc import analytics
 from dvc.cli import parse_args
-from dvc.lock import LockError
 from dvc.config import ConfigError
-from dvc.analytics import Analytics
-from dvc.exceptions import NotDvcRepoError, DvcParserError
-from dvc.remote.pool import close_pools
+from dvc.exceptions import DvcParserError
+from dvc.exceptions import NotDvcRepoError
 from dvc.external_repo import clean_repos
+from dvc.logger import FOOTER
+from dvc.remote.pool import close_pools
 from dvc.utils.compat import is_py2
 
 
+# Workaround for CPython bug. See [1] and [2] for more info.
+# [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
+# [2] https://bugs.python.org/issue29288
+"".encode("idna")
+
+
 logger = logging.getLogger("dvc")
 
 
@@ -41,9 +47,6 @@ def main(argv=None):
 
         cmd = args.func(args)
         ret = cmd.run()
-    except LockError:
-        logger.exception("failed to lock before running a command")
-        ret = 250
     except ConfigError:
         logger.exception("configuration error")
         ret = 251
@@ -75,6 +78,10 @@ def main(argv=None):
         # so won't be reused by any other subsequent run anyway.
         clean_repos()
 
-    Analytics().send_cmd(cmd, args, ret)
+    if ret != 0:
+        logger.info(FOOTER)
+
+    if analytics.is_enabled():
+        analytics.collect_and_send_report(args, ret)
 
     return ret

dvc/output/__init__.py

@@ -1,21 +1,20 @@
 from __future__ import unicode_literals
 
-import schema
-
-from dvc.scheme import Schemes
-from dvc.utils.compat import urlparse, str
+from voluptuous import Any, Required
 
 from dvc.output.base import OutputBase
-from dvc.output.s3 import OutputS3
 from dvc.output.gs import OutputGS
-from dvc.output.local import OutputLOCAL
 from dvc.output.hdfs import OutputHDFS
+from dvc.output.local import OutputLOCAL
+from dvc.output.s3 import OutputS3
 from dvc.output.ssh import OutputSSH
-
 from dvc.remote import Remote
-from dvc.remote.s3 import RemoteS3
 from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.local import RemoteLOCAL
+from dvc.remote.s3 import RemoteS3
+from dvc.scheme import Schemes
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
 
 OUTS = [
     OutputHDFS,
@@ -42,19 +41,19 @@ OUTS_MAP = {
 # so when a few types of outputs share the same name, we only need
 # specify it once.
 CHECKSUM_SCHEMA = {
-    schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None),
-    schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None),
-    schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
+    RemoteLOCAL.PARAM_CHECKSUM: Any(str, None),
+    RemoteS3.PARAM_CHECKSUM: Any(str, None),
+    RemoteHDFS.PARAM_CHECKSUM: Any(str, None),
 }
 
-TAGS_SCHEMA = {schema.Optional(str): CHECKSUM_SCHEMA}
+TAGS_SCHEMA = {str: CHECKSUM_SCHEMA}
 
 SCHEMA = CHECKSUM_SCHEMA.copy()
-SCHEMA[OutputBase.PARAM_PATH] = str
-SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)] = bool
-SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)] = OutputBase.METRIC_SCHEMA
-SCHEMA[schema.Optional(OutputBase.PARAM_TAGS)] = TAGS_SCHEMA
-SCHEMA[schema.Optional(OutputBase.PARAM_PERSIST)] = bool
+SCHEMA[Required(OutputBase.PARAM_PATH)] = str
+SCHEMA[OutputBase.PARAM_CACHE] = bool
+SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA
+SCHEMA[OutputBase.PARAM_TAGS] = TAGS_SCHEMA
+SCHEMA[OutputBase.PARAM_PERSIST] = bool
 
 
 def _get(stage, p, info, cache, metric, persist=False, tags=None):

dvc/output/base.py

@@ -3,13 +3,15 @@ from __future__ import unicode_literals
 import logging
 from copy import copy
 
-from schema import Or, Optional
+from voluptuous import Any
 
-from dvc.cache import NamedCache
 import dvc.prompt as prompt
-from dvc.exceptions import DvcException, CollectCacheError
-from dvc.utils.compat import str, urlparse
+from dvc.cache import NamedCache
+from dvc.exceptions import CollectCacheError
+from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteBASE
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -52,12 +54,12 @@ class OutputBase(object):
     PARAM_METRIC_XPATH = "xpath"
     PARAM_PERSIST = "persist"
 
-    METRIC_SCHEMA = Or(
+    METRIC_SCHEMA = Any(
         None,
         bool,
         {
-            Optional(PARAM_METRIC_TYPE): Or(str, None),
-            Optional(PARAM_METRIC_XPATH): Or(str, None),
+            PARAM_METRIC_TYPE: Any(str, None),
+            PARAM_METRIC_XPATH: Any(str, None),
         },
     )
 
@@ -168,12 +170,7 @@ class OutputBase(object):
         return self.remote.exists(self.path_info)
 
     def changed_checksum(self):
-        return (
-            self.checksum
-            != self.remote.save_info(self.path_info)[
-                self.remote.PARAM_CHECKSUM
-            ]
-        )
+        return self.checksum != self.remote.get_checksum(self.path_info)
 
     def changed_cache(self):
         if not self.use_cache or not self.checksum:
@@ -287,7 +284,8 @@ class OutputBase(object):
 
     def checkout(self, force=False, progress_callback=None, tag=None):
         if not self.use_cache:
-            progress_callback(str(self.path_info), self.get_files_number())
+            if progress_callback:
+                progress_callback(str(self.path_info), self.get_files_number())
             return None
 
         if tag:

dvc/output/gs.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.remote.gs import RemoteGS
 from dvc.output.s3 import OutputS3
+from dvc.remote.gs import RemoteGS
 
 
 class OutputGS(OutputS3):

dvc/output/local.py

@@ -1,13 +1,17 @@
 from __future__ import unicode_literals
 
-import os
 import logging
+import os
 
-from dvc.utils.compat import urlparse, fspath_py35, str
-from dvc.istextfile import istextfile
 from dvc.exceptions import DvcException
-from dvc.remote.local import RemoteLOCAL
+from dvc.istextfile import istextfile
 from dvc.output.base import OutputBase
+from dvc.remote.local import RemoteLOCAL
+from dvc.utils import relpath
+from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
+from dvc.utils.fs import path_isin
 
 
 logger = logging.getLogger(__name__)
@@ -26,7 +30,7 @@ class OutputLOCAL(OutputBase):
             # so we should expect both posix and windows style paths.
             # PathInfo accepts both, i.e. / works everywhere, \ only on win.
             #
-            # FIXME: if we have Windows path containig / or posix one with \
+            # FIXME: if we have Windows path containing / or posix one with \
             # then we have #2059 bug and can't really handle that.
             p = self.REMOTE.path_cls(path)
             if not p.is_absolute():
@@ -36,7 +40,14 @@ class OutputLOCAL(OutputBase):
         return self.REMOTE.path_cls(abs_p)
 
     def __str__(self):
-        return str(self.path_info)
+        if not self.is_in_repo:
+            return str(self.def_path)
+
+        cur_dir = os.getcwd()
+        if path_isin(cur_dir, self.repo.root_dir):
+            return relpath(self.path_info, cur_dir)
+
+        return relpath(self.path_info, self.repo.root_dir)
 
     @property
     def fspath(self):

dvc/output/s3.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.remote.s3 import RemoteS3
 from dvc.output.base import OutputBase
+from dvc.remote.s3 import RemoteS3
 
 
 class OutputS3(OutputBase):

dvc/path_info.py

@@ -1,20 +1,24 @@
-from __future__ import unicode_literals, absolute_import
+from __future__ import absolute_import
+from __future__ import unicode_literals
 
-import sys
 import os
 import posixpath
+import sys
 
 from funcy import cached_property
 
-from dvc.utils.compat import str, builtin_str, basestring, is_py2
-from dvc.utils.compat import pathlib, urlparse
+from dvc.utils import relpath
+from dvc.utils.compat import basestring
+from dvc.utils.compat import builtin_str
+from dvc.utils.compat import is_py2
+from dvc.utils.compat import pathlib
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
 
 
 # On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
 # which is lossy, thus we can't use that,
 # see https://github.com/mcmtroffaes/pathlib2/issues/56.
-from dvc.utils import relpath
-
 if is_py2:
     fs_encoding = "utf-8"
 
@@ -39,7 +43,7 @@ class PathInfo(pathlib.PurePath):
     def as_posix(self):
         f = self._flavour
         # Unlike original implementation [1] that uses `str()` we actually need
-        # to use `fspath`, because we've overriden `__str__` method to return
+        # to use `fspath`, because we've overridden `__str__` method to return
         # relative paths, which will break original `as_posix`.
         #
         # [1] https://github.com/python/cpython/blob/v3.7.0/Lib/pathlib.py#L692
@@ -112,52 +116,80 @@ class PosixPathInfo(PathInfo, pathlib.PurePosixPath):
     pass
 
 
+class _URLPathInfo(PosixPathInfo):
+    def __str__(self):
+        return self.__fspath__()
+
+    __unicode__ = __str__
+
+
 class _URLPathParents(object):
-    def __init__(self, pathcls, scheme, netloc, path):
-        self._scheme = scheme
-        self._netloc = netloc
-        self._parents = path.parents
-        self._pathcls = pathcls
+    def __init__(self, src):
+        self.src = src
+        self._parents = self.src._path.parents
 
     def __len__(self):
         return len(self._parents)
 
     def __getitem__(self, idx):
-        return self._pathcls.from_parts(
-            scheme=self._scheme,
-            netloc=self._netloc,
-            path=self._parents[idx].fspath,
-        )
+        return self.src.replace(path=self._parents[idx])
 
     def __repr__(self):
-        return "<{}.parents>".format(self._pathcls.__name__)
+        return "<{}.parents>".format(self.src)
 
 
 class URLInfo(object):
     DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22, "hdfs": 0}
 
     def __init__(self, url):
-        self.parsed = urlparse(url)
-        assert self.parsed.scheme != "remote"
+        p = urlparse(url)
+        assert not p.query and not p.params and not p.fragment
+        assert p.password is None
+
+        self.fill_parts(p.scheme, p.hostname, p.username, p.port, p.path)
 
     @classmethod
     def from_parts(
-        cls, scheme=None, netloc=None, host=None, user=None, port=None, path=""
+        cls, scheme=None, host=None, user=None, port=None, path="", netloc=None
     ):
-        assert scheme and (bool(host) ^ bool(netloc))
+        assert bool(host) ^ bool(netloc)
 
-        if netloc is None:
-            netloc = host
-            if user:
-                netloc = user + "@" + host
-            if port:
-                netloc += ":" + str(port)
-        return cls("{}://{}{}".format(scheme, netloc, path))
+        if netloc is not None:
+            return cls("{}://{}{}".format(scheme, netloc, path))
+
+        obj = cls.__new__(cls)
+        obj.fill_parts(scheme, host, user, port, path)
+        return obj
+
+    def fill_parts(self, scheme, host, user, port, path):
+        assert scheme != "remote"
+        assert isinstance(path, (basestring, _URLPathInfo))
+
+        self.scheme, self.host, self.user = scheme, host, user
+        self.port = int(port) if port else self.DEFAULT_PORTS.get(self.scheme)
+
+        if isinstance(path, _URLPathInfo):
+            self._spath = builtin_str(path)
+            self._path = path
+        else:
+            if path and path[0] != "/":
+                path = "/" + path
+            self._spath = path
+
+    @property
+    def _base_parts(self):
+        return (self.scheme, self.host, self.user, self.port)
+
+    @property
+    def parts(self):
+        return self._base_parts + self._path.parts
+
+    def replace(self, path=None):
+        return self.from_parts(*self._base_parts, path=path)
 
     @cached_property
     def url(self):
-        p = self.parsed
-        return "{}://{}{}".format(p.scheme, self.netloc, p.path)
+        return "{}://{}{}".format(self.scheme, self.netloc, self._spath)
 
     def __str__(self):
         return self.url
@@ -170,92 +202,60 @@ class URLInfo(object):
             other = self.__class__(other)
         return (
             self.__class__ == other.__class__
-            and self.scheme == other.scheme
-            and self.netloc == other.netloc
+            and self._base_parts == other._base_parts
             and self._path == other._path
         )
 
     def __hash__(self):
-        return hash(self.url)
+        return hash(self.parts)
 
     def __div__(self, other):
-        p = self.parsed
-        new_path = posixpath.join(p.path, str(other))
-        if not new_path.startswith("/"):
-            new_path = "/" + new_path
-        new_url = "{}://{}{}".format(p.scheme, p.netloc, new_path)
-        return self.__class__(new_url)
+        return self.replace(path=posixpath.join(self._spath, other))
 
     __truediv__ = __div__
 
-    def __getattr__(self, name):
-        # When deepcopy is called, it creates and object without __init__,
-        # self.parsed is not initialized and it causes infinite recursion.
-        # More on this special casing here:
-        # https://stackoverflow.com/a/47300262/298182
-        if name.startswith("__"):
-            raise AttributeError(name)
-        return getattr(self.parsed, name)
-
-    @cached_property
-    def netloc(self):
-        p = self.parsed
-        netloc = p.hostname
-        if p.username:
-            netloc = p.username + "@" + netloc
-        if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme):
-            netloc += ":" + str(p.port)
-        return netloc
-
     @property
-    def port(self):
-        return self.parsed.port or self.DEFAULT_PORTS.get(self.parsed.scheme)
-
-    @property
-    def host(self):
-        return self.parsed.hostname
-
-    @property
-    def user(self):
-        return self.parsed.username
+    def path(self):
+        return self._spath
 
     @cached_property
     def _path(self):
-        return PosixPathInfo(self.parsed.path)
+        return _URLPathInfo(self._spath)
 
     @property
     def name(self):
         return self._path.name
 
-    @property
-    def parts(self):
-        return (self.scheme, self.netloc) + self._path.parts
+    @cached_property
+    def netloc(self):
+        netloc = self.host
+        if self.user:
+            netloc = self.user + "@" + netloc
+        if self.port and int(self.port) != self.DEFAULT_PORTS.get(self.scheme):
+            netloc += ":" + str(self.port)
+        return netloc
 
     @property
     def bucket(self):
-        return self.parsed.netloc
+        return self.netloc
 
     @property
     def parent(self):
-        return self.from_parts(
-            scheme=self.scheme,
-            netloc=self.parsed.netloc,
-            path=self._path.parent.fspath,
-        )
+        return self.replace(path=self._path.parent)
 
     @property
     def parents(self):
-        return _URLPathParents(
-            type(self), self.scheme, self.parsed.netloc, self._path
-        )
+        return _URLPathParents(self)
 
     def relative_to(self, other):
-        if isinstance(other, str):
-            other = URLInfo(other)
-        if self.scheme != other.scheme or self.netloc != other.netloc:
-            raise ValueError(
-                "'{}' does not start with '{}'".format(self, other)
-            )
+        if isinstance(other, basestring):
+            other = self.__class__(other)
+        if self.__class__ != other.__class__:
+            msg = "'{}' has incompatible class with '{}'".format(self, other)
+            raise ValueError(msg)
+        if self._base_parts != other._base_parts:
+            msg = "'{}' does not start with '{}'".format(self, other)
+            raise ValueError(msg)
         return self._path.relative_to(other._path)
 
     def isin(self, other):
@@ -263,14 +263,12 @@ class URLInfo(object):
             other = self.__class__(other)
         elif self.__class__ != other.__class__:
             return False
-        return (
-            self.scheme == other.scheme
-            and self.netloc == other.netloc
-            and self._path.isin(other._path)
+        return self._base_parts == other._base_parts and self._path.isin(
+            other._path
         )
 
 
 class CloudURLInfo(URLInfo):
     @property
     def path(self):
-        return self.parsed.path.lstrip("/")
+        return self._spath.lstrip("/")

dvc/progress.py

@@ -1,35 +1,17 @@
 """Manages progress bars for dvc repo."""
 from __future__ import print_function
+
 import logging
 import sys
-from tqdm import tqdm
-from concurrent.futures import ThreadPoolExecutor
+from threading import RLock
+
 from funcy import merge
+from tqdm import tqdm
+
 from dvc.utils import env2bool
 
 logger = logging.getLogger(__name__)
-
-
-class TqdmThreadPoolExecutor(ThreadPoolExecutor):
-    """
-    Ensure worker progressbars are cleared away properly.
-    """
-
-    def __enter__(self):
-        """
-        Creates a blank initial dummy progress bar if needed so that workers
-        are forced to create "nested" bars.
-        """
-        blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False)
-        if blank_bar.pos > 0:
-            # already nested - don't need a placeholder bar
-            blank_bar.close()
-        self.bar = blank_bar
-        return super(TqdmThreadPoolExecutor, self).__enter__()
-
-    def __exit__(self, *a, **k):
-        super(TqdmThreadPoolExecutor, self).__exit__(*a, **k)
-        self.bar.close()
+tqdm.set_lock(RLock())
 
 
 class Tqdm(tqdm):
@@ -38,12 +20,18 @@ class Tqdm(tqdm):
     """
 
     BAR_FMT_DEFAULT = (
-        "{percentage:3.0f}%|{bar:10}|"
-        "{desc:{ncols_desc}.{ncols_desc}}{n}/{total}"
+        "{percentage:3.0f}% {desc}|{bar}|"
+        "{n_fmt}/{total_fmt}"
+        " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
+    )
+    # nested bars should have fixed bar widths to align nicely
+    BAR_FMT_DEFAULT_NESTED = (
+        "{percentage:3.0f}%|{bar:10}|{desc:{ncols_desc}.{ncols_desc}}"
+        "{n_fmt}/{total_fmt}"
         " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
     )
     BAR_FMT_NOTOTAL = (
-        "{desc:{ncols_desc}.{ncols_desc}}{n}"
+        "{desc:{ncols_desc}.{ncols_desc}}{n_fmt}"
         " [{elapsed}<??:??, {rate_fmt:>11}{postfix}]"
     )
 
@@ -96,11 +84,16 @@ class Tqdm(tqdm):
             leave=leave,
             desc=desc,
             bar_format="!",
+            lock_args=(False,),
             **kwargs
         )
         if bar_format is None:
             if self.__len__():
-                self.bar_format = self.BAR_FMT_DEFAULT
+                self.bar_format = (
+                    self.BAR_FMT_DEFAULT_NESTED
+                    if self.pos
+                    else self.BAR_FMT_DEFAULT
+                )
             else:
                 self.bar_format = self.BAR_FMT_NOTOTAL
         else:

dvc/prompt.py

@@ -1,10 +1,9 @@
 """Manages user prompts."""
-
-from __future__ import unicode_literals
 from __future__ import print_function
+from __future__ import unicode_literals
 
-import sys
 import logging
+import sys
 from getpass import getpass
 
 from dvc.utils.compat import input

dvc/remote/__init__.py

@@ -1,20 +1,21 @@
 from __future__ import unicode_literals
 
+from .config import RemoteConfig
 from dvc.remote.azure import RemoteAZURE
+from dvc.remote.gdrive import RemoteGDrive
 from dvc.remote.gs import RemoteGS
 from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
-from dvc.remote.ssh import RemoteSSH
 from dvc.remote.http import RemoteHTTP
 from dvc.remote.https import RemoteHTTPS
+from dvc.remote.local import RemoteLOCAL
 from dvc.remote.oss import RemoteOSS
-
-from .config import RemoteConfig
+from dvc.remote.s3 import RemoteS3
+from dvc.remote.ssh import RemoteSSH
 
 
 REMOTES = [
     RemoteAZURE,
+    RemoteGDrive,
     RemoteGS,
     RemoteHDFS,
     RemoteHTTP,

dvc/remote/azure.py

@@ -1,20 +1,20 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+import logging
 import os
 import re
-import logging
 from datetime import datetime, timedelta
+import threading
 
-from funcy import cached_property
+from funcy import cached_property, wrap_prop
 
-from dvc.scheme import Schemes
-
-from dvc.utils.compat import urlparse
-from dvc.progress import Tqdm
 from dvc.config import Config
-from dvc.remote.base import RemoteBASE
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.scheme import Schemes
+from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -64,6 +64,7 @@ class RemoteAZURE(RemoteBASE):
             else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
         )
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def blob_service(self):
         from azure.storage.blob import BlockBlobService

dvc/remote/base.py

@@ -1,31 +1,36 @@
 from __future__ import unicode_literals
 
-from operator import itemgetter
-from multiprocessing import cpu_count
-
+import itertools
 import json
 import logging
 import tempfile
-import itertools
-from functools import partial
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
+from functools import partial
+from multiprocessing import cpu_count
+from operator import itemgetter
 
-from dvc.ignore import DvcIgnore
-from dvc.utils.compat import str, basestring, urlparse, FileNotFoundError
-from dvc.remote.slow_link_detection import slow_link_guard
+from shortuuid import uuid
 
 import dvc.prompt as prompt
 from dvc.config import Config
-from dvc.exceptions import (
-    DvcException,
-    ConfirmRemoveError,
-    DvcIgnoreInCollectedDirError,
-)
+from dvc.exceptions import ConfirmRemoveError
+from dvc.exceptions import DvcException
+from dvc.exceptions import DvcIgnoreInCollectedDirError
+from dvc.ignore import DvcIgnore
+from dvc.path_info import PathInfo
+from dvc.path_info import URLInfo
 from dvc.progress import Tqdm
-from dvc.utils import tmp_fname, move, relpath, makedirs
+from dvc.remote.slow_link_detection import slow_link_guard
 from dvc.state import StateNoop
-from dvc.path_info import PathInfo, URLInfo
+from dvc.utils import makedirs
+from dvc.utils import relpath
+from dvc.utils import tmp_fname
+from dvc.utils.compat import basestring
+from dvc.utils.compat import FileNotFoundError
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
+from dvc.utils.fs import move
 from dvc.utils.http import open_url
 
 logger = logging.getLogger(__name__)
@@ -290,6 +295,8 @@ class RemoteBASE(object):
         return checksum.endswith(cls.CHECKSUM_DIR_SUFFIX)
 
     def get_checksum(self, path_info):
+        assert path_info.scheme == self.scheme
+
         if not self.exists(path_info):
             return None
 
@@ -319,7 +326,6 @@ class RemoteBASE(object):
         return checksum
 
     def save_info(self, path_info):
-        assert path_info.scheme == self.scheme
         return {self.PARAM_CHECKSUM: self.get_checksum(path_info)}
 
     def changed(self, path_info, checksum_info):
@@ -360,7 +366,7 @@ class RemoteBASE(object):
             )
             return True
 
-        actual = self.save_info(path_info)[self.PARAM_CHECKSUM]
+        actual = self.get_checksum(path_info)
         if checksum != actual:
             logger.debug(
                 "checksum '{}'(actual '{}') for '{}' has changed.".format(
@@ -421,10 +427,16 @@ class RemoteBASE(object):
         cache_info = self.checksum_to_path_info(checksum)
         if self.changed_cache(checksum):
             self.move(path_info, cache_info)
+            self.link(cache_info, path_info)
+        elif self.iscopy(path_info) and self._cache_is_copy(path_info):
+            # Default relink procedure involves unneeded copy
+            if self.protected:
+                self.protect(path_info)
+            else:
+                self.unprotect(path_info)
         else:
             self.remove(path_info)
-
-        self.link(cache_info, path_info)
+            self.link(cache_info, path_info)
 
         if save_link:
             self.state.save_link(path_info)
@@ -435,6 +447,28 @@ class RemoteBASE(object):
         self.state.save(path_info, checksum)
         self.state.save(cache_info, checksum)
 
+    def _cache_is_copy(self, path_info):
+        """Checks whether cache uses copies."""
+        if self.cache_type_confirmed:
+            return self.cache_types[0] == "copy"
+
+        if set(self.cache_types) <= {"copy"}:
+            return True
+
+        workspace_file = path_info.with_name("." + uuid())
+        test_cache_file = self.path_info / ".cache_type_test_file"
+        if not self.exists(test_cache_file):
+            with self.open(test_cache_file, "wb") as fobj:
+                fobj.write(bytes(1))
+        try:
+            self.link(test_cache_file, workspace_file)
+        finally:
+            self.remove(workspace_file)
+            self.remove(test_cache_file)
+
+        self.cache_type_confirmed = True
+        return self.cache_types[0] == "copy"
+
     def _save_dir(self, path_info, checksum):
         cache_info = self.checksum_to_path_info(checksum)
         dir_info = self.get_dir_cache(checksum)
@@ -463,6 +497,10 @@ class RemoteBASE(object):
         """
         return False
 
+    def iscopy(self, path_info):
+        """Check if this file is an independent copy."""
+        return False  # We can't be sure by default
+
     def walk_files(self, path_info):
         """Return a generator with `PathInfo`s to all the files"""
         raise NotImplementedError
@@ -479,10 +517,6 @@ class RemoteBASE(object):
             )
 
         checksum = checksum_info[self.PARAM_CHECKSUM]
-        if not self.changed_cache(checksum):
-            self._checkout(path_info, checksum)
-            return
-
         self._save(path_info, checksum)
 
     def _save(self, path_info, checksum):
@@ -543,11 +577,48 @@ class RemoteBASE(object):
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
+        if self.isdir(from_info):
+            return self._download_dir(
+                from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+            )
+        return self._download_file(
+            from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+        )
 
+    def _download_dir(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        from_infos = list(self.walk_files(from_info))
+        to_infos = (
+            to_info / info.relative_to(from_info) for info in from_infos
+        )
+
+        with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
+            download_files = partial(
+                self._download_file,
+                name=name,
+                no_progress_bar=True,
+                file_mode=file_mode,
+                dir_mode=dir_mode,
+            )
+            futures = executor.map(download_files, from_infos, to_infos)
+            with Tqdm(
+                futures,
+                total=len(from_infos),
+                desc="Downloading directory",
+                unit="Files",
+                disable=no_progress_bar,
+            ) as futures:
+                return sum(futures)
+
+    def _download_file(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
+
+        logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
         name = name or to_info.name
 
-        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
         tmp_file = tmp_fname(to_info)
 
         try:
@@ -754,45 +825,21 @@ class RemoteBASE(object):
         self.remove(path_info)
 
     def _checkout_file(
-        self,
-        path_info,
-        checksum,
-        force,
-        progress_callback=None,
-        save_link=True,
+        self, path_info, checksum, force, progress_callback=None
     ):
-        # NOTE: In case if path_info is already cached and path_info's
-        # link type matches cache link type, we would like to avoid
-        # relinking.
-        if self.changed(
-            path_info, {self.PARAM_CHECKSUM: checksum}
-        ) or not self._link_matches(path_info):
+        """The file is changed we need to checkout a new copy"""
+        cache_info = self.checksum_to_path_info(checksum)
+        if self.exists(path_info):
+            msg = "data '{}' exists. Removing before checkout."
+            logger.warning(msg.format(str(path_info)))
             self.safe_remove(path_info, force=force)
 
-            cache_info = self.checksum_to_path_info(checksum)
-            self.link(cache_info, path_info)
-
-            if save_link:
-                self.state.save_link(path_info)
-
-            self.state.save(path_info, checksum)
-        else:
-            # NOTE: performing (un)protection costs us +/- the same as checking
-            # if path_info is protected. Instead of implementing logic,
-            # just (un)protect according to self.protected.
-            if self.protected:
-                self.protect(path_info)
-            else:
-                # NOTE dont allow copy, because we checked before that link
-                # type matches cache, and we don't want data duplication
-                self.unprotect(path_info, allow_copy=False)
-
+        self.link(cache_info, path_info)
+        self.state.save_link(path_info)
+        self.state.save(path_info, checksum)
         if progress_callback:
             progress_callback(str(path_info))
 
-    def _link_matches(self, path_info):
-        return True
-
     def makedirs(self, path_info):
         """Optional: Implement only if the remote needs to create
         directories before copying/linking/moving data
@@ -814,14 +861,17 @@ class RemoteBASE(object):
         for entry in dir_info:
             relative_path = entry[self.PARAM_RELPATH]
             entry_checksum = entry[self.PARAM_CHECKSUM]
+            entry_cache_info = self.checksum_to_path_info(entry_checksum)
             entry_info = path_info / relative_path
-            self._checkout_file(
-                entry_info,
-                entry_checksum,
-                force,
-                progress_callback,
-                save_link=False,
-            )
+
+            entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
+            if self.changed(entry_info, entry_checksum_info):
+                if self.exists(entry_info):
+                    self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+                self.state.save(entry_info, entry_checksum)
+            if progress_callback:
+                progress_callback(str(entry_info))
 
         self._remove_redundant_files(path_info, dir_info, force)
 
@@ -846,6 +896,7 @@ class RemoteBASE(object):
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
         failed = None
+        skip = False
         if not checksum:
             logger.warning(
                 "No checksum info found for '{}'. "
@@ -854,13 +905,18 @@ class RemoteBASE(object):
             self.safe_remove(path_info, force=force)
             failed = path_info
 
+        elif not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            logger.debug(msg.format(str(path_info)))
+            skip = True
+
         elif self.changed_cache(checksum):
             msg = "Cache '{}' not found. File '{}' won't be created."
             logger.warning(msg.format(checksum, str(path_info)))
             self.safe_remove(path_info, force=force)
             failed = path_info
 
-        if failed:
+        if failed or skip:
             if progress_callback:
                 progress_callback(
                     str(path_info), self.get_files_number(checksum)
@@ -894,7 +950,7 @@ class RemoteBASE(object):
         return 1
 
     @staticmethod
-    def unprotect(path_info, allow_copy=True):
+    def unprotect(path_info):
         pass
 
     def _get_unpacked_dir_names(self, checksums):

dvc/remote/config.py

@@ -1,10 +1,11 @@
-import os
 import logging
+import os
 import posixpath
 
+from dvc.config import Config
+from dvc.config import ConfigError
 from dvc.utils import relpath
 from dvc.utils.compat import urlparse
-from dvc.config import Config, ConfigError
 
 
 logger = logging.getLogger(__name__)
@@ -104,7 +105,6 @@ class RemoteConfig(object):
             level=level,
         )
         if default:
-            logger.info("Setting '{}' as a default remote.".format(name))
             self.config.set(
                 Config.SECTION_CORE,
                 Config.SECTION_CORE_REMOTE,
@@ -150,3 +150,8 @@ class RemoteConfig(object):
         self.config.set(
             Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level
         )
+
+    def get_default(self, level=None):
+        return self.config.get(
+            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level
+        )

dvc/remote/gdrive.py

@@ -0,0 +1,306 @@
+from __future__ import unicode_literals
+
+import os
+import posixpath
+import logging
+import threading
+
+from funcy import retry, compose, decorator, wrap_with
+from funcy.py3 import cat
+
+from dvc.progress import Tqdm
+from dvc.scheme import Schemes
+from dvc.path_info import CloudURLInfo
+from dvc.remote.base import RemoteBASE
+from dvc.config import Config
+from dvc.exceptions import DvcException
+from dvc.utils import tmp_fname
+
+logger = logging.getLogger(__name__)
+FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
+
+
+class GDriveRetriableError(DvcException):
+    def __init__(self, msg):
+        super(GDriveRetriableError, self).__init__(msg)
+
+
+@decorator
+def _wrap_pydrive_retriable(call):
+    from apiclient import errors
+    from pydrive.files import ApiRequestError
+
+    try:
+        result = call()
+    except (ApiRequestError, errors.HttpError) as exception:
+        retry_codes = ["403", "500", "502", "503", "504"]
+        if any(
+            "HttpError {}".format(code) in str(exception)
+            for code in retry_codes
+        ):
+            raise GDriveRetriableError(msg="Google API request failed")
+        raise
+    return result
+
+
+gdrive_retry = compose(
+    # 8 tries, start at 0.5s, multiply by golden ratio, cap at 10s
+    retry(
+        8, GDriveRetriableError, timeout=lambda a: min(0.5 * 1.618 ** a, 10)
+    ),
+    _wrap_pydrive_retriable,
+)
+
+
+class RemoteGDrive(RemoteBASE):
+    scheme = Schemes.GDRIVE
+    path_cls = CloudURLInfo
+    REQUIRES = {"pydrive": "pydrive"}
+    GDRIVE_USER_CREDENTIALS_DATA = "GDRIVE_USER_CREDENTIALS_DATA"
+    DEFAULT_USER_CREDENTIALS_FILE = ".dvc/tmp/gdrive-user-credentials.json"
+
+    def __init__(self, repo, config):
+        super(RemoteGDrive, self).__init__(repo, config)
+        self.no_traverse = False
+        self.path_info = self.path_cls(config[Config.SECTION_REMOTE_URL])
+        self.config = config
+        self.init_drive()
+
+    def init_drive(self):
+        self.client_id = self.config.get(Config.SECTION_GDRIVE_CLIENT_ID, None)
+        self.client_secret = self.config.get(
+            Config.SECTION_GDRIVE_CLIENT_SECRET, None
+        )
+        if not self.client_id or not self.client_secret:
+            raise DvcException(
+                "Please specify Google Drive's client id and "
+                "secret in DVC's config. Learn more at "
+                "https://man.dvc.org/remote/add."
+            )
+        self.gdrive_user_credentials_path = (
+            tmp_fname(".dvc/tmp/")
+            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
+            else self.config.get(
+                Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE,
+                self.DEFAULT_USER_CREDENTIALS_FILE,
+            )
+        )
+
+    @gdrive_retry
+    def gdrive_upload_file(
+        self, args, no_progress_bar=True, from_file="", progress_name=""
+    ):
+        item = self.drive.CreateFile(
+            {"title": args["title"], "parents": [{"id": args["parent_id"]}]}
+        )
+
+        with open(from_file, "rb") as fobj:
+            total = os.path.getsize(from_file)
+            with Tqdm.wrapattr(
+                fobj,
+                "read",
+                desc=progress_name,
+                total=total,
+                disable=no_progress_bar,
+            ) as wrapped:
+                # PyDrive doesn't like content property setting for empty files
+                # https://github.com/gsuitedevs/PyDrive/issues/121
+                if total:
+                    item.content = wrapped
+                item.Upload()
+        return item
+
+    @gdrive_retry
+    def gdrive_download_file(
+        self, file_id, to_file, progress_name, no_progress_bar
+    ):
+        gdrive_file = self.drive.CreateFile({"id": file_id})
+        bar_format = (
+            "Donwloading {desc:{ncols_desc}.{ncols_desc}}... "
+            + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
+        )
+        with Tqdm(
+            bar_format=bar_format, desc=progress_name, disable=no_progress_bar
+        ):
+            gdrive_file.GetContentFile(to_file)
+
+    def gdrive_list_item(self, query):
+        file_list = self.drive.ListFile({"q": query, "maxResults": 1000})
+
+        # Isolate and decorate fetching of remote drive items in pages
+        get_list = gdrive_retry(lambda: next(file_list, None))
+
+        # Fetch pages until None is received, lazily flatten the thing
+        return cat(iter(get_list, None))
+
+    def cache_root_dirs(self):
+        cached_dirs = {}
+        cached_ids = {}
+        for dir1 in self.gdrive_list_item(
+            "'{}' in parents and trashed=false".format(self.root_id)
+        ):
+            remote_path = posixpath.join(self.path_info.path, dir1["title"])
+            cached_dirs.setdefault(remote_path, []).append(dir1["id"])
+            cached_ids[dir1["id"]] = dir1["title"]
+        return cached_dirs, cached_ids
+
+    @property
+    def cached_dirs(self):
+        if not hasattr(self, "_cached_dirs"):
+            self.drive
+        return self._cached_dirs
+
+    @property
+    def cached_ids(self):
+        if not hasattr(self, "_cached_ids"):
+            self.drive
+        return self._cached_ids
+
+    @property
+    @wrap_with(threading.RLock())
+    def drive(self):
+        if not hasattr(self, "_gdrive"):
+            from pydrive.auth import GoogleAuth
+            from pydrive.drive import GoogleDrive
+
+            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                with open(
+                    self.gdrive_user_credentials_path, "w"
+                ) as credentials_file:
+                    credentials_file.write(
+                        os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
+                    )
+
+            GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
+            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                "token_uri": "https://oauth2.googleapis.com/token",
+                "revoke_uri": "https://oauth2.googleapis.com/revoke",
+                "redirect_uri": "",
+            }
+            GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
+            GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
+            GoogleAuth.DEFAULT_SETTINGS[
+                "save_credentials_file"
+            ] = self.gdrive_user_credentials_path
+            GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
+            GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
+                "https://www.googleapis.com/auth/drive",
+                "https://www.googleapis.com/auth/drive.appdata",
+            ]
+
+            # Pass non existent settings path to force DEFAULT_SETTINGS loading
+            gauth = GoogleAuth(settings_file="")
+            gauth.CommandLineAuth()
+
+            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                os.remove(self.gdrive_user_credentials_path)
+
+            self._gdrive = GoogleDrive(gauth)
+
+            self.root_id = self.get_remote_id(self.path_info, create=True)
+            self._cached_dirs, self._cached_ids = self.cache_root_dirs()
+
+        return self._gdrive
+
+    @gdrive_retry
+    def create_remote_dir(self, parent_id, title):
+        item = self.drive.CreateFile(
+            {
+                "title": title,
+                "parents": [{"id": parent_id}],
+                "mimeType": FOLDER_MIME_TYPE,
+            }
+        )
+        item.Upload()
+        return item
+
+    @gdrive_retry
+    def get_remote_item(self, name, parents_ids):
+        if not parents_ids:
+            return None
+        query = " or ".join(
+            "'{}' in parents".format(parent_id) for parent_id in parents_ids
+        )
+
+        query += " and trashed=false and title='{}'".format(name)
+
+        # Limit found remote items count to 1 in response
+        item_list = self.drive.ListFile(
+            {"q": query, "maxResults": 1}
+        ).GetList()
+        return next(iter(item_list), None)
+
+    def resolve_remote_item_from_path(self, path_parts, create):
+        parents_ids = ["root"]
+        current_path = ""
+        for path_part in path_parts:
+            current_path = posixpath.join(current_path, path_part)
+            remote_ids = self.get_remote_id_from_cache(current_path)
+            if remote_ids:
+                parents_ids = remote_ids
+                continue
+            item = self.get_remote_item(path_part, parents_ids)
+            if not item and create:
+                item = self.create_remote_dir(parents_ids[0], path_part)
+            elif not item:
+                return None
+            parents_ids = [item["id"]]
+        return item
+
+    def get_remote_id_from_cache(self, remote_path):
+        if hasattr(self, "_cached_dirs"):
+            return self.cached_dirs.get(remote_path, [])
+        return []
+
+    def get_remote_id(self, path_info, create=False):
+        remote_ids = self.get_remote_id_from_cache(path_info.path)
+
+        if remote_ids:
+            return remote_ids[0]
+
+        file1 = self.resolve_remote_item_from_path(
+            path_info.path.split("/"), create
+        )
+        return file1["id"] if file1 else ""
+
+    def exists(self, path_info):
+        return self.get_remote_id(path_info) != ""
+
+    def _upload(self, from_file, to_info, name, no_progress_bar):
+        dirname = to_info.parent
+        if dirname:
+            parent_id = self.get_remote_id(dirname, True)
+        else:
+            parent_id = to_info.bucket
+
+        self.gdrive_upload_file(
+            {"title": to_info.name, "parent_id": parent_id},
+            no_progress_bar,
+            from_file,
+            name,
+        )
+
+    def _download(self, from_info, to_file, name, no_progress_bar):
+        file_id = self.get_remote_id(from_info)
+        self.gdrive_download_file(file_id, to_file, name, no_progress_bar)
+
+    def all(self):
+        if not self.cached_ids:
+            return
+
+        query = " or ".join(
+            "'{}' in parents".format(dir_id) for dir_id in self.cached_ids
+        )
+
+        query += " and trashed=false"
+        for file1 in self.gdrive_list_item(query):
+            parent_id = file1["parents"][0]["id"]
+            path = posixpath.join(self.cached_ids[parent_id], file1["title"])
+            try:
+                yield self.path_to_checksum(path)
+            except ValueError:
+                # We ignore all the non-cache looking files
+                logger.debug('Ignoring path as "non-cache looking"')

dvc/remote/gs.py

@@ -1,17 +1,21 @@
-from __future__ import unicode_literals
+from __future__ import unicode_literals, division
 
 import logging
 from datetime import timedelta
 from functools import wraps
+import io
+import os.path
+import threading
 
-from funcy import cached_property
+from funcy import cached_property, wrap_prop
 
-from dvc.utils.compat import FileNotFoundError  # skipcq: PYL-W0622
-from dvc.remote.base import RemoteBASE
 from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
+from dvc.utils.compat import FileNotFoundError  # skipcq: PYL-W0622
 
 logger = logging.getLogger(__name__)
 
@@ -20,25 +24,24 @@ def dynamic_chunk_size(func):
     @wraps(func)
     def wrapper(*args, **kwargs):
         import requests
-        from google.cloud.storage.blob import Blob, _DEFAULT_CHUNKSIZE
+        from google.cloud.storage.blob import Blob
 
-        # Default chunk size for gs is 100M, which might be too much for
-        # particular network (see [1]). So if we are getting ConnectionError,
-        # we should try lowering the chunk size until we reach the minimum
-        # allowed chunk size of 256K. Also note that `chunk_size` must be a
-        # multiple of 256K per the API specification.
+        # `ConnectionError` may be due to too large `chunk_size`
+        # (see [#2572]) so try halving on error.
+        # Note: start with 40 * [default: 256K] = 10M.
+        # Note: must be multiple of 256K.
         #
-        # [1] https://github.com/iterative/dvc/issues/2572
+        # [#2572]: https://github.com/iterative/dvc/issues/2572
 
         # skipcq: PYL-W0212
-        multiplier = int(_DEFAULT_CHUNKSIZE / Blob._CHUNK_SIZE_MULTIPLE)
+        multiplier = 40
         while True:
             try:
                 # skipcq: PYL-W0212
                 chunk_size = Blob._CHUNK_SIZE_MULTIPLE * multiplier
                 return func(*args, chunk_size=chunk_size, **kwargs)
             except requests.exceptions.ConnectionError:
-                multiplier = int(multiplier / 2)
+                multiplier //= 2
                 if not multiplier:
                     raise
 
@@ -46,9 +49,24 @@ def dynamic_chunk_size(func):
 
 
 @dynamic_chunk_size
-def _upload_to_bucket(bucket, from_file, to_info, **kwargs):
-    blob = bucket.blob(to_info.path, **kwargs)
-    blob.upload_from_filename(from_file)
+def _upload_to_bucket(
+    bucket,
+    from_file,
+    to_info,
+    chunk_size=None,
+    name=None,
+    no_progress_bar=True,
+):
+    blob = bucket.blob(to_info.path, chunk_size=chunk_size)
+    with io.open(from_file, mode="rb") as fobj:
+        with Tqdm.wrapattr(
+            fobj,
+            "read",
+            desc=name or to_info.path,
+            total=os.path.getsize(from_file),
+            disable=no_progress_bar,
+        ) as wrapped:
+            blob.upload_from_file(wrapped)
 
 
 class RemoteGS(RemoteBASE):
@@ -60,13 +78,13 @@ class RemoteGS(RemoteBASE):
     def __init__(self, repo, config):
         super(RemoteGS, self).__init__(repo, config)
 
-        storagepath = "gs://" + config.get(Config.SECTION_GCP_STORAGEPATH, "/")
-        url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        url = config.get(Config.SECTION_REMOTE_URL, "gs:///")
         self.path_info = self.path_cls(url)
 
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
         self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def gs(self):
         from google.cloud.storage import Client
@@ -112,25 +130,68 @@ class RemoteGS(RemoteBASE):
 
         blob.delete()
 
-    def _list_paths(self, bucket, prefix):
-        for blob in self.gs.bucket(bucket).list_blobs(prefix=prefix):
+    def _list_paths(self, path_info, max_items=None):
+        for blob in self.gs.bucket(path_info.bucket).list_blobs(
+            prefix=path_info.path, max_results=max_items
+        ):
             yield blob.name
 
     def list_cache_paths(self):
-        return self._list_paths(self.path_info.bucket, self.path_info.path)
+        return self._list_paths(self.path_info)
+
+    def walk_files(self, path_info):
+        for fname in self._list_paths(path_info / ""):
+            # skip nested empty directories
+            if fname.endswith("/"):
+                continue
+            yield path_info.replace(fname)
+
+    def makedirs(self, path_info):
+        self.gs.bucket(path_info.bucket).blob(
+            (path_info / "").path
+        ).upload_from_string("")
+
+    def isdir(self, path_info):
+        dir_path = path_info / ""
+        return bool(list(self._list_paths(dir_path, max_items=1)))
+
+    def isfile(self, path_info):
+        if path_info.path.endswith("/"):
+            return False
+
+        blob = self.gs.bucket(path_info.bucket).blob(path_info.path)
+        return blob.exists()
 
     def exists(self, path_info):
-        paths = set(self._list_paths(path_info.bucket, path_info.path))
-        return any(path_info.path == path for path in paths)
+        """Check if the blob exists. If it does not exist,
+        it could be a part of a directory path.
 
-    def _upload(self, from_file, to_info, **_kwargs):
+        eg: if `data/file.txt` exists, check for `data` should return True
+        """
+        return self.isfile(path_info) or self.isdir(path_info)
+
+    def _upload(self, from_file, to_info, name=None, no_progress_bar=True):
         bucket = self.gs.bucket(to_info.bucket)
-        _upload_to_bucket(bucket, from_file, to_info)
+        _upload_to_bucket(
+            bucket,
+            from_file,
+            to_info,
+            name=name,
+            no_progress_bar=no_progress_bar,
+        )
 
-    def _download(self, from_info, to_file, **_kwargs):
+    def _download(self, from_info, to_file, name=None, no_progress_bar=True):
         bucket = self.gs.bucket(from_info.bucket)
         blob = bucket.get_blob(from_info.path)
-        blob.download_to_filename(to_file)
+        with io.open(to_file, mode="wb") as fobj:
+            with Tqdm.wrapattr(
+                fobj,
+                "write",
+                desc=name or from_info.path,
+                total=blob.size,
+                disable=no_progress_bar,
+            ) as wrapped:
+                blob.download_to_file(wrapped)
 
     def _generate_download_url(self, path_info, expires=3600):
         expiration = timedelta(seconds=int(expires))

dvc/remote/hdfs.py

@@ -1,22 +1,21 @@
 from __future__ import unicode_literals
 
-import os
-import re
+from dvc.utils.compat import FileNotFoundError, urlparse
 import io
-import posixpath
 import logging
-from subprocess import Popen, PIPE
-from contextlib import contextmanager, closing
+import os
+import posixpath
+import re
+from collections import deque
+from contextlib import closing, contextmanager
+import subprocess
 
+from .base import RemoteBASE, RemoteCmdError
+from .pool import get_connection
 from dvc.config import Config
 from dvc.scheme import Schemes
-
-from dvc.utils.compat import urlparse, FileNotFoundError
 from dvc.utils import fix_env, tmp_fname
 
-from .pool import get_connection
-from .base import RemoteBASE, RemoteCmdError
-
 logger = logging.getLogger(__name__)
 
 
@@ -65,15 +64,15 @@ class RemoteHDFS(RemoteBASE):
         close_fds = os.name != "nt"
 
         executable = os.getenv("SHELL") if os.name != "nt" else None
-        p = Popen(
+        p = subprocess.Popen(
             cmd,
             shell=True,
             close_fds=close_fds,
             executable=executable,
             env=fix_env(os.environ),
-            stdin=PIPE,
-            stdout=PIPE,
-            stderr=PIPE,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
         )
         out, err = p.communicate()
         if p.returncode != 0:
@@ -153,10 +152,9 @@ class RemoteHDFS(RemoteBASE):
 
     def list_cache_paths(self):
         if not self.exists(self.path_info):
-            return []
+            return
 
-        files = []
-        dirs = [self.path_info.path]
+        dirs = deque([self.path_info.path])
 
         with self.hdfs(self.path_info) as hdfs:
             while dirs:
@@ -164,6 +162,4 @@ class RemoteHDFS(RemoteBASE):
                     if entry["kind"] == "directory":
                         dirs.append(urlparse(entry["name"]).path)
                     elif entry["kind"] == "file":
-                        files.append(urlparse(entry["name"]).path)
-
-        return files
+                        yield urlparse(entry["name"]).path

dvc/remote/http.py

@@ -1,15 +1,17 @@
 from __future__ import unicode_literals
 
 import logging
+import threading
 
-from funcy import cached_property
+from funcy import cached_property, wrap_prop
 
+from dvc.config import Config
+from dvc.config import ConfigError
+from dvc.exceptions import DvcException, HTTPError
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
 from dvc.utils.compat import open
-from dvc.progress import Tqdm
-from dvc.exceptions import DvcException
-from dvc.config import Config, ConfigError
-from dvc.remote.base import RemoteBASE
 
 logger = logging.getLogger(__name__)
 
@@ -35,30 +37,26 @@ class RemoteHTTP(RemoteBASE):
             )
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        request = self._request("GET", from_info.url, stream=True)
+        response = self._request("GET", from_info.url, stream=True)
+        if response.status_code != 200:
+            raise HTTPError(response.status_code, response.reason)
         with Tqdm(
-            total=None if no_progress_bar else self._content_length(from_info),
+            total=None if no_progress_bar else self._content_length(response),
             leave=False,
             bytes=True,
             desc=from_info.url if name is None else name,
             disable=no_progress_bar,
         ) as pbar:
             with open(to_file, "wb") as fd:
-                for chunk in request.iter_content(chunk_size=self.CHUNK_SIZE):
+                for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE):
                     fd.write(chunk)
-                    fd.flush()
                     pbar.update(len(chunk))
 
     def exists(self, path_info):
         return bool(self._request("HEAD", path_info.url))
 
-    def _content_length(self, url_or_request):
-        headers = getattr(
-            url_or_request,
-            "headers",
-            self._request("HEAD", url_or_request).headers,
-        )
-        res = headers.get("Content-Length")
+    def _content_length(self, response):
+        res = response.headers.get("Content-Length")
         return int(res) if res else None
 
     def get_file_checksum(self, path_info):
@@ -80,6 +78,7 @@ class RemoteHTTP(RemoteBASE):
 
         return etag
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def _session(self):
         import requests

dvc/remote/https.py

@@ -1,8 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.scheme import Schemes
-
 from .http import RemoteHTTP
+from dvc.scheme import Schemes
 
 
 class RemoteHTTPS(RemoteHTTP):

dvc/remote/local.py

@@ -1,37 +1,38 @@
 from __future__ import unicode_literals
-import os
-import stat
+
 import errno
 import logging
+import os
+import stat
+from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 
 from shortuuid import uuid
 
-from dvc.scheme import Schemes
-from dvc.utils.compat import str, fspath_py35, open
-from dvc.system import System
-from dvc.remote.base import (
-    RemoteBASE,
-    STATUS_MAP,
-    STATUS_NEW,
-    STATUS_DELETED,
-    STATUS_MISSING,
-)
-from dvc.utils import (
-    remove,
-    move,
-    copyfile,
-    tmp_fname,
-    file_md5,
-    walk_files,
-    relpath,
-    makedirs,
-)
 from dvc.config import Config
-from dvc.exceptions import DvcException, DownloadError, UploadError
-from dvc.progress import Tqdm, TqdmThreadPoolExecutor
-
+from dvc.exceptions import DownloadError
+from dvc.exceptions import DvcException
+from dvc.exceptions import UploadError
 from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.remote.base import STATUS_DELETED
+from dvc.remote.base import STATUS_MAP
+from dvc.remote.base import STATUS_MISSING
+from dvc.remote.base import STATUS_NEW
+from dvc.scheme import Schemes
+from dvc.system import System
+from dvc.utils import copyfile
+from dvc.utils import file_md5
+from dvc.utils import makedirs
+from dvc.utils import relpath
+from dvc.utils import tmp_fname
+from dvc.utils import walk_files
+from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import open
+from dvc.utils.compat import str
+from dvc.utils.fs import move
+from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
@@ -59,9 +60,7 @@ class RemoteLOCAL(RemoteBASE):
             # cache files are set to be read-only for everyone
             self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
 
-        # A clunky way to detect cache dir
-        storagepath = config.get(Config.SECTION_LOCAL_STORAGEPATH, None)
-        cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        cache_dir = config.get(Config.SECTION_REMOTE_URL)
 
         if cache_dir is not None and not os.path.isabs(cache_dir):
             cwd = config[Config.PRIVATE_CWD]
@@ -84,17 +83,7 @@ class RemoteLOCAL(RemoteBASE):
 
     def list_cache_paths(self):
         assert self.path_info is not None
-
-        clist = []
-        for entry in os.listdir(fspath_py35(self.path_info)):
-            subdir = self.path_info / entry
-            if not os.path.isdir(fspath_py35(subdir)):
-                continue
-            clist.extend(
-                subdir / cache for cache in os.listdir(fspath_py35(subdir))
-            )
-
-        return clist
+        return walk_files(self.path_info, None)
 
     def get(self, md5):
         if not md5:
@@ -139,6 +128,11 @@ class RemoteLOCAL(RemoteBASE):
     def isdir(path_info):
         return os.path.isdir(fspath_py35(path_info))
 
+    def iscopy(self, path_info):
+        return not (
+            System.is_symlink(path_info) or System.is_hardlink(path_info)
+        )
+
     @staticmethod
     def getsize(path_info):
         return os.path.getsize(fspath_py35(path_info))
@@ -148,7 +142,7 @@ class RemoteLOCAL(RemoteBASE):
             yield PathInfo(fname)
 
     def get_file_checksum(self, path_info):
-        return file_md5(fspath_py35(path_info))[0]
+        return file_md5(path_info)[0]
 
     def remove(self, path_info):
         if path_info.scheme != "local":
@@ -184,7 +178,7 @@ class RemoteLOCAL(RemoteBASE):
         # will get something like: `too many links error`
         #
         # This is because all those empty files will have the same checksum
-        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therfore, they will be
+        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
         # linked to the same file in the cache.
         #
         # From https://en.wikipedia.org/wiki/Hard_link
@@ -223,7 +217,7 @@ class RemoteLOCAL(RemoteBASE):
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        makedirs(fspath_py35(to_info.parent), exist_ok=True)
+        makedirs(to_info.parent, exist_ok=True)
 
         tmp_file = tmp_fname(to_info)
         copyfile(
@@ -235,10 +229,7 @@ class RemoteLOCAL(RemoteBASE):
         self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
     ):
         copyfile(
-            fspath_py35(from_info),
-            to_file,
-            no_progress_bar=no_progress_bar,
-            name=name,
+            from_info, to_file, no_progress_bar=no_progress_bar, name=name
         )
 
     @staticmethod
@@ -263,7 +254,7 @@ class RemoteLOCAL(RemoteBASE):
 
         # This is a performance optimization. We can safely assume that,
         # if the resources that we want to fetch are already cached,
-        # there's no need to check the remote storage for the existance of
+        # there's no need to check the remote storage for the existence of
         # those files.
         if download and sorted(local_exists) == sorted(md5s):
             remote_exists = local_exists
@@ -359,7 +350,7 @@ class RemoteLOCAL(RemoteBASE):
             return 0
 
         if jobs > 1:
-            with TqdmThreadPoolExecutor(max_workers=jobs) as executor:
+            with ThreadPoolExecutor(max_workers=jobs) as executor:
                 fails = sum(executor.map(func, *plans))
         else:
             fails = sum(map(func, *plans))
@@ -408,22 +399,19 @@ class RemoteLOCAL(RemoteBASE):
             logger.warning(msg)
 
     @staticmethod
-    def _unprotect_file(path, allow_copy=True):
+    def _unprotect_file(path):
         if System.is_symlink(path) or System.is_hardlink(path):
-            if allow_copy:
-                logger.debug("Unprotecting '{}'".format(path))
-                tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
+            logger.debug("Unprotecting '{}'".format(path))
+            tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
 
-                # The operations order is important here - if some application
-                # would access the file during the process of copyfile then it
-                # would get only the part of file. So, at first, the file
-                # should be copied with the temporary name, and then
-                # original file should be replaced by new.
-                copyfile(
-                    path, tmp, name="Unprotecting '{}'".format(relpath(path))
-                )
-                remove(path)
-                os.rename(tmp, path)
+            # The operations order is important here - if some application
+            # would access the file during the process of copyfile then it
+            # would get only the part of file. So, at first, the file should be
+            # copied with the temporary name, and then original file should be
+            # replaced by new.
+            copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
+            remove(path)
+            os.rename(tmp, path)
 
         else:
             logger.debug(
@@ -433,11 +421,11 @@ class RemoteLOCAL(RemoteBASE):
 
         os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
 
-    def _unprotect_dir(self, path, allow_copy=True):
+    def _unprotect_dir(self, path):
         for fname in walk_files(path, self.repo.dvcignore):
-            self._unprotect_file(fname, allow_copy)
+            RemoteLOCAL._unprotect_file(fname)
 
-    def unprotect(self, path_info, allow_copy=True):
+    def unprotect(self, path_info):
         path = path_info.fspath
         if not os.path.exists(path):
             raise DvcException(
@@ -445,9 +433,9 @@ class RemoteLOCAL(RemoteBASE):
             )
 
         if os.path.isdir(path):
-            self._unprotect_dir(path, allow_copy)
+            self._unprotect_dir(path)
         else:
-            self._unprotect_file(path, allow_copy)
+            RemoteLOCAL._unprotect_file(path)
 
     @staticmethod
     def protect(path_info):
@@ -521,36 +509,3 @@ class RemoteLOCAL(RemoteBASE):
             if self.is_dir_checksum(c):
                 unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
         return unpacked
-
-    def _get_cache_type(self, path_info):
-        if self.cache_type_confirmed:
-            return self.cache_types[0]
-
-        workspace_file = path_info.with_name("." + uuid())
-        test_cache_file = self.path_info / ".cache_type_test_file"
-        if not self.exists(test_cache_file):
-            with open(fspath_py35(test_cache_file), "wb") as fobj:
-                fobj.write(bytes(1))
-        try:
-            self.link(test_cache_file, workspace_file)
-        finally:
-            self.remove(workspace_file)
-            self.remove(test_cache_file)
-
-        self.cache_type_confirmed = True
-        return self.cache_types[0]
-
-    def _link_matches(self, path_info):
-        is_hardlink = System.is_hardlink(path_info)
-        is_symlink = System.is_symlink(path_info)
-        is_copy_or_reflink = not is_hardlink and not is_symlink
-
-        cache_type = self._get_cache_type(path_info)
-
-        if cache_type == "symlink":
-            return is_symlink
-
-        if cache_type == "hardlink":
-            return is_hardlink
-
-        return is_copy_or_reflink

dvc/remote/oss.py

@@ -1,15 +1,17 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-import os
 import logging
+import os
+import threading
 
-from dvc.scheme import Schemes
+from funcy import cached_property, wrap_prop
 
 from dvc.config import Config
-from dvc.remote.base import RemoteBASE
-from dvc.progress import Tqdm
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.scheme import Schemes
 
 
 logger = logging.getLogger(__name__)
@@ -62,30 +64,29 @@ class RemoteOSS(RemoteBASE):
             or "defaultSecret"
         )
 
-        self._bucket = None
-
-    @property
+    @wrap_prop(threading.Lock())
+    @cached_property
     def oss_service(self):
         import oss2
 
-        if self._bucket is None:
-            logger.debug("URL {}".format(self.path_info))
-            logger.debug("key id {}".format(self.key_id))
-            logger.debug("key secret {}".format(self.key_secret))
-            auth = oss2.Auth(self.key_id, self.key_secret)
-            self._bucket = oss2.Bucket(
-                auth, self.endpoint, self.path_info.bucket
+        logger.debug("URL {}".format(self.path_info))
+        logger.debug("key id {}".format(self.key_id))
+        logger.debug("key secret {}".format(self.key_secret))
+
+        auth = oss2.Auth(self.key_id, self.key_secret)
+        bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)
+
+        # Ensure bucket exists
+        try:
+            bucket.get_bucket_info()
+        except oss2.exceptions.NoSuchBucket:
+            bucket.create_bucket(
+                oss2.BUCKET_ACL_PUBLIC_READ,
+                oss2.models.BucketCreateConfig(
+                    oss2.BUCKET_STORAGE_CLASS_STANDARD
+                ),
             )
-            try:  # verify that bucket exists
-                self._bucket.get_bucket_info()
-            except oss2.exceptions.NoSuchBucket:
-                self._bucket.create_bucket(
-                    oss2.BUCKET_ACL_PUBLIC_READ,
-                    oss2.models.BucketCreateConfig(
-                        oss2.BUCKET_STORAGE_CLASS_STANDARD
-                    ),
-                )
-        return self._bucket
+        return bucket
 
     def remove(self, path_info):
         if path_info.scheme != self.scheme:

dvc/remote/pool.py

@@ -1,6 +1,8 @@
 from collections import deque
 from contextlib import contextmanager
-from funcy import memoize
+import threading
+
+from funcy import memoize, wrap_with
 
 
 @contextmanager
@@ -16,6 +18,7 @@ def get_connection(conn_func, *args, **kwargs):
         pool.release(conn)
 
 
+@wrap_with(threading.Lock())
 @memoize
 def get_pool(conn_func, *args, **kwargs):
     return Pool(conn_func, *args, **kwargs)

dvc/remote/s3.py

@@ -1,17 +1,18 @@
 # -*- coding: utf-8 -*-
-
 from __future__ import unicode_literals
 
-import os
 import logging
-import posixpath
-from funcy import cached_property
+import os
+import threading
+
+from funcy import cached_property, wrap_prop
 
-from dvc.progress import Tqdm
 from dvc.config import Config
-from dvc.remote.base import RemoteBASE
-from dvc.exceptions import DvcException, ETagMismatchError
+from dvc.exceptions import DvcException
+from dvc.exceptions import ETagMismatchError
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
@@ -26,11 +27,7 @@ class RemoteS3(RemoteBASE):
     def __init__(self, repo, config):
         super(RemoteS3, self).__init__(repo, config)
 
-        storagepath = "s3://{}".format(
-            config.get(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
-        )
-
-        url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        url = config.get(Config.SECTION_REMOTE_URL, "s3://")
         self.path_info = self.path_cls(url)
 
         self.region = config.get(Config.SECTION_AWS_REGION)
@@ -60,6 +57,7 @@ class RemoteS3(RemoteBASE):
         if shared_creds:
             os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
 
+    @wrap_prop(threading.Lock())
     @cached_property
     def s3(self):
         import boto3
@@ -145,13 +143,13 @@ class RemoteS3(RemoteBASE):
         # multipart.
         #
         # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
-        # then it was transfered as a single part, which means that the chunk
+        # then it was transferred as a single part, which means that the chunk
         # size used to transfer it was greater or equal to the ContentLength
         # of that object. So to preserve that tag over the next transfer, we
         # could use any value >= ContentLength.
         #
         # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
-        # then it was transfered as a multipart, which means that the chunk
+        # then it was transferred as a multipart, which means that the chunk
         # size used to transfer it was less than ContentLength of that object.
         # Unfortunately, in general, it doesn't mean that the chunk size was
         # the same throughout the transfer, so it means that in order to
@@ -209,10 +207,37 @@ class RemoteS3(RemoteBASE):
     def list_cache_paths(self):
         return self._list_paths(self.path_info)
 
+    def isfile(self, path_info):
+        from botocore.exceptions import ClientError
+
+        if path_info.path.endswith("/"):
+            return False
+
+        try:
+            self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
+        except ClientError as exc:
+            if exc.response["Error"]["Code"] != "404":
+                raise
+            return False
+
+        return True
+
     def exists(self, path_info):
+        """Check if the blob exists. If it does not exist,
+        it could be a part of a directory path.
+
+        eg: if `data/file.txt` exists, check for `data` should return True
+        """
+        return self.isfile(path_info) or self.isdir(path_info)
+
+    def makedirs(self, path_info):
+        # We need to support creating empty directories, which means
+        # creating an object with an empty body and a trailing slash `/`.
+        #
+        # We are not creating directory objects for every parent prefix,
+        # as it is not required.
         dir_path = path_info / ""
-        fname = next(self._list_paths(path_info, max_items=1), "")
-        return path_info.path == fname or fname.startswith(dir_path.path)
+        self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
 
     def isdir(self, path_info):
         # S3 doesn't have a concept for directories.
@@ -272,5 +297,8 @@ class RemoteS3(RemoteBASE):
         )
 
     def walk_files(self, path_info, max_items=None):
-        for fname in self._list_paths(path_info, max_items):
-            yield path_info / posixpath.relpath(fname, path_info.path)
+        for fname in self._list_paths(path_info / "", max_items):
+            if fname.endswith("/"):
+                continue
+
+            yield path_info.replace(path=fname)

dvc/remote/slow_link_detection.py

@@ -1,9 +1,10 @@
 import logging
-import time
 import sys
+import time
 from functools import wraps
 
 import colorama
+
 from dvc.config import Config
 
 

dvc/remote/ssh/__init__.py

@@ -1,30 +1,39 @@
 from __future__ import unicode_literals
 
 import errno
-import itertools
-import io
-import os
-import posixpath
 import getpass
+import io
+import itertools
 import logging
+import os
 import threading
 from concurrent.futures import ThreadPoolExecutor
-from contextlib import contextmanager, closing
+from contextlib import closing, contextmanager
+
+from funcy import memoize, wrap_with
 
 import dvc.prompt as prompt
 from dvc.config import Config
-from dvc.utils import to_chunks
-from dvc.utils.compat import urlparse, StringIO
-from dvc.remote.base import RemoteBASE
-from dvc.scheme import Schemes
-from dvc.remote.pool import get_connection
 from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.remote.pool import get_connection
+from dvc.scheme import Schemes
+from dvc.utils import to_chunks
+from dvc.utils.compat import StringIO
+from dvc.utils.compat import urlparse
 
 logger = logging.getLogger(__name__)
 
 
-saved_passwords = {}
-saved_passwords_lock = threading.Lock()
+@wrap_with(threading.Lock())
+@memoize
+def ask_password(host, user, port):
+    return prompt.password(
+        "Enter a private key passphrase or a password for "
+        "host '{host}' port '{port}' user '{user}'".format(
+            host=host, port=port, user=user
+        )
+    )
 
 
 class RemoteSSH(RemoteBASE):
@@ -119,21 +128,11 @@ class RemoteSSH(RemoteBASE):
     def ensure_credentials(self, path_info=None):
         if path_info is None:
             path_info = self.path_info
-        host, user, port = path_info.host, path_info.user, path_info.port
+
         # NOTE: we use the same password regardless of the server :(
         if self.ask_password and self.password is None:
-            with saved_passwords_lock:
-                server_key = (host, user, port)
-                password = saved_passwords.get(server_key)
-
-                if password is None:
-                    saved_passwords[server_key] = password = prompt.password(
-                        "Enter a private key passphrase or a password for "
-                        "host '{host}' port '{port}' user '{user}'".format(
-                            host=host, port=port, user=user
-                        )
-                    )
-                self.password = password
+            host, user, port = path_info.host, path_info.user, path_info.port
+            self.password = ask_password(host, user, port)
 
     def ssh(self, path_info):
         self.ensure_credentials(path_info)
@@ -263,12 +262,14 @@ class RemoteSSH(RemoteBASE):
 
     def list_cache_paths(self):
         with self.ssh(self.path_info) as ssh:
-            return list(ssh.walk_files(self.path_info.path))
+            # If we simply return an iterator then with above closes instantly
+            for path in ssh.walk_files(self.path_info.path):
+                yield path
 
     def walk_files(self, path_info):
         with self.ssh(path_info) as ssh:
             for fname in ssh.walk_files(path_info.path):
-                yield path_info / posixpath.relpath(fname, path_info.path)
+                yield path_info.replace(path=fname)
 
     def makedirs(self, path_info):
         with self.ssh(path_info) as ssh:

dvc/remote/ssh/connection.py

@@ -1,8 +1,9 @@
+import errno
+import logging
 import os
 import posixpath
-import logging
-import errno
 import stat
+
 from funcy import cached_property
 
 try:
@@ -185,8 +186,26 @@ class SSHConnection:
             self.sftp.get(src, dest, callback=pbar.update_to)
 
     def move(self, src, dst):
+        """Rename src to dst, if it is not possible (in case src and dst are
+        on different filesystems) and actual physical copying of data is
+        happening.
+        """
         self.makedirs(posixpath.dirname(dst))
-        self.sftp.rename(src, dst)
+
+        try:
+            self.sftp.rename(src, dst)
+        except OSError:
+            self._atomic_copy(src, dst)
+            self.remove(src)
+
+    def _atomic_copy(self, src, dst):
+        tmp = tmp_fname(dst)
+
+        try:
+            self.copy(src, tmp)
+            self.sftp.rename(tmp, dst)
+        finally:
+            self.remove(tmp)
 
     def upload(self, src, dest, no_progress_bar=False, progress_title=None):
         self.makedirs(posixpath.dirname(dest))

dvc/repo/__init__.py

@@ -1,26 +1,29 @@
 from __future__ import unicode_literals
 
-import os
 import logging
+import os
 from contextlib import contextmanager
+from functools import wraps
 from itertools import chain
 
-from functools import wraps
 from funcy import cached_property
 
+from .graph import check_acyclic
+from .graph import get_pipeline
+from .graph import get_pipelines
+from .graph import get_stages
 from dvc.config import Config
-from dvc.exceptions import (
-    NotDvcRepoError,
-    OutputNotFoundError,
-    FileMissingError,
-)
+from dvc.exceptions import FileMissingError
+from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import OutputNotFoundError
 from dvc.ignore import DvcIgnoreFilter
 from dvc.path_info import PathInfo
 from dvc.remote.base import RemoteActionNotImplemented
-from dvc.utils.compat import open as _open, fspath_py35, FileNotFoundError
 from dvc.utils import relpath
-
-from .graph import check_acyclic, get_pipeline, get_pipelines, get_stages
+from dvc.utils.fs import path_isin
+from dvc.utils.compat import FileNotFoundError
+from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import open as _open
 
 
 logger = logging.getLogger(__name__)
@@ -66,7 +69,7 @@ class Repo(object):
 
     def __init__(self, root_dir=None):
         from dvc.state import State
-        from dvc.lock import Lock
+        from dvc.lock import make_lock
         from dvc.scm import SCM
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
@@ -85,9 +88,12 @@ class Repo(object):
 
         self.tree = WorkingTree(self.root_dir)
 
-        self.lock = Lock(
+        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        self.lock = make_lock(
             os.path.join(self.dvc_dir, "lock"),
             tmp_dir=os.path.join(self.dvc_dir, "tmp"),
+            hardlink_lock=hardlink_lock,
+            friendly=True,
         )
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
@@ -164,7 +170,7 @@ class Repo(object):
             + updater.lock.files
         )
 
-        if self.cache.local.cache_dir.startswith(self.root_dir + os.sep):
+        if path_isin(self.cache.local.cache_dir, self.root_dir):
             flist += [self.cache.local.cache_dir]
 
         self.scm.ignore_list(flist)
@@ -191,7 +197,7 @@ class Repo(object):
             ret = []
             for node in nodes:
                 stage = attrs[node]
-                if stage.path.startswith(target + os.sep):
+                if path_isin(stage.path, target):
                     ret.append(stage)
             return ret
 
@@ -433,7 +439,7 @@ class Repo(object):
 
         matched = list(filter(func, outs))
         if not matched:
-            raise OutputNotFoundError(path)
+            raise OutputNotFoundError(path, self)
 
         return matched
 
@@ -494,7 +500,7 @@ class Repo(object):
 
     @cached_property
     def dvcignore(self):
-        return DvcIgnoreFilter(self.root_dir)
+        return DvcIgnoreFilter(self.root_dir, self.tree)
 
     def close(self):
         self.scm.close()

dvc/repo/add.py

@@ -1,16 +1,18 @@
 from __future__ import unicode_literals
 
-import os
 import logging
+import os
+
 import colorama
 from six import string_types
 
-from dvc.repo.scm_context import scm_context
-from dvc.stage import Stage
-from dvc.utils import walk_files, LARGE_DIR_SIZE
+from . import locked
 from dvc.exceptions import RecursiveAddingWhileUsingFilename
 from dvc.progress import Tqdm
-from . import locked
+from dvc.repo.scm_context import scm_context
+from dvc.stage import Stage
+from dvc.utils import LARGE_DIR_SIZE
+from dvc.utils import walk_files
 
 logger = logging.getLogger(__name__)
 

dvc/repo/brancher.py

@@ -1,4 +1,5 @@
 from funcy import group_by
+
 from dvc.scm.tree import WorkingTree
 
 
@@ -22,7 +23,7 @@ def brancher(  # noqa: E302
         str: the display name for the currently selected tree, it could be:
             - a git revision identifier
             - empty string it there is no branches to iterate over
-            - "Working Tree" if there are uncommited changes in the SCM repo
+            - "Working Tree" if there are uncommitted changes in the SCM repo
     """
     if not any([branches, all_branches, tags, all_tags, all_commits]):
         yield ""

dvc/repo/checkout.py

@@ -2,21 +2,22 @@ from __future__ import unicode_literals
 
 import logging
 
-from dvc.exceptions import CheckoutErrorSuggestGit, CheckoutError
+from dvc.exceptions import CheckoutError
+from dvc.exceptions import CheckoutErrorSuggestGit
 from dvc.progress import Tqdm
 
 
 logger = logging.getLogger(__name__)
 
 
-def _cleanup_unused_links(self, all_stages):
+def _cleanup_unused_links(repo):
     used = [
         out.fspath
-        for stage in all_stages
+        for stage in repo.stages
         for out in stage.outs
         if out.scheme == "local"
     ]
-    self.state.remove_unused_links(used)
+    repo.state.remove_unused_links(used)
 
 
 def get_all_files_numbers(stages):
@@ -29,7 +30,11 @@ def _checkout(
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
     stages = set()
-    targets = targets or [None]
+
+    if not targets:
+        targets = [None]
+        _cleanup_unused_links(self)
+
     for target in targets:
         try:
             new = self.collect(
@@ -41,7 +46,6 @@ def _checkout(
                 raise
             raise CheckoutErrorSuggestGit(target, exc)
 
-    _cleanup_unused_links(self, self.stages)
     total = get_all_files_numbers(stages)
     if total == 0:
         logger.info("Nothing to do")
@@ -50,12 +54,6 @@ def _checkout(
         total=total, unit="file", desc="Checkout", disable=total == 0
     ) as pbar:
         for stage in stages:
-            if stage.locked:
-                logger.warning(
-                    "DVC-file '{path}' is locked. Its dependencies are"
-                    " not going to be checked out.".format(path=stage.relpath)
-                )
-
             failed.extend(
                 stage.checkout(force=force, progress_callback=pbar.update_desc)
             )

dvc/repo/destroy.py

@@ -1,4 +1,4 @@
-from dvc.utils import remove
+from dvc.utils.fs import remove
 
 
 def destroy(self):

dvc/repo/diff.py

@@ -3,18 +3,14 @@ from __future__ import unicode_literals
 import os
 from errno import ENOENT
 
-
 import dvc.logger as logger
-from dvc.scm.base import FileNotInCommitError
-from dvc.scm.git import (
-    DIFF_A_TREE,
-    DIFF_B_TREE,
-    DIFF_A_REF,
-    DIFF_B_REF,
-    DIFF_EQUAL,
-)
-
 from . import locked
+from dvc.scm.base import FileNotInCommitError
+from dvc.scm.git import DIFF_A_REF
+from dvc.scm.git import DIFF_A_TREE
+from dvc.scm.git import DIFF_B_REF
+from dvc.scm.git import DIFF_B_TREE
+from dvc.scm.git import DIFF_EQUAL
 
 
 DIFF_TARGET = "target"
@@ -224,7 +220,7 @@ def _diff_royal(self, target, diff_dct):
 
 @locked
 def diff(self, a_ref, target=None, b_ref=None):
-    """Gerenates diff message string output
+    """Generates diff message string output
 
     Args:
         target(str) - file/directory to check diff of

dvc/repo/fetch.py

@@ -2,11 +2,12 @@ from __future__ import unicode_literals
 
 import logging
 
-from dvc.config import NoRemoteError
-from dvc.exceptions import DownloadError, OutputNotFoundError
-from dvc.scm.base import CloneError
-from dvc.external_repo import external_repo
 from dvc.cache import NamedCache
+from dvc.config import NoRemoteError
+from dvc.exceptions import DownloadError
+from dvc.exceptions import OutputNotFoundError
+from dvc.external_repo import external_repo
+from dvc.scm.base import CloneError
 
 
 logger = logging.getLogger(__name__)
@@ -26,7 +27,7 @@ def _fetch(
     """Download data items from a cloud and imported repositories
 
     Returns:
-        int: number of succesfully downloaded files
+        int: number of successfully downloaded files
 
     Raises:
         DownloadError: thrown when there are failed downloads, either