dvc: bump to 0.66.2

* pull: fix-2684 - failing while pulling imported directories

* pull: fix-2684 - fix code indentation

* minor imported directories pull test update

Co-Authored-By: Ruslan Kuprieiev <[email protected]>

.deepsource.toml

@@ -5,7 +5,7 @@ test_patterns = [
 ]
 
 exclude_patterns = [
-  
+  'dvc/utils/compat.py'
 ]
 
 [[analyzers]]

.dvc/.gitignore

@@ -8,3 +8,4 @@
 /cache
 /pkg
 /repos
+/tmp

.dvc/config

@@ -1,4 +1,4 @@
 ['remote "dvc"']
-url = https://dvc.org/s3/dvc
+url = https://remote.dvc.org/dvc
 [core]
 remote = dvc

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,5 @@
 * [ ] Have you followed the guidelines in our
-      [Contributing document](https://dvc.org/doc/user-guide/contributing)?
+      [Contributing document](https://dvc.org/doc/user-guide/contributing/core)?
 
 * [ ] Does your PR affect documented changes or does it add new functionality
       that should be documented? If yes, have you created a PR for

.gitignore

@@ -23,6 +23,7 @@ innosetup/config.ini
 *.exe
 
 .coverage
+.coverage.*
 
 *.swp
 

.mailmap

@@ -0,0 +1,6 @@
+Paweł Redzyński <[email protected]>
+Dmitry Petrov <[email protected]>
+Earl Hathaway <[email protected]>
+Nabanita Dash <[email protected]>
+Kurian Benoy <[email protected]>
+Sritanu Chakraborty <[email protected]>

.pylintrc

@@ -139,7 +139,9 @@ disable=print-statement,
         deprecated-sys-function,
         exception-escape,
         comprehension-escape,
-        useless-object-inheritance
+        useless-object-inheritance,
+        missing-docstring,
+        bad-continuation
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -314,7 +316,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=100
+max-line-length=79
 
 # Maximum number of lines in a module.
 max-module-lines=1000

.travis.yml

@@ -1,20 +1,17 @@
-language: python
 cache:
   directories:
     - $HOME/.cache/pip
-    - /opt/python
     - $HOME/.pyenv
-    - c/ProgramData/chocolatey/bin
-    - c/ProgramData/chocolatey/lib
 sudo: true
 dist: xenial
-python: "3.7"
 env:
   global:
+    - HOMEBREW_NO_AUTO_UPDATE=1
     - PYENV_ROOT="$HOME/.pyenv"
     - PATH="$PYENV_ROOT/bin:$PATH"
     - PIP_CACHE_DIR="$HOME/.cache/pip"  # unify pip cache location for all platforms
 stages:
+  - check
   - test
   - build
   - name: deploy
@@ -22,93 +19,91 @@ stages:
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
-# Note that we specify language as ruby-2.4.0 to help us satisfy
+# Note that we specify language as ruby-2.4.3 to help us satisfy
 # fpm dependencies, as travis runs el capitan with an old ruby
 # version, that causes fpm to crash.
 jobs:
   include:
+  # patch formatting checks
+  - name: "Check patch formatting"
+    stage: "check"
+    os: linux
+    language: python
+    python: 3.7
+    before_install:
+    install:
+    script: ./scripts/ci/check_patch.sh
+  # test jobs
   - name: "2.7 on Windows"
     stage: test
     os: windows           # Windows 10.0.17134 N/A Build 17134
     language: shell       # 'language: python' is an error on Travis CI Windows
     env:
-    - PYTHON_VER=2.7
+    - PY2=1
   - name: "3.7 on Windows"
     stage: test
     os: windows           # Windows 10.0.17134 N/A Build 17134
     language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-    - PYTHON_VER=3.7
-  - name: "2.7"
+  - name: "2.7 on Linux"
     stage: test
+    language: python
+    python: 2.7
     env:
-    - PYTHON_VER=2.7.13
-  - name: "3.5"
+    - PY2=1
+  - name: "3.5 on Linux"
     stage: test
-    env:
-    - PYTHON_VER=3.5.3
-  - name: "3.6"
+    language: python
+    python: 3.5
+  - name: "3.6 on Linux"
     stage: test
-    env:
-    - PYTHON_VER=3.6.2
-  - name: "3.7"
+    language: python
+    python: 3.6
+  - name: "3.7 on Linux"
     stage: test
-    env:
-    # NOTE: using default python
-    - PYTHON_VER=
+    language: python
+    python: 3.7
   - name: "2.7 on Mac"
     stage: test
     os: osx
-    osx_image: xcode7.3
-    language: ruby
-    rvm: 2.4.0
+    osx_image: xcode9.4
     env:
-    - PYTHON_VER=2.7.10
-    - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
-    - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
-    - PYTHON_CONFIGURE_OPTS="--enable-framework"
-  - name: "3.7 on Mac"
+    - PY2=1
+  - name: "3.6 on Mac"
     stage: test
     os: osx
-    osx_image: xcode7.3
-    language: ruby
-    rvm: 2.4.0
-    env:
-    - PYTHON_VER=3.7.3
-    - PYTHON_CONFIGURE_OPTS="--enable-framework"
+    osx_image: xcode9.4
   # build jobs
   - name: "Mac pkg"
     stage: build
     os: osx
-    osx_image: xcode7.3
+    osx_image: xcode9.4
     language: ruby
-    rvm: 2.4.0
-    env:
-    - PYTHON_VER=3.7.3
-    - PYTHON_CONFIGURE_OPTS="--enable-framework"
+    rvm: 2.4.3
+    install:
     script: ./scripts/build_posix.sh
   - name: "Windows pkg"
     stage: build
     os: windows
     language: shell
-    env:
-    - PYTHON_VER=3.7
+    install:
     script: ./scripts/build_windows.cmd
   - name: "Linux pkgs"
     stage: build
-    env:
-    # NOTE: using default python
-    - PYTHON_VER=
+    language: python
+    python: 3.7
+    before_install:
+    install:
     script: ./scripts/build_posix.sh
   - name: "Pypi pkgs"
     stage: build
-    env:
-    # NOTE: using default python
-    - PYTHON_VER=
+    language: python
+    python: 3.7
     script: ./scripts/build_package.sh
   # deploy jobs
   - name: "Pypi pkgs"
     stage: deploy
+    language: python
+    python: 3.7
     script: ./scripts/build_package.sh
     deploy:
     - provider: pypi

CONTRIBUTING.md

@@ -1 +1 @@
-### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing#contributing).
+### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing/core).

MANIFEST.in

@@ -0,0 +1,2 @@
+include fastentrypoints.py
+include LICENSE

README.rst

@@ -26,6 +26,10 @@
   :target: https://www.patreon.com/DVCorg/overview
   :alt: Donate
 
+.. image:: https://anaconda.org/conda-forge/dvc/badges/version.svg
+  :target: https://anaconda.org/conda-forge/dvc
+  :alt: Conda-forge
+
 |
 
 **Data Version Control** or **DVC** is an **open-source** tool for data science and machine
@@ -42,7 +46,7 @@ learning projects. Key features:
 
 #. it helps manage experiments with Git tags or branches and **metrics** tracking;
 
-It aims to replace tools like Excel and Docs that are being commonly used as a knowledge repo and
+**DVC** aims to replace tools like Excel and Google Docs that are being commonly used as a knowledge repo and
 a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
 data file suffixes and prefixes.
 
@@ -115,7 +119,7 @@ pip (PyPI)
    pip install dvc
 
 Depending on the remote storage type you plan to use to keep and share your data, you might need to specify
-one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all_remotes`` to include them all.
+one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all`` to include them all.
 The command should look like this: ``pip install dvc[s3]`` - it installs the ``boto3`` library along with
 DVC to support the AWS S3 storage.
 
@@ -130,14 +134,9 @@ Homebrew
 
 .. code-block:: bash
 
-   brew install iterative/homebrew-dvc/dvc
+   brew install dvc
 
-or:
 
-.. code-block:: bash
-
-   brew cask install iterative/homebrew-dvc/dvc
-   
 Conda (Anaconda)
 ----------------
 
@@ -198,7 +197,7 @@ Related technologies
 
 Contributing
 ============
-Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/>`_ for more
+Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0

dvc/__init__.py

@@ -6,10 +6,12 @@ Make your data science projects reproducible and shareable.
 
 from __future__ import unicode_literals
 
-from dvc.version import __version__  # noqa: F401
 import warnings
+
 import dvc.logger
 
+from dvc.version import __version__  # noqa: F401
+
 
 dvc.logger.setup()
 

dvc/analytics.py

@@ -10,6 +10,7 @@ import errno
 import logging
 
 from dvc import __version__
+from dvc.utils import env2bool
 
 
 logger = logging.getLogger(__name__)
@@ -66,7 +67,7 @@ class Analytics(object):
                 raise
 
         self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
-        self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + ".lock")
+        self.user_id_file_lock = Lock(self.user_id_file + ".lock")
 
     @staticmethod
     def load(path):
@@ -113,7 +114,7 @@ class Analytics(object):
                 return user_id
         except LockError:
             msg = "Failed to acquire '{}'"
-            logger.debug(msg.format(self.user_id_file_lock.lock_file))
+            logger.debug(msg.format(self.user_id_file_lock.lockfile))
 
     def _collect_windows(self):
         import sys
@@ -229,7 +230,7 @@ class Analytics(object):
     def is_enabled(cmd=None):
         from dvc.command.daemon import CmdDaemonBase
 
-        if os.getenv("DVC_TEST"):
+        if env2bool("DVC_TEST"):
             return False
 
         if isinstance(cmd, CmdDaemonBase):

dvc/cache.py

@@ -3,7 +3,10 @@
 from __future__ import unicode_literals
 
 import os
+from collections import defaultdict
+from funcy import cached_property
 
+from dvc.utils.compat import builtin_str
 from dvc.config import Config
 
 
@@ -21,6 +24,42 @@ class CacheConfig(object):
         )
 
 
+def _make_remote_property(name):
+    """
+    The config file is stored in a way that allows you to have a
+    cache for each remote.
+
+    This is needed when specifying external outputs
+    (as they require you to have an external cache location).
+
+    Imagine a config file like the following:
+
+            ['remote "dvc-storage"']
+            url = ssh://localhost/tmp
+            ask_password = true
+
+            [cache]
+            ssh = dvc-storage
+
+    This method creates a cached property, containing cache named `name`:
+
+        self.config == {'ssh': 'dvc-storage'}
+        self.ssh  # a RemoteSSH instance
+    """
+
+    def getter(self):
+        from dvc.remote import Remote
+
+        remote = self.config.get(name)
+        if not remote:
+            return None
+
+        return Remote(self.repo, name=remote)
+
+    getter.__name__ = builtin_str(name)
+    return cached_property(getter)
+
+
 class Cache(object):
     """Class that manages cache locations of a dvc repo.
 
@@ -35,7 +74,7 @@ class Cache(object):
 
         self.repo = repo
 
-        config = repo.config.config[Config.SECTION_CACHE]
+        self.config = config = repo.config.config[Config.SECTION_CACHE]
         local = config.get(Config.SECTION_CACHE_LOCAL)
 
         if local:
@@ -46,6 +85,7 @@ class Cache(object):
             cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
             cache_type = config.get(Config.SECTION_CACHE_TYPE)
             protected = config.get(Config.SECTION_CACHE_PROTECTED)
+            shared = config.get(Config.SECTION_CACHE_SHARED)
 
             settings = {
                 Config.PRIVATE_CWD: config.get(
@@ -54,51 +94,46 @@ class Cache(object):
                 Config.SECTION_REMOTE_URL: cache_dir,
                 Config.SECTION_CACHE_TYPE: cache_type,
                 Config.SECTION_CACHE_PROTECTED: protected,
+                Config.SECTION_CACHE_SHARED: shared,
             }
 
         self.local = Remote(repo, **settings)
-        self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
-        self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
-        self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
-        self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
-        self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
 
-    def _get_remote(self, config, name):
-        """
-        The config file is stored in a way that allows you to have a
-        cache for each remote.
+    s3 = _make_remote_property(Config.SECTION_CACHE_S3)
+    gs = _make_remote_property(Config.SECTION_CACHE_GS)
+    ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
+    hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
+    azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
 
-        This is needed when specifying external outputs
-        (as they require you to have an external cache location).
 
-        Imagine a config file like the following:
+class NamedCache(object):
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(set))
+        self.external = defaultdict(set)
 
-                ['remote "dvc-storage"']
-                url = ssh://localhost/tmp
-                ask_password = true
+    @classmethod
+    def make(cls, scheme, checksum, name):
+        cache = cls()
+        cache.add(scheme, checksum, name)
+        return cache
 
-                [cache]
-                ssh = dvc-storage
+    def __getitem__(self, key):
+        return self._items[key]
 
-        This method resolves the name under the cache section into the
-        correct Remote instance.
+    def add(self, scheme, checksum, name):
+        self._items[scheme][checksum].add(name)
 
-        Args:
-            config (dict): The cache section on the config file
-            name (str): Name of the section we are interested in to retrieve
+    def add_external(self, url, rev, path):
+        self.external[url, rev].add(path)
 
-        Returns:
-            remote (dvc.Remote): Remote instance that the section is referring.
-                None when there's no remote with that name.
+    def update(self, cache, suffix=""):
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme]
+            for checksum, names in src.items():
+                if suffix:
+                    dst[checksum].update(n + suffix for n in names)
+                else:
+                    dst[checksum].update(names)
 
-        Example:
-            >>> _get_remote(config={'ssh': 'dvc-storage'}, name='ssh')
-        """
-        from dvc.remote import Remote
-
-        remote = config.get(name)
-
-        if not remote:
-            return None
-
-        return Remote(self.repo, name=remote)
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)

dvc/cli.py

@@ -144,6 +144,19 @@ def parse_args(argv=None):
         description=desc,
         parents=[parent_parser],
         formatter_class=argparse.RawTextHelpFormatter,
+        add_help=False,
+    )
+
+    # NOTE: We are doing this to capitalize help message.
+    # Unfortunately, there is no easier and clearer way to do it,
+    # as adding this argument in get_parent_parser() either in
+    # log_level_group or on parent_parser itself will cause unexpected error.
+    parser.add_argument(
+        "-h",
+        "--help",
+        action="help",
+        default=argparse.SUPPRESS,
+        help="Show this help message and exit.",
     )
 
     # NOTE: On some python versions action='version' prints to stderr

dvc/command/add.py

@@ -5,6 +5,7 @@ import logging
 
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import RecursiveAddingWhileUsingFilename
 
 
 logger = logging.getLogger(__name__)
@@ -14,49 +15,50 @@ class CmdAdd(CmdBase):
     def run(self):
         try:
             if len(self.args.targets) > 1 and self.args.file:
-                raise DvcException("can't use '--file' with multiple targets")
+                raise RecursiveAddingWhileUsingFilename()
 
-            for target in self.args.targets:
-                self.repo.add(
-                    target,
-                    recursive=self.args.recursive,
-                    no_commit=self.args.no_commit,
-                    fname=self.args.file,
-                )
+            self.repo.add(
+                self.args.targets,
+                recursive=self.args.recursive,
+                no_commit=self.args.no_commit,
+                fname=self.args.file,
+            )
 
         except DvcException:
-            logger.exception("failed to add file")
+            logger.exception("")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    ADD_HELP = "Take data files or directories under DVC control."
+    ADD_HELP = "Track data files or directories with DVC."
 
-    add_parser = subparsers.add_parser(
+    parser = subparsers.add_parser(
         "add",
         parents=[parent_parser],
         description=append_doc_link(ADD_HELP, "add"),
         help=ADD_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    add_parser.add_argument(
+    parser.add_argument(
         "-R",
         "--recursive",
         action="store_true",
         default=False,
-        help="Recursively add each file under the directory.",
+        help="Recursively add files under directory targets.",
     )
-    add_parser.add_argument(
+    parser.add_argument(
         "--no-commit",
         action="store_true",
         default=False,
         help="Don't put files/directories into cache.",
     )
-    add_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC-file it generates."
+    parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
     )
-    add_parser.add_argument(
+    parser.add_argument(
         "targets", nargs="+", help="Input files/directories to add."
     )
-    add_parser.set_defaults(func=CmdAdd)
+    parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -26,7 +26,7 @@ def append_doc_link(help_message, path):
     if not path:
         return help_message
     doc_base = "https://man.dvc.org/"
-    return "{message}\ndocumentation: {blue}{base}{path}{nc}".format(
+    return "{message}\nDocumentation: <{blue}{base}{path}{nc}>".format(
         message=help_message,
         base=doc_base,
         path=path,
@@ -55,24 +55,11 @@ class CmdBase(object):
         logger.warning(msg)
         return [Stage.STAGE_FILE]
 
-    def run_cmd(self):
-        from dvc.lock import LockError
-
-        try:
-            with self.repo.lock:
-                return self.run()
-        except LockError:
-            logger.exception("failed to lock before running a command")
-            return 1
-
     # Abstract methods that have to be implemented by any inheritance class
     def run(self):
-        pass
+        raise NotImplementedError
 
 
 class CmdBaseNoRepo(CmdBase):
     def __init__(self, args):
         self.args = args
-
-    def run_cmd(self):
-        return self.run()

dvc/command/cache.py

@@ -44,7 +44,7 @@ def add_parser(subparsers, parent_parser):
     cache_dir_parser = cache_subparsers.add_parser(
         "dir",
         parents=[parent_parser, parent_cache_config_parser],
-        description=append_doc_link(CACHE_HELP, "cache-dir"),
+        description=append_doc_link(CACHE_HELP, "cache/dir"),
         help=CACHE_DIR_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )

dvc/command/checkout.py

@@ -7,16 +7,12 @@ from dvc.command.base import CmdBase, append_doc_link
 
 class CmdCheckout(CmdBase):
     def run(self):
-        if not self.args.targets:
-            self.repo.checkout(force=self.args.force)
-        else:
-            for target in self.args.targets:
-                self.repo.checkout(
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    force=self.args.force,
-                    recursive=self.args.recursive,
-                )
+        self.repo.checkout(
+            targets=self.args.targets,
+            with_deps=self.args.with_deps,
+            force=self.args.force,
+            recursive=self.args.recursive,
+        )
         return 0
 
 

dvc/command/destroy.py

@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)
 
 
 class CmdDestroy(CmdBase):
-    def run_cmd(self):
+    def run(self):
         try:
             statement = (
                 "This will destroy all information about your pipelines,"
@@ -24,7 +24,7 @@ class CmdDestroy(CmdBase):
             if not self.args.force and not prompt.confirm(statement):
                 raise DvcException(
                     "cannot destroy without a confirmation from the user."
-                    " Use '-f' to force."
+                    " Use `-f` to force."
                 )
 
             self.repo.destroy()

dvc/command/diff.py

@@ -9,7 +9,6 @@ from funcy import compact
 
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase, append_doc_link
-import dvc.repo.diff as diff
 
 
 logger = logging.getLogger(__name__)
@@ -37,9 +36,11 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_dir_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         engine = inflect.engine()
         changes_msg = (
-            "{} {} not changed, {} {} modified, {} {} added, "
+            "{} {} untouched, {} {} modified, {} {} added, "
             "{} {} deleted, size was {}"
         )
         changes_msg = changes_msg.format(
@@ -57,6 +58,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_file_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         if (
             dct.get(diff.DIFF_OLD_FILE)
             and dct.get(diff.DIFF_NEW_FILE)
@@ -79,6 +82,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_royal_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
             if dct.get("is_dir"):
                 return cls._get_dir_changes(dct)
@@ -88,6 +93,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _show(cls, diff_dct):
+        import dvc.repo.diff as diff
+
         msg = "dvc diff from {} to {}".format(
             diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
         )

dvc/command/gc.py

@@ -13,17 +13,19 @@ logger = logging.getLogger(__name__)
 
 class CmdGC(CmdBase):
     def run(self):
-        msg = "this will remove all cache except the cache that is used in "
-        if not self.args.all_branches and not self.args.all_tags:
-            msg += "the current git branch"
-        elif self.args.all_branches and not self.args.all_tags:
-            msg += "all git branches"
-        elif not self.args.all_branches and self.args.all_tags:
-            msg += "all git tags"
-        else:
-            msg += "all git branches and all git tags"
+        msg = "This will remove all cache except items used in "
 
-        if self.args.repos is not None and len(self.args.repos) > 0:
+        msg += "the working tree"
+        if self.args.all_commits:
+            msg += " and all git commits"
+        elif self.args.all_branches and self.args.all_tags:
+            msg += " and all git branches and tags"
+        elif self.args.all_branches:
+            msg += " and all git branches"
+        elif self.args.all_tags:
+            msg += " and all git tags"
+
+        if self.args.repos:
             msg += " of the current and the following repos:"
 
             for repo_path in self.args.repos:
@@ -40,6 +42,7 @@ class CmdGC(CmdBase):
         self.repo.gc(
             all_branches=self.args.all_branches,
             all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
             cloud=self.args.cloud,
             remote=self.args.remote,
             force=self.args.force,
@@ -76,6 +79,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Keep data files for all git tags.",
     )
+    gc_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     gc_parser.add_argument(
         "-c",
         "--cloud",

dvc/command/get.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
 from dvc.exceptions import DvcException
 from .base import CmdBaseNoRepo, append_doc_link
 
@@ -13,6 +12,8 @@ logger = logging.getLogger(__name__)
 
 class CmdGet(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         try:
             Repo.get(
                 self.args.url,
@@ -40,7 +41,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     get_parser.add_argument(
-        "url", help="DVC repository URL to download data from."
+        "url", help="URL of Git repository with DVC project to download from."
     )
     get_parser.add_argument("path", help="Path to data within DVC repository.")
     get_parser.add_argument(

dvc/command/get_url.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
 from dvc.exceptions import DvcException
 from .base import CmdBaseNoRepo, append_doc_link
 
@@ -13,6 +12,8 @@ logger = logging.getLogger(__name__)
 
 class CmdGetUrl(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         try:
             Repo.get_url(self.args.url, out=self.args.out)
             return 0

dvc/command/imp.py

@@ -41,7 +41,9 @@ def add_parser(subparsers, parent_parser):
         help=IMPORT_HELP,
         formatter_class=argparse.RawTextHelpFormatter,
     )
-    import_parser.add_argument("url", help="DVC repository URL.")
+    import_parser.add_argument(
+        "url", help="URL of Git repository with DVC project to download from."
+    )
     import_parser.add_argument(
         "path", help="Path to data within DVC repository."
     )

dvc/command/imp_url.py

@@ -19,7 +19,7 @@ class CmdImportUrl(CmdBase):
         except DvcException:
             logger.exception(
                 "failed to import {}. You could also try downloading "
-                "it manually and adding it with `dvc add` command.".format(
+                "it manually, and adding it with `dvc add`.".format(
                     self.args.url
                 )
             )

dvc/command/init.py

@@ -53,7 +53,7 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
         help=(
-            "Overwrite existing '.dvc' directory. "
+            "Overwrite existing '.dvc/' directory. "
             "This operation removes local cache."
         ),
     )

dvc/command/install.py

@@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
 
 
 class CmdInstall(CmdBase):
-    def run_cmd(self):
+    def run(self):
         try:
             self.repo.install()
         except Exception:

dvc/command/metrics.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, BadMetricError
 from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
 
 
@@ -16,6 +16,10 @@ def show_metrics(metrics, all_branches=False, all_tags=False):
         metrics (list): Where each element is either a `list`
             if an xpath was specified, otherwise a `str`
     """
+    # When `metrics` contains a `None` key, it means that some files
+    # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
+    missing = metrics.pop(None, None)
+
     for branch, val in metrics.items():
         if all_branches or all_tags:
             logger.info("{branch}:".format(branch=branch))
@@ -32,16 +36,17 @@ def show_metrics(metrics, all_branches=False, all_tags=False):
             else:
                 logger.info("\t{}: {}".format(fname, metric))
 
+    if missing:
+        raise BadMetricError(missing)
+
 
 class CmdMetricsShow(CmdBase):
     def run(self):
-        typ = self.args.type
-        xpath = self.args.xpath
         try:
             metrics = self.repo.metrics.show(
-                self.args.path,
-                typ=typ,
-                xpath=xpath,
+                self.args.targets,
+                typ=self.args.type,
+                xpath=self.args.xpath,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
                 recursive=self.args.recursive,
@@ -116,12 +121,15 @@ def add_parser(subparsers, parent_parser):
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
         parents=[parent_parser],
-        description=append_doc_link(METRICS_SHOW_HELP, "metrics-show"),
+        description=append_doc_link(METRICS_SHOW_HELP, "metrics/show"),
         help=METRICS_SHOW_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_show_parser.add_argument(
-        "path", nargs="?", help="Path to a metric file or a directory."
+        "targets",
+        nargs="*",
+        help="Metric files or directories (see -R) to show "
+        "(leave empty to display all)",
     )
     metrics_show_parser.add_argument(
         "-t",
@@ -155,8 +163,8 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
         help=(
-            "If path is a directory, recursively search and process metric "
-            "files in path."
+            "If any target is a directory, recursively search and process "
+            "metric files."
         ),
     )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
@@ -165,7 +173,7 @@ def add_parser(subparsers, parent_parser):
     metrics_add_parser = metrics_subparsers.add_parser(
         "add",
         parents=[parent_parser],
-        description=append_doc_link(METRICS_ADD_HELP, "metrics-add"),
+        description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
         help=METRICS_ADD_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -182,7 +190,7 @@ def add_parser(subparsers, parent_parser):
     metrics_modify_parser = metrics_subparsers.add_parser(
         "modify",
         parents=[parent_parser],
-        description=append_doc_link(METRICS_MODIFY_HELP, "metrics-modify"),
+        description=append_doc_link(METRICS_MODIFY_HELP, "metrics/modify"),
         help=METRICS_MODIFY_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -199,7 +207,7 @@ def add_parser(subparsers, parent_parser):
     metrics_remove_parser = metrics_subparsers.add_parser(
         "remove",
         parents=[parent_parser],
-        description=append_doc_link(METRICS_REMOVE_HELP, "metrics-remove"),
+        description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
         help=METRICS_REMOVE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )

dvc/command/pipeline.py

@@ -19,7 +19,7 @@ class CmdPipelineShow(CmdBase):
         from dvc.stage import Stage
 
         stage = Stage.load(self.repo, target)
-        G = self.repo.graph()[0]
+        G = self.repo.graph
         stages = networkx.get_node_attributes(G, "stage")
         node = relpath(stage.path, self.repo.root_dir)
         nodes = networkx.dfs_postorder_nodes(G, node)
@@ -41,20 +41,16 @@ class CmdPipelineShow(CmdBase):
     def __build_graph(self, target, commands, outs):
         import networkx
         from dvc.stage import Stage
+        from dvc.repo.graph import get_pipeline
 
         stage = Stage.load(self.repo, target)
         node = relpath(stage.path, self.repo.root_dir)
 
-        pipelines = list(
-            filter(lambda g: node in g.nodes(), self.repo.pipelines())
-        )
-
-        assert len(pipelines) == 1
-        G = pipelines[0]
+        G = get_pipeline(self.repo.pipelines, node)
         stages = networkx.get_node_attributes(G, "stage")
 
         nodes = []
-        for n in G.nodes():
+        for n in G:
             stage = stages[n]
             if commands:
                 if stage.cmd is None:
@@ -168,7 +164,7 @@ class CmdPipelineList(CmdBase):
     def run(self):
         import networkx
 
-        pipelines = self.repo.pipelines()
+        pipelines = self.repo.pipelines
         for p in pipelines:
             stages = networkx.get_node_attributes(p, "stage")
             for stage in stages:
@@ -201,7 +197,7 @@ def add_parser(subparsers, parent_parser):
     pipeline_show_parser = pipeline_subparsers.add_parser(
         "show",
         parents=[parent_parser],
-        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline-show"),
+        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline/show"),
         help=PIPELINE_SHOW_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -257,7 +253,7 @@ def add_parser(subparsers, parent_parser):
     pipeline_list_parser = pipeline_subparsers.add_parser(
         "list",
         parents=[parent_parser],
-        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline-list"),
+        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
         help=PIPELINE_LIST_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )

dvc/command/remote.py

@@ -85,7 +85,7 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser = remote_subparsers.add_parser(
         "add",
         parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_ADD_HELP, "remote-add"),
+        description=append_doc_link(REMOTE_ADD_HELP, "remote/add"),
         help=REMOTE_ADD_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -114,7 +114,7 @@ def add_parser(subparsers, parent_parser):
     remote_default_parser = remote_subparsers.add_parser(
         "default",
         parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_DEFAULT_HELP, "remote-default"),
+        description=append_doc_link(REMOTE_DEFAULT_HELP, "remote/default"),
         help=REMOTE_DEFAULT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -134,7 +134,7 @@ def add_parser(subparsers, parent_parser):
     remote_remove_parser = remote_subparsers.add_parser(
         "remove",
         parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_REMOVE_HELP, "remote-remove"),
+        description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
         help=REMOTE_REMOVE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -147,7 +147,7 @@ def add_parser(subparsers, parent_parser):
     remote_modify_parser = remote_subparsers.add_parser(
         "modify",
         parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_MODIFY_HELP, "remote-modify"),
+        description=append_doc_link(REMOTE_MODIFY_HELP, "remote/modify"),
         help=REMOTE_MODIFY_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -171,7 +171,7 @@ def add_parser(subparsers, parent_parser):
     remote_list_parser = remote_subparsers.add_parser(
         "list",
         parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_LIST_HELP, "remote-list"),
+        description=append_doc_link(REMOTE_LIST_HELP, "remote/list"),
         help=REMOTE_LIST_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )

dvc/command/remove.py

@@ -28,7 +28,7 @@ class CmdRemove(CmdBase):
 
         raise DvcException(
             "Cannot purge without a confirmation from the user."
-            " Use '-f' to force."
+            " Use `-f` to force."
         )
 
     def run(self):
@@ -74,9 +74,6 @@ def add_parser(subparsers, parent_parser):
         help="Force purge.",
     )
     remove_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="DVC-files to remove. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
+        "targets", nargs="+", help="DVC-files to remove."
     )
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/root.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.utils import relpath
 
@@ -12,6 +11,8 @@ logger = logging.getLogger(__name__)
 
 class CmdRoot(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         logger.info(relpath(Repo.find_root()))
         return 0
 

dvc/command/run.py

@@ -27,9 +27,9 @@ class CmdRun(CmdBase):
             ]
         ):  # pragma: no cover
             logger.error(
-                "too few arguments. Specify at least one: '-d', '-o', '-O',"
-                " '-m', '-M', '--outs-persist', '--outs-persist-no-cache',"
-                " 'command'."
+                "too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
+                "`-m`, `-M`, `--outs-persist`, `--outs-persist-no-cache`, "
+                "`command`."
             )
             return 1
 
@@ -51,6 +51,7 @@ class CmdRun(CmdBase):
                 no_commit=self.args.no_commit,
                 outs_persist=self.args.outs_persist,
                 outs_persist_no_cache=self.args.outs_persist_no_cache,
+                always_changed=self.args.always_changed,
             )
         except DvcException:
             logger.exception("failed to run command")
@@ -189,6 +190,12 @@ def add_parser(subparsers, parent_parser):
         help="Declare output file or directory that will not be "
         "removed upon repro (do not put into DVC cache).",
     )
+    run_parser.add_argument(
+        "--always-changed",
+        action="store_true",
+        default=False,
+        help="Always consider this DVC-file as changed.",
+    )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."
     )

dvc/command/status.py

@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)
 class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 20
     STATUS_INDENT = "\t"
-    UP_TO_DATE_MSG = "Pipelines are up to date. Nothing to reproduce."
+    UP_TO_DATE_MSG = "Data and pipelines are up to date."
 
     def _normalize(self, s):
         s += ":"

dvc/command/version.py

@@ -5,15 +5,15 @@ import platform
 import argparse
 import logging
 import uuid
+import itertools
 
 try:
     import psutil
 except ImportError:
     psutil = None
 
-from dvc.utils import is_binary
+from dvc.utils import is_binary, relpath
 from dvc.utils.compat import pathlib
-from dvc.repo import Repo
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
 from dvc.exceptions import DvcException, NotDvcRepoError
@@ -25,6 +25,8 @@ logger = logging.getLogger(__name__)
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         dvc_version = __version__
         python_version = platform.python_version()
         platform_type = platform.platform()
@@ -46,9 +48,22 @@ class CmdVersion(CmdBaseNoRepo):
             repo = Repo()
             root_directory = repo.root_dir
 
-            info += "Cache: {cache}\n".format(
-                cache=self.get_linktype_support_info(repo)
-            )
+            # cache_dir might not exist yet (e.g. after `dvc init`), and we
+            # can't auto-create it, as it might cause issues if the user
+            # later decides to enable shared cache mode with
+            # `dvc config cache.shared group`.
+            if os.path.exists(repo.cache.local.cache_dir):
+                info += "Cache: {cache}\n".format(
+                    cache=self.get_linktype_support_info(repo)
+                )
+            else:
+                logger.warning(
+                    "Unable to detect supported link types, as cache "
+                    "directory '{}' doesn't exist. It is usually auto-created "
+                    "by commands such as `dvc add/fetch/pull/run/import`, "
+                    "but you could create it manually to enable this "
+                    "check.".format(relpath(repo.cache.local.cache_dir))
+                )
 
             if psutil:
                 info += (
@@ -69,9 +84,12 @@ class CmdVersion(CmdBaseNoRepo):
     def get_fs_type(path):
         partition = {
             pathlib.Path(part.mountpoint): (part.fstype, part.device)
-            for part in psutil.disk_partitions()
+            for part in psutil.disk_partitions(all=True)
         }
-        for parent in pathlib.Path(path).parents:
+
+        path = pathlib.Path(path)
+
+        for parent in itertools.chain([path], path.parents):
             if parent in partition:
                 return partition[parent]
         return ("unkown", "none")
@@ -109,7 +127,7 @@ class CmdVersion(CmdBaseNoRepo):
 
 
 def add_parser(subparsers, parent_parser):
-    VERSION_HELP = "Show DVC version and system/environment informaion."
+    VERSION_HELP = "Show DVC version and system/environment information."
 
     version_parser = subparsers.add_parser(
         "version",

dvc/config.py

@@ -31,6 +31,18 @@ class ConfigError(DvcException):
         )
 
 
+class NoRemoteError(ConfigError):
+    def __init__(self, command, cause=None):
+        msg = (
+            "no remote specified. Setup default remote with\n"
+            "    dvc config core.remote <name>\n"
+            "or use:\n"
+            "    dvc {} -r <name>\n".format(command)
+        )
+
+        super(NoRemoteError, self).__init__(msg, cause=cause)
+
+
 def supported_cache_type(types):
     """Checks if link type config option has a valid value.
 
@@ -160,6 +172,8 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_CACHE_TYPE = "type"
     SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
     SECTION_CACHE_PROTECTED = "protected"
+    SECTION_CACHE_SHARED = "shared"
+    SECTION_CACHE_SHARED_SCHEMA = And(Use(str.lower), Choices("group"))
     SECTION_CACHE_LOCAL = "local"
     SECTION_CACHE_S3 = "s3"
     SECTION_CACHE_GS = "gs"
@@ -177,6 +191,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_CACHE_DIR): str,
         Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
         Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
+        Optional(SECTION_CACHE_SHARED): SECTION_CACHE_SHARED_SCHEMA,
         Optional(PRIVATE_CWD): str,
         Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): BOOL_SCHEMA,
     }
@@ -207,6 +222,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_AWS_PROFILE = "profile"
     SECTION_AWS_USE_SSL = "use_ssl"
     SECTION_AWS_SSE = "sse"
+    SECTION_AWS_ACL = "acl"
     SECTION_AWS_SCHEMA = {
         SECTION_AWS_STORAGEPATH: str,
         Optional(SECTION_AWS_REGION): str,
@@ -216,6 +232,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
         Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
         Optional(SECTION_AWS_SSE): str,
+        Optional(SECTION_AWS_ACL): str,
     }
 
     # backward compatibility
@@ -248,6 +265,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_REMOTE_TIMEOUT = "timeout"
     SECTION_REMOTE_PASSWORD = "password"
     SECTION_REMOTE_ASK_PASSWORD = "ask_password"
+    SECTION_REMOTE_GSS_AUTH = "gss_auth"
     SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
     SECTION_REMOTE_SCHEMA = {
         SECTION_REMOTE_URL: str,
@@ -258,6 +276,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
         Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
         Optional(SECTION_AWS_SSE): str,
+        Optional(SECTION_AWS_ACL): str,
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
         Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
@@ -267,6 +286,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_REMOTE_TIMEOUT): Use(int),
         Optional(SECTION_REMOTE_PASSWORD): str,
         Optional(SECTION_REMOTE_ASK_PASSWORD): BOOL_SCHEMA,
+        Optional(SECTION_REMOTE_GSS_AUTH): BOOL_SCHEMA,
         Optional(SECTION_AZURE_CONNECTION_STRING): str,
         Optional(SECTION_OSS_ACCESS_KEY_ID): str,
         Optional(SECTION_OSS_ACCESS_KEY_SECRET): str,

dvc/daemon.py

@@ -94,7 +94,7 @@ def daemon(args):
 
     cmd = [sys.executable]
     if not is_binary():
-        cmd += ["-m", "dvc"]
+        cmd += [sys.argv[0]]
     cmd += ["daemon", "-q"] + args
 
     env = fix_env()

dvc/data_cloud.py

@@ -4,16 +4,8 @@ from __future__ import unicode_literals
 
 import logging
 
-from dvc.config import Config, ConfigError
+from dvc.config import Config, NoRemoteError
 from dvc.remote import Remote
-from dvc.remote.s3 import RemoteS3
-from dvc.remote.gs import RemoteGS
-from dvc.remote.azure import RemoteAZURE
-from dvc.remote.oss import RemoteOSS
-from dvc.remote.ssh import RemoteSSH
-from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.http import RemoteHTTP
 
 
 logger = logging.getLogger(__name__)
@@ -30,18 +22,6 @@ class DataCloud(object):
         config.ConfigError: thrown when config has invalid format.
     """
 
-    CLOUD_MAP = {
-        "aws": RemoteS3,
-        "gcp": RemoteGS,
-        "azure": RemoteAZURE,
-        "oss": RemoteOSS,
-        "ssh": RemoteSSH,
-        "hdfs": RemoteHDFS,
-        "local": RemoteLOCAL,
-        "http": RemoteHTTP,
-        "https": RemoteHTTP,
-    }
-
     def __init__(self, repo):
         self.repo = repo
 
@@ -60,21 +40,16 @@ class DataCloud(object):
         if remote:
             return self._init_remote(remote)
 
-        raise ConfigError(
-            "No remote repository specified. Setup default repository with\n"
-            "    dvc config core.remote <name>\n"
-            "or use:\n"
-            "    dvc {} -r <name>\n".format(command)
-        )
+        raise NoRemoteError(command)
 
     def _init_remote(self, remote):
         return Remote(self.repo, name=remote)
 
-    def push(self, targets, jobs=None, remote=None, show_checksums=False):
+    def push(self, cache, jobs=None, remote=None, show_checksums=False):
         """Push data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to push to the cloud.
+            cache (NamedCache): named checksums to push to the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to push to.
                 By default remote from core.remote config option is used.
@@ -82,17 +57,17 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.push(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "push"),
             show_checksums=show_checksums,
         )
 
-    def pull(self, targets, jobs=None, remote=None, show_checksums=False):
+    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
         """Pull data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to pull from the cloud.
+            cache (NamedCache): named checksums to pull from the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
                 By default remote from core.remote config option is used.
@@ -100,25 +75,25 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.pull(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "pull"),
             show_checksums=show_checksums,
         )
 
-    def status(self, targets, jobs=None, remote=None, show_checksums=False):
+    def status(self, cache, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to check status for.
+            cache (NamedCache): named checksums to check status for.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to compare
-                targets to. By default remote from core.remote config option
+                cache to. By default remote from core.remote config option
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
         remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            targets, jobs=jobs, remote=remote, show_checksums=show_checksums
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )

dvc/dependency/base.py

@@ -15,11 +15,19 @@ class DependencyIsNotFileOrDirError(DvcException):
         super(DependencyIsNotFileOrDirError, self).__init__(msg)
 
 
+class DependencyIsStageFileError(DvcException):
+    def __init__(self, path):
+        super(DependencyIsStageFileError, self).__init__(
+            "Stage file '{}' cannot be a dependency.".format(path)
+        )
+
+
 class DependencyBase(object):
     IS_DEPENDENCY = True
 
     DoesNotExistError = DependencyDoesNotExistError
     IsNotFileOrDirError = DependencyIsNotFileOrDirError
+    IsStageFileError = DependencyIsStageFileError
 
     def update(self):
         pass

dvc/dependency/repo.py

@@ -35,6 +35,11 @@ class DependencyREPO(DependencyLOCAL):
     def is_in_repo(self):
         return False
 
+    @property
+    def repo_pair(self):
+        d = self.def_repo
+        return d[self.PARAM_URL], d[self.PARAM_REV_LOCK] or d[self.PARAM_REV]
+
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 
@@ -61,7 +66,7 @@ class DependencyREPO(DependencyLOCAL):
     def dumpd(self):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
-    def download(self, to, resume=False):
+    def fetch(self):
         with self._make_repo(
             cache_dir=self.repo.cache.local.cache_dir
         ) as repo:
@@ -70,8 +75,13 @@ class DependencyREPO(DependencyLOCAL):
             out = repo.find_out_by_relpath(self.def_path)
             with repo.state:
                 repo.cloud.pull(out.get_used_cache())
-            to.info = copy.copy(out.info)
-            to.checkout()
+
+        return out
+
+    def download(self, to):
+        out = self.fetch()
+        to.info = copy.copy(out.info)
+        to.checkout()
 
     def update(self):
         with self._make_repo(rev_lock=None) as repo:

dvc/exceptions.py

@@ -47,7 +47,7 @@ class OutputDuplicationError(DvcException):
             for stage in stages
         )
         msg = (
-            "file/directory '{}' is specified as an output in more than one"
+            "file/directory '{}' is specified as an output in more than one "
             "stage: {}"
         ).format(output, "\n    ".join(stages))
         super(OutputDuplicationError, self).__init__(msg)
@@ -82,8 +82,8 @@ class StagePathAsOutputError(DvcException):
         assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
         assert isinstance(fname, str) or isinstance(fname, builtin_str)
         msg = (
-            "current working directory '{cwd}' is specified as an output in"
-            " '{fname}'. Use another CWD to prevent any data removal.".format(
+            "current working directory '{cwd}' is specified as an output in "
+            "'{fname}'. Use another CWD to prevent any data removal.".format(
                 cwd=wdir, fname=fname
             )
         )
@@ -167,8 +167,8 @@ class CyclicGraphError(DvcException):
         assert isinstance(stages, list)
         stages = "\n".join("\t- {}".format(stage) for stage in stages)
         msg = (
-            "you've introduced a cycle in your pipeline that involves"
-            " the following stages:"
+            "you've introduced a cycle in your pipeline that involves "
+            "the following stages:"
             "\n"
             "{stages}".format(stages=stages)
         )
@@ -179,7 +179,7 @@ class ConfirmRemoveError(DvcException):
     def __init__(self, path):
         super(ConfirmRemoveError, self).__init__(
             "unable to remove '{}' without a confirmation from the user. Use "
-            "'-f' to force.".format(path)
+            "`-f` to force.".format(path)
         )
 
 
@@ -196,10 +196,11 @@ class ReproductionError(DvcException):
 
 
 class BadMetricError(DvcException):
-    def __init__(self, path):
+    def __init__(self, paths):
         super(BadMetricError, self).__init__(
-            "'{}' does not exist, not a metric or is malformed".format(
-                relpath(path)
+            "the following metrics do not exists, "
+            "are not metric files or are malformed: {paths}".format(
+                paths=", ".join("'{}'".format(path) for path in paths)
             )
         )
 
@@ -208,7 +209,7 @@ class NoMetricsError(DvcException):
     def __init__(self):
         super(NoMetricsError, self).__init__(
             "no metric files in this repository. "
-            "Use 'dvc metrics add' to add a metric file to track."
+            "Use `dvc metrics add` to add a metric file to track."
         )
 
 
@@ -225,7 +226,7 @@ class StageFileCorruptedError(DvcException):
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
         super(RecursiveAddingWhileUsingFilename, self).__init__(
-            "using fname with recursive is not allowed."
+            "cannot use `fname` with multiple targets or `-R|--recursive`"
         )
 
 
@@ -243,17 +244,10 @@ class OverlappingOutputPathsError(DvcException):
         )
 
 
-class TargetNotDirectoryError(DvcException):
-    def __init__(self, path):
-        super(TargetNotDirectoryError, self).__init__(
-            "Target: {} is not a directory".format(path)
-        )
-
-
 class CheckoutErrorSuggestGit(DvcException):
     def __init__(self, target, cause):
         super(CheckoutErrorSuggestGit, self).__init__(
-            "Did you mean 'git checkout {}'?".format(target), cause=cause
+            "Did you mean `git checkout {}`?".format(target), cause=cause
         )
 
 
@@ -265,10 +259,11 @@ class ETagMismatchError(DvcException):
         )
 
 
-class OutputFileMissingError(DvcException):
-    def __init__(self, path):
-        super(OutputFileMissingError, self).__init__(
-            "Can't find {} neither locally nor on remote".format(path)
+class FileMissingError(DvcException):
+    def __init__(self, path, cause=None):
+        super(FileMissingError, self).__init__(
+            "Can't find '{}' neither locally nor on remote".format(path),
+            cause=cause,
         )
 
 
@@ -278,3 +273,59 @@ class DvcIgnoreInCollectedDirError(DvcException):
             ".dvcignore file should not be in collected dir path: "
             "'{}'".format(ignore_dirname)
         )
+
+
+class UrlNotDvcRepoError(DvcException):
+    def __init__(self, url):
+        super(UrlNotDvcRepoError, self).__init__(
+            "URL '{}' is not a dvc repository.".format(url)
+        )
+
+
+class GetDVCFileError(DvcException):
+    def __init__(self):
+        super(GetDVCFileError, self).__init__(
+            "the given path is a DVC-file, you must specify a data file "
+            "or a directory"
+        )
+
+
+class GitHookAlreadyExistsError(DvcException):
+    def __init__(self, hook_name):
+        super(GitHookAlreadyExistsError, self).__init__(
+            "Hook '{}' already exists. Please refer to "
+            "https://man.dvc.org/install "
+            "for more info.".format(hook_name)
+        )
+
+
+class DownloadError(DvcException):
+    def __init__(self, amount):
+        self.amount = amount
+
+        super(DownloadError, self).__init__(
+            "{amount} files failed to download".format(amount=amount)
+        )
+
+
+class UploadError(DvcException):
+    def __init__(self, amount):
+        self.amount = amount
+
+        super(UploadError, self).__init__(
+            "{amount} files failed to upload".format(amount=amount)
+        )
+
+
+class CheckoutError(DvcException):
+    def __init__(self, target_infos):
+        targets = [str(t) for t in target_infos]
+        m = (
+            "Checkout failed for following targets:\n {}\nDid you "
+            "forget to fetch?".format("\n".join(targets))
+        )
+        super(CheckoutError, self).__init__(m)
+
+
+class CollectCacheError(DvcException):
+    pass

dvc/external_repo.py

@@ -1,87 +1,91 @@
 from __future__ import unicode_literals
 
 import os
-import logging
 import tempfile
+from distutils.dir_util import copy_tree
 
-from funcy import retry
 from contextlib import contextmanager
+from funcy import retry
 
-from dvc.config import Config
-from dvc.cache import CacheConfig
-from dvc.exceptions import DvcException
 from dvc.utils import remove
 
 
-logger = logging.getLogger(__name__)
-
-
-class ExternalRepoError(DvcException):
-    pass
-
-
-class CloneError(ExternalRepoError):
-    def __init__(self, url, path, cause):
-        super(CloneError, self).__init__(
-            "Failed to clone repo '{}' to '{}'".format(url, path), cause=cause
-        )
-
-
-class RevError(ExternalRepoError):
-    def __init__(self, url, rev, cause):
-        super(RevError, self).__init__(
-            "Failed to access revision '{}' for repo '{}'".format(rev, url),
-            cause=cause,
-        )
-
-
-def _clone(cache_dir=None, **kwargs):
-    from dvc.repo import Repo
-
-    _path = tempfile.mkdtemp("dvc-repo")
-
-    _clone_git_repo(_path, **kwargs)
-
-    if cache_dir:
-        repo = Repo(_path)
-        cache_config = CacheConfig(repo.config)
-        cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
-        repo.scm.git.close()
-
-    return Repo(_path)
-
-
-def _clone_git_repo(to_path, url=None, rev=None, rev_lock=None):
-    import git
-
-    try:
-        repo = git.Repo.clone_from(url, to_path, no_single_branch=True)
-    except git.exc.GitCommandError as exc:
-        raise CloneError(url, to_path, exc)
-    try:
-        revision = rev_lock or rev
-        if revision:
-            try:
-                repo.git.checkout(revision)
-            except git.exc.GitCommandError as exc:
-                raise RevError(url, revision, exc)
-    finally:
-        repo.close()
-
-
-def _remove(repo):
-    repo.scm.git.close()
-
-    if os.name == "nt":
-        # git.exe may hang for a while not permitting to remove temp dir
-        os_retry = retry(5, errors=OSError, timeout=0.1)
-        os_retry(remove)(repo.root_dir)
-    else:
-        remove(repo.root_dir)
+REPO_CACHE = {}
 
 
 @contextmanager
-def external_repo(**kwargs):
-    repo = _clone(**kwargs)
+def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
+    from dvc.repo import Repo
+
+    path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
+    repo = Repo(path)
     yield repo
-    _remove(repo)
+    repo.close()
+
+
+def _external_repo(url=None, rev=None, cache_dir=None):
+    from dvc.config import Config
+    from dvc.cache import CacheConfig
+    from dvc.repo import Repo
+
+    key = (url, rev, cache_dir)
+    if key in REPO_CACHE:
+        return REPO_CACHE[key]
+
+    new_path = tempfile.mkdtemp("dvc-erepo")
+
+    # Copy and adjust existing clone
+    if (url, None, None) in REPO_CACHE:
+        old_path = REPO_CACHE[url, None, None]
+
+        # This one unlike shutil.copytree() works with an existing dir
+        copy_tree(old_path, new_path)
+    else:
+        # Create a new clone
+        _clone_repo(url, new_path)
+
+        # Save clean clone dir so that we will have access to a default branch
+        clean_clone_path = tempfile.mkdtemp("dvc-erepo")
+        copy_tree(new_path, clean_clone_path)
+        REPO_CACHE[url, None, None] = clean_clone_path
+
+    # Adjust new clone/copy to fit rev and cache_dir
+    repo = Repo(new_path)
+    try:
+        if rev is not None:
+            repo.scm.checkout(rev)
+
+        if cache_dir is not None:
+            cache_config = CacheConfig(repo.config)
+            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
+    finally:
+        # Need to close/reopen repo to force config reread
+        repo.close()
+
+    REPO_CACHE[key] = new_path
+    return new_path
+
+
+def clean_repos():
+    # Outside code should not see cache while we are removing
+    repo_paths = list(REPO_CACHE.values())
+    REPO_CACHE.clear()
+
+    for path in repo_paths:
+        _remove(path)
+
+
+def _remove(path):
+    if os.name == "nt":
+        # git.exe may hang for a while not permitting to remove temp dir
+        os_retry = retry(5, errors=OSError, timeout=0.1)
+        os_retry(remove)(path)
+    else:
+        remove(path)
+
+
+def _clone_repo(url, path):
+    from dvc.scm.git import Git
+
+    git = Git.clone(url, path)
+    git.close()

dvc/lock.py

@@ -3,58 +3,143 @@
 from __future__ import unicode_literals
 
 import os
+import hashlib
 import time
-import zc.lockfile
+from datetime import timedelta
+
+from funcy.py3 import lkeep
 
 from dvc.exceptions import DvcException
+from dvc.utils import makedirs
+from dvc.utils.compat import is_py3
+
+
+DEFAULT_TIMEOUT = 5
 
 
 class LockError(DvcException):
     """Thrown when unable to acquire the lock for dvc repo."""
 
 
-class Lock(object):
-    """Class for dvc repo lock.
+if is_py3:
+    import flufl.lock
 
-    Args:
-        dvc_dir (str): path to the directory that the lock should be created
-            in.
-        name (str): name of the lock file.
-    """
+    class Lock(flufl.lock.Lock):
+        """Class for dvc repo lock.
 
-    LOCK_FILE = "lock"
-    TIMEOUT = 5
+        Args:
+            lockfile (str): the lock filename
+                in.
+            tmp_dir (str): a directory to store claim files.
+        """
 
-    def __init__(self, dvc_dir, name=LOCK_FILE):
-        self.lock_file = os.path.join(dvc_dir, name)
-        self._lock = None
+        def __init__(self, lockfile, tmp_dir=None):
+            import socket
 
-    def _do_lock(self):
-        try:
-            self._lock = zc.lockfile.LockFile(self.lock_file)
-        except zc.lockfile.LockError:
-            raise LockError(
-                "cannot perform the cmd since DVC is busy and "
-                "locked. Please retry the cmd later."
-            )
+            self._tmp_dir = tmp_dir
+            if self._tmp_dir is not None:
+                makedirs(self._tmp_dir, exist_ok=True)
+
+            # NOTE: this is basically Lock.__init__ copy-paste, except that
+            # instead of using `socket.getfqdn()` we use `socket.gethostname()`
+            # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
+            # anything, which is way too slow. `gethostname()` is actually a
+            # fallback for `getfqdn()` when it is not able to resolve a
+            # canonical hostname through network. The claimfile that uses
+            # `self._hostname` is still usable, as it uses `pid` and random
+            # number to generate the resulting lock file name, which is unique
+            # enough for our application.
+            #
+            # [1] https://github.com/iterative/dvc/issues/2582
+            self._hostname = socket.gethostname()
+
+            self._lockfile = lockfile
+            self._lifetime = timedelta(days=365)  # Lock for good by default
+            self._separator = flufl.lock.SEP
+            self._set_claimfile()
+            self._owned = True
+            self._retry_errnos = []
+
+        @property
+        def lockfile(self):
+            return self._lockfile
+
+        @property
+        def files(self):
+            return lkeep([self._lockfile, self._tmp_dir])
+
+        def lock(self):
+            try:
+                super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
+            except flufl.lock.TimeOutError:
+                raise LockError(
+                    "cannot perform the cmd since DVC is busy and "
+                    "locked. Please retry the cmd later."
+                )
+
+        def _set_claimfile(self, pid=None):
+            super(Lock, self)._set_claimfile(pid)
+
+            if self._tmp_dir is not None:
+                # Under Windows file path length is limited so we hash it
+                filename = hashlib.md5(self._claimfile.encode()).hexdigest()
+                self._claimfile = os.path.join(
+                    self._tmp_dir, filename + ".lock"
+                )
+
+        # Fix for __del__ bug in flufl.lock [1] which is causing errors on
+        # Python shutdown [2].
+        # [1] https://gitlab.com/warsaw/flufl.lock/issues/7
+        # [2] https://github.com/iterative/dvc/issues/2573
+        def __del__(self):
+            try:
+                if self._owned:
+                    self.finalize()
+            except ImportError:
+                pass
+
+
+else:
+    import zc.lockfile
+
+    class Lock(object):
+        """Class for dvc repo lock.
+
+        Uses zc.lockfile as backend.
+        """
+
+        def __init__(self, lockfile, tmp_dir=None):
+            self.lockfile = lockfile
+            self._lock = None
+
+        @property
+        def files(self):
+            return [self.lockfile]
+
+        def _do_lock(self):
+            try:
+                self._lock = zc.lockfile.LockFile(self.lockfile)
+            except zc.lockfile.LockError:
+                raise LockError(
+                    "cannot perform the cmd since DVC is busy and "
+                    "locked. Please retry the cmd later."
+                )
+
+        def lock(self):
+            try:
+                self._do_lock()
+                return
+            except LockError:
+                time.sleep(DEFAULT_TIMEOUT)
 
-    def lock(self):
-        """Acquire lock for dvc repo."""
-        try:
             self._do_lock()
-            return
-        except LockError:
-            time.sleep(self.TIMEOUT)
 
-        self._do_lock()
+        def unlock(self):
+            self._lock.close()
+            self._lock = None
 
-    def unlock(self):
-        """Release lock for dvc repo."""
-        self._lock.close()
-        self._lock = None
+        def __enter__(self):
+            self.lock()
 
-    def __enter__(self):
-        self.lock()
-
-    def __exit__(self, typ, value, tbck):
-        self.unlock()
+        def __exit__(self, typ, value, tbck):
+            self.unlock()

dvc/logger.py

@@ -2,7 +2,8 @@
 
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, StringIO
+from dvc.utils.compat import str, StringIO, RecursionError
+from dvc.progress import Tqdm
 
 import logging
 import logging.handlers
@@ -18,7 +19,12 @@ class LoggingException(Exception):
 
 class ExcludeErrorsFilter(logging.Filter):
     def filter(self, record):
-        return record.levelno < logging.ERROR
+        return record.levelno < logging.WARNING
+
+
+class ExcludeInfoFilter(logging.Filter):
+    def filter(self, record):
+        return record.levelno < logging.INFO
 
 
 class ColorFormatter(logging.Formatter):
@@ -43,7 +49,7 @@ class ColorFormatter(logging.Formatter):
     }
 
     footer = (
-        "{yellow}Having any troubles?{nc}."
+        "{yellow}Having any troubles?{nc}"
         " Hit us up at {blue}https://dvc.org/support{nc},"
         " we are always happy to help!"
     ).format(
@@ -53,9 +59,6 @@ class ColorFormatter(logging.Formatter):
     )
 
     def format(self, record):
-        if self._is_visible(record):
-            self._progress_aware()
-
         if record.levelname == "INFO":
             return record.msg
 
@@ -146,20 +149,25 @@ class ColorFormatter(logging.Formatter):
 
         return (exception, stack_trace)
 
-    def _progress_aware(self):
-        """Add a new line if progress bar hasn't finished"""
-        from dvc.progress import progress
-
-        if not progress.is_finished:
-            progress._print()
-        progress.clearln()
-
 
 class LoggerHandler(logging.StreamHandler):
     def handleError(self, record):
         super(LoggerHandler, self).handleError(record)
         raise LoggingException(record)
 
+    def emit(self, record):
+        """Write to Tqdm's stream so as to not break progressbars"""
+        try:
+            msg = self.format(record)
+            Tqdm.write(
+                msg, file=self.stream, end=getattr(self, "terminator", "\n")
+            )
+            self.flush()
+        except RecursionError:
+            raise
+        except Exception:
+            self.handleError(record)
+
 
 def setup(level=logging.INFO):
     colorama.init()
@@ -167,19 +175,29 @@ def setup(level=logging.INFO):
     logging.config.dictConfig(
         {
             "version": 1,
-            "filters": {"exclude_errors": {"()": ExcludeErrorsFilter}},
+            "filters": {
+                "exclude_errors": {"()": ExcludeErrorsFilter},
+                "exclude_info": {"()": ExcludeInfoFilter},
+            },
             "formatters": {"color": {"()": ColorFormatter}},
             "handlers": {
-                "console": {
+                "console_info": {
                     "class": "dvc.logger.LoggerHandler",
-                    "level": "DEBUG",
+                    "level": "INFO",
                     "formatter": "color",
                     "stream": "ext://sys.stdout",
                     "filters": ["exclude_errors"],
                 },
+                "console_debug": {
+                    "class": "dvc.logger.LoggerHandler",
+                    "level": "DEBUG",
+                    "formatter": "color",
+                    "stream": "ext://sys.stdout",
+                    "filters": ["exclude_info"],
+                },
                 "console_errors": {
                     "class": "dvc.logger.LoggerHandler",
-                    "level": "ERROR",
+                    "level": "WARNING",
                     "formatter": "color",
                     "stream": "ext://sys.stderr",
                 },
@@ -187,11 +205,27 @@ def setup(level=logging.INFO):
             "loggers": {
                 "dvc": {
                     "level": level,
-                    "handlers": ["console", "console_errors"],
+                    "handlers": [
+                        "console_info",
+                        "console_debug",
+                        "console_errors",
+                    ],
                 },
                 "paramiko": {
                     "level": logging.CRITICAL,
-                    "handlers": ["console", "console_errors"],
+                    "handlers": [
+                        "console_info",
+                        "console_debug",
+                        "console_errors",
+                    ],
+                },
+                "flufl.lock": {
+                    "level": logging.CRITICAL,
+                    "handlers": [
+                        "console_info",
+                        "console_debug",
+                        "console_errors",
+                    ],
                 },
             },
         }

dvc/main.py

@@ -5,10 +5,13 @@ from __future__ import unicode_literals
 import logging
 
 from dvc.cli import parse_args
+from dvc.lock import LockError
 from dvc.config import ConfigError
 from dvc.analytics import Analytics
 from dvc.exceptions import NotDvcRepoError, DvcParserError
 from dvc.remote.pool import close_pools
+from dvc.external_repo import clean_repos
+from dvc.utils.compat import is_py2
 
 
 logger = logging.getLogger("dvc")
@@ -37,7 +40,10 @@ def main(argv=None):
             logger.setLevel(logging.DEBUG)
 
         cmd = args.func(args)
-        ret = cmd.run_cmd()
+        ret = cmd.run()
+    except LockError:
+        logger.exception("failed to lock before running a command")
+        ret = 250
     except ConfigError:
         logger.exception("configuration error")
         ret = 251
@@ -49,15 +55,26 @@ def main(argv=None):
         ret = 253
     except DvcParserError:
         ret = 254
-    except Exception:  # pylint: disable=broad-except
-        logger.exception("unexpected error")
+    except Exception as exc:  # pylint: disable=broad-except
+        if isinstance(exc, UnicodeError) and is_py2:
+            logger.exception(
+                "unicode is not supported in DVC for Python 2 "
+                "(end-of-life January 1, 2020), please upgrade to Python 3"
+            )
+        else:
+            logger.exception("unexpected error")
         ret = 255
     finally:
         logger.setLevel(outerLogLevel)
+
         # Python 2 fails to close these clean occasionally and users see
         # weird error messages, so we do it manually
         close_pools()
 
+        # Remove cached repos in the end of the call, these are anonymous
+        # so won't be reused by any other subsequent run anyway.
+        clean_repos()
+
     Analytics().send_cmd(cmd, args, ret)
 
     return ret

dvc/output/base.py

@@ -5,8 +5,9 @@ from copy import copy
 
 from schema import Or, Optional
 
+from dvc.cache import NamedCache
 import dvc.prompt as prompt
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, CollectCacheError
 from dvc.utils.compat import str, urlparse
 from dvc.remote.base import RemoteBASE
 
@@ -32,6 +33,13 @@ class OutputAlreadyTrackedError(DvcException):
         super(OutputAlreadyTrackedError, self).__init__(msg)
 
 
+class OutputIsStageFileError(DvcException):
+    def __init__(self, path):
+        super(OutputIsStageFileError, self).__init__(
+            "Stage file '{}' cannot be an output.".format(path)
+        )
+
+
 class OutputBase(object):
     IS_DEPENDENCY = False
 
@@ -57,6 +65,7 @@ class OutputBase(object):
 
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
+    IsStageFileError = OutputIsStageFileError
 
     sep = "/"
 
@@ -71,6 +80,7 @@ class OutputBase(object):
         persist=False,
         tags=None,
     ):
+        self._validate_output_path(path)
         # This output (and dependency) objects have too many paths/urls
         # here is a list and comments:
         #
@@ -104,8 +114,7 @@ class OutputBase(object):
         if remote:
             parsed = urlparse(path)
             return remote.path_info / parsed.path.lstrip("/")
-        else:
-            return self.REMOTE.path_cls(path)
+        return self.REMOTE.path_cls(path)
 
     def __repr__(self):
         return "{class_name}: '{def_path}'".format(
@@ -138,11 +147,6 @@ class OutputBase(object):
     def dir_cache(self):
         return self.cache.get_dir_cache(self.checksum)
 
-    def assign_to_stage_file(self, target_repo):
-        raise DvcException(
-            "change repo is not supported for {}".format(self.scheme)
-        )
-
     @classmethod
     def supported(cls, url):
         return cls.REMOTE.supported(url)
@@ -283,14 +287,15 @@ class OutputBase(object):
 
     def checkout(self, force=False, progress_callback=None, tag=None):
         if not self.use_cache:
-            return
+            progress_callback(str(self.path_info), self.get_files_number())
+            return None
 
         if tag:
             info = self.tags[tag]
         else:
             info = self.info
 
-        self.cache.checkout(
+        return self.cache.checkout(
             self.path_info,
             info,
             force=force,
@@ -319,20 +324,17 @@ class OutputBase(object):
             self.repo.scm.ignore(self.fspath)
 
     def get_files_number(self):
-        if not self.use_cache or not self.checksum:
+        if not self.use_cache:
             return 0
 
-        if self.is_dir_checksum:
-            return len(self.dir_cache)
-
-        return 1
+        return self.cache.get_files_number(self.checksum)
 
     def unprotect(self):
         if self.exists:
             self.remote.unprotect(self.path_info)
 
     def _collect_used_dir_cache(self, remote=None, force=False, jobs=None):
-        """Get a list of `info`s retaled to the given directory.
+        """Get a list of `info`s related to the given directory.
 
         - Pull the directory entry from the remote cache if it was changed.
 
@@ -344,25 +346,19 @@ class OutputBase(object):
             $ echo "bar" > directory/bar
             $ dvc add directory
 
-            It will return something similar to the following list:
+            It will return a NamedCache like:
 
-            [
-                { 'path': 'directory/foo', 'md5': 'c157a79031e1', ... },
-                { 'path': 'directory/bar', 'md5': 'd3b07384d113', ... },
-            ]
+            nc = NamedCache()
+            nc.add(self.scheme, 'c157a79031e1', 'directory/foo')
+            nc.add(self.scheme, 'd3b07384d113', 'directory/bar')
         """
 
-        ret = []
+        cache = NamedCache()
 
         if self.cache.changed_cache_file(self.checksum):
             try:
                 self.repo.cloud.pull(
-                    [
-                        {
-                            self.remote.PARAM_CHECKSUM: self.checksum,
-                            "name": str(self),
-                        }
-                    ],
+                    NamedCache.make("local", self.checksum, str(self)),
                     jobs=jobs,
                     remote=remote,
                     show_checksums=False,
@@ -377,20 +373,19 @@ class OutputBase(object):
                 "Would you like to continue? Use '-f' to force."
             )
             if not force and not prompt.confirm(msg.format(self.path_info)):
-                raise DvcException(
+                raise CollectCacheError(
                     "unable to fully collect used cache"
                     " without cache for directory '{}'".format(self)
                 )
             else:
-                return ret
+                return cache
 
         for entry in self.dir_cache:
-            info = copy(entry)
+            checksum = entry[self.remote.PARAM_CHECKSUM]
             path_info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            info["name"] = str(path_info)
-            ret.append(info)
+            cache.add(self.scheme, checksum, str(path_info))
 
-        return ret
+        return cache
 
     def get_used_cache(self, **kwargs):
         """Get a dumpd of the given `out`, with an entry including the branch.
@@ -401,11 +396,8 @@ class OutputBase(object):
         include the `info` of its files.
         """
 
-        if self.stage.is_repo_import:
-            return []
-
         if not self.use_cache:
-            return []
+            return NamedCache()
 
         if not self.info:
             logger.warning(
@@ -413,13 +405,20 @@ class OutputBase(object):
                 "not be collected. Use dvc repro to get your pipeline up to "
                 "date.".format(self, self.stage)
             )
-            return []
+            return NamedCache()
 
-        ret = [{self.remote.PARAM_CHECKSUM: self.checksum, "name": str(self)}]
+        ret = NamedCache.make(self.scheme, self.checksum, str(self))
 
         if not self.is_dir_checksum:
             return ret
 
-        ret.extend(self._collect_used_dir_cache(**kwargs))
+        ret.update(self._collect_used_dir_cache(**kwargs))
 
         return ret
+
+    @classmethod
+    def _validate_output_path(cls, path):
+        from dvc.stage import Stage
+
+        if Stage.is_valid_filename(path):
+            raise cls.IsStageFileError(path)

dvc/output/local.py

@@ -47,20 +47,6 @@ class OutputLOCAL(OutputBase):
         def_scheme = urlparse(self.def_path).scheme
         return def_scheme != "remote" and not os.path.isabs(self.def_path)
 
-    def assign_to_stage_file(self, stage):
-        from dvc.repo import Repo
-
-        # Make path info the same but relative to passed stage
-        rel_path = self.path_info.relpath(self.stage.wdir)
-        self.path_info = (
-            self.REMOTE.path_cls(os.path.abspath(stage.wdir)) / rel_path
-        )
-
-        self.repo = Repo(self.fspath)
-
-        self.stage = stage
-        return self
-
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
         if self.is_in_repo:

dvc/path_info.py

@@ -36,6 +36,15 @@ class PathInfo(pathlib.PurePath):
     def from_posix(cls, s):
         return cls(PosixPathInfo(s))
 
+    def as_posix(self):
+        f = self._flavour
+        # Unlike original implementation [1] that uses `str()` we actually need
+        # to use `fspath`, because we've overriden `__str__` method to return
+        # relative paths, which will break original `as_posix`.
+        #
+        # [1] https://github.com/python/cpython/blob/v3.7.0/Lib/pathlib.py#L692
+        return self.fspath.replace(f.sep, "/")
+
     def __str__(self):
         path = self.__fspath__()
         return relpath(path)
@@ -94,10 +103,6 @@ class PathInfo(pathlib.PurePath):
         def with_name(self, name):
             return pathlib.PurePath.with_name(self, name.encode(fs_encoding))
 
-        def as_posix(self):
-            f = self._flavour
-            return str(self).replace(f.sep, "/")
-
 
 class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
     pass

dvc/progress.py

@@ -1,154 +1,140 @@
 """Manages progress bars for dvc repo."""
-
 from __future__ import print_function
-from __future__ import unicode_literals
-
-from dvc.utils.compat import str
-
+import logging
 import sys
-import threading
+from tqdm import tqdm
+from concurrent.futures import ThreadPoolExecutor
+from funcy import merge
+from dvc.utils import env2bool
 
-CLEARLINE_PATTERN = "\r\x1b[K"
+logger = logging.getLogger(__name__)
 
 
-class Progress(object):
+class TqdmThreadPoolExecutor(ThreadPoolExecutor):
     """
-    Simple multi-target progress bar.
+    Ensure worker progressbars are cleared away properly.
     """
 
-    def __init__(self):
-        self._n_total = 0
-        self._n_finished = 0
-        self._lock = threading.Lock()
-        self._line = None
-
-    def set_n_total(self, total):
-        """Sets total number of targets."""
-        self._n_total = total
-        self._n_finished = 0
-
-    @property
-    def is_finished(self):
-        """Returns if all targets have finished."""
-        return self._n_total == self._n_finished
-
-    def clearln(self):
-        self._print(CLEARLINE_PATTERN, end="")
-
-    def _writeln(self, line):
-        self.clearln()
-        self._print(line, end="")
-        sys.stdout.flush()
-
-    def reset(self):
-        with self._lock:
-            self._n_total = 0
-            self._n_finished = 0
-            self._line = None
-
-    def refresh(self, line=None):
-        """Refreshes progress bar."""
-        # Just go away if it is locked. Will update next time
-        if not self._lock.acquire(False):
-            return
-
-        if line is None:
-            line = self._line
-
-        if sys.stdout.isatty() and line is not None:
-            self._writeln(line)
-            self._line = line
-
-        self._lock.release()
-
-    def update_target(self, name, current, total):
-        """Updates progress bar for a specified target."""
-        self.refresh(self._bar(name, current, total))
-
-    def finish_target(self, name):
-        """Finishes progress bar for a specified target."""
-        # We have to write a msg about finished target
-        with self._lock:
-            pbar = self._bar(name, 100, 100)
-
-            if sys.stdout.isatty():
-                self.clearln()
-
-            self._print(pbar)
-
-            self._n_finished += 1
-            self._line = None
-
-    def _bar(self, target_name, current, total):
-        """
-        Make a progress bar out of info, which looks like:
-        (1/2): [########################################] 100% master.zip
-        """
-        bar_len = 30
-
-        if total is None:
-            state = 0
-            percent = "?% "
-        else:
-            total = int(total)
-            state = int((100 * current) / total) if current < total else 100
-            percent = str(state) + "% "
-
-        if self._n_total > 1:
-            num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
-        else:
-            num = ""
-
-        n_sh = int((state * bar_len) / 100)
-        n_sp = bar_len - n_sh
-        pbar = "[" + "#" * n_sh + " " * n_sp + "] "
-
-        return num + pbar + percent + target_name
-
-    @staticmethod
-    def _print(*args, **kwargs):
-        import logging
-
-        logger = logging.getLogger(__name__)
-
-        if logger.getEffectiveLevel() == logging.CRITICAL:
-            return
-
-        print(*args, **kwargs)
-
     def __enter__(self):
-        self._lock.acquire(True)
-        if self._line is not None:
-            self.clearln()
+        """
+        Creates a blank initial dummy progress bar if needed so that workers
+        are forced to create "nested" bars.
+        """
+        blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False)
+        if blank_bar.pos > 0:
+            # already nested - don't need a placeholder bar
+            blank_bar.close()
+        self.bar = blank_bar
+        return super(TqdmThreadPoolExecutor, self).__enter__()
 
-    def __exit__(self, typ, value, tbck):
-        if self._line is not None:
-            self.refresh()
-        self._lock.release()
-
-    def __call__(self, seq, name="", total=None):
-        if total is None:
-            total = len(seq)
-
-        self.update_target(name, 0, total)
-        for done, item in enumerate(seq, start=1):
-            yield item
-            self.update_target(name, done, total)
-        self.finish_target(name)
+    def __exit__(self, *a, **k):
+        super(TqdmThreadPoolExecutor, self).__exit__(*a, **k)
+        self.bar.close()
 
 
-class ProgressCallback(object):
-    def __init__(self, total):
-        self.total = total
-        self.current = 0
-        progress.reset()
+class Tqdm(tqdm):
+    """
+    maximum-compatibility tqdm-based progressbars
+    """
 
-    def update(self, name, progress_to_add=1):
-        self.current += progress_to_add
-        progress.update_target(name, self.current, self.total)
+    BAR_FMT_DEFAULT = (
+        "{percentage:3.0f}%|{bar:10}|"
+        "{desc:{ncols_desc}.{ncols_desc}}{n}/{total}"
+        " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
+    )
+    BAR_FMT_NOTOTAL = (
+        "{desc:{ncols_desc}.{ncols_desc}}{n}"
+        " [{elapsed}<??:??, {rate_fmt:>11}{postfix}]"
+    )
 
-    def finish(self, name):
-        progress.finish_target(name)
+    def __init__(
+        self,
+        iterable=None,
+        disable=None,
+        level=logging.ERROR,
+        desc=None,
+        leave=False,
+        bar_format=None,
+        bytes=False,  # pylint: disable=W0622
+        file=None,
+        **kwargs
+    ):
+        """
+        bytes   : shortcut for
+            `unit='B', unit_scale=True, unit_divisor=1024, miniters=1`
+        desc  : persists after `close()`
+        level  : effective logging level for determining `disable`;
+            used only if `disable` is unspecified
+        disable  : If (default: None), will be determined by logging level.
+            May be overridden to `True` due to non-TTY status.
+            Skip override by specifying env var `DVC_IGNORE_ISATTY`.
+        kwargs  : anything accepted by `tqdm.tqdm()`
+        """
+        kwargs = kwargs.copy()
+        kwargs.setdefault("unit_scale", True)
+        if bytes:
+            bytes_defaults = dict(
+                unit="B", unit_scale=True, unit_divisor=1024, miniters=1
+            )
+            kwargs = merge(bytes_defaults, kwargs)
+        if file is None:
+            file = sys.stderr
+        self.desc_persist = desc
+        # auto-disable based on `logger.level`
+        if disable is None:
+            disable = logger.getEffectiveLevel() > level
+        # auto-disable based on TTY
+        if (
+            not disable
+            and not env2bool("DVC_IGNORE_ISATTY")
+            and hasattr(file, "isatty")
+        ):
+            disable = not file.isatty()
+        super(Tqdm, self).__init__(
+            iterable=iterable,
+            disable=disable,
+            leave=leave,
+            desc=desc,
+            bar_format="!",
+            **kwargs
+        )
+        if bar_format is None:
+            if self.__len__():
+                self.bar_format = self.BAR_FMT_DEFAULT
+            else:
+                self.bar_format = self.BAR_FMT_NOTOTAL
+        else:
+            self.bar_format = bar_format
+        self.refresh()
 
+    def update_desc(self, desc, n=1):
+        """
+        Calls `set_description_str(desc)` and `update(n)`
+        """
+        self.set_description_str(desc, refresh=False)
+        self.update(n)
 
-progress = Progress()  # pylint: disable=invalid-name
+    def update_to(self, current, total=None):
+        if total:
+            self.total = total  # pylint: disable=W0613,W0201
+        self.update(current - self.n)
+
+    def close(self):
+        if self.desc_persist is not None:
+            self.set_description_str(self.desc_persist, refresh=False)
+        super(Tqdm, self).close()
+
+    @property
+    def format_dict(self):
+        """inject `ncols_desc` to fill the display width (`ncols`)"""
+        d = super(Tqdm, self).format_dict
+        ncols = d["ncols"] or 80
+        ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1
+        ncols_desc = max(ncols_desc, 0)
+        if ncols_desc:
+            d["ncols_desc"] = ncols_desc
+        else:
+            # work-around for zero-width description
+            d["ncols_desc"] = 1
+            d["prefix"] = ""
+        return d

dvc/remote/azure.py

@@ -4,17 +4,14 @@ from __future__ import unicode_literals
 import os
 import re
 import logging
+from datetime import datetime, timedelta
+
+from funcy import cached_property
 
 from dvc.scheme import Schemes
 
-try:
-    from azure.storage.blob import BlockBlobService
-    from azure.common import AzureMissingResourceHttpError
-except ImportError:
-    BlockBlobService = None
-
 from dvc.utils.compat import urlparse
-from dvc.progress import progress
+from dvc.progress import Tqdm
 from dvc.config import Config
 from dvc.remote.base import RemoteBASE
 from dvc.path_info import CloudURLInfo
@@ -23,14 +20,6 @@ from dvc.path_info import CloudURLInfo
 logger = logging.getLogger(__name__)
 
 
-class Callback(object):
-    def __init__(self, name):
-        self.name = name
-
-    def __call__(self, current, total):
-        progress.update_target(self.name, current, total)
-
-
 class RemoteAZURE(RemoteBASE):
     scheme = Schemes.AZURE
     path_cls = CloudURLInfo
@@ -40,7 +29,7 @@ class RemoteAZURE(RemoteBASE):
         r"(ContainerName=(?P<container_name>[^;]+);?)?"
         r"(?P<connection_string>.+)?)?)$"
     )
-    REQUIRES = {"azure-storage-blob": BlockBlobService}
+    REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
 
@@ -69,29 +58,30 @@ class RemoteAZURE(RemoteBASE):
         if not self.connection_string:
             raise ValueError("azure storage connection string missing")
 
-        self.__blob_service = None
         self.path_info = (
             self.path_cls(url)
             if path
             else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
         )
 
-    @property
+    @cached_property
     def blob_service(self):
-        if self.__blob_service is None:
-            logger.debug("URL {}".format(self.path_info))
-            logger.debug("Connection string {}".format(self.connection_string))
-            self.__blob_service = BlockBlobService(
-                connection_string=self.connection_string
+        from azure.storage.blob import BlockBlobService
+        from azure.common import AzureMissingResourceHttpError
+
+        logger.debug("URL {}".format(self.path_info))
+        logger.debug("Connection string {}".format(self.connection_string))
+        blob_service = BlockBlobService(
+            connection_string=self.connection_string
+        )
+        logger.debug("Container name {}".format(self.path_info.bucket))
+        try:  # verify that container exists
+            blob_service.list_blobs(
+                self.path_info.bucket, delimiter="/", num_results=1
             )
-            logger.debug("Container name {}".format(self.path_info.bucket))
-            try:  # verify that container exists
-                self.__blob_service.list_blobs(
-                    self.path_info.bucket, delimiter="/", num_results=1
-                )
-            except AzureMissingResourceHttpError:
-                self.__blob_service.create_container(self.path_info.bucket)
-        return self.__blob_service
+        except AzureMissingResourceHttpError:
+            blob_service.create_container(self.path_info.bucket)
+        return blob_service
 
     def remove(self, path_info):
         if path_info.scheme != self.scheme:
@@ -122,15 +112,41 @@ class RemoteAZURE(RemoteBASE):
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        cb = None if no_progress_bar else Callback(name)
-        self.blob_service.create_blob_from_path(
-            to_info.bucket, to_info.path, from_file, progress_callback=cb
-        )
+        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
+            self.blob_service.create_blob_from_path(
+                to_info.bucket,
+                to_info.path,
+                from_file,
+                progress_callback=pbar.update_to,
+            )
 
     def _download(
         self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
     ):
-        cb = None if no_progress_bar else Callback(name)
-        self.blob_service.get_blob_to_path(
-            from_info.bucket, from_info.path, to_file, progress_callback=cb
+        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
+            self.blob_service.get_blob_to_path(
+                from_info.bucket,
+                from_info.path,
+                to_file,
+                progress_callback=pbar.update_to,
+            )
+
+    def exists(self, path_info):
+        paths = self._list_paths(path_info.bucket, path_info.path)
+        return any(path_info.path == path for path in paths)
+
+    def _generate_download_url(self, path_info, expires=3600):
+        from azure.storage.blob import BlobPermissions
+
+        expires_at = datetime.utcnow() + timedelta(seconds=expires)
+
+        sas_token = self.blob_service.generate_blob_shared_access_signature(
+            path_info.bucket,
+            path_info.path,
+            permission=BlobPermissions.READ,
+            expiry=expires_at,
         )
+        download_url = self.blob_service.make_blob_url(
+            path_info.bucket, path_info.path, sas_token=sas_token
+        )
+        return download_url

dvc/remote/base.py

@@ -1,16 +1,19 @@
 from __future__ import unicode_literals
 
-from dvc.ignore import DvcIgnore
-from dvc.utils.compat import str, basestring, urlparse, fspath_py35, makedirs
+from operator import itemgetter
+from multiprocessing import cpu_count
 
-import os
 import json
 import logging
 import tempfile
 import itertools
-from operator import itemgetter
-from multiprocessing import cpu_count
+from functools import partial
 from concurrent.futures import ThreadPoolExecutor
+from copy import copy
+
+from dvc.ignore import DvcIgnore
+from dvc.utils.compat import str, basestring, urlparse, FileNotFoundError
+from dvc.remote.slow_link_detection import slow_link_guard
 
 import dvc.prompt as prompt
 from dvc.config import Config
@@ -19,21 +22,19 @@ from dvc.exceptions import (
     ConfirmRemoveError,
     DvcIgnoreInCollectedDirError,
 )
-from dvc.progress import progress, ProgressCallback
-from dvc.utils import LARGE_DIR_SIZE, tmp_fname, to_chunks, move, relpath
+from dvc.progress import Tqdm
+from dvc.utils import tmp_fname, move, relpath, makedirs
 from dvc.state import StateNoop
 from dvc.path_info import PathInfo, URLInfo
-
+from dvc.utils.http import open_url
 
 logger = logging.getLogger(__name__)
 
-
 STATUS_OK = 1
 STATUS_MISSING = 2
 STATUS_NEW = 3
 STATUS_DELETED = 4
 
-
 STATUS_MAP = {
     # (local_exists, remote_exists)
     (True, True): STATUS_OK,
@@ -68,6 +69,14 @@ class RemoteMissingDepsError(DvcException):
     pass
 
 
+class DirCacheError(DvcException):
+    def __init__(self, checksum, cause=None):
+        super(DirCacheError, self).__init__(
+            "Failed to load dir cache for checksum: '{}'.".format(checksum),
+            cause=cause,
+        )
+
+
 class RemoteBASE(object):
     scheme = "base"
     path_cls = URLInfo
@@ -77,36 +86,14 @@ class RemoteBASE(object):
     PARAM_RELPATH = "relpath"
     CHECKSUM_DIR_SUFFIX = ".dir"
     CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
+    DEFAULT_CACHE_TYPES = ["copy"]
 
     state = StateNoop()
 
     def __init__(self, repo, config):
         self.repo = repo
-        deps_ok = all(self.REQUIRES.values())
-        if not deps_ok:
-            missing = [k for k, v in self.REQUIRES.items() if v is None]
-            url = config.get(
-                Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
-            )
-            msg = (
-                "URL '{}' is supported but requires these missing "
-                "dependencies: {}. If you have installed dvc using pip, "
-                "choose one of these options to proceed: \n"
-                "\n"
-                "    1) Install specific missing dependencies:\n"
-                "        pip install {}\n"
-                "    2) Install dvc package that includes those missing "
-                "dependencies: \n"
-                "        pip install dvc[{}]\n"
-                "    3) Install dvc package with all possible "
-                "dependencies included: \n"
-                "        pip install dvc[all]\n"
-                "\n"
-                "If you have installed dvc from a binary package and you "
-                "are still seeing this message, please report it to us "
-                "using https://github.com/iterative/dvc/issues. Thank you!"
-            ).format(url, missing, " ".join(missing), self.scheme)
-            raise RemoteMissingDepsError(msg)
+
+        self._check_requires(config)
 
         core = config.get(Config.SECTION_CORE, {})
         self.checksum_jobs = core.get(
@@ -114,9 +101,55 @@ class RemoteBASE(object):
         )
 
         self.protected = False
-        self.no_traverse = config.get(Config.SECTION_REMOTE_NO_TRAVERSE)
+        self.no_traverse = config.get(Config.SECTION_REMOTE_NO_TRAVERSE, True)
         self._dir_info = {}
 
+        types = config.get(Config.SECTION_CACHE_TYPE, None)
+        if types:
+            if isinstance(types, str):
+                types = [t.strip() for t in types.split(",")]
+            self.cache_types = types
+        else:
+            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
+        self.cache_type_confirmed = False
+
+    def _check_requires(self, config):
+        import importlib
+
+        missing = []
+
+        for package, module in self.REQUIRES.items():
+            try:
+                importlib.import_module(module)
+            except ImportError:
+                missing.append(package)
+
+        if not missing:
+            return
+
+        url = config.get(
+            Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
+        )
+        msg = (
+            "URL '{}' is supported but requires these missing "
+            "dependencies: {}. If you have installed dvc using pip, "
+            "choose one of these options to proceed: \n"
+            "\n"
+            "    1) Install specific missing dependencies:\n"
+            "        pip install {}\n"
+            "    2) Install dvc package that includes those missing "
+            "dependencies: \n"
+            "        pip install 'dvc[{}]'\n"
+            "    3) Install dvc package with all possible "
+            "dependencies included: \n"
+            "        pip install 'dvc[all]'\n"
+            "\n"
+            "If you have installed dvc from a binary package and you "
+            "are still seeing this message, please report it to us "
+            "using https://github.com/iterative/dvc/issues. Thank you!"
+        ).format(url, missing, " ".join(missing), self.scheme)
+        raise RemoteMissingDepsError(msg)
+
     def __repr__(self):
         return "{class_name}: '{path_info}'".format(
             class_name=type(self).__name__,
@@ -145,29 +178,23 @@ class RemoteBASE(object):
         file_infos = list(file_infos)
         with ThreadPoolExecutor(max_workers=self.checksum_jobs) as executor:
             tasks = executor.map(self.get_file_checksum, file_infos)
-
-            if len(file_infos) > LARGE_DIR_SIZE:
-                msg = (
-                    "Computing md5 for a large number of files. "
-                    "This is only done once."
-                )
-                logger.info(msg)
-                tasks = progress(tasks, total=len(file_infos))
-
-        checksums = {
-            file_infos[index]: task for index, task in enumerate(tasks)
-        }
+            with Tqdm(
+                tasks,
+                total=len(file_infos),
+                unit="md5",
+                desc="Computing hashes (only done once)",
+            ) as tasks:
+                checksums = dict(zip(file_infos, tasks))
         return checksums
 
     def _collect_dir(self, path_info):
-
         file_infos = set()
-        for root, _dirs, files in self.walk(path_info):
 
-            if DvcIgnore.DVCIGNORE_FILE in files:
-                raise DvcIgnoreInCollectedDirError(root)
+        for fname in self.walk_files(path_info):
+            if DvcIgnore.DVCIGNORE_FILE == fname.name:
+                raise DvcIgnoreInCollectedDirError(fname.parent)
 
-            file_infos.update(path_info / root / fname for fname in files)
+            file_infos.add(fname)
 
         checksums = {fi: self.state.get(fi) for fi in file_infos}
         not_in_state = {
@@ -202,6 +229,7 @@ class RemoteBASE(object):
         checksum, tmp_info = self._get_dir_info_checksum(dir_info)
         new_info = self.cache.checksum_to_path_info(checksum)
         if self.cache.changed_cache_file(checksum):
+            self.cache.makedirs(new_info.parent)
             self.cache.move(tmp_info, new_info)
 
         self.state.save(path_info, checksum)
@@ -228,30 +256,26 @@ class RemoteBASE(object):
         if dir_info:
             return dir_info
 
-        dir_info = self.load_dir_cache(checksum)
+        try:
+            dir_info = self.load_dir_cache(checksum)
+        except DirCacheError:
+            dir_info = []
+
         self._dir_info[checksum] = dir_info
         return dir_info
 
     def load_dir_cache(self, checksum):
         path_info = self.checksum_to_path_info(checksum)
 
-        fobj = tempfile.NamedTemporaryFile(delete=False)
-        path = fobj.name
-        to_info = PathInfo(path)
-        self.cache.download(path_info, to_info, no_progress_bar=True)
-
         try:
-            with open(path, "r") as fobj:
+            with self.cache.open(path_info, "r") as fobj:
                 d = json.load(fobj)
-        except ValueError:
-            logger.exception("Failed to load dir cache '{}'".format(path_info))
-            return []
-        finally:
-            os.unlink(path)
+        except (ValueError, FileNotFoundError) as exc:
+            raise DirCacheError(checksum, cause=exc)
 
         if not isinstance(d, list):
             msg = "dir cache file format error '{}' [skipping the file]"
-            logger.error(msg.format(relpath(path)))
+            logger.error(msg.format(relpath(path_info)))
             return []
 
         for info in d:
@@ -348,8 +372,48 @@ class RemoteBASE(object):
         logger.debug("'{}' hasn't changed.".format(path_info))
         return False
 
-    def link(self, from_info, to_info, link_type=None):
-        self.copy(from_info, to_info)
+    def link(self, from_info, to_info):
+        self._link(from_info, to_info, self.cache_types)
+
+    def _link(self, from_info, to_info, link_types):
+        assert self.isfile(from_info)
+
+        self.makedirs(to_info.parent)
+
+        self._try_links(from_info, to_info, link_types)
+
+    @slow_link_guard
+    def _try_links(self, from_info, to_info, link_types):
+        while link_types:
+            link_method = getattr(self, link_types[0])
+            try:
+                self._do_link(from_info, to_info, link_method)
+                self.cache_type_confirmed = True
+                return
+
+            except DvcException as exc:
+                msg = "Cache type '{}' is not supported: {}"
+                logger.debug(msg.format(link_types[0], str(exc)))
+                del link_types[0]
+
+        raise DvcException("no possible cache types left to try out.")
+
+    def _do_link(self, from_info, to_info, link_method):
+        if self.exists(to_info):
+            raise DvcException("Link '{}' already exists!".format(to_info))
+
+        link_method(from_info, to_info)
+
+        if self.protected:
+            self.protect(to_info)
+
+        msg = "Created {}'{}': {} -> {}".format(
+            "protected " if self.protected else "",
+            self.cache_types[0],
+            from_info,
+            to_info,
+        )
+        logger.debug(msg)
 
     def _save_file(self, path_info, checksum, save_link=True):
         assert checksum
@@ -388,12 +452,19 @@ class RemoteBASE(object):
         return False
 
     def isfile(self, path_info):
-        raise NotImplementedError
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return True
 
     def isdir(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
         return False
 
-    def walk(self, path_info):
+    def walk_files(self, path_info):
+        """Return a generator with `PathInfo`s to all the files"""
         raise NotImplementedError
 
     @staticmethod
@@ -416,7 +487,7 @@ class RemoteBASE(object):
 
     def _save(self, path_info, checksum):
         to_info = self.checksum_to_path_info(checksum)
-        logger.info("Saving '{}' to '{}'.".format(path_info, to_info))
+        logger.debug("Saving '{}' to '{}'.".format(path_info, to_info))
         if self.isdir(path_info):
             self._save_dir(path_info, checksum)
             return
@@ -436,9 +507,6 @@ class RemoteBASE(object):
 
         name = name or from_info.name
 
-        if not no_progress_bar:
-            progress.update_target(name, 0, None)
-
         try:
             self._upload(
                 from_info.fspath,
@@ -451,12 +519,17 @@ class RemoteBASE(object):
             logger.exception(msg.format(from_info, to_info))
             return 1  # 1 fail
 
-        if not no_progress_bar:
-            progress.finish_target(name)
-
         return 0
 
-    def download(self, from_info, to_info, name=None, no_progress_bar=False):
+    def download(
+        self,
+        from_info,
+        to_info,
+        name=None,
+        no_progress_bar=False,
+        file_mode=None,
+        dir_mode=None,
+    ):
         if not hasattr(self, "_download"):
             raise RemoteActionNotImplemented("download", self.scheme)
 
@@ -474,12 +547,7 @@ class RemoteBASE(object):
 
         name = name or to_info.name
 
-        if not no_progress_bar:
-            # real progress is not always available,
-            # lets at least show start and finish
-            progress.update_target(name, 0, None)
-
-        makedirs(fspath_py35(to_info.parent), exist_ok=True)
+        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
         tmp_file = tmp_fname(to_info)
 
         try:
@@ -491,13 +559,17 @@ class RemoteBASE(object):
             logger.exception(msg.format(from_info, to_info))
             return 1  # 1 fail
 
-        move(tmp_file, fspath_py35(to_info))
-
-        if not no_progress_bar:
-            progress.finish_target(name)
+        move(tmp_file, to_info, mode=file_mode)
 
         return 0
 
+    def open(self, path_info, mode="r", encoding=None):
+        if hasattr(self, "_generate_download_url"):
+            get_url = partial(self._generate_download_url, path_info)
+            return open_url(get_url, mode=mode, encoding=encoding)
+
+        raise RemoteActionNotImplemented("open", self.scheme)
+
     def remove(self, path_info):
         raise RemoteActionNotImplemented("remove", self.scheme)
 
@@ -508,11 +580,25 @@ class RemoteBASE(object):
     def copy(self, from_info, to_info):
         raise RemoteActionNotImplemented("copy", self.scheme)
 
+    def symlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("symlink", self.scheme)
+
+    def hardlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("hardlink", self.scheme)
+
+    def reflink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("reflink", self.scheme)
+
     def exists(self, path_info):
         raise NotImplementedError
 
     def path_to_checksum(self, path):
-        return "".join(self.path_cls(path).parts[-2:])
+        parts = self.path_cls(path).parts[-2:]
+
+        if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
+            raise ValueError("Bad cache file path")
+
+        return "".join(parts)
 
     def checksum_to_path_info(self, checksum):
         return self.path_info / checksum[0:2] / checksum[2:]
@@ -525,18 +611,18 @@ class RemoteBASE(object):
         # is 32 bytes, so ~3200Mb list) and we don't really need all of it at
         # the same time, so it makes sense to use a generator to gradually
         # iterate over it, without keeping all of it in memory.
-        return (
-            self.path_to_checksum(path) for path in self.list_cache_paths()
-        )
+        for path in self.list_cache_paths():
+            try:
+                yield self.path_to_checksum(path)
+            except ValueError:
+                # We ignore all the non-cache looking files
+                pass
 
-    def gc(self, cinfos):
-        used = self.extract_used_local_checksums(cinfos)
+    def gc(self, named_cache):
+        used = self.extract_used_local_checksums(named_cache)
 
         if self.scheme != "":
-            used |= {
-                info[self.PARAM_CHECKSUM]
-                for info in cinfos.get(self.scheme, [])
-            }
+            used.update(named_cache[self.scheme])
 
         removed = False
         for checksum in self.all():
@@ -600,7 +686,7 @@ class RemoteBASE(object):
             return self._changed_dir_cache(checksum)
         return self.changed_cache_file(checksum)
 
-    def cache_exists(self, checksums, jobs=None):
+    def cache_exists(self, checksums, jobs=None, name=None):
         """Check if the given checksums are stored in the remote.
 
         There are two ways of performing this check:
@@ -618,27 +704,31 @@ class RemoteBASE(object):
         take much shorter time to just retrieve everything they have under
         a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
         check if particular file exists much quicker, use their own
-        implementation of cache_exists (see http, local).
+        implementation of cache_exists (see ssh, local).
 
         Returns:
             A list with checksums that were found in the remote
         """
-        progress_callback = ProgressCallback(len(checksums))
+        if not self.no_traverse:
+            return list(set(checksums) & set(self.all()))
 
-        def exists_with_progress(chunks):
-            return self.batch_exists(chunks, callback=progress_callback)
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(checksums),
+            unit="file",
+        ) as pbar:
 
-        if self.no_traverse and hasattr(self, "batch_exists"):
-            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
-                path_infos = [self.checksum_to_path_info(x) for x in checksums]
-                chunks = to_chunks(path_infos, num_chunks=self.JOBS)
-                results = executor.map(exists_with_progress, chunks)
-                in_remote = itertools.chain.from_iterable(results)
-                ret = list(itertools.compress(checksums, in_remote))
-                progress_callback.finish("")
+            def exists_with_progress(path_info):
+                ret = self.exists(path_info)
+                pbar.update_desc(str(path_info))
                 return ret
 
-        return list(set(checksums) & set(self.all()))
+            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
+                path_infos = map(self.checksum_to_path_info, checksums)
+                in_remote = executor.map(exists_with_progress, path_infos)
+                ret = list(itertools.compress(checksums, in_remote))
+                return ret
 
     def already_cached(self, path_info):
         current = self.get_checksum(path_info)
@@ -664,22 +754,50 @@ class RemoteBASE(object):
         self.remove(path_info)
 
     def _checkout_file(
-        self, path_info, checksum, force, progress_callback=None
+        self,
+        path_info,
+        checksum,
+        force,
+        progress_callback=None,
+        save_link=True,
     ):
-        cache_info = self.checksum_to_path_info(checksum)
-        if self.exists(path_info):
-            msg = "data '{}' exists. Removing before checkout."
-            logger.warning(msg.format(str(path_info)))
+        # NOTE: In case if path_info is already cached and path_info's
+        # link type matches cache link type, we would like to avoid
+        # relinking.
+        if self.changed(
+            path_info, {self.PARAM_CHECKSUM: checksum}
+        ) or not self._link_matches(path_info):
             self.safe_remove(path_info, force=force)
 
-        self.link(cache_info, path_info)
-        self.state.save_link(path_info)
-        self.state.save(path_info, checksum)
+            cache_info = self.checksum_to_path_info(checksum)
+            self.link(cache_info, path_info)
+
+            if save_link:
+                self.state.save_link(path_info)
+
+            self.state.save(path_info, checksum)
+        else:
+            # NOTE: performing (un)protection costs us +/- the same as checking
+            # if path_info is protected. Instead of implementing logic,
+            # just (un)protect according to self.protected.
+            if self.protected:
+                self.protect(path_info)
+            else:
+                # NOTE dont allow copy, because we checked before that link
+                # type matches cache, and we don't want data duplication
+                self.unprotect(path_info, allow_copy=False)
+
         if progress_callback:
-            progress_callback.update(str(path_info))
+            progress_callback(str(path_info))
+
+    def _link_matches(self, path_info):
+        return True
 
     def makedirs(self, path_info):
-        raise NotImplementedError
+        """Optional: Implement only if the remote needs to create
+        directories before copying/linking/moving data
+        """
+        pass
 
     def _checkout_dir(
         self, path_info, checksum, force, progress_callback=None
@@ -696,17 +814,14 @@ class RemoteBASE(object):
         for entry in dir_info:
             relative_path = entry[self.PARAM_RELPATH]
             entry_checksum = entry[self.PARAM_CHECKSUM]
-            entry_cache_info = self.checksum_to_path_info(entry_checksum)
             entry_info = path_info / relative_path
-
-            entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
-            if self.changed(entry_info, entry_checksum_info):
-                if self.exists(entry_info):
-                    self.safe_remove(entry_info, force=force)
-                self.link(entry_cache_info, entry_info)
-                self.state.save(entry_info, entry_checksum)
-            if progress_callback:
-                progress_callback.update(str(entry_info))
+            self._checkout_file(
+                entry_info,
+                entry_checksum,
+                force,
+                progress_callback,
+                save_link=False,
+            )
 
         self._remove_redundant_files(path_info, dir_info, force)
 
@@ -714,11 +829,7 @@ class RemoteBASE(object):
         self.state.save(path_info, checksum)
 
     def _remove_redundant_files(self, path_info, dir_info, force):
-        existing_files = set(
-            path_info / root / fname
-            for root, _, files in self.walk(path_info)
-            for fname in files
-        )
+        existing_files = set(self.walk_files(path_info))
 
         needed_files = {
             path_info / entry[self.PARAM_RELPATH] for entry in dir_info
@@ -734,29 +845,33 @@ class RemoteBASE(object):
             raise NotImplementedError
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
+        failed = None
         if not checksum:
             logger.warning(
                 "No checksum info found for '{}'. "
                 "It won't be created.".format(str(path_info))
             )
             self.safe_remove(path_info, force=force)
-            return
+            failed = path_info
 
-        if not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            logger.debug(msg.format(str(path_info)))
-            return
-
-        if self.changed_cache(checksum):
+        elif self.changed_cache(checksum):
             msg = "Cache '{}' not found. File '{}' won't be created."
             logger.warning(msg.format(checksum, str(path_info)))
             self.safe_remove(path_info, force=force)
-            return
+            failed = path_info
+
+        if failed:
+            if progress_callback:
+                progress_callback(
+                    str(path_info), self.get_files_number(checksum)
+                )
+            return failed
 
         msg = "Checking out '{}' with cache '{}'."
         logger.debug(msg.format(str(path_info), checksum))
 
-        return self._checkout(path_info, checksum, force, progress_callback)
+        self._checkout(path_info, checksum, force, progress_callback)
+        return None
 
     def _checkout(
         self, path_info, checksum, force=False, progress_callback=None
@@ -769,22 +884,29 @@ class RemoteBASE(object):
             path_info, checksum, force, progress_callback=progress_callback
         )
 
+    def get_files_number(self, checksum):
+        if not checksum:
+            return 0
+
+        if self.is_dir_checksum(checksum):
+            return len(self.get_dir_cache(checksum))
+
+        return 1
+
     @staticmethod
-    def unprotect(path_info):
+    def unprotect(path_info, allow_copy=True):
         pass
 
     def _get_unpacked_dir_names(self, checksums):
         return set()
 
-    def extract_used_local_checksums(self, cinfos):
-        from dvc.remote import RemoteLOCAL
-
-        used = {info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]}
+    def extract_used_local_checksums(self, named_cache):
+        used = set(named_cache["local"])
         unpacked = self._get_unpacked_dir_names(used)
         return used | unpacked
 
     def _changed_unpacked_dir(self, checksum):
         return True
 
-    def _update_unpacked_dir(self, checksum, progress_callback=None):
+    def _update_unpacked_dir(self, checksum):
         pass

dvc/remote/gs.py

@@ -1,28 +1,60 @@
 from __future__ import unicode_literals
 
 import logging
-import itertools
+from datetime import timedelta
+from functools import wraps
+
 from funcy import cached_property
 
-try:
-    from google.cloud import storage
-except ImportError:
-    storage = None
-
+from dvc.utils.compat import FileNotFoundError  # skipcq: PYL-W0622
 from dvc.remote.base import RemoteBASE
 from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.scheme import Schemes
 
-
 logger = logging.getLogger(__name__)
 
 
+def dynamic_chunk_size(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        import requests
+        from google.cloud.storage.blob import Blob, _DEFAULT_CHUNKSIZE
+
+        # Default chunk size for gs is 100M, which might be too much for
+        # particular network (see [1]). So if we are getting ConnectionError,
+        # we should try lowering the chunk size until we reach the minimum
+        # allowed chunk size of 256K. Also note that `chunk_size` must be a
+        # multiple of 256K per the API specification.
+        #
+        # [1] https://github.com/iterative/dvc/issues/2572
+
+        # skipcq: PYL-W0212
+        multiplier = int(_DEFAULT_CHUNKSIZE / Blob._CHUNK_SIZE_MULTIPLE)
+        while True:
+            try:
+                # skipcq: PYL-W0212
+                chunk_size = Blob._CHUNK_SIZE_MULTIPLE * multiplier
+                return func(*args, chunk_size=chunk_size, **kwargs)
+            except requests.exceptions.ConnectionError:
+                multiplier = int(multiplier / 2)
+                if not multiplier:
+                    raise
+
+    return wrapper
+
+
+@dynamic_chunk_size
+def _upload_to_bucket(bucket, from_file, to_info, **kwargs):
+    blob = bucket.blob(to_info.path, **kwargs)
+    blob.upload_from_filename(from_file)
+
+
 class RemoteGS(RemoteBASE):
     scheme = Schemes.GS
     path_cls = CloudURLInfo
-    REQUIRES = {"google.cloud.storage": storage}
+    REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
     PARAM_CHECKSUM = "md5"
 
     def __init__(self, repo, config):
@@ -37,10 +69,12 @@ class RemoteGS(RemoteBASE):
 
     @cached_property
     def gs(self):
+        from google.cloud.storage import Client
+
         return (
-            storage.Client.from_service_account_json(self.credentialpath)
+            Client.from_service_account_json(self.credentialpath)
             if self.credentialpath
-            else storage.Client(self.projectname)
+            else Client(self.projectname)
         )
 
     def get_file_checksum(self, path_info):
@@ -89,23 +123,20 @@ class RemoteGS(RemoteBASE):
         paths = set(self._list_paths(path_info.bucket, path_info.path))
         return any(path_info.path == path for path in paths)
 
-    def batch_exists(self, path_infos, callback):
-        paths = []
-
-        for path_info in path_infos:
-            paths.append(self._list_paths(path_info.bucket, path_info.path))
-            callback.update(str(path_info))
-
-        paths = set(itertools.chain.from_iterable(paths))
-
-        return [path_info.path in paths for path_info in path_infos]
-
     def _upload(self, from_file, to_info, **_kwargs):
         bucket = self.gs.bucket(to_info.bucket)
-        blob = bucket.blob(to_info.path)
-        blob.upload_from_filename(from_file)
+        _upload_to_bucket(bucket, from_file, to_info)
 
     def _download(self, from_info, to_file, **_kwargs):
         bucket = self.gs.bucket(from_info.bucket)
         blob = bucket.get_blob(from_info.path)
         blob.download_to_filename(to_file)
+
+    def _generate_download_url(self, path_info, expires=3600):
+        expiration = timedelta(seconds=int(expires))
+
+        bucket = self.gs.bucket(path_info.bucket)
+        blob = bucket.get_blob(path_info.path)
+        if blob is None:
+            raise FileNotFoundError
+        return blob.generate_signed_url(expiration=expiration)

dvc/remote/hdfs.py

@@ -2,20 +2,16 @@ from __future__ import unicode_literals
 
 import os
 import re
-import getpass
+import io
 import posixpath
 import logging
 from subprocess import Popen, PIPE
-
-try:
-    import pyarrow
-except ImportError:
-    pyarrow = None
+from contextlib import contextmanager, closing
 
 from dvc.config import Config
 from dvc.scheme import Schemes
 
-from dvc.utils.compat import urlparse
+from dvc.utils.compat import urlparse, FileNotFoundError
 from dvc.utils import fix_env, tmp_fname
 
 from .pool import get_connection
@@ -28,7 +24,7 @@ class RemoteHDFS(RemoteBASE):
     scheme = Schemes.HDFS
     REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
     PARAM_CHECKSUM = "checksum"
-    REQUIRES = {"pyarrow": pyarrow}
+    REQUIRES = {"pyarrow": "pyarrow"}
 
     def __init__(self, repo, config):
         super(RemoteHDFS, self).__init__(repo, config)
@@ -39,11 +35,7 @@ class RemoteHDFS(RemoteBASE):
 
         parsed = urlparse(url)
 
-        user = (
-            parsed.username
-            or config.get(Config.SECTION_REMOTE_USER)
-            or getpass.getuser()
-        )
+        user = parsed.username or config.get(Config.SECTION_REMOTE_USER)
 
         self.path_info = self.path_cls.from_parts(
             scheme=self.scheme,
@@ -53,8 +45,9 @@ class RemoteHDFS(RemoteBASE):
             path=parsed.path,
         )
 
-    @staticmethod
-    def hdfs(path_info):
+    def hdfs(self, path_info):
+        import pyarrow
+
         return get_connection(
             pyarrow.hdfs.connect,
             path_info.host,
@@ -139,6 +132,25 @@ class RemoteHDFS(RemoteBASE):
             with open(to_file, "wb+") as fobj:
                 hdfs.download(from_info.path, fobj)
 
+    @contextmanager
+    def open(self, path_info, mode="r", encoding=None):
+        assert mode in {"r", "rt", "rb"}
+
+        try:
+            with self.hdfs(path_info) as hdfs, closing(
+                hdfs.open(path_info.path, mode="rb")
+            ) as fd:
+                if mode == "rb":
+                    yield fd
+                else:
+                    yield io.TextIOWrapper(fd, encoding=encoding)
+        except IOError as e:
+            # Empty .errno and not specific enough error class in pyarrow,
+            # see https://issues.apache.org/jira/browse/ARROW-6248
+            if "file does not exist" in str(e):
+                raise FileNotFoundError(*e.args)
+            raise
+
     def list_cache_paths(self):
         if not self.exists(self.path_info):
             return []

dvc/remote/http.py

@@ -1,37 +1,23 @@
 from __future__ import unicode_literals
 
-from dvc.scheme import Schemes
-
-from dvc.utils.compat import open
-
-import threading
-import requests
 import logging
 
-from dvc.progress import progress
-from dvc.exceptions import DvcException
-from dvc.config import Config
-from dvc.remote.base import RemoteBASE
+from funcy import cached_property
 
+from dvc.scheme import Schemes
+from dvc.utils.compat import open
+from dvc.progress import Tqdm
+from dvc.exceptions import DvcException
+from dvc.config import Config, ConfigError
+from dvc.remote.base import RemoteBASE
 
 logger = logging.getLogger(__name__)
 
 
-class ProgressBarCallback(object):
-    def __init__(self, name, total):
-        self.name = name
-        self.total = total
-        self.current = 0
-        self.lock = threading.Lock()
-
-    def __call__(self, byts):
-        with self.lock:
-            self.current += byts
-            progress.update_target(self.name, self.current, self.total)
-
-
 class RemoteHTTP(RemoteBASE):
     scheme = Schemes.HTTP
+    SESSION_RETRIES = 5
+    SESSION_BACKOFF_FACTOR = 0.1
     REQUEST_TIMEOUT = 10
     CHUNK_SIZE = 2 ** 16
     PARAM_CHECKSUM = "etag"
@@ -42,46 +28,43 @@ class RemoteHTTP(RemoteBASE):
         url = config.get(Config.SECTION_REMOTE_URL)
         self.path_info = self.path_cls(url) if url else None
 
+        if not self.no_traverse:
+            raise ConfigError(
+                "HTTP doesn't support traversing the remote to list existing "
+                "files. Use: `dvc remote modify <name> no_traverse true`"
+            )
+
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        callback = None
-        if not no_progress_bar:
-            total = self._content_length(from_info.url)
-            if total:
-                callback = ProgressBarCallback(name, total)
-
         request = self._request("GET", from_info.url, stream=True)
-
-        with open(to_file, "wb") as fd:
-            transferred_bytes = 0
-
-            for chunk in request.iter_content(chunk_size=self.CHUNK_SIZE):
-                fd.write(chunk)
-                fd.flush()
-                transferred_bytes += len(chunk)
-
-                if callback:
-                    callback(transferred_bytes)
+        with Tqdm(
+            total=None if no_progress_bar else self._content_length(from_info),
+            leave=False,
+            bytes=True,
+            desc=from_info.url if name is None else name,
+            disable=no_progress_bar,
+        ) as pbar:
+            with open(to_file, "wb") as fd:
+                for chunk in request.iter_content(chunk_size=self.CHUNK_SIZE):
+                    fd.write(chunk)
+                    fd.flush()
+                    pbar.update(len(chunk))
 
     def exists(self, path_info):
         return bool(self._request("HEAD", path_info.url))
 
-    def batch_exists(self, path_infos, callback):
-        results = []
-
-        for path_info in path_infos:
-            results.append(self.exists(path_info))
-            callback.update(str(path_info))
-
-        return results
-
-    def _content_length(self, url):
-        return self._request("HEAD", url).headers.get("Content-Length")
+    def _content_length(self, url_or_request):
+        headers = getattr(
+            url_or_request,
+            "headers",
+            self._request("HEAD", url_or_request).headers,
+        )
+        res = headers.get("Content-Length")
+        return int(res) if res else None
 
     def get_file_checksum(self, path_info):
         url = path_info.url
-        etag = self._request("HEAD", url).headers.get("ETag") or self._request(
-            "HEAD", url
-        ).headers.get("Content-MD5")
+        headers = self._request("HEAD", url).headers
+        etag = headers.get("ETag") or headers.get("Content-MD5")
 
         if not etag:
             raise DvcException(
@@ -97,12 +80,32 @@ class RemoteHTTP(RemoteBASE):
 
         return etag
 
+    @cached_property
+    def _session(self):
+        import requests
+        from requests.adapters import HTTPAdapter
+        from urllib3.util.retry import Retry
+
+        session = requests.Session()
+
+        retries = Retry(
+            total=self.SESSION_RETRIES,
+            backoff_factor=self.SESSION_BACKOFF_FACTOR,
+        )
+
+        session.mount("http://", HTTPAdapter(max_retries=retries))
+        session.mount("https://", HTTPAdapter(max_retries=retries))
+
+        return session
+
     def _request(self, method, url, **kwargs):
+        import requests
+
         kwargs.setdefault("allow_redirects", True)
         kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
 
         try:
-            return requests.request(method, url, **kwargs)
+            return self._session.request(method, url, **kwargs)
         except requests.exceptions.RequestException:
             raise DvcException("could not perform a {} request".format(method))
 

dvc/remote/local.py

@@ -1,17 +1,14 @@
 from __future__ import unicode_literals
-
-from copy import copy
-
-from dvc.scheme import Schemes
-from dvc.remote.local.slow_link_detection import slow_link_guard
-from dvc.utils.compat import str, makedirs, fspath_py35
-
 import os
 import stat
-from shortuuid import uuid
-import shutil
+import errno
 import logging
+from functools import partial
 
+from shortuuid import uuid
+
+from dvc.scheme import Schemes
+from dvc.utils.compat import str, fspath_py35, open
 from dvc.system import System
 from dvc.remote.base import (
     RemoteBASE,
@@ -28,12 +25,11 @@ from dvc.utils import (
     file_md5,
     walk_files,
     relpath,
-    dvc_walk,
+    makedirs,
 )
 from dvc.config import Config
-from dvc.exceptions import DvcException
-from dvc.progress import progress
-from concurrent.futures import ThreadPoolExecutor
+from dvc.exceptions import DvcException, DownloadError, UploadError
+from dvc.progress import Tqdm, TqdmThreadPoolExecutor
 
 from dvc.path_info import PathInfo
 
@@ -49,24 +45,19 @@ class RemoteLOCAL(RemoteBASE):
     UNPACKED_DIR_SUFFIX = ".unpacked"
 
     DEFAULT_CACHE_TYPES = ["reflink", "copy"]
-    CACHE_TYPE_MAP = {
-        "copy": shutil.copyfile,
-        "symlink": System.symlink,
-        "hardlink": System.hardlink,
-        "reflink": System.reflink,
-    }
+
+    SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
 
     def __init__(self, repo, config):
         super(RemoteLOCAL, self).__init__(repo, config)
         self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
 
-        types = config.get(Config.SECTION_CACHE_TYPE, None)
-        if types:
-            if isinstance(types, str):
-                types = [t.strip() for t in types.split(",")]
-            self.cache_types = types
-        else:
-            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
+        shared = config.get(Config.SECTION_CACHE_SHARED)
+        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
+
+        if self.protected:
+            # cache files are set to be read-only for everyone
+            self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
 
         # A clunky way to detect cache dir
         storagepath = config.get(Config.SECTION_LOCAL_STORAGEPATH, None)
@@ -76,9 +67,6 @@ class RemoteLOCAL(RemoteBASE):
             cwd = config[Config.PRIVATE_CWD]
             cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
 
-        if cache_dir is not None and not os.path.exists(cache_dir):
-            os.mkdir(cache_dir)
-
         self.path_info = PathInfo(cache_dir) if cache_dir else None
         self._dir_info = {}
 
@@ -114,81 +102,13 @@ class RemoteLOCAL(RemoteBASE):
 
         return self.checksum_to_path_info(md5).url
 
-    def exists(self, path_info):
+    @staticmethod
+    def exists(path_info):
         assert path_info.scheme == "local"
         return os.path.lexists(fspath_py35(path_info))
 
     def makedirs(self, path_info):
-        if not self.exists(path_info):
-            os.makedirs(fspath_py35(path_info))
-
-    def link(self, from_info, to_info, link_type=None):
-        from_path = from_info.fspath
-        to_path = to_info.fspath
-
-        assert os.path.isfile(from_path)
-
-        dname = os.path.dirname(to_path)
-        if not os.path.exists(dname):
-            os.makedirs(dname)
-
-        # NOTE: just create an empty file for an empty cache
-        if os.path.getsize(from_path) == 0:
-            open(to_path, "w+").close()
-
-            msg = "Created empty file: {} -> {}".format(from_path, to_path)
-            logger.debug(msg)
-            return
-
-        if not link_type:
-            link_types = self.cache_types
-        else:
-            link_types = [link_type]
-
-        self._try_links(from_info, to_info, link_types)
-
-    @classmethod
-    def _get_link_method(cls, link_type):
-        try:
-            return cls.CACHE_TYPE_MAP[link_type]
-        except KeyError:
-            raise DvcException(
-                "Cache type: '{}' not supported!".format(link_type)
-            )
-
-    def _link(self, from_info, to_info, link_method):
-        if self.exists(to_info):
-            raise DvcException("Link '{}' already exists!".format(to_info))
-        else:
-            link_method(from_info.fspath, to_info.fspath)
-
-        if self.protected:
-            self.protect(to_info)
-
-        msg = "Created {}'{}': {} -> {}".format(
-            "protected " if self.protected else "",
-            self.cache_types[0],
-            from_info,
-            to_info,
-        )
-        logger.debug(msg)
-
-    @slow_link_guard
-    def _try_links(self, from_info, to_info, link_types):
-        i = len(link_types)
-        while i > 0:
-            link_method = self._get_link_method(link_types[0])
-            try:
-                self._link(from_info, to_info, link_method)
-                return
-
-            except DvcException as exc:
-                msg = "Cache type '{}' is not supported: {}"
-                logger.debug(msg.format(link_types[0], str(exc)))
-                del link_types[0]
-                i -= 1
-
-        raise DvcException("no possible cache types left to try out.")
+        makedirs(path_info, exist_ok=True, mode=self._dir_mode)
 
     def already_cached(self, path_info):
         assert path_info.scheme in ["", "local"]
@@ -211,14 +131,21 @@ class RemoteLOCAL(RemoteBASE):
 
         return False
 
-    def isfile(self, path_info):
+    @staticmethod
+    def isfile(path_info):
         return os.path.isfile(fspath_py35(path_info))
 
-    def isdir(self, path_info):
+    @staticmethod
+    def isdir(path_info):
         return os.path.isdir(fspath_py35(path_info))
 
-    def walk(self, path_info):
-        return dvc_walk(path_info, self.repo.dvcignore)
+    @staticmethod
+    def getsize(path_info):
+        return os.path.getsize(fspath_py35(path_info))
+
+    def walk_files(self, path_info):
+        for fname in walk_files(path_info, self.repo.dvcignore):
+            yield PathInfo(fname)
 
     def get_file_checksum(self, path_info):
         return file_md5(fspath_py35(path_info))[0]
@@ -234,20 +161,62 @@ class RemoteLOCAL(RemoteBASE):
         if from_info.scheme != "local" or to_info.scheme != "local":
             raise NotImplementedError
 
-        inp = from_info.fspath
-        outp = to_info.fspath
+        self.makedirs(to_info.parent)
 
-        # moving in two stages to make the whole operation atomic in
-        # case inp and outp are in different filesystems and actual
-        # physical copying of data is happening
-        tmp = "{}.{}".format(outp, str(uuid()))
-        move(inp, tmp)
-        move(tmp, outp)
+        if self.isfile(from_info):
+            mode = self._file_mode
+        else:
+            mode = self._dir_mode
 
-    def cache_exists(self, md5s, jobs=None):
+        move(from_info, to_info, mode=mode)
+
+    @staticmethod
+    def copy(from_info, to_info):
+        System.copy(from_info, to_info)
+
+    @staticmethod
+    def symlink(from_info, to_info):
+        System.symlink(from_info, to_info)
+
+    def hardlink(self, from_info, to_info):
+        # If there are a lot of empty files (which happens a lot in datasets),
+        # and the cache type is `hardlink`, we might reach link limits and
+        # will get something like: `too many links error`
+        #
+        # This is because all those empty files will have the same checksum
+        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therfore, they will be
+        # linked to the same file in the cache.
+        #
+        # From https://en.wikipedia.org/wiki/Hard_link
+        #   * ext4 limits the number of hard links on a file to 65,000
+        #   * Windows with NTFS has a limit of 1024 hard links on a file
+        #
+        # That's why we simply create an empty file rather than a link.
+        if self.getsize(from_info) == 0:
+            self.open(to_info, "w").close()
+
+            logger.debug(
+                "Created empty file: {src} -> {dest}".format(
+                    src=str(from_info), dest=str(to_info)
+                )
+            )
+            return
+
+        System.hardlink(from_info, to_info)
+
+    @staticmethod
+    def reflink(from_info, to_info):
+        System.reflink(from_info, to_info)
+
+    def cache_exists(self, checksums, jobs=None, name=None):
         return [
             checksum
-            for checksum in progress(md5s)
+            for checksum in Tqdm(
+                checksums,
+                unit="file",
+                desc="Querying "
+                + ("cache in " + name if name else "local cache"),
+            )
             if not self.changed_cache_file(checksum)
         ]
 
@@ -272,44 +241,25 @@ class RemoteLOCAL(RemoteBASE):
             name=name,
         )
 
-    def _group(self, checksum_infos, show_checksums=False):
-        by_md5 = {}
-
-        for info in checksum_infos:
-            md5 = info[self.PARAM_CHECKSUM]
-
-            if show_checksums:
-                by_md5[md5] = {"name": md5}
-                continue
-
-            name = info["name"]
-            branch = info.get("branch")
-            if branch:
-                name += "({})".format(branch)
-
-            if md5 not in by_md5.keys():
-                by_md5[md5] = {"name": name}
-            else:
-                by_md5[md5]["name"] += " " + name
-
-        return by_md5
+    @staticmethod
+    def open(path_info, mode="r", encoding=None):
+        return open(fspath_py35(path_info), mode=mode, encoding=encoding)
 
     def status(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
         download=False,
     ):
-        logger.info(
+        logger.debug(
             "Preparing to collect status from {}".format(remote.path_info)
         )
-        ret = self._group(checksum_infos, show_checksums=show_checksums) or {}
-        md5s = list(ret)
+        md5s = list(named_cache[self.scheme])
 
-        logger.info("Collecting information from local cache...")
-        local_exists = self.cache_exists(md5s, jobs=jobs)
+        logger.debug("Collecting information from local cache...")
+        local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
 
         # This is a performance optimization. We can safely assume that,
         # if the resources that we want to fetch are already cached,
@@ -318,16 +268,25 @@ class RemoteLOCAL(RemoteBASE):
         if download and sorted(local_exists) == sorted(md5s):
             remote_exists = local_exists
         else:
-            logger.info("Collecting information from remote cache...")
-            remote_exists = list(remote.cache_exists(md5s, jobs=jobs))
+            logger.debug("Collecting information from remote cache...")
+            remote_exists = list(
+                remote.cache_exists(
+                    md5s, jobs=jobs, name=str(remote.path_info)
+                )
+            )
 
+        ret = {
+            checksum: {"name": checksum if show_checksums else " ".join(names)}
+            for checksum, names in named_cache[self.scheme].items()
+        }
         self._fill_statuses(ret, local_exists, remote_exists)
 
         self._log_missing_caches(ret)
 
         return ret
 
-    def _fill_statuses(self, checksum_info_dir, local_exists, remote_exists):
+    @staticmethod
+    def _fill_statuses(checksum_info_dir, local_exists, remote_exists):
         # Using sets because they are way faster for lookups
         local = set(local_exists)
         remote = set(remote_exists)
@@ -340,8 +299,8 @@ class RemoteLOCAL(RemoteBASE):
         cache = []
         path_infos = []
         names = []
-        for md5, info in progress(
-            status_info.items(), name="Analysing status"
+        for md5, info in Tqdm(
+            status_info.items(), desc="Analysing status", unit="file"
         ):
             if info["status"] == status:
                 cache.append(self.checksum_to_path_info(md5))
@@ -359,13 +318,13 @@ class RemoteLOCAL(RemoteBASE):
 
     def _process(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
         download=False,
     ):
-        logger.info(
+        logger.debug(
             "Preparing to {} '{}'".format(
                 "download data from" if download else "upload data to",
                 remote.path_info,
@@ -373,7 +332,11 @@ class RemoteLOCAL(RemoteBASE):
         )
 
         if download:
-            func = remote.download
+            func = partial(
+                remote.download,
+                dir_mode=self._dir_mode,
+                file_mode=self._file_mode,
+            )
             status = STATUS_DELETED
         else:
             func = remote.upload
@@ -383,7 +346,7 @@ class RemoteLOCAL(RemoteBASE):
             jobs = remote.JOBS
 
         status_info = self.status(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
@@ -396,38 +359,38 @@ class RemoteLOCAL(RemoteBASE):
             return 0
 
         if jobs > 1:
-            with ThreadPoolExecutor(max_workers=jobs) as executor:
+            with TqdmThreadPoolExecutor(max_workers=jobs) as executor:
                 fails = sum(executor.map(func, *plans))
         else:
             fails = sum(map(func, *plans))
 
         if fails:
-            msg = "{} files failed to {}"
-            raise DvcException(
-                msg.format(fails, "download" if download else "upload")
-            )
+            if download:
+                raise DownloadError(fails)
+            raise UploadError(fails)
 
         return len(plans[0])
 
-    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def push(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
             download=False,
         )
 
-    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
             download=True,
         )
 
-    def _log_missing_caches(self, checksum_info_dict):
+    @staticmethod
+    def _log_missing_caches(checksum_info_dict):
         missing_caches = [
             (md5, info)
             for md5, info in checksum_info_dict.items()
@@ -435,10 +398,8 @@ class RemoteLOCAL(RemoteBASE):
         ]
         if missing_caches:
             missing_desc = "".join(
-                [
-                    "\nname: {}, md5: {}".format(info["name"], md5)
-                    for md5, info in missing_caches
-                ]
+                "\nname: {}, md5: {}".format(info["name"], md5)
+                for md5, info in missing_caches
             )
             msg = (
                 "Some of the cache files do not exist neither locally "
@@ -447,19 +408,22 @@ class RemoteLOCAL(RemoteBASE):
             logger.warning(msg)
 
     @staticmethod
-    def _unprotect_file(path):
+    def _unprotect_file(path, allow_copy=True):
         if System.is_symlink(path) or System.is_hardlink(path):
-            logger.debug("Unprotecting '{}'".format(path))
-            tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
+            if allow_copy:
+                logger.debug("Unprotecting '{}'".format(path))
+                tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
 
-            # The operations order is important here - if some application
-            # would access the file during the process of copyfile then it
-            # would get only the part of file. So, at first, the file should be
-            # copied with the temporary name, and then original file should be
-            # replaced by new.
-            copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
-            remove(path)
-            os.rename(tmp, path)
+                # The operations order is important here - if some application
+                # would access the file during the process of copyfile then it
+                # would get only the part of file. So, at first, the file
+                # should be copied with the temporary name, and then
+                # original file should be replaced by new.
+                copyfile(
+                    path, tmp, name="Unprotecting '{}'".format(relpath(path))
+                )
+                remove(path)
+                os.rename(tmp, path)
 
         else:
             logger.debug(
@@ -469,11 +433,11 @@ class RemoteLOCAL(RemoteBASE):
 
         os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
 
-    def _unprotect_dir(self, path):
-        for path in walk_files(path, self.repo.dvcignore):
-            RemoteLOCAL._unprotect_file(path)
+    def _unprotect_dir(self, path, allow_copy=True):
+        for fname in walk_files(path, self.repo.dvcignore):
+            self._unprotect_file(fname, allow_copy)
 
-    def unprotect(self, path_info):
+    def unprotect(self, path_info, allow_copy=True):
         path = path_info.fspath
         if not os.path.exists(path):
             raise DvcException(
@@ -481,15 +445,26 @@ class RemoteLOCAL(RemoteBASE):
             )
 
         if os.path.isdir(path):
-            self._unprotect_dir(path)
+            self._unprotect_dir(path, allow_copy)
         else:
-            RemoteLOCAL._unprotect_file(path)
+            self._unprotect_file(path, allow_copy)
 
     @staticmethod
     def protect(path_info):
-        os.chmod(
-            fspath_py35(path_info), stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
-        )
+        path = fspath_py35(path_info)
+        mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
+
+        try:
+            os.chmod(path, mode)
+        except OSError as exc:
+            # In share cache scenario, we might not own the cache file, so we
+            # need to check if cache file is already protected.
+            if exc.errno not in [errno.EPERM, errno.EACCES]:
+                raise
+
+            actual = os.stat(path).st_mode
+            if actual & mode != mode:
+                raise
 
     def _get_unpacked_dir_path_info(self, checksum):
         info = self.checksum_to_path_info(checksum)
@@ -519,13 +494,18 @@ class RemoteLOCAL(RemoteBASE):
     def _create_unpacked_dir(self, checksum, dir_info, unpacked_dir_info):
         self.makedirs(unpacked_dir_info)
 
-        for entry in progress(dir_info, name="Created unpacked dir"):
+        for entry in Tqdm(dir_info, desc="Creating unpacked dir", unit="file"):
             entry_cache_info = self.checksum_to_path_info(
                 entry[self.PARAM_CHECKSUM]
             )
             relative_path = entry[self.PARAM_RELPATH]
-            self.link(
-                entry_cache_info, unpacked_dir_info / relative_path, "hardlink"
+            # In shared cache mode some cache files might not be owned by the
+            # user, so we need to use symlinks because, unless
+            # /proc/sys/fs/protected_hardlinks is disabled, the user is not
+            # allowed to create hardlinks to files that he doesn't own.
+            link_types = ["hardlink", "symlink"]
+            self._link(
+                entry_cache_info, unpacked_dir_info / relative_path, link_types
             )
 
         self.state.save(unpacked_dir_info, checksum)
@@ -541,3 +521,36 @@ class RemoteLOCAL(RemoteBASE):
             if self.is_dir_checksum(c):
                 unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
         return unpacked
+
+    def _get_cache_type(self, path_info):
+        if self.cache_type_confirmed:
+            return self.cache_types[0]
+
+        workspace_file = path_info.with_name("." + uuid())
+        test_cache_file = self.path_info / ".cache_type_test_file"
+        if not self.exists(test_cache_file):
+            with open(fspath_py35(test_cache_file), "wb") as fobj:
+                fobj.write(bytes(1))
+        try:
+            self.link(test_cache_file, workspace_file)
+        finally:
+            self.remove(workspace_file)
+            self.remove(test_cache_file)
+
+        self.cache_type_confirmed = True
+        return self.cache_types[0]
+
+    def _link_matches(self, path_info):
+        is_hardlink = System.is_hardlink(path_info)
+        is_symlink = System.is_symlink(path_info)
+        is_copy_or_reflink = not is_hardlink and not is_symlink
+
+        cache_type = self._get_cache_type(path_info)
+
+        if cache_type == "symlink":
+            return is_symlink
+
+        if cache_type == "hardlink":
+            return is_hardlink
+
+        return is_copy_or_reflink

dvc/remote/local/slow_link_detection.py

@@ -1,62 +0,0 @@
-import logging
-import time
-
-import colorama
-from dvc.config import Config
-
-logger = logging.getLogger(__name__)
-
-
-class SlowLinkDetectorDecorator(object):
-    LINKING_TIMEOUT_SECONDS = 10.0
-    was_displayed = False
-
-    @classmethod
-    def should_display(cls):
-        if not cls.was_displayed:
-            cls.was_displayed = True
-            return True
-        return False
-
-    def __init__(self, method):
-        self.method = method
-
-    def __call__(self, *args, **kwargs):
-        start = time.time()
-        result = self.method(*args, **kwargs)
-        execution_time_seconds = time.time() - start
-
-        if (
-            execution_time_seconds >= self.LINKING_TIMEOUT_SECONDS
-            and self.should_display()
-        ):
-            msg = (
-                "You can cut execution time considerably by using a different"
-                " `cache.type` configuration.\nSee "
-                "{blue}https://dvc.org/doc/commands-reference/config#cache{"
-                "reset} for more information.\n"
-                "To disable this message, run:\n"
-                "'dvc config cache.slow_link_warning false'".format(
-                    blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
-                )
-            )
-            logger.warning(msg)
-
-        return result
-
-
-def slow_link_guard(method):
-    def call(remote_local, *args, **kwargs):
-        cache_config = remote_local.repo.config.config.get(
-            Config.SECTION_CACHE, {}
-        )
-        should_warn = cache_config.get(
-            Config.SECTION_CACHE_SLOW_LINK_WARNING, True
-        ) and not cache_config.get(Config.SECTION_CACHE_TYPE, None)
-
-        if should_warn:
-            decorated = SlowLinkDetectorDecorator(method)
-            return decorated(remote_local, *args, **kwargs)
-        return method(remote_local, *args, **kwargs)
-
-    return call

dvc/remote/oss.py

@@ -6,14 +6,9 @@ import logging
 
 from dvc.scheme import Schemes
 
-try:
-    import oss2
-except ImportError:
-    oss2 = None
-
 from dvc.config import Config
 from dvc.remote.base import RemoteBASE
-from dvc.remote.azure import Callback
+from dvc.progress import Tqdm
 from dvc.path_info import CloudURLInfo
 
 
@@ -41,7 +36,7 @@ class RemoteOSS(RemoteBASE):
 
     scheme = Schemes.OSS
     path_cls = CloudURLInfo
-    REQUIRES = {"oss2": oss2}
+    REQUIRES = {"oss2": "oss2"}
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
 
@@ -71,6 +66,8 @@ class RemoteOSS(RemoteBASE):
 
     @property
     def oss_service(self):
+        import oss2
+
         if self._bucket is None:
             logger.debug("URL {}".format(self.path_info))
             logger.debug("key id {}".format(self.key_id))
@@ -98,6 +95,8 @@ class RemoteOSS(RemoteBASE):
         self.oss_service.delete_object(path_info.path)
 
     def _list_paths(self, prefix):
+        import oss2
+
         for blob in oss2.ObjectIterator(self.oss_service, prefix=prefix):
             yield blob.key
 
@@ -107,15 +106,24 @@ class RemoteOSS(RemoteBASE):
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        cb = None if no_progress_bar else Callback(name)
-        self.oss_service.put_object_from_file(
-            to_info.path, from_file, progress_callback=cb
-        )
+        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
+            self.oss_service.put_object_from_file(
+                to_info.path, from_file, progress_callback=pbar.update_to
+            )
 
     def _download(
         self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
     ):
-        cb = None if no_progress_bar else Callback(name)
-        self.oss_service.get_object_to_file(
-            from_info.path, to_file, progress_callback=cb
-        )
+        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
+            self.oss_service.get_object_to_file(
+                from_info.path, to_file, progress_callback=pbar.update_to
+            )
+
+    def _generate_download_url(self, path_info, expires=3600):
+        assert path_info.bucket == self.path_info.bucket
+
+        return self.oss_service.sign_url("GET", path_info.path, expires)
+
+    def exists(self, path_info):
+        paths = self._list_paths(path_info.path)
+        return any(path_info.path == path for path in paths)

dvc/remote/s3.py

@@ -1,17 +1,13 @@
+# -*- coding: utf-8 -*-
+
 from __future__ import unicode_literals
 
 import os
-import threading
 import logging
-import itertools
+import posixpath
 from funcy import cached_property
 
-try:
-    import boto3
-except ImportError:
-    boto3 = None
-
-from dvc.progress import progress
+from dvc.progress import Tqdm
 from dvc.config import Config
 from dvc.remote.base import RemoteBASE
 from dvc.exceptions import DvcException, ETagMismatchError
@@ -21,23 +17,10 @@ from dvc.scheme import Schemes
 logger = logging.getLogger(__name__)
 
 
-class Callback(object):
-    def __init__(self, name, total):
-        self.name = name
-        self.total = total
-        self.current = 0
-        self.lock = threading.Lock()
-
-    def __call__(self, byts):
-        with self.lock:
-            self.current += byts
-            progress.update_target(self.name, self.current, self.total)
-
-
 class RemoteS3(RemoteBASE):
     scheme = Schemes.S3
     path_cls = CloudURLInfo
-    REQUIRES = {"boto3": boto3}
+    REQUIRES = {"boto3": "boto3"}
     PARAM_CHECKSUM = "etag"
 
     def __init__(self, repo, config):
@@ -56,7 +39,10 @@ class RemoteS3(RemoteBASE):
 
         self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
 
-        self.list_objects = config.get(Config.SECTION_AWS_LIST_OBJECTS)
+        if config.get(Config.SECTION_AWS_LIST_OBJECTS):
+            self.list_objects_api = "list_objects"
+        else:
+            self.list_objects_api = "list_objects_v2"
 
         self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
 
@@ -66,12 +52,18 @@ class RemoteS3(RemoteBASE):
         if self.sse:
             self.extra_args["ServerSideEncryption"] = self.sse
 
+        self.acl = config.get(Config.SECTION_AWS_ACL, "")
+        if self.acl:
+            self.extra_args["ACL"] = self.acl
+
         shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
         if shared_creds:
             os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
 
     @cached_property
     def s3(self):
+        import boto3
+
         session = boto3.session.Session(
             profile_name=self.profile, region_name=self.region
         )
@@ -194,58 +186,91 @@ class RemoteS3(RemoteBASE):
         logger.debug("Removing {}".format(path_info))
         self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
 
-    def _list_paths(self, bucket, prefix):
+    def _list_objects(self, path_info, max_items=None):
         """ Read config for list object api, paginate through list objects."""
-        kwargs = {"Bucket": bucket, "Prefix": prefix}
-        if self.list_objects:
-            list_objects_api = "list_objects"
-        else:
-            list_objects_api = "list_objects_v2"
-        paginator = self.s3.get_paginator(list_objects_api)
+        kwargs = {
+            "Bucket": path_info.bucket,
+            "Prefix": path_info.path,
+            "PaginationConfig": {"MaxItems": max_items},
+        }
+        paginator = self.s3.get_paginator(self.list_objects_api)
         for page in paginator.paginate(**kwargs):
             contents = page.get("Contents", None)
             if not contents:
                 continue
             for item in contents:
-                yield item["Key"]
+                yield item
+
+    def _list_paths(self, path_info, max_items=None):
+        return (
+            item["Key"] for item in self._list_objects(path_info, max_items)
+        )
 
     def list_cache_paths(self):
-        return self._list_paths(self.path_info.bucket, self.path_info.path)
+        return self._list_paths(self.path_info)
 
     def exists(self, path_info):
-        paths = self._list_paths(path_info.bucket, path_info.path)
-        return any(path_info.path == path for path in paths)
+        dir_path = path_info / ""
+        fname = next(self._list_paths(path_info, max_items=1), "")
+        return path_info.path == fname or fname.startswith(dir_path.path)
 
-    def batch_exists(self, path_infos, callback):
-        paths = []
-
-        for path_info in path_infos:
-            paths.append(self._list_paths(path_info.bucket, path_info.path))
-            callback.update(str(path_info))
-
-        paths = set(itertools.chain.from_iterable(paths))
-        return [path_info.path in paths for path_info in path_infos]
+    def isdir(self, path_info):
+        # S3 doesn't have a concept for directories.
+        #
+        # Using `head_object` with a path pointing to a directory
+        # will throw a 404 error.
+        #
+        # A reliable way to know if a given path is a directory is by
+        # checking if there are more files sharing the same prefix
+        # with a `list_objects` call.
+        #
+        # We need to make sure that the path ends with a forward slash,
+        # since we can end with false-positives like the following example:
+        #
+        #       bucket
+        #       └── data
+        #          ├── alice
+        #          └── alpha
+        #
+        # Using `data/al` as prefix will return `[data/alice, data/alpha]`,
+        # While `data/al/` will return nothing.
+        #
+        dir_path = path_info / ""
+        return bool(list(self._list_paths(dir_path, max_items=1)))
 
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
         total = os.path.getsize(from_file)
-        cb = None if no_progress_bar else Callback(name, total)
-        self.s3.upload_file(
-            from_file,
-            to_info.bucket,
-            to_info.path,
-            Callback=cb,
-            ExtraArgs=self.extra_args,
-        )
+        with Tqdm(
+            disable=no_progress_bar, total=total, bytes=True, desc=name
+        ) as pbar:
+            self.s3.upload_file(
+                from_file,
+                to_info.bucket,
+                to_info.path,
+                Callback=pbar.update,
+                ExtraArgs=self.extra_args,
+            )
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
         if no_progress_bar:
-            cb = None
+            total = None
         else:
             total = self.s3.head_object(
                 Bucket=from_info.bucket, Key=from_info.path
             )["ContentLength"]
-            cb = Callback(name, total)
+        with Tqdm(
+            disable=no_progress_bar, total=total, bytes=True, desc=name
+        ) as pbar:
+            self.s3.download_file(
+                from_info.bucket, from_info.path, to_file, Callback=pbar.update
+            )
 
-        self.s3.download_file(
-            from_info.bucket, from_info.path, to_file, Callback=cb
+    def _generate_download_url(self, path_info, expires=3600):
+        params = {"Bucket": path_info.bucket, "Key": path_info.path}
+        return self.s3.generate_presigned_url(
+            ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
         )
+
+    def walk_files(self, path_info, max_items=None):
+        for fname in self._list_paths(path_info, max_items):
+            yield path_info / posixpath.relpath(fname, path_info.path)

dvc/remote/slow_link_detection.py

@@ -0,0 +1,50 @@
+import logging
+import time
+import sys
+from functools import wraps
+
+import colorama
+from dvc.config import Config
+
+
+logger = logging.getLogger(__name__)
+this = sys.modules[__name__]
+
+this.timeout_seconds = 10.0
+this.already_displayed = False
+this.message = (
+    "You can cut execution time considerably by using a different "
+    "`cache.type` configuration.\n"
+    "See {blue}https://dvc.org/doc/commands-reference/config#cache{reset} "
+    "for more information.\n"
+    "To disable this message, run:\n"
+    "`dvc config cache.slow_link_warning false`".format(
+        blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
+    )
+)
+
+
+def slow_link_guard(f):
+    @wraps(f)
+    def wrapper(remote, *args, **kwargs):
+        if this.already_displayed:
+            return f(remote, *args, **kwargs)
+
+        config = remote.repo.config.config.get(Config.SECTION_CACHE, {})
+        cache_type = config.get(Config.SECTION_CACHE_TYPE)
+        should_warn = config.get(Config.SECTION_CACHE_SLOW_LINK_WARNING, True)
+
+        if not should_warn or cache_type:
+            return f(remote, *args, **kwargs)
+
+        start = time.time()
+        result = f(remote, *args, **kwargs)
+        delta = time.time() - start
+
+        if delta >= this.timeout_seconds:
+            logger.warning(this.message)
+            this.already_displayed = True
+
+        return result
+
+    return wrapper

dvc/remote/ssh/__init__.py

@@ -1,17 +1,15 @@
 from __future__ import unicode_literals
 
+import errno
+import itertools
+import io
 import os
+import posixpath
 import getpass
 import logging
-import itertools
-import errno
-from concurrent.futures import ThreadPoolExecutor
 import threading
-
-try:
-    import paramiko
-except ImportError:
-    paramiko = None
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager, closing
 
 import dvc.prompt as prompt
 from dvc.config import Config
@@ -20,9 +18,7 @@ from dvc.utils.compat import urlparse, StringIO
 from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
 from dvc.remote.pool import get_connection
-
-from .connection import SSHConnection
-
+from dvc.progress import Tqdm
 
 logger = logging.getLogger(__name__)
 
@@ -33,7 +29,7 @@ saved_passwords_lock = threading.Lock()
 
 class RemoteSSH(RemoteBASE):
     scheme = Schemes.SSH
-    REQUIRES = {"paramiko": paramiko}
+    REQUIRES = {"paramiko": "paramiko"}
 
     JOBS = 4
     PARAM_CHECKSUM = "md5"
@@ -44,9 +40,10 @@ class RemoteSSH(RemoteBASE):
     # We use conservative setting of 4 instead to not exhaust max sessions.
     CHECKSUM_JOBS = 4
 
+    DEFAULT_CACHE_TYPES = ["copy"]
+
     def __init__(self, repo, config):
         super(RemoteSSH, self).__init__(repo, config)
-
         url = config.get(Config.SECTION_REMOTE_URL)
         if url:
             parsed = urlparse(url)
@@ -84,6 +81,7 @@ class RemoteSSH(RemoteBASE):
         self.ask_password = config.get(
             Config.SECTION_REMOTE_ASK_PASSWORD, False
         )
+        self.gss_auth = config.get(Config.SECTION_REMOTE_GSS_AUTH, False)
 
     @staticmethod
     def ssh_config_filename():
@@ -91,8 +89,10 @@ class RemoteSSH(RemoteBASE):
 
     @staticmethod
     def _load_user_ssh_config(hostname):
+        import paramiko
+
         user_config_file = RemoteSSH.ssh_config_filename()
-        user_ssh_config = dict()
+        user_ssh_config = {}
         if hostname and os.path.exists(user_config_file):
             ssh_config = paramiko.SSHConfig()
             with open(user_config_file) as f:
@@ -116,9 +116,10 @@ class RemoteSSH(RemoteBASE):
             return identity_file[0]
         return None
 
-    def ssh(self, path_info):
+    def ensure_credentials(self, path_info=None):
+        if path_info is None:
+            path_info = self.path_info
         host, user, port = path_info.host, path_info.user, path_info.port
-
         # NOTE: we use the same password regardless of the server :(
         if self.ask_password and self.password is None:
             with saved_passwords_lock:
@@ -134,48 +135,26 @@ class RemoteSSH(RemoteBASE):
                     )
                 self.password = password
 
+    def ssh(self, path_info):
+        self.ensure_credentials(path_info)
+
+        from .connection import SSHConnection
+
         return get_connection(
             SSHConnection,
-            host,
-            username=user,
-            port=port,
+            path_info.host,
+            username=path_info.user,
+            port=path_info.port,
             key_filename=self.keyfile,
             timeout=self.timeout,
             password=self.password,
+            gss_auth=self.gss_auth,
         )
 
     def exists(self, path_info):
         with self.ssh(path_info) as ssh:
             return ssh.exists(path_info.path)
 
-    def batch_exists(self, path_infos, callback):
-        def _exists(chunk_and_channel):
-            chunk, channel = chunk_and_channel
-            ret = []
-            for path in chunk:
-                try:
-                    channel.stat(path)
-                    ret.append(True)
-                except IOError as exc:
-                    if exc.errno != errno.ENOENT:
-                        raise
-                    ret.append(False)
-                callback.update(path)
-            return ret
-
-        with self.ssh(path_infos[0]) as ssh:
-            channels = ssh.open_max_sftp_channels()
-            max_workers = len(channels)
-
-            with ThreadPoolExecutor(max_workers=max_workers) as executor:
-                paths = [path_info.path for path_info in path_infos]
-                chunks = to_chunks(paths, num_chunks=max_workers)
-                chunks_and_channels = zip(chunks, channels)
-                outcome = executor.map(_exists, chunks_and_channels)
-                results = list(itertools.chain.from_iterable(outcome))
-
-            return results
-
     def get_file_checksum(self, path_info):
         if path_info.scheme != self.scheme:
             raise NotImplementedError
@@ -187,12 +166,54 @@ class RemoteSSH(RemoteBASE):
         with self.ssh(path_info) as ssh:
             return ssh.isdir(path_info.path)
 
+    def isfile(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.isfile(path_info.path)
+
+    def getsize(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.getsize(path_info.path)
+
     def copy(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.copy(from_info.path, to_info.path)
+
+    def symlink(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.symlink(from_info.path, to_info.path)
+
+    def hardlink(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        # See dvc/remote/local/__init__.py - hardlink()
+        if self.getsize(from_info) == 0:
+
+            with self.ssh(to_info) as ssh:
+                ssh.sftp.open(to_info.path, "w").close()
+
+            logger.debug(
+                "Created empty file: {src} -> {dest}".format(
+                    src=str(from_info), dest=str(to_info)
+                )
+            )
+            return
+
+        with self.ssh(from_info) as ssh:
+            ssh.hardlink(from_info.path, to_info.path)
+
+    def reflink(self, from_info, to_info):
         if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
             raise NotImplementedError
 
         with self.ssh(from_info) as ssh:
-            ssh.cp(from_info.path, to_info.path)
+            ssh.reflink(from_info.path, to_info.path)
 
     def remove(self, path_info):
         if path_info.scheme != self.scheme:
@@ -228,15 +249,85 @@ class RemoteSSH(RemoteBASE):
                 no_progress_bar=no_progress_bar,
             )
 
+    @contextmanager
+    def open(self, path_info, mode="r", encoding=None):
+        assert mode in {"r", "rt", "rb", "wb"}
+
+        with self.ssh(path_info) as ssh, closing(
+            ssh.sftp.open(path_info.path, mode)
+        ) as fd:
+            if "b" in mode:
+                yield fd
+            else:
+                yield io.TextIOWrapper(fd, encoding=encoding)
+
     def list_cache_paths(self):
         with self.ssh(self.path_info) as ssh:
             return list(ssh.walk_files(self.path_info.path))
 
-    def walk(self, path_info):
+    def walk_files(self, path_info):
         with self.ssh(path_info) as ssh:
-            for entry in ssh.walk(path_info.path):
-                yield entry
+            for fname in ssh.walk_files(path_info.path):
+                yield path_info / posixpath.relpath(fname, path_info.path)
 
     def makedirs(self, path_info):
         with self.ssh(path_info) as ssh:
             ssh.makedirs(path_info.path)
+
+    def batch_exists(self, path_infos, callback):
+        def _exists(chunk_and_channel):
+            chunk, channel = chunk_and_channel
+            ret = []
+            for path in chunk:
+                try:
+                    channel.stat(path)
+                    ret.append(True)
+                except IOError as exc:
+                    if exc.errno != errno.ENOENT:
+                        raise
+                    ret.append(False)
+                callback(path)
+            return ret
+
+        with self.ssh(path_infos[0]) as ssh:
+            channels = ssh.open_max_sftp_channels()
+            max_workers = len(channels)
+
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                paths = [path_info.path for path_info in path_infos]
+                chunks = to_chunks(paths, num_chunks=max_workers)
+                chunks_and_channels = zip(chunks, channels)
+                outcome = executor.map(_exists, chunks_and_channels)
+                results = list(itertools.chain.from_iterable(outcome))
+
+            return results
+
+    def cache_exists(self, checksums, jobs=None, name=None):
+        """This is older implementation used in remote/base.py
+        We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
+        faster than current approach (relying on exists(path_info)) applied in
+        remote/base.
+        """
+        if not self.no_traverse:
+            return list(set(checksums) & set(self.all()))
+
+        # possibly prompt for credentials before "Querying" progress output
+        self.ensure_credentials()
+
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(checksums),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(chunks):
+                return self.batch_exists(chunks, callback=pbar.update_desc)
+
+            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
+                path_infos = [self.checksum_to_path_info(x) for x in checksums]
+                chunks = to_chunks(path_infos, num_chunks=self.JOBS)
+                results = executor.map(exists_with_progress, chunks)
+                in_remote = itertools.chain.from_iterable(results)
+                ret = list(itertools.compress(checksums, in_remote))
+                return ret

dvc/remote/ssh/connection.py

@@ -12,7 +12,7 @@ except ImportError:
 
 from dvc.utils import tmp_fname
 from dvc.utils.compat import ignore_file_not_found
-from dvc.progress import progress
+from dvc.progress import Tqdm
 from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteCmdError
 
@@ -29,21 +29,6 @@ def sizeof_fmt(num, suffix="B"):
     return "%.1f%s%s" % (num, "Y", suffix)
 
 
-def percent_cb(name, complete, total):
-    """ Callback for updating target progress """
-    logger.debug(
-        "{}: {} transferred out of {}".format(
-            name, sizeof_fmt(complete), sizeof_fmt(total)
-        )
-    )
-    progress.update_target(name, complete, total)
-
-
-def create_cb(name):
-    """ Create callback function for multipart object """
-    return lambda cur, tot: percent_cb(name, cur, tot)
-
-
 class SSHConnection:
     def __init__(self, host, *args, **kwargs):
         logger.debug(
@@ -75,7 +60,13 @@ class SSHConnection:
 
     def st_mode(self, path):
         with ignore_file_not_found():
-            return self.sftp.stat(path).st_mode
+            return self.sftp.lstat(path).st_mode
+
+        return 0
+
+    def getsize(self, path):
+        with ignore_file_not_found():
+            return self.sftp.lstat(path).st_size
 
         return 0
 
@@ -115,7 +106,10 @@ class SSHConnection:
                 # Since paramiko errors are very vague we need to recheck
                 # whether it's because path already exists or something else
                 if e.errno == errno.EACCES or not self.exists(path):
-                    raise
+                    raise DvcException(
+                        "unable to create remote directory '{}'".format(path),
+                        cause=e,
+                    )
 
     def walk(self, directory, topdown=True):
         # NOTE: original os.walk() implementation [1] with default options was
@@ -183,14 +177,12 @@ class SSHConnection:
             self._remove_file(path)
 
     def download(self, src, dest, no_progress_bar=False, progress_title=None):
-        if no_progress_bar:
-            self.sftp.get(src, dest)
-        else:
-            if not progress_title:
-                progress_title = os.path.basename(src)
-
-            self.sftp.get(src, dest, callback=create_cb(progress_title))
-            progress.finish_target(progress_title)
+        with Tqdm(
+            desc=progress_title or os.path.basename(src),
+            disable=no_progress_bar,
+            bytes=True,
+        ) as pbar:
+            self.sftp.get(src, dest, callback=pbar.update_to)
 
     def move(self, src, dst):
         self.makedirs(posixpath.dirname(dst))
@@ -199,15 +191,13 @@ class SSHConnection:
     def upload(self, src, dest, no_progress_bar=False, progress_title=None):
         self.makedirs(posixpath.dirname(dest))
         tmp_file = tmp_fname(dest)
+        if not progress_title:
+            progress_title = posixpath.basename(dest)
 
-        if no_progress_bar:
-            self.sftp.put(src, tmp_file)
-        else:
-            if not progress_title:
-                progress_title = posixpath.basename(dest)
-
-            self.sftp.put(src, tmp_file, callback=create_cb(progress_title))
-            progress.finish_target(progress_title)
+        with Tqdm(
+            desc=progress_title, disable=no_progress_bar, bytes=True
+        ) as pbar:
+            self.sftp.put(src, tmp_file, callback=pbar.update_to)
 
         self.sftp.rename(tmp_file, dest)
 
@@ -286,8 +276,7 @@ class SSHConnection:
         assert len(md5) == 32
         return md5
 
-    def cp(self, src, dest):
-        self.makedirs(posixpath.dirname(dest))
+    def copy(self, src, dest):
         self.execute("cp {} {}".format(src, dest))
 
     def open_max_sftp_channels(self):
@@ -301,3 +290,23 @@ class SSHConnection:
                         raise
                     break
         return self._sftp_channels
+
+    def open(self, *args, **kwargs):
+        return self.sftp.open(*args, **kwargs)
+
+    def symlink(self, src, dest):
+        self.sftp.symlink(src, dest)
+
+    def reflink(self, src, dest):
+        if self.uname == "Linux":
+            return self.execute("cp --reflink {} {}".format(src, dest))
+
+        if self.uname == "Darwin":
+            return self.execute("cp -c {} {}".format(src, dest))
+
+        raise DvcException(
+            "'{}' is not supported as a SSH remote".format(self.uname)
+        )
+
+    def hardlink(self, src, dest):
+        self.execute("ln {} {}".format(src, dest))

dvc/repo/__init__.py

@@ -2,26 +2,42 @@ from __future__ import unicode_literals
 
 import os
 import logging
-
+from contextlib import contextmanager
 from itertools import chain
 
+from functools import wraps
 from funcy import cached_property
 
 from dvc.config import Config
 from dvc.exceptions import (
     NotDvcRepoError,
     OutputNotFoundError,
-    TargetNotDirectoryError,
-    OutputFileMissingError,
+    FileMissingError,
 )
 from dvc.ignore import DvcIgnoreFilter
 from dvc.path_info import PathInfo
-from dvc.utils.compat import open as _open, fspath_py35
+from dvc.remote.base import RemoteActionNotImplemented
+from dvc.utils.compat import open as _open, fspath_py35, FileNotFoundError
 from dvc.utils import relpath
 
+from .graph import check_acyclic, get_pipeline, get_pipelines, get_stages
+
+
 logger = logging.getLogger(__name__)
 
 
+def locked(f):
+    @wraps(f)
+    def wrapper(repo, *args, **kwargs):
+        with repo.lock, repo.state:
+            ret = f(repo, *args, **kwargs)
+            # Our graph cache is no longer valid after we release the repo.lock
+            repo._reset()
+            return ret
+
+    return wrapper
+
+
 class Repo(object):
     DVC_DIR = ".dvc"
 
@@ -35,9 +51,9 @@ class Repo(object):
     from dvc.repo.imp import imp
     from dvc.repo.imp_url import imp_url
     from dvc.repo.reproduce import reproduce
-    from dvc.repo.checkout import checkout
+    from dvc.repo.checkout import _checkout
     from dvc.repo.push import push
-    from dvc.repo.fetch import fetch
+    from dvc.repo.fetch import _fetch
     from dvc.repo.pull import pull
     from dvc.repo.status import status
     from dvc.repo.gc import gc
@@ -65,11 +81,14 @@ class Repo(object):
 
         self.config = Config(self.dvc_dir)
 
-        self.scm = SCM(self.root_dir, repo=self)
+        self.scm = SCM(self.root_dir)
 
         self.tree = WorkingTree(self.root_dir)
 
-        self.lock = Lock(self.dvc_dir)
+        self.lock = Lock(
+            os.path.join(self.dvc_dir, "lock"),
+            tmp_dir=os.path.join(self.dvc_dir, "tmp"),
+        )
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
         self.state = State(self, self.config.config)
@@ -88,6 +107,17 @@ class Repo(object):
 
         self._ignore()
 
+    @property
+    def tree(self):
+        return self._tree
+
+    @tree.setter
+    def tree(self, tree):
+        self._tree = tree
+        # Our graph cache is no longer valid, as it was based on the previous
+        # tree.
+        self._reset()
+
     def __repr__(self):
         return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
 
@@ -127,64 +157,63 @@ class Repo(object):
 
         updater = Updater(self.dvc_dir)
 
-        flist = [
-            self.lock.lock_file,
-            self.config.config_local_file,
-            updater.updater_file,
-            updater.lock.lock_file,
-        ] + self.state.files
+        flist = (
+            [self.config.config_local_file, updater.updater_file]
+            + self.state.files
+            + self.lock.files
+            + updater.lock.files
+        )
 
-        if self.cache.local.cache_dir.startswith(self.root_dir):
+        if self.cache.local.cache_dir.startswith(self.root_dir + os.sep):
             flist += [self.cache.local.cache_dir]
 
         self.scm.ignore_list(flist)
 
-    def check_dag(self, stages):
+    def check_modified_graph(self, new_stages):
         """Generate graph including the new stage to check for errors"""
-        self.graph(stages=stages)
+        self._collect_graph(self.stages + new_stages)
 
-    @staticmethod
-    def _check_cyclic_graph(graph):
-        import networkx as nx
-        from dvc.exceptions import CyclicGraphError
-
-        cycles = list(nx.simple_cycles(graph))
-
-        if cycles:
-            raise CyclicGraphError(cycles[0])
-
-    def _get_pipeline(self, node):
-        pipelines = [i for i in self.pipelines() if i.has_node(node)]
-        assert len(pipelines) == 1
-        return pipelines[0]
-
-    def collect(self, target, with_deps=False, recursive=False):
+    def collect(self, target, with_deps=False, recursive=False, graph=None):
         import networkx as nx
         from dvc.stage import Stage
 
-        if not target or (recursive and os.path.isdir(target)):
-            return self.active_stages(target)
+        G = graph or self.graph
+
+        if not target:
+            return get_stages(G)
+
+        target = os.path.abspath(target)
+
+        if recursive and os.path.isdir(target):
+            attrs = nx.get_node_attributes(G, "stage")
+            nodes = [node for node in nx.dfs_postorder_nodes(G)]
+
+            ret = []
+            for node in nodes:
+                stage = attrs[node]
+                if stage.path.startswith(target + os.sep):
+                    ret.append(stage)
+            return ret
 
         stage = Stage.load(self, target)
         if not with_deps:
             return [stage]
 
         node = relpath(stage.path, self.root_dir)
-        G = self._get_pipeline(node)
+        pipeline = get_pipeline(get_pipelines(G), node)
 
-        ret = []
-        for n in nx.dfs_postorder_nodes(G, node):
-            ret.append(G.node[n]["stage"])
-
-        return ret
+        return [
+            pipeline.node[n]["stage"]
+            for n in nx.dfs_postorder_nodes(pipeline, node)
+        ]
 
     def used_cache(
         self,
         targets=None,
         all_branches=False,
-        active=True,
         with_deps=False,
         all_tags=False,
+        all_commits=False,
         remote=None,
         force=False,
         jobs=None,
@@ -203,54 +232,41 @@ class Repo(object):
             A dictionary with Schemes (representing output's location) as keys,
             and a list with the outputs' `dumpd` as values.
         """
+        from dvc.cache import NamedCache
 
-        cache = {}
-        cache["local"] = []
-        cache["s3"] = []
-        cache["gs"] = []
-        cache["hdfs"] = []
-        cache["ssh"] = []
-        cache["azure"] = []
+        cache = NamedCache()
 
         for branch in self.brancher(
-            all_branches=all_branches, all_tags=all_tags
+            all_branches=all_branches,
+            all_tags=all_tags,
+            all_commits=all_commits,
         ):
             if targets:
                 stages = []
                 for target in targets:
-                    if recursive and os.path.isdir(target):
-                        stages.extend(self.stages(target))
-                    else:
-                        stages.extend(
-                            self.collect(target, with_deps=with_deps)
-                        )
-            elif active:
-                stages = self.active_stages()
+                    collected = self.collect(
+                        target, recursive=recursive, with_deps=with_deps
+                    )
+                    stages.extend(collected)
             else:
-                stages = self.stages()
+                stages = self.stages
 
             for stage in stages:
-                if active and not targets and stage.locked:
-                    logger.warning(
-                        "DVC-file '{path}' is locked. Its dependencies are"
-                        " not going to be pushed/pulled/fetched.".format(
-                            path=stage.relpath
-                        )
-                    )
+                if stage.is_repo_import:
+                    dep, = stage.deps
+                    cache.external[dep.repo_pair].add(dep.def_path)
+                    continue
 
                 for out in stage.outs:
-                    scheme = out.path_info.scheme
                     used_cache = out.get_used_cache(
                         remote=remote, force=force, jobs=jobs
                     )
-
-                    cache[scheme].extend(
-                        dict(entry, branch=branch) for entry in used_cache
-                    )
+                    suffix = "({})".format(branch) if branch else ""
+                    cache.update(used_cache, suffix=suffix)
 
         return cache
 
-    def graph(self, stages=None, from_directory=None):
+    def _collect_graph(self, stages=None):
         """Generate a graph by using the given stages on the given directory
 
         The nodes of the graph are the stage's path relative to the root.
@@ -280,11 +296,8 @@ class Repo(object):
               (weakly connected components)
 
         Args:
-            stages (list): used to build a graph, if None given, use the ones
-                on the `from_directory`.
-
-            from_directory (str): directory where to look at for stages, if
-                None is given, use the current working directory
+            stages (list): used to build a graph, if None given, collect stages
+                in the repository.
 
         Raises:
             OutputDuplicationError: two outputs with the same path
@@ -300,8 +313,7 @@ class Repo(object):
         )
 
         G = nx.DiGraph()
-        G_active = nx.DiGraph()
-        stages = stages or self.stages(from_directory, check_dag=False)
+        stages = stages or self.collect_stages()
         stages = [stage for stage in stages if stage]
         outs = {}
 
@@ -328,7 +340,6 @@ class Repo(object):
             node = relpath(stage.path, self.root_dir)
 
             G.add_node(node, stage=stage)
-            G_active.add_node(node, stage=stage)
 
             for dep in stage.deps:
                 if dep.path_info is None:
@@ -344,22 +355,18 @@ class Repo(object):
                         dep_node = relpath(dep_stage.path, self.root_dir)
                         G.add_node(dep_node, stage=dep_stage)
                         G.add_edge(node, dep_node)
-                        if not stage.locked:
-                            G_active.add_node(dep_node, stage=dep_stage)
-                            G_active.add_edge(node, dep_node)
 
-        self._check_cyclic_graph(G)
+        check_acyclic(G)
 
-        return G, G_active
+        return G
 
-    def pipelines(self, from_directory=None):
-        import networkx as nx
+    @cached_property
+    def graph(self):
+        return self._collect_graph()
 
-        G, G_active = self.graph(from_directory=from_directory)
-
-        return [
-            G.subgraph(c).copy() for c in nx.weakly_connected_components(G)
-        ]
+    @cached_property
+    def pipelines(self):
+        return get_pipelines(self.graph)
 
     @staticmethod
     def _filter_out_dirs(dirs, outs, root_dir):
@@ -372,7 +379,7 @@ class Repo(object):
 
         return list(filter(filter_dirs, dirs))
 
-    def stages(self, from_directory=None, check_dag=True):
+    def collect_stages(self):
         """
         Walks down the root directory looking for Dvcfiles,
         skipping the directories that are related with
@@ -384,16 +391,11 @@ class Repo(object):
         """
         from dvc.stage import Stage
 
-        if not from_directory:
-            from_directory = self.root_dir
-        elif not os.path.isdir(from_directory):
-            raise TargetNotDirectoryError(from_directory)
-
         stages = []
         outs = []
 
         for root, dirs, files in self.tree.walk(
-            from_directory, dvcignore=self.dvcignore
+            self.root_dir, dvcignore=self.dvcignore
         ):
             for fname in files:
                 path = os.path.join(root, fname)
@@ -407,26 +409,15 @@ class Repo(object):
 
             dirs[:] = self._filter_out_dirs(dirs, outs, root)
 
-        if check_dag:
-            self.check_dag(stages)
-
         return stages
 
-    def active_stages(self, from_directory=None):
-        import networkx as nx
-
-        stages = []
-        for G in self.pipelines(from_directory):
-            stages.extend(list(nx.get_node_attributes(G, "stage").values()))
-        return stages
+    @cached_property
+    def stages(self):
+        return get_stages(self.graph)
 
     def find_outs_by_path(self, path, outs=None, recursive=False):
         if not outs:
-            # there is no `from_directory=path` argument because some data
-            # files might be generated to an upper level, and so it is
-            # needed to look at all the files (project root_dir)
-            stages = self.stages()
-            outs = [out for stage in stages for out in stage.outs]
+            outs = [out for stage in self.stages for out in stage.outs]
 
         abs_path = os.path.abspath(path)
         is_dir = self.tree.isdir(abs_path)
@@ -455,25 +446,69 @@ class Repo(object):
         path_parts = os.path.normpath(path).split(os.path.sep)
         return self.DVC_DIR in path_parts
 
+    @contextmanager
     def open(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
-        out, = self.find_outs_by_path(path)
+        cause = None
+        try:
+            out, = self.find_outs_by_path(path)
+        except OutputNotFoundError as e:
+            out = None
+            cause = e
+
+        if out and out.use_cache:
+            try:
+                with self._open_cached(out, remote, mode, encoding) as fd:
+                    yield fd
+                return
+            except FileNotFoundError as e:
+                raise FileMissingError(relpath(path, self.root_dir), cause=e)
+
+        if self.tree.exists(path):
+            with self.tree.open(path, mode, encoding) as fd:
+                yield fd
+            return
+
+        raise FileMissingError(relpath(path, self.root_dir), cause=cause)
+
+    def _open_cached(self, out, remote=None, mode="r", encoding=None):
         if out.isdir():
             raise ValueError("Can't open a dir")
 
         cache_file = self.cache.local.checksum_to_path_info(out.checksum)
         cache_file = fspath_py35(cache_file)
 
-        with self.state:
-            cache_info = out.get_used_cache(remote=remote)
-            self.cloud.pull(cache_info, remote=remote)
+        if os.path.exists(cache_file):
+            return _open(cache_file, mode=mode, encoding=encoding)
 
-        # Since pull may just skip with a warning, we need to check it here
-        if not os.path.exists(cache_file):
-            raise OutputFileMissingError(relpath(path, self.root_dir))
+        try:
+            remote_obj = self.cloud.get_remote(remote)
+            remote_info = remote_obj.checksum_to_path_info(out.checksum)
+            return remote_obj.open(remote_info, mode=mode, encoding=encoding)
+        except RemoteActionNotImplemented:
+            with self.state:
+                cache_info = out.get_used_cache(remote=remote)
+                self.cloud.pull(cache_info, remote=remote)
 
-        return _open(cache_file, mode=mode, encoding=encoding)
+            return _open(cache_file, mode=mode, encoding=encoding)
 
     @cached_property
     def dvcignore(self):
         return DvcIgnoreFilter(self.root_dir)
+
+    def close(self):
+        self.scm.close()
+
+    @locked
+    def checkout(self, *args, **kwargs):
+        return self._checkout(*args, **kwargs)
+
+    @locked
+    def fetch(self, *args, **kwargs):
+        return self._fetch(*args, **kwargs)
+
+    def _reset(self):
+        self.__dict__.pop("graph", None)
+        self.__dict__.pop("stages", None)
+        self.__dict__.pop("pipelines", None)
+        self.__dict__.pop("dvcignore", None)

dvc/repo/add.py

@@ -1,43 +1,65 @@
+from __future__ import unicode_literals
+
 import os
 import logging
 import colorama
+from six import string_types
 
 from dvc.repo.scm_context import scm_context
 from dvc.stage import Stage
 from dvc.utils import walk_files, LARGE_DIR_SIZE
 from dvc.exceptions import RecursiveAddingWhileUsingFilename
-
+from dvc.progress import Tqdm
+from . import locked
 
 logger = logging.getLogger(__name__)
 
 
+@locked
 @scm_context
-def add(repo, target, recursive=False, no_commit=False, fname=None):
+def add(repo, targets, recursive=False, no_commit=False, fname=None):
     if recursive and fname:
         raise RecursiveAddingWhileUsingFilename()
 
-    targets = _find_all_targets(repo, target, recursive)
+    if isinstance(targets, string_types):
+        targets = [targets]
 
-    if os.path.isdir(target) and len(targets) > LARGE_DIR_SIZE:
-        logger.warning(
-            "You are adding a large directory '{target}' recursively,"
-            " consider tracking it as a whole instead.\n"
-            "{purple}HINT:{nc} Remove the generated DVC-file and then"
-            " run {cyan}dvc add {target}{nc}".format(
-                purple=colorama.Fore.MAGENTA,
-                cyan=colorama.Fore.CYAN,
-                nc=colorama.Style.RESET_ALL,
-                target=target,
-            )
-        )
+    stages_list = []
+    with Tqdm(total=len(targets), desc="Add", unit="file", leave=True) as pbar:
+        for target in targets:
+            sub_targets = _find_all_targets(repo, target, recursive)
+            pbar.total += len(sub_targets) - 1
 
-    stages = _create_stages(repo, targets, fname, no_commit)
+            if os.path.isdir(target) and len(sub_targets) > LARGE_DIR_SIZE:
+                logger.warning(
+                    "You are adding a large directory '{target}' recursively,"
+                    " consider tracking it as a whole instead.\n"
+                    "{purple}HINT:{nc} Remove the generated DVC-file and then"
+                    " run {cyan}dvc add {target}{nc}".format(
+                        purple=colorama.Fore.MAGENTA,
+                        cyan=colorama.Fore.CYAN,
+                        nc=colorama.Style.RESET_ALL,
+                        target=target,
+                    )
+                )
 
-    repo.check_dag(repo.stages() + stages)
+            stages = _create_stages(repo, sub_targets, fname, pbar=pbar)
 
-    for stage in stages:
-        stage.dump()
-    return stages
+            repo.check_modified_graph(stages)
+
+            for stage in stages:
+                stage.save()
+
+                if not no_commit:
+                    stage.commit()
+
+                stage.dump()
+
+            stages_list += stages
+        # remove filled bar bit of progress, leaving stats
+        pbar.bar_format = pbar.BAR_FMT_DEFAULT.replace("|{bar:10}|", " ")
+
+    return stages_list
 
 
 def _find_all_targets(repo, target, recursive):
@@ -53,21 +75,19 @@ def _find_all_targets(repo, target, recursive):
     return [target]
 
 
-def _create_stages(repo, targets, fname, no_commit):
+def _create_stages(repo, targets, fname, pbar=None):
     stages = []
 
-    with repo.state:
-        for out in targets:
-            stage = Stage.create(repo, outs=[out], add=True, fname=fname)
+    for out in targets:
+        stage = Stage.create(repo, outs=[out], add=True, fname=fname)
 
-            if not stage:
-                continue
+        if not stage:
+            if pbar is not None:
+                pbar.total -= 1
+            continue
 
-            stage.save()
-
-            if not no_commit:
-                stage.commit()
-
-            stages.append(stage)
+        stages.append(stage)
+        if pbar is not None:
+            pbar.update_desc(out)
 
     return stages

dvc/repo/brancher.py

@@ -1,8 +1,14 @@
+from funcy import group_by
 from dvc.scm.tree import WorkingTree
 
 
 def brancher(  # noqa: E302
-    self, branches=None, all_branches=False, tags=None, all_tags=False
+    self,
+    branches=None,
+    all_branches=False,
+    tags=None,
+    all_tags=False,
+    all_commits=False,
 ):
     """Generator that iterates over specified revisions.
 
@@ -18,7 +24,7 @@ def brancher(  # noqa: E302
             - empty string it there is no branches to iterate over
             - "Working Tree" if there are uncommited changes in the SCM repo
     """
-    if not any([branches, all_branches, tags, all_tags]):
+    if not any([branches, all_branches, tags, all_tags, all_commits]):
         yield ""
         return
 
@@ -30,25 +36,28 @@ def brancher(  # noqa: E302
     self.tree = WorkingTree(self.root_dir)
     yield "working tree"
 
-    if all_branches:
-        branches = scm.list_branches()
+    if all_commits:
+        revs = scm.list_all_commits()
+    else:
+        if all_branches:
+            branches = scm.list_branches()
 
-    if all_tags:
-        tags = scm.list_tags()
+        if all_tags:
+            tags = scm.list_tags()
 
-    if branches is not None:
-        revs.extend(branches)
+        if branches is not None:
+            revs.extend(branches)
 
-    if tags is not None:
-        revs.extend(tags)
+        if tags is not None:
+            revs.extend(tags)
 
     # NOTE: it might be a good idea to wrap this loop in try/finally block
     # to don't leave the tree on some unexpected branch after the
     # `brancher()`, but this could cause problems on exception handling
     # code which might expect the tree on which exception was raised to
     # stay in place. This behavior is a subject to change.
-    for rev in revs:
-        self.tree = scm.get_tree(rev)
-        yield rev
+    for sha, names in group_by(scm.resolve_rev, revs).items():
+        self.tree = scm.get_tree(sha)
+        yield ", ".join(names)
 
     self.tree = saved_tree

dvc/repo/checkout.py

@@ -2,8 +2,8 @@ from __future__ import unicode_literals
 
 import logging
 
-from dvc.exceptions import CheckoutErrorSuggestGit
-from dvc.progress import ProgressCallback
+from dvc.exceptions import CheckoutErrorSuggestGit, CheckoutError
+from dvc.progress import Tqdm
 
 
 logger = logging.getLogger(__name__)
@@ -23,29 +23,32 @@ def get_all_files_numbers(stages):
     return sum(stage.get_all_files_number() for stage in stages)
 
 
-def get_progress_callback(stages):
-    total_files_num = get_all_files_numbers(stages)
-    if total_files_num == 0:
-        return None
-    return ProgressCallback(total_files_num)
-
-
-def checkout(self, target=None, with_deps=False, force=False, recursive=False):
+def _checkout(
+    self, targets=None, with_deps=False, force=False, recursive=False
+):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    all_stages = self.stages()
-
-    try:
-        stages = self.collect(target, with_deps=with_deps, recursive=recursive)
-    except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
-        if not target:
-            raise
-        raise CheckoutErrorSuggestGit(target, exc)
-
-    with self.state:
-        _cleanup_unused_links(self, all_stages)
-        progress_callback = get_progress_callback(stages)
+    stages = set()
+    targets = targets or [None]
+    for target in targets:
+        try:
+            new = self.collect(
+                target, with_deps=with_deps, recursive=recursive
+            )
+            stages.update(new)
+        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+            if not target:
+                raise
+            raise CheckoutErrorSuggestGit(target, exc)
 
+    _cleanup_unused_links(self, self.stages)
+    total = get_all_files_numbers(stages)
+    if total == 0:
+        logger.info("Nothing to do")
+    failed = []
+    with Tqdm(
+        total=total, unit="file", desc="Checkout", disable=total == 0
+    ) as pbar:
         for stage in stages:
             if stage.locked:
                 logger.warning(
@@ -53,6 +56,8 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False):
                     " not going to be checked out.".format(path=stage.relpath)
                 )
 
-            stage.checkout(force=force, progress_callback=progress_callback)
-        if progress_callback:
-            progress_callback.finish("Checkout finished!")
+            failed.extend(
+                stage.checkout(force=force, progress_callback=pbar.update_desc)
+            )
+    if failed:
+        raise CheckoutError(failed)

dvc/repo/commit.py

@@ -1,7 +1,10 @@
+from . import locked
+
+
+@locked
 def commit(self, target, with_deps=False, recursive=False, force=False):
     stages = self.collect(target, with_deps=with_deps, recursive=recursive)
-    with self.state:
-        for stage in stages:
-            stage.check_can_commit(force=force)
-            stage.commit()
-            stage.dump()
+    for stage in stages:
+        stage.check_can_commit(force=force)
+        stage.commit()
+        stage.dump()

dvc/repo/destroy.py

@@ -2,7 +2,7 @@ from dvc.utils import remove
 
 
 def destroy(self):
-    for stage in self.stages():
+    for stage in self.stages:
         stage.remove(remove_outs=False)
 
     remove(self.dvc_dir)

dvc/repo/diff.py

@@ -14,6 +14,8 @@ from dvc.scm.git import (
     DIFF_EQUAL,
 )
 
+from . import locked
+
 
 DIFF_TARGET = "target"
 DIFF_IS_DIR = "is_dir"
@@ -77,7 +79,7 @@ def _modified_files(self, a_entries, b_entries):
     modified_count = 0
     for key in keys:
         if a_entries[key] != b_entries[key]:
-            modified_count
+            modified_count += 1
             diff_size += os.path.getsize(
                 self.cache.local.get(b_entries[key])
             ) - os.path.getsize(self.cache.local.get(a_entries[key]))
@@ -140,9 +142,9 @@ def _is_dir(path, a_outs, b_outs):
 
 def _get_diff_outs(self, diff_dct):
     self.tree = diff_dct[DIFF_A_TREE]
-    a_outs = {str(out): out for st in self.stages() for out in st.outs}
+    a_outs = {str(out): out for st in self.stages for out in st.outs}
     self.tree = diff_dct[DIFF_B_TREE]
-    b_outs = {str(out): out for st in self.stages() for out in st.outs}
+    b_outs = {str(out): out for st in self.stages for out in st.outs}
     outs_paths = set(a_outs.keys())
     outs_paths.update(b_outs.keys())
     results = {}
@@ -220,6 +222,7 @@ def _diff_royal(self, target, diff_dct):
     return _diff_file(self, target, diff_dct)
 
 
+@locked
 def diff(self, a_ref, target=None, b_ref=None):
     """Gerenates diff message string output
 

dvc/repo/fetch.py

@@ -1,7 +1,18 @@
 from __future__ import unicode_literals
 
+import logging
 
-def fetch(
+from dvc.config import NoRemoteError
+from dvc.exceptions import DownloadError, OutputNotFoundError
+from dvc.scm.base import CloneError
+from dvc.external_repo import external_repo
+from dvc.cache import NamedCache
+
+
+logger = logging.getLogger(__name__)
+
+
+def _fetch(
     self,
     targets=None,
     jobs=None,
@@ -12,17 +23,81 @@ def fetch(
     all_tags=False,
     recursive=False,
 ):
-    with self.state:
-        used = self.used_cache(
-            targets,
-            all_branches=all_branches,
-            all_tags=all_tags,
-            with_deps=with_deps,
-            force=True,
-            remote=remote,
-            jobs=jobs,
-            recursive=recursive,
-        )["local"]
-        return self.cloud.pull(
+    """Download data items from a cloud and imported repositories
+
+    Returns:
+        int: number of succesfully downloaded files
+
+    Raises:
+        DownloadError: thrown when there are failed downloads, either
+            during `cloud.pull` or trying to fetch imported files
+
+        config.NoRemoteError: thrown when downloading only local files and no
+            remote is configured
+    """
+    used = self.used_cache(
+        targets,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        with_deps=with_deps,
+        force=True,
+        remote=remote,
+        jobs=jobs,
+        recursive=recursive,
+    )
+
+    downloaded = 0
+    failed = 0
+
+    try:
+        downloaded += self.cloud.pull(
             used, jobs, remote=remote, show_checksums=show_checksums
         )
+    except NoRemoteError:
+        if not used.external and used["local"]:
+            raise
+    except DownloadError as exc:
+        failed += exc.amount
+
+    for (repo_url, repo_rev), files in used.external.items():
+        d, f = _fetch_external(self, repo_url, repo_rev, files)
+        downloaded += d
+        failed += f
+
+    if failed:
+        raise DownloadError(failed)
+
+    return downloaded
+
+
+def _fetch_external(self, repo_url, repo_rev, files):
+    failed = 0
+
+    cache_dir = self.cache.local.cache_dir
+    try:
+        with external_repo(repo_url, repo_rev, cache_dir=cache_dir) as repo:
+            with repo.state:
+                cache = NamedCache()
+                for name in files:
+                    try:
+                        out = repo.find_out_by_relpath(name)
+                    except OutputNotFoundError:
+                        failed += 1
+                        logger.exception(
+                            "failed to fetch data for '{}'".format(name)
+                        )
+                        continue
+                    else:
+                        cache.update(out.get_used_cache())
+
+                try:
+                    return repo.cloud.pull(cache), failed
+                except DownloadError as exc:
+                    failed += exc.amount
+    except CloneError:
+        failed += 1
+        logger.exception(
+            "failed to fetch data for '{}'".format(", ".join(files))
+        )
+
+    return 0, failed

dvc/repo/gc.py

@@ -1,61 +1,21 @@
 from __future__ import unicode_literals
 
-import collections
 import logging
 
+from dvc.cache import NamedCache
+from . import locked
+
 
 logger = logging.getLogger(__name__)
 
 
-def _merge_cache_lists(clists):
-    merged_cache = collections.defaultdict(list)
-
-    for cache_list in clists:
-        for scheme, cache in cache_list.items():
-            for item in cache:
-                if item not in merged_cache[scheme]:
-                    merged_cache[scheme].append(item)
-
-    return merged_cache
-
-
-def _load_all_used_cache(
-    repos,
-    target=None,
-    all_branches=False,
-    active=True,
-    with_deps=False,
-    all_tags=False,
-    remote=None,
-    force=False,
-    jobs=None,
-):
-    clists = []
-
-    for repo in repos:
-        with repo.state:
-            repo_clist = repo.used_cache(
-                targets=None,
-                all_branches=all_branches,
-                active=False,
-                with_deps=with_deps,
-                all_tags=all_tags,
-                remote=remote,
-                force=force,
-                jobs=jobs,
-            )
-
-            clists.append(repo_clist)
-
-    return clists
-
-
 def _do_gc(typ, func, clist):
     removed = func(clist)
     if not removed:
         logger.info("No unused {} cache to remove.".format(typ))
 
 
+@locked
 def gc(
     self,
     all_branches=False,
@@ -63,51 +23,54 @@ def gc(
     remote=None,
     with_deps=False,
     all_tags=False,
+    all_commits=False,
     force=False,
     jobs=None,
     repos=None,
 ):
+    from dvc.utils.compat import ExitStack
     from dvc.repo import Repo
 
-    all_repos = [self]
+    all_repos = []
 
     if repos:
-        all_repos.extend(Repo(path) for path in repos)
+        all_repos = [Repo(path) for path in repos]
 
-    all_clists = _load_all_used_cache(
-        all_repos,
-        target=None,
-        all_branches=all_branches,
-        active=False,
-        with_deps=with_deps,
-        all_tags=all_tags,
-        remote=remote,
-        force=force,
-        jobs=jobs,
-    )
+    with ExitStack() as stack:
+        for repo in all_repos:
+            stack.enter_context(repo.lock)
+            stack.enter_context(repo.state)
 
-    if len(all_clists) > 1:
-        clist = _merge_cache_lists(all_clists)
-    else:
-        clist = all_clists[0]
+        used = NamedCache()
+        for repo in all_repos + [self]:
+            used.update(
+                repo.used_cache(
+                    all_branches=all_branches,
+                    with_deps=with_deps,
+                    all_tags=all_tags,
+                    all_commits=all_commits,
+                    remote=remote,
+                    force=force,
+                    jobs=jobs,
+                )
+            )
 
-    with self.state:
-        _do_gc("local", self.cache.local.gc, clist)
+    _do_gc("local", self.cache.local.gc, used)
 
-        if self.cache.s3:
-            _do_gc("s3", self.cache.s3.gc, clist)
+    if self.cache.s3:
+        _do_gc("s3", self.cache.s3.gc, used)
 
-        if self.cache.gs:
-            _do_gc("gs", self.cache.gs.gc, clist)
+    if self.cache.gs:
+        _do_gc("gs", self.cache.gs.gc, used)
 
-        if self.cache.ssh:
-            _do_gc("ssh", self.cache.ssh.gc, clist)
+    if self.cache.ssh:
+        _do_gc("ssh", self.cache.ssh.gc, used)
 
-        if self.cache.hdfs:
-            _do_gc("hdfs", self.cache.hdfs.gc, clist)
+    if self.cache.hdfs:
+        _do_gc("hdfs", self.cache.hdfs.gc, used)
 
-        if self.cache.azure:
-            _do_gc("azure", self.cache.azure.gc, clist)
+    if self.cache.azure:
+        _do_gc("azure", self.cache.azure.gc, used)
 
-        if cloud:
-            _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, clist)
+    if cloud:
+        _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, used)

dvc/repo/get.py

@@ -1,17 +1,29 @@
+import logging
 import os
 import shortuuid
 
+from dvc.stage import Stage
 from dvc.config import Config
+from dvc.exceptions import (
+    NotDvcRepoError,
+    UrlNotDvcRepoError,
+    GetDVCFileError,
+    OutputNotFoundError,
+)
 from dvc.path_info import PathInfo
 from dvc.external_repo import external_repo
 from dvc.state import StateNoop
-from dvc.utils import remove
-from dvc.utils.compat import urlparse
+from dvc.utils import remove, resolve_output
+
+logger = logging.getLogger(__name__)
 
 
 @staticmethod
 def get(url, path, out=None, rev=None):
-    out = out or os.path.basename(urlparse(path).path)
+    out = resolve_output(path, out)
+
+    if Stage.is_valid_filename(out):
+        raise GetDVCFileError()
 
     # Creating a directory right beside the output to make sure that they
     # are on the same filesystem, so we could take the advantage of
@@ -48,5 +60,10 @@ def get(url, path, out=None, rev=None):
             o.path_info = PathInfo(os.path.abspath(out))
             with o.repo.state:
                 o.checkout()
+
+    except NotDvcRepoError:
+        raise UrlNotDvcRepoError(url)
+    except OutputNotFoundError:
+        raise OutputNotFoundError(path)
     finally:
         remove(tmp_dir)

dvc/repo/get_url.py

@@ -3,12 +3,12 @@ import os
 import dvc.output as output
 import dvc.dependency as dependency
 
-from dvc.utils.compat import urlparse
+from dvc.utils import resolve_output
 
 
 @staticmethod
 def get_url(url, out=None):
-    out = out or os.path.basename(urlparse(url).path)
+    out = resolve_output(url, out)
 
     if os.path.exists(url):
         url = os.path.abspath(url)

dvc/repo/graph.py

@@ -0,0 +1,36 @@
+from __future__ import unicode_literals
+
+
+def check_acyclic(graph):
+    import networkx as nx
+    from dvc.exceptions import CyclicGraphError
+
+    try:
+        edges = nx.find_cycle(graph, orientation="original")
+    except nx.NetworkXNoCycle:
+        return
+
+    stages = set()
+    for from_node, to_node, _ in edges:
+        stages.add(from_node)
+        stages.add(to_node)
+
+    raise CyclicGraphError(list(stages))
+
+
+def get_pipeline(pipelines, node):
+    found = [i for i in pipelines if i.has_node(node)]
+    assert len(found) == 1
+    return found[0]
+
+
+def get_pipelines(G):
+    import networkx as nx
+
+    return [G.subgraph(c).copy() for c in nx.weakly_connected_components(G)]
+
+
+def get_stages(G):
+    import networkx
+
+    return list(networkx.get_node_attributes(G, "stage").values())

dvc/repo/imp_url.py

@@ -1,25 +1,26 @@
-from dvc.utils.compat import pathlib
+from dvc.utils import resolve_output
 from dvc.repo.scm_context import scm_context
 
+from . import locked as locked_repo
 
+
+@locked_repo
 @scm_context
 def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
     from dvc.stage import Stage
 
-    out = out or pathlib.PurePath(url).name
+    out = resolve_output(url, out)
 
-    with self.state:
-        stage = Stage.create(
-            self, cmd=None, deps=[url], outs=[out], fname=fname, erepo=erepo
-        )
+    stage = Stage.create(
+        self, cmd=None, deps=[url], outs=[out], fname=fname, erepo=erepo
+    )
 
     if stage is None:
         return None
 
-    self.check_dag(self.stages() + [stage])
+    self.check_modified_graph([stage])
 
-    with self.state:
-        stage.run()
+    stage.run()
 
     stage.locked = locked
 

dvc/repo/init.py

@@ -65,7 +65,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
     if isinstance(scm, NoSCM) and not no_scm:
         raise InitError(
             "{repo} is not tracked by any supported scm tool (e.g. git). "
-            "Use '--no-scm' if you don't want to use any scm.".format(
+            "Use `--no-scm` if you don't want to use any scm.".format(
                 repo=root_dir
             )
         )
@@ -73,7 +73,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
     if os.path.isdir(dvc_dir):
         if not force:
             raise InitError(
-                "'{repo}' exists. Use '-f' to force.".format(
+                "'{repo}' exists. Use `-f` to force.".format(
                     repo=relpath(dvc_dir)
                 )
             )

dvc/repo/lock.py

@@ -1,3 +1,7 @@
+from . import locked
+
+
+@locked
 def lock(self, target, unlock=False):
     from dvc.stage import Stage
 

dvc/repo/metrics/modify.py

@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
+from dvc.repo import locked
 from dvc.exceptions import DvcException
 
 
+@locked
 def modify(repo, path, typ=None, xpath=None, delete=False):
     supported_types = ["raw", "json", "csv", "tsv", "hcsv", "htsv"]
     outs = repo.find_outs_by_path(path)

dvc/repo/metrics/show.py

@@ -8,7 +8,8 @@ import logging
 
 from jsonpath_ng.ext import parse
 
-from dvc.exceptions import OutputNotFoundError, BadMetricError, NoMetricsError
+from dvc.repo import locked
+from dvc.exceptions import OutputNotFoundError, NoMetricsError
 from dvc.utils.compat import builtin_str, open, StringIO, csv_reader
 
 NO_METRICS_FILE_AT_REFERENCE_WARNING = (
@@ -159,8 +160,8 @@ def _collect_metrics(repo, path, recursive, typ, xpath, branch):
         path (str): Path to a metric file or a directory.
         recursive (bool): If path is a directory, do a recursive search for
             metrics on the given path.
-        typ (str): The type of metric to search for, could be one of the
-            following (raw|json|tsv|htsv|csv|hcsv).
+        typ (str): The type that will be used to interpret the metric file,
+            one of the followings - (raw|json|tsv|htsv|csv|hcsv).
         xpath (str): Path to search for.
         branch (str): Branch to look up for metrics.
 
@@ -170,7 +171,7 @@ def _collect_metrics(repo, path, recursive, typ, xpath, branch):
             - typ:
             - xpath:
     """
-    outs = [out for stage in repo.stages() for out in stage.outs]
+    outs = [out for stage in repo.stages for out in stage.outs]
 
     if path:
         try:
@@ -188,16 +189,15 @@ def _collect_metrics(repo, path, recursive, typ, xpath, branch):
         if not o.metric:
             continue
 
-        # NOTE this case assumes that metric has been modified with
-        # `dvc metric modify -t {} -x {}`, so stage file should contain
-        # info what type metric is and what is its xpath
+        # NOTE this case assumes that typ has not been provided in CLI call
+        # and one of the following cases:
+        # - stage file contains metric type
+        # - typ will be read from file extension later
         if not typ and isinstance(o.metric, dict):
             t = o.metric.get(o.PARAM_METRIC_TYPE, typ)
             # NOTE user might want to check different xpath, hence xpath first
             x = xpath or o.metric.get(o.PARAM_METRIC_XPATH)
         else:
-            # NOTE if metric has not been modified, user needs to provide
-            # both metric type and xpath
             t = typ
             x = xpath
 
@@ -258,9 +258,10 @@ def _read_metrics(repo, metrics, branch):
     return res
 
 
+@locked
 def show(
     repo,
-    path=None,
+    targets=None,
     typ=None,
     xpath=None,
     all_branches=False,
@@ -268,16 +269,34 @@ def show(
     recursive=False,
 ):
     res = {}
+    found = set()
+
+    if not targets:
+        # Iterate once to call `_collect_metrics` on all the stages
+        targets = [None]
 
     for branch in repo.brancher(all_branches=all_branches, all_tags=all_tags):
-        entries = _collect_metrics(repo, path, recursive, typ, xpath, branch)
-        metrics = _read_metrics(repo, entries, branch)
+        metrics = {}
+
+        for target in targets:
+            entries = _collect_metrics(
+                repo, target, recursive, typ, xpath, branch
+            )
+            metric = _read_metrics(repo, entries, branch)
+
+            if metric:
+                found.add(target)
+                metrics.update(metric)
+
         if metrics:
             res[branch] = metrics
 
-    if not res:
-        if path:
-            raise BadMetricError(path)
+    if not res and not any(targets):
         raise NoMetricsError()
 
+    missing = set(targets) - found
+
+    if missing:
+        res[None] = missing
+
     return res

dvc/repo/move.py

@@ -3,6 +3,8 @@ import os
 from dvc.exceptions import MoveNotDataSourceError
 from dvc.repo.scm_context import scm_context
 
+from . import locked
+
 
 def _expand_target_path(from_path, to_path):
     if os.path.isdir(to_path):
@@ -10,6 +12,7 @@ def _expand_target_path(from_path, to_path):
     return to_path
 
 
+@locked
 @scm_context
 def move(self, from_path, to_path):
     """
@@ -59,11 +62,11 @@ def move(self, from_path, to_path):
             os.path.join(os.curdir, os.path.dirname(to_path))
         )
 
-    to_out = Output.loads_from(
-        stage, [os.path.basename(to_path)], out.use_cache, out.metric
-    )[0]
+    to_path = os.path.relpath(to_path, stage.wdir)
 
-    with self.state:
-        out.move(to_out)
+    to_out = Output.loads_from(stage, [to_path], out.use_cache, out.metric)[0]
+
+    out.move(to_out)
+    stage.save()
 
     stage.dump()

dvc/repo/pull.py

@@ -1,6 +1,9 @@
 from __future__ import unicode_literals
 
+from . import locked
 
+
+@locked
 def pull(
     self,
     targets=None,
@@ -12,7 +15,7 @@ def pull(
     force=False,
     recursive=False,
 ):
-    processed_files_count = self.fetch(
+    processed_files_count = self._fetch(
         targets,
         jobs,
         remote=remote,
@@ -21,11 +24,7 @@ def pull(
         with_deps=with_deps,
         recursive=recursive,
     )
-    for target in targets or [None]:
-        self.checkout(
-            target=target,
-            with_deps=with_deps,
-            force=force,
-            recursive=recursive,
-        )
+    self._checkout(
+        targets=targets, with_deps=with_deps, force=force, recursive=recursive
+    )
     return processed_files_count

dvc/repo/push.py

@@ -1,6 +1,9 @@
 from __future__ import unicode_literals
 
+from . import locked
 
+
+@locked
 def push(
     self,
     targets=None,
@@ -11,15 +14,14 @@ def push(
     all_tags=False,
     recursive=False,
 ):
-    with self.state:
-        used = self.used_cache(
-            targets,
-            all_branches=all_branches,
-            all_tags=all_tags,
-            with_deps=with_deps,
-            force=True,
-            remote=remote,
-            jobs=jobs,
-            recursive=recursive,
-        )["local"]
-        return self.cloud.push(used, jobs, remote=remote)
+    used = self.used_cache(
+        targets,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        with_deps=with_deps,
+        force=True,
+        remote=remote,
+        jobs=jobs,
+        recursive=recursive,
+    )
+    return self.cloud.push(used, jobs, remote=remote)

dvc/repo/remove.py

@@ -1,3 +1,7 @@
+from . import locked
+
+
+@locked
 def remove(self, target, outs_only=False):
     from dvc.stage import Stage
 

dvc/repo/reproduce.py

@@ -1,12 +1,15 @@
 from __future__ import unicode_literals
 
-import os
 import logging
 
 from dvc.exceptions import ReproductionError
 from dvc.repo.scm_context import scm_context
 from dvc.utils import relpath
 
+from . import locked
+from .graph import get_pipeline, get_pipelines
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -29,6 +32,24 @@ def _reproduce_stage(stages, node, **kwargs):
     return [stage]
 
 
+def _get_active_graph(G):
+    import networkx as nx
+
+    active = G.copy()
+    stages = nx.get_node_attributes(G, "stage")
+    for node in G:
+        stage = stages[node]
+        if not stage.locked:
+            continue
+        for n in nx.dfs_postorder_nodes(G, node):
+            if n == node:
+                continue
+            if n in active:
+                active.remove_node(n)
+    return active
+
+
+@locked
 @scm_context
 def reproduce(
     self,
@@ -52,43 +73,37 @@ def reproduce(
             config.SECTION_CORE_INTERACTIVE, False
         )
 
-    targets = []
-    if recursive and os.path.isdir(target):
-        G = self.graph(from_directory=target)[1]
-        dir_targets = [
-            os.path.join(self.root_dir, n) for n in nx.dfs_postorder_nodes(G)
-        ]
-        targets.extend(dir_targets)
-    elif pipeline or all_pipelines:
-        if pipeline:
+    active_graph = _get_active_graph(self.graph)
+    active_pipelines = get_pipelines(active_graph)
+
+    if pipeline or all_pipelines:
+        if all_pipelines:
+            pipelines = active_pipelines
+        else:
             stage = Stage.load(self, target)
             node = relpath(stage.path, self.root_dir)
-            pipelines = [self._get_pipeline(node)]
-        else:
-            pipelines = self.pipelines()
+            pipelines = [get_pipeline(active_pipelines, node)]
 
+        targets = []
         for G in pipelines:
-            for node in G.nodes():
+            attrs = nx.get_node_attributes(G, "stage")
+            for node in G:
                 if G.in_degree(node) == 0:
-                    targets.append(os.path.join(self.root_dir, node))
+                    targets.append(attrs[node])
     else:
-        targets.append(target)
+        targets = self.collect(target, recursive=recursive, graph=active_graph)
 
     ret = []
-    with self.state:
-        for target in targets:
-            stages = _reproduce(self, target, **kwargs)
-            ret.extend(stages)
+    for target in targets:
+        stages = _reproduce(self, active_graph, target, **kwargs)
+        ret.extend(stages)
 
     return ret
 
 
-def _reproduce(self, target, **kwargs):
+def _reproduce(self, G, stage, **kwargs):
     import networkx as nx
-    from dvc.stage import Stage
 
-    stage = Stage.load(self, target)
-    G = self.graph()[1]
     stages = nx.get_node_attributes(G, "stage")
     node = relpath(stage.path, self.root_dir)
 

dvc/repo/run.py

@@ -1,23 +1,23 @@
 from __future__ import unicode_literals
 
-from dvc.repo.scm_context import scm_context
+from . import locked
+from .scm_context import scm_context
 
 
+@locked
 @scm_context
 def run(self, no_exec=False, **kwargs):
     from dvc.stage import Stage
 
-    with self.state:
-        stage = Stage.create(self, **kwargs)
+    stage = Stage.create(self, **kwargs)
 
     if stage is None:
         return None
 
-    self.check_dag(self.stages() + [stage])
+    self.check_modified_graph([stage])
 
-    with self.state:
-        if not no_exec:
-            stage.run(no_commit=kwargs.get("no_commit", False))
+    if not no_exec:
+        stage.run(no_commit=kwargs.get("no_commit", False))
 
     stage.dump()
 

dvc/repo/status.py

@@ -3,6 +3,8 @@ from __future__ import unicode_literals
 import logging
 from funcy.py3 import cat
 
+from . import locked
+
 
 logger = logging.getLogger(__name__)
 
@@ -74,7 +76,7 @@ def _cloud_status(
         force=True,
         remote=remote,
         jobs=jobs,
-    )["local"]
+    )
 
     ret = {}
     status_info = self.cloud.status(used, jobs, remote=remote)
@@ -91,6 +93,7 @@ def _cloud_status(
     return ret
 
 
+@locked
 def status(
     self,
     targets=None,
@@ -101,15 +104,14 @@ def status(
     with_deps=False,
     all_tags=False,
 ):
-    with self.state:
-        if cloud or remote:
-            return _cloud_status(
-                self,
-                targets,
-                jobs,
-                all_branches=all_branches,
-                with_deps=with_deps,
-                remote=remote,
-                all_tags=all_tags,
-            )
-        return _local_status(self, targets, with_deps=with_deps)
+    if cloud or remote:
+        return _cloud_status(
+            self,
+            targets,
+            jobs,
+            all_branches=all_branches,
+            with_deps=with_deps,
+            remote=remote,
+            all_tags=all_tags,
+        )
+    return _local_status(self, targets, with_deps=with_deps)

dvc/repo/tag/add.py

@@ -1,9 +1,13 @@
 import logging
 from copy import copy
 
+from dvc.repo import locked
+
+
 logger = logging.getLogger(__name__)
 
 
+@locked
 def add(self, tag, target=None, with_deps=False, recursive=False):
     stages = self.collect(target, with_deps=with_deps, recursive=recursive)
     for stage in stages:

dvc/repo/tag/list.py

@@ -1,3 +1,7 @@
+from dvc.repo import locked
+
+
+@locked
 def list(self, target=None, with_deps=False, recursive=False):
     ret = {}
     stages = self.collect(target, with_deps=with_deps, recursive=recursive)

dvc/repo/tag/remove.py

@@ -1,9 +1,12 @@
 import logging
 
+from dvc.repo import locked
+
 
 logger = logging.getLogger(__name__)
 
 
+@locked
 def remove(self, tag, target=None, with_deps=False, recursive=False):
     stages = self.collect(target, with_deps=with_deps, recursive=recursive)
     for stage in stages:

dvc/repo/update.py

@@ -1,9 +1,11 @@
+from . import locked
+
+
+@locked
 def update(self, target):
     from dvc.stage import Stage
 
     stage = Stage.load(self, target)
-    with self.state:
-        stage.update()
-        stage.save()
+    stage.update()
 
     stage.dump()

dvc/scm/__init__.py

@@ -12,7 +12,7 @@ class NoSCM(Base):
     pass
 
 
-def SCM(root_dir, repo=None):  # pylint: disable=invalid-name
+def SCM(root_dir):  # pylint: disable=invalid-name
     """Returns SCM instance that corresponds to a repo at the specified
     path.
 
@@ -24,6 +24,6 @@ def SCM(root_dir, repo=None):  # pylint: disable=invalid-name
         dvc.scm.base.Base: SCM instance.
     """
     if Git.is_repo(root_dir) or Git.is_submodule(root_dir):
-        return Git(root_dir, repo=repo)
+        return Git(root_dir)
 
-    return NoSCM(root_dir, repo=repo)
+    return NoSCM(root_dir)

dvc/scm/base.py

@@ -11,28 +11,37 @@ class SCMError(DvcException):
     """Base class for source control management errors."""
 
 
-class FileNotInRepoError(DvcException):
+class FileNotInRepoError(SCMError):
     """Thrown when trying to find .gitignore for a file that is not in a scm
     repository.
     """
 
 
-class FileNotInCommitError(DvcException):
+class FileNotInCommitError(SCMError):
     """Thrown when trying to find a file/directory that is not
     in the specified commit in the repository.
     """
 
 
-class FileNotInTargetSubdirError(DvcException):
-    """Thrown when trying to place .gitignore for a file that not in
-    the file subdirectory."""
+class CloneError(SCMError):
+    def __init__(self, url, path, cause):
+        super(CloneError, self).__init__(
+            "Failed to clone repo '{}' to '{}'".format(url, path), cause=cause
+        )
+
+
+class RevError(SCMError):
+    def __init__(self, url, rev, cause):
+        super(RevError, self).__init__(
+            "Failed to access revision '{}' for repo '{}'".format(rev, url),
+            cause=cause,
+        )
 
 
 class Base(object):
     """Base class for source control management driver implementations."""
 
-    def __init__(self, root_dir=os.curdir, repo=None):
-        self.repo = repo
+    def __init__(self, root_dir=os.curdir):
         self.root_dir = os.path.realpath(root_dir)
 
     def __repr__(self):
@@ -111,6 +120,10 @@ class Base(object):
         """Returns a list of available tags in the repo."""
         return []
 
+    def list_all_commits(self):  # pylint: disable=no-self-use
+        """Returns a list of commits in the repo."""
+        return []
+
     def install(self):
         """Adds dvc commands to SCM hooks for the repo."""
 
@@ -144,3 +157,6 @@ class Base(object):
 
     def belongs_to_scm(self, path):
         """Return boolean whether file belongs to scm"""
+
+    def close(self):
+        """ Method to close the files """

dvc/scm/git/__init__.py

@@ -5,13 +5,18 @@ from __future__ import unicode_literals
 import os
 import logging
 
-from dvc.utils.compat import str, open
-from dvc.utils import fix_env, relpath
+from funcy import cached_property
+from pathspec.patterns import GitWildMatchPattern
+
+from dvc.exceptions import GitHookAlreadyExistsError
+from dvc.utils.compat import str, open, cast_bytes_py2
+from dvc.utils import fix_env, relpath, is_binary
 from dvc.scm.base import (
     Base,
     SCMError,
     FileNotInRepoError,
-    FileNotInTargetSubdirError,
+    CloneError,
+    RevError,
 )
 from dvc.scm.git.tree import GitTree
 
@@ -32,17 +37,17 @@ class Git(Base):
     GITIGNORE = ".gitignore"
     GIT_DIR = ".git"
 
-    def __init__(self, root_dir=os.curdir, repo=None):
+    def __init__(self, root_dir=os.curdir):
         """Git class constructor.
         Requires `Repo` class from `git` module (from gitpython package).
         """
-        super(Git, self).__init__(root_dir, repo=repo)
+        super(Git, self).__init__(root_dir)
 
         import git
         from git.exc import InvalidGitRepositoryError
 
         try:
-            self.git = git.Repo(self.root_dir)
+            self.repo = git.Repo(self.root_dir)
         except InvalidGitRepositoryError:
             msg = "{} is not a git repository"
             raise SCMError(msg.format(self.root_dir))
@@ -51,11 +56,50 @@ class Git(Base):
         # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
         env = fix_env(None)
         libpath = env.get("LD_LIBRARY_PATH", None)
-        self.git.git.update_environment(LD_LIBRARY_PATH=libpath)
+        self.repo.git.update_environment(LD_LIBRARY_PATH=libpath)
 
         self.ignored_paths = []
         self.files_to_track = set()
 
+    @staticmethod
+    def clone(url, to_path, rev=None):
+        import git
+
+        ld_key = "LD_LIBRARY_PATH"
+
+        env = fix_env(None)
+        if is_binary() and ld_key not in env.keys():
+            # In fix_env, we delete LD_LIBRARY_PATH key if it was empty before
+            # PyInstaller modified it. GitPython, in git.Repo.clone_from, uses
+            # env to update its own internal state. When there is no key in
+            # env, this value is not updated and GitPython re-uses
+            # LD_LIBRARY_PATH that has been set by PyInstaller.
+            # See [1] for more info.
+            # [1] https://github.com/gitpython-developers/GitPython/issues/924
+            env[cast_bytes_py2(ld_key)] = ""
+
+        try:
+            tmp_repo = git.Repo.clone_from(
+                url,
+                to_path,
+                env=env,  # needed before we can fix it in __init__
+                no_single_branch=True,
+            )
+            tmp_repo.close()
+        except git.exc.GitCommandError as exc:
+            raise CloneError(url, to_path, exc)
+
+        # NOTE: using our wrapper to make sure that env is fixed in __init__
+        repo = Git(to_path)
+
+        if rev:
+            try:
+                repo.checkout(rev)
+            except git.exc.GitCommandError as exc:
+                raise RevError(url, rev, exc)
+
+        return repo
+
     @staticmethod
     def is_repo(root_dir):
         return os.path.isdir(Git._get_git_dir(root_dir))
@@ -70,28 +114,18 @@ class Git(Base):
 
     @property
     def dir(self):
-        return self.git.git_dir
+        return self.repo.git_dir
 
     @property
     def ignore_file(self):
         return self.GITIGNORE
 
-    def _get_gitignore(self, path, ignore_file_dir=None):
-        if not ignore_file_dir:
-            ignore_file_dir = os.path.dirname(os.path.realpath(path))
+    def _get_gitignore(self, path):
+        ignore_file_dir = os.path.dirname(path)
 
         assert os.path.isabs(path)
         assert os.path.isabs(ignore_file_dir)
 
-        if not path.startswith(ignore_file_dir):
-            msg = (
-                "{} file has to be located in one of '{}' subdirectories"
-                ", not outside '{}'"
-            )
-            raise FileNotInTargetSubdirError(
-                msg.format(self.GITIGNORE, path, ignore_file_dir)
-            )
-
         entry = relpath(path, ignore_file_dir).replace(os.sep, "/")
         # NOTE: using '/' prefix to make path unambiguous
         if len(entry) > 0 and entry[0] != "/":
@@ -114,19 +148,14 @@ class Git(Base):
             )
         return False
 
-    def ignore(self, path, in_curr_dir=False):
-        base_dir = (
-            os.path.realpath(os.curdir)
-            if in_curr_dir
-            else os.path.dirname(path)
-        )
-        entry, gitignore = self._get_gitignore(path, base_dir)
+    def ignore(self, path):
+        entry, gitignore = self._get_gitignore(path)
 
         if self._ignored(entry, gitignore):
             return
 
         msg = "Adding '{}' to '{}'.".format(relpath(path), relpath(gitignore))
-        logger.info(msg)
+        logger.debug(msg)
 
         self._add_entry_to_gitignore(entry, gitignore)
 
@@ -134,8 +163,9 @@ class Git(Base):
 
         self.ignored_paths.append(path)
 
-    @staticmethod
-    def _add_entry_to_gitignore(entry, gitignore):
+    def _add_entry_to_gitignore(self, entry, gitignore):
+        entry = GitWildMatchPattern.escape(entry)
+
         with open(gitignore, "a+", encoding="utf-8") as fobj:
             fobj.seek(0, os.SEEK_END)
             if fobj.tell() == 0:
@@ -167,58 +197,65 @@ class Git(Base):
         # NOTE: GitPython is not currently able to handle index version >= 3.
         # See https://github.com/iterative/dvc/issues/610 for more details.
         try:
-            self.git.index.add(paths)
+            self.repo.index.add(paths)
         except AssertionError:
             msg = (
-                "failed to add '{}' to git. You can add those files"
-                " manually using 'git add'."
-                " See 'https://github.com/iterative/dvc/issues/610'"
-                " for more details.".format(str(paths))
+                "failed to add '{}' to git. You can add those files "
+                "manually using `git add`. See "
+                "https://github.com/iterative/dvc/issues/610 for more "
+                "details.".format(str(paths))
             )
 
             logger.exception(msg)
 
     def commit(self, msg):
-        self.git.index.commit(msg)
+        self.repo.index.commit(msg)
 
     def checkout(self, branch, create_new=False):
         if create_new:
-            self.git.git.checkout("HEAD", b=branch)
+            self.repo.git.checkout("HEAD", b=branch)
         else:
-            self.git.git.checkout(branch)
+            self.repo.git.checkout(branch)
 
     def branch(self, branch):
-        self.git.git.branch(branch)
+        self.repo.git.branch(branch)
 
     def tag(self, tag):
-        self.git.git.tag(tag)
+        self.repo.git.tag(tag)
 
     def untracked_files(self):
-        files = self.git.untracked_files
-        return [os.path.join(self.git.working_dir, fname) for fname in files]
+        files = self.repo.untracked_files
+        return [os.path.join(self.repo.working_dir, fname) for fname in files]
 
     def is_tracked(self, path):
-        # it is equivalent to `bool(self.git.git.ls_files(path))` by
+        # it is equivalent to `bool(self.repo.git.ls_files(path))` by
         # functionality, but ls_files fails on unicode filenames
         path = relpath(path, self.root_dir)
-        return path in [i[0] for i in self.git.index.entries]
+        return path in [i[0] for i in self.repo.index.entries]
 
     def is_dirty(self):
-        return self.git.is_dirty()
+        return self.repo.is_dirty()
 
     def active_branch(self):
-        return self.git.active_branch.name
+        return self.repo.active_branch.name
 
     def list_branches(self):
-        return [h.name for h in self.git.heads]
+        return [h.name for h in self.repo.heads]
 
     def list_tags(self):
-        return [t.name for t in self.git.tags]
+        return [t.name for t in self.repo.tags]
+
+    def list_all_commits(self):
+        return [c.hexsha for c in self.repo.iter_commits("--all")]
 
     def _install_hook(self, name, cmd):
-        command = '[ -z "$(git ls-files .dvc)" ] || exec dvc {}'.format(cmd)
+        command = (
+            '[ "$3" = "0" ]'
+            ' || [ -z "$(git ls-files .dvc)" ]'
+            " || exec dvc {}".format(cmd)
+        )
 
-        hook = os.path.join(self.root_dir, self.GIT_DIR, "hooks", name)
+        hook = self._hook_path(name)
 
         if os.path.isfile(hook):
             with open(hook, "r+") as fobj:
@@ -231,6 +268,8 @@ class Git(Base):
         os.chmod(hook, 0o777)
 
     def install(self):
+        self._verify_dvc_hooks()
+
         self._install_hook("post-checkout", "checkout")
         self._install_hook("pre-commit", "status")
         self._install_hook("pre-push", "push")
@@ -266,7 +305,7 @@ class Git(Base):
         return basename == self.ignore_file or Git.GIT_DIR in path_parts
 
     def get_tree(self, rev):
-        return GitTree(self.git, rev)
+        return GitTree(self.repo, rev)
 
     def _get_diff_trees(self, a_ref, b_ref):
         """Private method for getting the trees and commit hashes of 2 git
@@ -284,10 +323,10 @@ class Git(Base):
         trees = {DIFF_A_TREE: None, DIFF_B_TREE: None}
         commits = []
         if b_ref is None:
-            b_ref = self.git.head.commit
+            b_ref = self.repo.head.commit
         try:
-            a_commit = self.git.git.rev_parse(a_ref, short=True)
-            b_commit = self.git.git.rev_parse(b_ref, short=True)
+            a_commit = self.repo.git.rev_parse(a_ref, short=True)
+            b_commit = self.repo.git.rev_parse(b_ref, short=True)
             # See https://gitpython.readthedocs.io
             # /en/2.1.11/reference.html#git.objects.base.Object.__str__
             commits.append(a_commit)
@@ -322,4 +361,26 @@ class Git(Base):
         return diff_dct
 
     def get_rev(self):
-        return self.git.git.rev_parse("HEAD")
+        return self.repo.git.rev_parse("HEAD")
+
+    def resolve_rev(self, rev):
+        return self.repo.git.rev_parse(rev)
+
+    def close(self):
+        self.repo.close()
+
+    @cached_property
+    def _hooks_home(self):
+        return os.path.join(self.root_dir, self.GIT_DIR, "hooks")
+
+    def _hook_path(self, name):
+        return os.path.join(self._hooks_home, name)
+
+    def _verify_hook(self, name):
+        if os.path.exists(self._hook_path(name)):
+            raise GitHookAlreadyExistsError(name)
+
+    def _verify_dvc_hooks(self):
+        self._verify_hook("post-checkout")
+        self._verify_hook("pre-commit")
+        self._verify_hook("pre-push")

dvc/scm/git/tree.py

@@ -45,7 +45,8 @@ class GitTree(BaseTree):
     def tree_root(self):
         return self.git.working_dir
 
-    def open(self, path, binary=False):
+    def open(self, path, mode="r", encoding="utf-8"):
+        assert mode in {"r", "rb"}
 
         relative_path = relpath(path, self.git.working_dir)
 
@@ -61,9 +62,9 @@ class GitTree(BaseTree):
         # the `open()` behavior (since data_stream.read() returns bytes,
         # and `open` with default "r" mode returns str)
         data = obj.data_stream.read()
-        if binary:
+        if mode == "rb":
             return BytesIO(data)
-        return StringIO(data.decode("utf-8"))
+        return StringIO(data.decode(encoding))
 
     def exists(self, path):
         return self.git_object_by_path(path) is not None

dvc/scm/tree.py

@@ -11,7 +11,7 @@ class BaseTree(object):
     def tree_root(self):
         pass
 
-    def open(self, path, binary=False):
+    def open(self, path, mode="r", encoding="utf-8"):
         """Open file and return a stream."""
 
     def exists(self, path):
@@ -42,11 +42,9 @@ class WorkingTree(BaseTree):
     def tree_root(self):
         return self.repo_root
 
-    def open(self, path, binary=False):
+    def open(self, path, mode="r", encoding="utf-8"):
         """Open file and return a stream."""
-        if binary:
-            return open(path, "rb")
-        return open(path, encoding="utf-8")
+        return open(path, mode=mode, encoding=encoding)
 
     def exists(self, path):
         """Test whether a path exists."""

dvc/stage.py

@@ -9,6 +9,7 @@ import subprocess
 import logging
 import signal
 import threading
+from itertools import chain
 
 from dvc.utils import relpath
 from dvc.utils.compat import pathlib
@@ -138,6 +139,7 @@ class Stage(object):
     PARAM_OUTS = "outs"
     PARAM_LOCKED = "locked"
     PARAM_META = "meta"
+    PARAM_ALWAYS_CHANGED = "always_changed"
 
     SCHEMA = {
         Optional(PARAM_MD5): Or(str, None),
@@ -147,6 +149,7 @@ class Stage(object):
         Optional(PARAM_OUTS): Or(And(list, Schema([output.SCHEMA])), None),
         Optional(PARAM_LOCKED): bool,
         Optional(PARAM_META): object,
+        Optional(PARAM_ALWAYS_CHANGED): bool,
     }
 
     TAG_REGEX = r"^(?P<path>.*)@(?P<tag>[^\\/@:]*)$"
@@ -163,6 +166,7 @@ class Stage(object):
         locked=False,
         tag=None,
         state=None,
+        always_changed=False,
     ):
         if deps is None:
             deps = []
@@ -178,6 +182,7 @@ class Stage(object):
         self.md5 = md5
         self.locked = locked
         self.tag = tag
+        self.always_changed = always_changed
         self._state = state or {}
 
     def __repr__(self):
@@ -237,12 +242,15 @@ class Stage(object):
 
         if self.is_callback:
             logger.warning(
-                "DVC-file '{fname}' is a 'callback' stage "
+                "DVC-file '{fname}' is a \"callback\" stage "
                 "(has a command and no dependencies) and thus always "
                 "considered as changed.".format(fname=self.relpath)
             )
             return True
 
+        if self.always_changed:
+            return True
+
         for dep in self.deps:
             status = dep.status()
             if status:
@@ -285,7 +293,7 @@ class Stage(object):
         if ret:
             logger.warning("Stage '{}' changed.".format(self.relpath))
         else:
-            logger.info("Stage '{}' didn't change.".format(self.relpath))
+            logger.debug("Stage '{}' didn't change.".format(self.relpath))
 
         return ret
 
@@ -326,8 +334,6 @@ class Stage(object):
         if interactive and not prompt.confirm(msg):
             raise DvcException("reproduction aborted by the user")
 
-        logger.info("Reproducing '{stage}'".format(stage=self.relpath))
-
         self.run(**kwargs)
 
         logger.debug("'{stage}' was reproduced".format(stage=self.relpath))
@@ -438,25 +444,27 @@ class Stage(object):
         wdir = kwargs.get("wdir", None)
         cwd = kwargs.get("cwd", None)
         fname = kwargs.get("fname", None)
+        add = kwargs.get("add", False)
 
         # Backward compatibility for `cwd` option
         if wdir is None and cwd is not None:
             if fname is not None and os.path.basename(fname) != fname:
                 raise StageFileBadNameError(
                     "DVC-file name '{fname}' may not contain subdirectories"
-                    " if '-c|--cwd' (deprecated) is specified. Use '-w|--wdir'"
-                    " along with '-f' to specify DVC-file path with working"
+                    " if `-c|--cwd` (deprecated) is specified. Use `-w|--wdir`"
+                    " along with `-f` to specify DVC-file path with working"
                     " directory.".format(fname=fname)
                 )
             wdir = cwd
-        else:
-            wdir = os.curdir if wdir is None else wdir
+        elif wdir is None:
+            wdir = os.curdir
 
         stage = Stage(
             repo=repo,
             wdir=wdir,
             cmd=kwargs.get("cmd", None),
             locked=kwargs.get("locked", False),
+            always_changed=kwargs.get("always_changed", False),
         )
 
         Stage._fill_stage_outputs(stage, **kwargs)
@@ -468,11 +476,18 @@ class Stage(object):
         stage._check_duplicated_arguments()
 
         if not fname:
-            fname = Stage._stage_fname(
-                stage.outs, add=kwargs.get("add", False)
-            )
+            fname = Stage._stage_fname(stage.outs, add)
         stage._check_dvc_filename(fname)
 
+        # Autodetecting wdir for add, we need to create outs first to do that,
+        # so we start with wdir = . and remap out paths later.
+        if add and kwargs.get("wdir") is None and cwd is None:
+            wdir = os.path.dirname(fname)
+
+            for out in chain(stage.outs, stage.deps):
+                if out.is_in_repo:
+                    out.def_path = relpath(out.path_info, wdir)
+
         wdir = os.path.abspath(wdir)
 
         if cwd is not None:
@@ -498,8 +513,6 @@ class Stage(object):
             stage.remove_outs(ignore_remove=False)
             logger.warning("Build cache is ignored when using --remove-outs.")
             ignore_build_cache = True
-        else:
-            stage.unprotect_outs()
 
         if os.path.exists(path) and any(out.persist for out in stage.outs):
             logger.warning("Build cache is ignored when persisting outputs.")
@@ -510,6 +523,7 @@ class Stage(object):
                 not ignore_build_cache
                 and stage.is_cached
                 and not stage.is_callback
+                and not stage.always_changed
             ):
                 logger.info("Stage is cached, skipping.")
                 return None
@@ -557,8 +571,8 @@ class Stage(object):
     def _check_dvc_filename(fname):
         if not Stage.is_valid_filename(fname):
             raise StageFileBadNameError(
-                "bad DVC-file name '{}'. DVC-files should be named"
-                " 'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
+                "bad DVC-file name '{}'. DVC-files should be named "
+                "'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
                     relpath(fname), os.path.basename(fname)
                 )
             )
@@ -614,6 +628,7 @@ class Stage(object):
             md5=d.get(Stage.PARAM_MD5),
             locked=d.get(Stage.PARAM_LOCKED, False),
             tag=tag,
+            always_changed=d.get(Stage.PARAM_ALWAYS_CHANGED, False),
             state=state,
         )
 
@@ -624,16 +639,21 @@ class Stage(object):
 
     def dumpd(self):
         rel_wdir = relpath(self.wdir, os.path.dirname(self.path))
+
+        wdir = pathlib.PurePath(rel_wdir).as_posix()
+        wdir = wdir if wdir != "." else None
+
         return {
             key: value
             for key, value in {
                 Stage.PARAM_MD5: self.md5,
                 Stage.PARAM_CMD: self.cmd,
-                Stage.PARAM_WDIR: pathlib.PurePath(rel_wdir).as_posix(),
+                Stage.PARAM_WDIR: wdir,
                 Stage.PARAM_LOCKED: self.locked,
                 Stage.PARAM_DEPS: [d.dumpd() for d in self.deps],
                 Stage.PARAM_OUTS: [o.dumpd() for o in self.outs],
                 Stage.PARAM_META: self._state.get("meta"),
+                Stage.PARAM_ALWAYS_CHANGED: self.always_changed,
             }.items()
             if value
         }
@@ -643,7 +663,7 @@ class Stage(object):
 
         self._check_dvc_filename(fname)
 
-        logger.info(
+        logger.debug(
             "Saving information to '{file}'.".format(file=relpath(fname))
         )
         d = self.dumpd()
@@ -711,13 +731,12 @@ class Stage(object):
             msg += " and " if (changed_deps and changed_outs) else ""
             msg += "outputs {}".format(changed_outs) if changed_outs else ""
             msg += "md5" if not (changed_deps or changed_outs) else ""
-            msg += " of '{}' changed. Are you sure you commit it?".format(
-                self.relpath
-            )
+            msg += " of '{}' changed.".format(self.relpath)
+            msg += "Are you sure you want to commit it?"
             if not force and not prompt.confirm(msg):
                 raise StageCommitError(
                     "unable to commit changed '{}'. Use `-f|--force` to "
-                    "force.`".format(self.relpath)
+                    "force.".format(self.relpath)
                 )
             self.save()
 
@@ -769,28 +788,48 @@ class Stage(object):
 
     def _run(self):
         self._check_missing_deps()
-        executable = os.getenv("SHELL") if os.name != "nt" else None
-        self._warn_if_fish(executable)
+
+        kwargs = {"cwd": self.wdir, "env": fix_env(None), "close_fds": True}
+
+        if os.name == "nt":
+            kwargs["shell"] = True
+            cmd = self.cmd
+        else:
+            # NOTE: when you specify `shell=True`, `Popen` [1] will default to
+            # `/bin/sh` on *nix and will add ["/bin/sh", "-c"] to your command.
+            # But we actually want to run the same shell that we are running
+            # from right now, which is usually determined by the `SHELL` env
+            # var. So instead, we compose our command on our own, making sure
+            # to include special flags to prevent shell from reading any
+            # configs and modifying env, which may change the behavior or the
+            # command we are running. See [2] for more info.
+            #
+            # [1] https://github.com/python/cpython/blob/3.7/Lib/subprocess.py
+            #                                                            #L1426
+            # [2] https://github.com/iterative/dvc/issues/2506
+            #                                           #issuecomment-535396799
+            kwargs["shell"] = False
+            executable = os.getenv("SHELL") or "/bin/sh"
+
+            self._warn_if_fish(executable)
+
+            opts = {"zsh": ["--no-rcs"], "bash": ["--noprofile", "--norc"]}
+            name = os.path.basename(executable).lower()
+            cmd = [executable] + opts.get(name, []) + ["-c", self.cmd]
 
         main_thread = isinstance(
             threading.current_thread(), threading._MainThread
         )
+        old_handler = None
         p = None
 
         try:
-            p = subprocess.Popen(
-                self.cmd,
-                cwd=self.wdir,
-                shell=True,
-                env=fix_env(os.environ),
-                executable=executable,
-                close_fds=True,
-            )
+            p = subprocess.Popen(cmd, **kwargs)
             if main_thread:
                 old_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
             p.communicate()
         finally:
-            if main_thread:
+            if old_handler:
                 signal.signal(signal.SIGINT, old_handler)
 
         if (p is None) or (p.returncode != 0):
@@ -833,6 +872,7 @@ class Stage(object):
                 if (
                     not force
                     and not self.is_callback
+                    and not self.always_changed
                     and self._already_cached()
                 ):
                     self.checkout()
@@ -850,10 +890,14 @@ class Stage(object):
             raise MissingDataSource(paths)
 
     def checkout(self, force=False, progress_callback=None):
+        failed_checkouts = []
         for out in self.outs:
-            out.checkout(
+            failed = out.checkout(
                 force=force, tag=self.tag, progress_callback=progress_callback
             )
+            if failed:
+                failed_checkouts.append(failed)
+        return failed_checkouts
 
     @staticmethod
     def _status(entries):
@@ -879,7 +923,7 @@ class Stage(object):
         if self.changed_md5():
             ret.append("changed checksum")
 
-        if self.is_callback:
+        if self.is_callback or self.always_changed:
             ret.append("always changed")
 
         if ret:

dvc/state.py

@@ -3,6 +3,7 @@
 from __future__ import unicode_literals
 
 import os
+import re
 import sqlite3
 import logging
 
@@ -10,7 +11,8 @@ from dvc.config import Config
 from dvc.utils import remove, current_timestamp, relpath, to_chunks
 from dvc.exceptions import DvcException
 from dvc.utils.fs import get_mtime_and_size, get_inode
-from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import fspath_py35, urlencode, urlunparse, is_py2
+
 
 SQLITE_MAX_VARIABLES_NUMBER = 999
 
@@ -213,7 +215,11 @@ class State(object):  # pylint: disable=too-many-instance-attributes
             assert self.cursor is None
             assert self.inserts == 0
             empty = not os.path.exists(self.state_file)
-            self.database = sqlite3.connect(self.state_file)
+            # NOTE: we use nolock option because fcntl() lock sqlite uses
+            # doesn't work on some older NFS/CIFS filesystems.
+            # This opens a possibility of data corruption by concurrent writes,
+            # which is prevented by repo lock.
+            self.database = _connect_sqlite(self.state_file, {"nolock": 1})
             self.cursor = self.database.cursor()
 
             # Try loading once to check that the file is indeed a database
@@ -473,3 +479,25 @@ class State(object):  # pylint: disable=too-many-instance-attributes
                 self.LINK_STATE_TABLE, ",".join(["?"] * len(chunk_unused))
             )
             self._execute(cmd, tuple(chunk_unused))
+
+
+def _connect_sqlite(filename, options):
+    # Connect by URI was added in Python 3.4 and sqlite 3.7.7,
+    # we ignore options, which should be fine unless repo is on old NFS/CIFS
+    if is_py2 or sqlite3.sqlite_version_info < (3, 7, 7):
+        return sqlite3.connect(filename)
+
+    uri = _build_sqlite_uri(filename, options)
+    return sqlite3.connect(uri, uri=True)
+
+
+def _build_sqlite_uri(filename, options):
+    # Convert filename to uri according to https://www.sqlite.org/uri.html, 3.1
+    uri_path = filename.replace("?", "%3f").replace("#", "%23")
+    if os.name == "nt":
+        uri_path = uri_path.replace("\\", "/")
+        uri_path = re.sub(r"^([a-z]:)", "/\\1", uri_path, flags=re.I)
+    uri_path = re.sub(r"/+", "/", uri_path)
+
+    # Empty netloc, params and fragment
+    return urlunparse(("file", "", uri_path, "", urlencode(options), ""))

dvc/system.py

@@ -1,9 +1,9 @@
 from __future__ import unicode_literals
-
-from dvc.utils.compat import str, open, fspath
-
 import os
 import errno
+import shutil
+
+from dvc.utils.compat import str, open, fspath
 
 
 class System(object):
@@ -11,6 +11,11 @@ class System(object):
     def is_unix():
         return os.name != "nt"
 
+    @staticmethod
+    def copy(src, dest):
+        src, dest = fspath(src), fspath(dest)
+        return shutil.copyfile(src, dest)
+
     @staticmethod
     def hardlink(source, link_name):
         import ctypes

dvc/updater.py

@@ -4,13 +4,12 @@ import sys
 import os
 import time
 import logging
-import requests
 import colorama
-from pkg_resources import parse_version
+from packaging import version
 
 from dvc import __version__
 from dvc.lock import Lock, LockError
-from dvc.utils import is_binary, boxify
+from dvc.utils import is_binary, boxify, env2bool
 
 
 logger = logging.getLogger(__name__)
@@ -25,8 +24,10 @@ class Updater(object):  # pragma: no cover
     def __init__(self, dvc_dir):
         self.dvc_dir = dvc_dir
         self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
-        self.lock = Lock(dvc_dir, self.updater_file + ".lock")
-        self.current = parse_version(__version__).base_version
+        self.lock = Lock(
+            self.updater_file + ".lock", tmp_dir=os.path.join(dvc_dir, "tmp")
+        )
+        self.current = version.parse(__version__).base_version
 
     def _is_outdated_file(self):
         ctime = os.path.getmtime(self.updater_file)
@@ -41,10 +42,10 @@ class Updater(object):  # pragma: no cover
                 func()
         except LockError:
             msg = "Failed to acquire '{}' before {} updates"
-            logger.debug(msg.format(self.lock.lock_file, action))
+            logger.debug(msg.format(self.lock.lockfile, action))
 
     def check(self):
-        if os.getenv("CI") or os.getenv("DVC_TEST"):
+        if os.getenv("CI") or env2bool("DVC_TEST"):
             return
 
         self._with_lock(self._check, "checking")
@@ -80,6 +81,7 @@ class Updater(object):  # pragma: no cover
 
     def _get_latest_version(self):
         import json
+        import requests
 
         try:
             r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
@@ -93,7 +95,7 @@ class Updater(object):  # pragma: no cover
             json.dump(info, fobj)
 
     def _is_outdated(self):
-        return parse_version(self.current) < parse_version(self.latest)
+        return version.parse(self.current) < version.parse(self.latest)
 
     def _notify(self):
         if not sys.stdout.isatty():

dvc/utils/__init__.py

@@ -9,8 +9,9 @@ from dvc.utils.compat import (
     cast_bytes_py2,
     StringIO,
     fspath_py35,
+    fspath,
+    makedirs as _makedirs,
 )
-from dvc.utils.compat import fspath
 
 import os
 import sys
@@ -26,13 +27,14 @@ import colorama
 import re
 import logging
 
+from shortuuid import uuid
 from ruamel.yaml import YAML
 
 
 logger = logging.getLogger(__name__)
 
-LOCAL_CHUNK_SIZE = 1024 * 1024
-LARGE_FILE_SIZE = 1024 * 1024 * 1024
+LOCAL_CHUNK_SIZE = 2 ** 20  # 1 MB
+LARGE_FILE_SIZE = 2 ** 30  # 1 GB
 LARGE_DIR_SIZE = 100
 
 
@@ -42,44 +44,44 @@ def dos2unix(data):
 
 def file_md5(fname):
     """ get the (md5 hexdigest, md5 digest) of a file """
-    from dvc.progress import progress
+    from dvc.progress import Tqdm
     from dvc.istextfile import istextfile
 
     if os.path.exists(fname):
         hash_md5 = hashlib.md5()
         binary = not istextfile(fname)
         size = os.path.getsize(fname)
-        bar = False
+        no_progress_bar = True
         if size >= LARGE_FILE_SIZE:
-            bar = True
+            no_progress_bar = False
             msg = "Computing md5 for a large file {}. This is only done once."
             logger.info(msg.format(relpath(fname)))
-            name = relpath(fname)
-            total = 0
+        name = relpath(fname)
 
-        with open(fname, "rb") as fobj:
-            while True:
-                data = fobj.read(LOCAL_CHUNK_SIZE)
-                if not data:
-                    break
+        with Tqdm(
+            desc=name,
+            disable=no_progress_bar,
+            total=size,
+            bytes=True,
+            leave=False,
+        ) as pbar:
+            with open(fname, "rb") as fobj:
+                while True:
+                    data = fobj.read(LOCAL_CHUNK_SIZE)
+                    if not data:
+                        break
 
-                if bar:
-                    total += len(data)
-                    progress.update_target(name, total, size)
+                    if binary:
+                        chunk = data
+                    else:
+                        chunk = dos2unix(data)
 
-                if binary:
-                    chunk = data
-                else:
-                    chunk = dos2unix(data)
-
-                hash_md5.update(chunk)
-
-        if bar:
-            progress.finish_target(name)
+                    hash_md5.update(chunk)
+                    pbar.update(len(data))
 
         return (hash_md5.hexdigest(), hash_md5.digest())
-    else:
-        return (None, None)
+
+    return (None, None)
 
 
 def bytes_md5(byts):
@@ -99,14 +101,13 @@ def dict_filter(d, exclude=()):
     if isinstance(d, list):
         return [dict_filter(e, exclude) for e in d]
 
-    elif isinstance(d, dict):
+    if isinstance(d, dict):
         items = ((fix_key(k), v) for k, v in d.items())
         return {
             k: dict_filter(v, exclude) for k, v in items if k not in exclude
         }
 
-    else:
-        return d
+    return d
 
 
 def dict_md5(d, exclude=()):
@@ -118,10 +119,9 @@ def dict_md5(d, exclude=()):
 def copyfile(src, dest, no_progress_bar=False, name=None):
     """Copy file with progress bar"""
     from dvc.exceptions import DvcException
-    from dvc.progress import progress
+    from dvc.progress import Tqdm
     from dvc.system import System
 
-    copied = 0
     name = name if name else os.path.basename(dest)
     total = os.stat(src).st_size
 
@@ -131,32 +131,56 @@ def copyfile(src, dest, no_progress_bar=False, name=None):
     try:
         System.reflink(src, dest)
     except DvcException:
-        with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
-            while True:
-                buf = fsrc.read(LOCAL_CHUNK_SIZE)
-                if not buf:
-                    break
-                fdest.write(buf)
-                copied += len(buf)
-                if not no_progress_bar:
-                    progress.update_target(name, copied, total)
-
-    if not no_progress_bar:
-        progress.finish_target(name)
+        with Tqdm(
+            desc=name, disable=no_progress_bar, total=total, bytes=True
+        ) as pbar:
+            with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
+                while True:
+                    buf = fsrc.read(LOCAL_CHUNK_SIZE)
+                    if not buf:
+                        break
+                    fdest.write(buf)
+                    pbar.update(len(buf))
 
 
-def move(src, dst):
-    dst = os.path.abspath(dst)
-    dname = os.path.dirname(dst)
-    if not os.path.exists(dname):
-        os.makedirs(dname)
+def makedirs(path, exist_ok=False, mode=None):
+    path = fspath_py35(path)
 
-    if os.path.islink(src):
-        shutil.copy(os.readlink(src), dst)
-        os.unlink(src)
+    if mode is None:
+        _makedirs(path, exist_ok=exist_ok)
         return
 
-    shutil.move(src, dst)
+    umask = os.umask(0)
+    try:
+        _makedirs(path, exist_ok=exist_ok, mode=mode)
+    finally:
+        os.umask(umask)
+
+
+def move(src, dst, mode=None):
+    """Atomically move src to dst and chmod it with mode.
+
+    Moving is performed in two stages to make the whole operation atomic in
+    case src and dst are on different filesystems and actual physical copying
+    of data is happening.
+    """
+
+    src = fspath_py35(src)
+    dst = fspath_py35(dst)
+
+    dst = os.path.abspath(dst)
+    tmp = "{}.{}".format(dst, str(uuid()))
+
+    if os.path.islink(src):
+        shutil.copy(os.readlink(src), tmp)
+        os.unlink(src)
+    else:
+        shutil.move(src, tmp)
+
+    if mode is not None:
+        os.chmod(tmp, mode)
+
+    shutil.move(tmp, dst)
 
 
 def _chmod(func, p, excinfo):
@@ -166,8 +190,8 @@ def _chmod(func, p, excinfo):
     try:
         os.chmod(p, perm)
     except OSError as exc:
-        # broken symlink
-        if exc.errno != errno.ENOENT:
+        # broken symlink or file is not owned by us
+        if exc.errno not in [errno.ENOENT, errno.EPERM]:
             raise
 
     func(p)
@@ -208,7 +232,7 @@ def _to_chunks_by_chunks_number(list_to_split, num_chunks):
 def to_chunks(list_to_split, num_chunks=None, chunk_size=None):
     if (num_chunks and chunk_size) or (not num_chunks and not chunk_size):
         raise ValueError(
-            "One and only one of 'num_chunks', 'chunk_size' must be defined"
+            "Only one among `num_chunks` or `chunk_size` must be defined."
         )
     if chunk_size:
         return _split(list_to_split, chunk_size)
@@ -221,9 +245,11 @@ def is_binary():
     return getattr(sys, "frozen", False)
 
 
-# NOTE: Fix env variables modified by PyInstaller
-# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
 def fix_env(env=None):
+    """Fix env variables modified by PyInstaller [1] and pyenv [2].
+    [1] http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
+    [2] https://github.com/pyenv/pyenv/issues/985
+    """
     if env is None:
         env = os.environ.copy()
     else:
@@ -238,24 +264,70 @@ def fix_env(env=None):
         else:
             env.pop(lp_key, None)
 
+    # Unlike PyInstaller, pyenv doesn't leave backups of original env vars
+    # when it modifies them. If we look into the shim, pyenv and pyenv-exec,
+    # we can figure out that the PATH is modified like this:
+    #
+    #     PATH=$PYENV_BIN_PATH:${bin_path}:${plugin_bin}:$PATH
+    #
+    # where
+    #
+    #     PYENV_BIN_PATH - might not start with $PYENV_ROOT if we are running
+    #         `system` version of the command, see pyenv-exec source code.
+    #     bin_path - might not start with $PYENV_ROOT as it runs realpath on
+    #         it, but always has `libexec` part in it, see pyenv source code.
+    #     plugin_bin - might contain more than 1 entry, which start with
+    #         $PYENV_ROOT, see pyenv source code.
+    #
+    # Also, we know that whenever pyenv is running, it exports these env vars:
+    #
+    #     PYENV_DIR
+    #     PYENV_HOOK_PATH
+    #     PYENV_VERSION
+    #     PYENV_ROOT
+    #
+    # So having this, we can make a rightful assumption about what parts of the
+    # PATH we need to remove in order to get the original PATH.
+    path = env.get("PATH", "")
+    parts = path.split(":")
+    bin_path = parts[1] if len(parts) > 2 else ""
+    pyenv_dir = env.get("PYENV_DIR")
+    pyenv_hook_path = env.get("PYENV_HOOK_PATH")
+    pyenv_version = env.get("PYENV_VERSION")
+    pyenv_root = env.get("PYENV_ROOT")
+
+    env_matches = all([pyenv_dir, pyenv_hook_path, pyenv_version, pyenv_root])
+
+    bin_path_matches = os.path.basename(bin_path) == "libexec"
+
+    # NOTE: we don't support pyenv-win
+    if os.name != "nt" and env_matches and bin_path_matches:
+        # removing PYENV_BIN_PATH and bin_path
+        parts = parts[2:]
+
+        if parts:
+            # removing plugin_bin from the left
+            plugin_bin = os.path.join(pyenv_root, "plugins")
+            while parts[0].startswith(plugin_bin):
+                del parts[0]
+
+        env["PATH"] = ":".join(parts)
+
     return env
 
 
 def convert_to_unicode(data):
     if isinstance(data, builtin_str):
         return str(data)
-    elif isinstance(data, dict):
+    if isinstance(data, dict):
         return dict(map(convert_to_unicode, data.items()))
-    elif isinstance(data, list) or isinstance(data, tuple):
+    if isinstance(data, (list, tuple)):
         return type(data)(map(convert_to_unicode, data))
-    else:
-        return data
+    return data
 
 
 def tmp_fname(fname):
     """ Temporary name for a partial download """
-    from shortuuid import uuid
-
     return fspath(fname) + "." + str(uuid()) + ".tmp"
 
 
@@ -388,3 +460,24 @@ def relpath(path, start=os.curdir):
     ):
         return path
     return os.path.relpath(path, start)
+
+
+def env2bool(var, undefined=False):
+    """
+    undefined: return value if env var is unset
+    """
+    var = os.getenv(var, None)
+    if var is None:
+        return undefined
+    return bool(re.search("1|y|yes|true", var, flags=re.I))
+
+
+def resolve_output(inp, out):
+    from dvc.utils.compat import urlparse
+
+    name = os.path.basename(urlparse(inp).path)
+    if not out:
+        return name
+    if os.path.isdir(out):
+        return os.path.join(out, name)
+    return out