dvc: bump to 0.68.0

Add better heuristic for build detection

.deepsource.toml

@@ -5,7 +5,7 @@ test_patterns = [
 ]
 
 exclude_patterns = [
-  
+  'dvc/utils/compat.py'
 ]
 
 [[analyzers]]

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,9 +1,8 @@
-* [ ] Have you followed the guidelines in our
-      [Contributing document](https://dvc.org/doc/user-guide/contributing)?
+* [ ] ❗ Have you followed the guidelines in the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) list?
 
-* [ ] Does your PR affect documented changes or does it add new functionality
-      that should be documented? If yes, have you created a PR for
-      [dvc.org](https://github.com/iterative/dvc.org) documenting it or at
-      least opened an issue for it? If so, please add a link to it.
+* [ ] 📖 Check this box if this PR **does not** require [documentation](https://dvc.org/doc) updates, or if it does **and** you have created a separate PR in [dvc.org](https://github.com/iterative/dvc.org) with such updates (or at least opened an issue about it in that repo). Please link below to your PR (or issue) in the [dvc.org](https://github.com/iterative/dvc.org) repo.
+
+* [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect everything to be addresses. Please review them carefully and fix those that actually improve code or fix bugs.
+
+Thank you for the contribution - we'll try to review it as soon as possible. 🙏
 
------

.github/release-drafter.yml

@@ -0,0 +1,5 @@
+# Config for https://github.com/apps/release-drafter
+branches:
+  - master
+template: |
+  $CHANGES

.gitignore

@@ -4,6 +4,7 @@ neatlynx/__pycache__
 cache
 *.pyc
 .env/
+.env2.7/
 
 cache
 .dvc.conf.lock

.pre-commit-config.yaml

@@ -9,3 +9,8 @@ repos:
     hooks:
     - id: flake8
       language_version: python3
+-   repo: https://github.com/asottile/reorder_python_imports
+    rev: v1.8.0
+    hooks:
+    - id: reorder-python-imports
+      language_version: python3

.restyled.yaml

@@ -0,0 +1,21 @@
+---
+restylers:
+  - name: black
+    image: restyled/restyler-black:v19.3b0
+    command:
+    - black
+    arguments: []
+    include:
+    - "**/*.py"
+    interpreters:
+    - python
+  - name: reorder-python-imports
+    image: restyled/restyler-reorder-python-imports:v1.6.0
+    command:
+    - reorder-python-imports
+    - "--exit-zero-even-if-changed"
+    arguments: []
+    include:
+    - "**/*.py"
+    interpreters:
+    - python

CONTRIBUTING.md

@@ -1 +1 @@
-### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing#contributing).
+### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing/core).

README.rst

@@ -134,14 +134,9 @@ Homebrew
 
 .. code-block:: bash
 
-   brew install iterative/homebrew-dvc/dvc
+   brew install dvc
 
-or:
 
-.. code-block:: bash
-
-   brew cask install iterative/homebrew-dvc/dvc
-   
 Conda (Anaconda)
 ----------------
 
@@ -173,15 +168,6 @@ Fedora / CentOS (rpm)
    sudo yum update
    sudo yum install dvc
 
-Arch Linux (AUR)
-^^^^^^^^^^^^^^^^
-*Unofficial package*, any inquiries regarding the AUR package,
-`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
-
-.. code-block:: bash
-
-   yay -S dvc
-
 Related technologies
 ====================
 
@@ -202,7 +188,7 @@ Related technologies
 
 Contributing
 ============
-Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/>`_ for more
+Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0

dvc/__init__.py

@@ -3,13 +3,11 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-
 from __future__ import unicode_literals
 
 import warnings
 
 import dvc.logger
-
 from dvc.version import __version__  # noqa: F401
 
 

dvc/__main__.py

@@ -1,5 +1,4 @@
 """Main entry point for dvc command line tool."""
-
 from __future__ import unicode_literals
 
 import sys

dvc/analytics.py

@@ -1,15 +1,14 @@
 """Collect and send usage analytics"""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str
-
-import os
-import json
 import errno
+import json
 import logging
+import os
 
 from dvc import __version__
+from dvc.utils import env2bool
+from dvc.utils.compat import str
 
 
 logger = logging.getLogger(__name__)
@@ -205,45 +204,19 @@ class Analytics(object):
             json.dump(self.info, fobj)
             return fobj.name
 
-    @staticmethod
-    def _is_enabled_config(config):
-        from dvc.config import Config
-
-        core = config.config.get(Config.SECTION_CORE, {})
-        return core.get(Config.SECTION_CORE_ANALYTICS, True)
-
-    @staticmethod
-    def _get_current_config():
-        from dvc.config import Config
-        from dvc.repo import Repo
-        from dvc.exceptions import NotDvcRepoError
-
-        try:
-            dvc_dir = Repo.find_dvc_dir()
-            config = Config(dvc_dir)
-        except NotDvcRepoError:
-            config = Config(validate=False)
-        return config
-
     @staticmethod
     def is_enabled(cmd=None):
+        from dvc.config import Config, to_bool
         from dvc.command.daemon import CmdDaemonBase
 
-        if os.getenv("DVC_TEST"):
+        if env2bool("DVC_TEST"):
             return False
 
         if isinstance(cmd, CmdDaemonBase):
             return False
 
-        config = (
-            Analytics._get_current_config()
-            if cmd is None or not hasattr(cmd, "config")
-            else cmd.config
-        )
-
-        assert config is not None
-
-        enabled = Analytics._is_enabled_config(config)
+        core = Config(validate=False).config.get(Config.SECTION_CORE, {})
+        enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true"))
         logger.debug(
             "Analytics is {}.".format("enabled" if enabled else "disabled")
         )

dvc/api.py

@@ -1,5 +1,5 @@
-from contextlib import contextmanager
 import os
+from contextlib import contextmanager
 
 try:
     from contextlib import _GeneratorContextManager as GCM

dvc/cache.py

@@ -1,12 +1,13 @@
 """Manages cache of a dvc repo."""
-
 from __future__ import unicode_literals
 
 import os
+from collections import defaultdict
+
 from funcy import cached_property
 
-from dvc.utils.compat import builtin_str
 from dvc.config import Config
+from dvc.utils.compat import builtin_str
 
 
 class CacheConfig(object):
@@ -103,3 +104,36 @@ class Cache(object):
     ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
     hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
     azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
+
+
+class NamedCache(object):
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(set))
+        self.external = defaultdict(set)
+
+    @classmethod
+    def make(cls, scheme, checksum, name):
+        cache = cls()
+        cache.add(scheme, checksum, name)
+        return cache
+
+    def __getitem__(self, key):
+        return self._items[key]
+
+    def add(self, scheme, checksum, name):
+        self._items[scheme][checksum].add(name)
+
+    def add_external(self, url, rev, path):
+        self.external[url, rev].add(path)
+
+    def update(self, cache, suffix=""):
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme]
+            for checksum, names in src.items():
+                if suffix:
+                    dst[checksum].update(n + suffix for n in names)
+                else:
+                    dst[checksum].update(names)
+
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)

dvc/cli.py

@@ -1,42 +1,41 @@
 """Command line interface for dvc."""
-
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import sys
 import argparse
 import logging
+import sys
 
-from dvc.command.base import fix_subparsers
-import dvc.command.init as init
+import dvc.command.add as add
+import dvc.command.cache as cache
+import dvc.command.checkout as checkout
+import dvc.command.commit as commit
+import dvc.command.config as config
+import dvc.command.daemon as daemon
+import dvc.command.data_sync as data_sync
+import dvc.command.destroy as destroy
+import dvc.command.diff as diff
+import dvc.command.gc as gc
 import dvc.command.get as get
 import dvc.command.get_url as get_url
-import dvc.command.destroy as destroy
-import dvc.command.remove as remove
-import dvc.command.move as move
-import dvc.command.unprotect as unprotect
-import dvc.command.run as run
-import dvc.command.repro as repro
-import dvc.command.data_sync as data_sync
-import dvc.command.gc as gc
-import dvc.command.add as add
 import dvc.command.imp as imp
 import dvc.command.imp_url as imp_url
-import dvc.command.config as config
-import dvc.command.checkout as checkout
-import dvc.command.remote as remote
-import dvc.command.cache as cache
-import dvc.command.metrics as metrics
+import dvc.command.init as init
 import dvc.command.install as install
-import dvc.command.root as root
 import dvc.command.lock as lock
+import dvc.command.metrics as metrics
+import dvc.command.move as move
 import dvc.command.pipeline as pipeline
-import dvc.command.daemon as daemon
-import dvc.command.commit as commit
+import dvc.command.remote as remote
+import dvc.command.remove as remove
+import dvc.command.repro as repro
+import dvc.command.root as root
+import dvc.command.run as run
 import dvc.command.tag as tag
-import dvc.command.diff as diff
-import dvc.command.version as version
+import dvc.command.unprotect as unprotect
 import dvc.command.update as update
+import dvc.command.version as version
+from dvc.command.base import fix_subparsers
 from dvc.exceptions import DvcParserError
 
 

dvc/command/add.py

@@ -3,8 +3,10 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import RecursiveAddingWhileUsingFilename
 
 
 logger = logging.getLogger(__name__)
@@ -14,49 +16,50 @@ class CmdAdd(CmdBase):
     def run(self):
         try:
             if len(self.args.targets) > 1 and self.args.file:
-                raise DvcException("can't use '--file' with multiple targets")
+                raise RecursiveAddingWhileUsingFilename()
 
-            for target in self.args.targets:
-                self.repo.add(
-                    target,
-                    recursive=self.args.recursive,
-                    no_commit=self.args.no_commit,
-                    fname=self.args.file,
-                )
+            self.repo.add(
+                self.args.targets,
+                recursive=self.args.recursive,
+                no_commit=self.args.no_commit,
+                fname=self.args.file,
+            )
 
         except DvcException:
-            logger.exception("failed to add file")
+            logger.exception("")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    ADD_HELP = "Take data files or directories under DVC control."
+    ADD_HELP = "Track data files or directories with DVC."
 
-    add_parser = subparsers.add_parser(
+    parser = subparsers.add_parser(
         "add",
         parents=[parent_parser],
         description=append_doc_link(ADD_HELP, "add"),
         help=ADD_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    add_parser.add_argument(
+    parser.add_argument(
         "-R",
         "--recursive",
         action="store_true",
         default=False,
-        help="Recursively add each file under the directory.",
+        help="Recursively add files under directory targets.",
     )
-    add_parser.add_argument(
+    parser.add_argument(
         "--no-commit",
         action="store_true",
         default=False,
         help="Don't put files/directories into cache.",
     )
-    add_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC-file it generates."
+    parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
     )
-    add_parser.add_argument(
+    parser.add_argument(
         "targets", nargs="+", help="Input files/directories to add."
     )
-    add_parser.set_defaults(func=CmdAdd)
+    parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -1,8 +1,9 @@
 from __future__ import unicode_literals
 
-import colorama
 import logging
 
+import colorama
+
 
 logger = logging.getLogger(__name__)
 
@@ -26,7 +27,7 @@ def append_doc_link(help_message, path):
     if not path:
         return help_message
     doc_base = "https://man.dvc.org/"
-    return "{message}\ndocumentation: {blue}{base}{path}{nc}".format(
+    return "{message}\nDocumentation: <{blue}{base}{path}{nc}>".format(
         message=help_message,
         base=doc_base,
         path=path,
@@ -57,7 +58,7 @@ class CmdBase(object):
 
     # Abstract methods that have to be implemented by any inheritance class
     def run(self):
-        pass
+        raise NotImplementedError
 
 
 class CmdBaseNoRepo(CmdBase):

dvc/command/cache.py

@@ -2,9 +2,10 @@ from __future__ import unicode_literals
 
 import argparse
 
-from dvc.command.base import fix_subparsers, append_doc_link
-from dvc.command.config import CmdConfig
 from dvc.cache import CacheConfig
+from dvc.command.base import append_doc_link
+from dvc.command.base import fix_subparsers
+from dvc.command.config import CmdConfig
 
 
 class CmdCacheDir(CmdConfig):

dvc/command/checkout.py

@@ -2,21 +2,18 @@ from __future__ import unicode_literals
 
 import argparse
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 
 
 class CmdCheckout(CmdBase):
     def run(self):
-        if not self.args.targets:
-            self.repo.checkout(force=self.args.force)
-        else:
-            for target in self.args.targets:
-                self.repo.checkout(
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    force=self.args.force,
-                    recursive=self.args.recursive,
-                )
+        self.repo.checkout(
+            targets=self.args.targets,
+            with_deps=self.args.with_deps,
+            force=self.args.force,
+            recursive=self.args.recursive,
+        )
         return 0
 
 

dvc/command/commit.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/config.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 from dvc.config import Config
 
 

dvc/command/daemon.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.command.base import CmdBaseNoRepo
+from dvc.command.base import fix_subparsers
 
 
 class CmdDaemonBase(CmdBaseNoRepo):

dvc/command/data_sync.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 

dvc/command/destroy.py

@@ -4,7 +4,8 @@ import argparse
 import logging
 
 import dvc.prompt as prompt
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 
@@ -24,7 +25,7 @@ class CmdDestroy(CmdBase):
             if not self.args.force and not prompt.confirm(statement):
                 raise DvcException(
                     "cannot destroy without a confirmation from the user."
-                    " Use '-f' to force."
+                    " Use `-f` to force."
                 )
 
             self.repo.destroy()

dvc/command/diff.py

@@ -1,14 +1,15 @@
 from __future__ import unicode_literals
 
 import argparse
+import logging
 
 import humanize
 import inflect
-import logging
 from funcy import compact
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/gc.py

@@ -1,11 +1,12 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
-import dvc.prompt as prompt
 import logging
+import os
 
-from dvc.command.base import CmdBase, append_doc_link
+import dvc.prompt as prompt
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 
 
 logger = logging.getLogger(__name__)
@@ -13,17 +14,19 @@ logger = logging.getLogger(__name__)
 
 class CmdGC(CmdBase):
     def run(self):
-        msg = "this will remove all cache except the cache that is used in "
-        if not self.args.all_branches and not self.args.all_tags:
-            msg += "the current git branch"
-        elif self.args.all_branches and not self.args.all_tags:
-            msg += "all git branches"
-        elif not self.args.all_branches and self.args.all_tags:
-            msg += "all git tags"
-        else:
-            msg += "all git branches and all git tags"
+        msg = "This will remove all cache except items used in "
 
-        if self.args.repos is not None and len(self.args.repos) > 0:
+        msg += "the working tree"
+        if self.args.all_commits:
+            msg += " and all git commits"
+        elif self.args.all_branches and self.args.all_tags:
+            msg += " and all git branches and tags"
+        elif self.args.all_branches:
+            msg += " and all git branches"
+        elif self.args.all_tags:
+            msg += " and all git tags"
+
+        if self.args.repos:
             msg += " of the current and the following repos:"
 
             for repo_path in self.args.repos:
@@ -40,6 +43,7 @@ class CmdGC(CmdBase):
         self.repo.gc(
             all_branches=self.args.all_branches,
             all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
             cloud=self.args.cloud,
             remote=self.args.remote,
             force=self.args.force,
@@ -76,6 +80,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Keep data files for all git tags.",
     )
+    gc_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     gc_parser.add_argument(
         "-c",
         "--cloud",

dvc/command/get.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from .base import append_doc_link
+from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
-from .base import CmdBaseNoRepo, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/get_url.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from .base import append_doc_link
+from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
-from .base import CmdBaseNoRepo, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/imp.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/imp_url.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)
@@ -19,7 +20,7 @@ class CmdImportUrl(CmdBase):
         except DvcException:
             logger.exception(
                 "failed to import {}. You could also try downloading "
-                "it manually and adding it with `dvc add` command.".format(
+                "it manually, and adding it with `dvc add`.".format(
                     self.args.url
                 )
             )

dvc/command/init.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 
 
 logger = logging.getLogger(__name__)
@@ -53,7 +54,7 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
         help=(
-            "Overwrite existing '.dvc' directory. "
+            "Overwrite existing '.dvc/' directory. "
             "This operation removes local cache."
         ),
     )

dvc/command/install.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 
 
 logger = logging.getLogger(__name__)

dvc/command/lock.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/metrics.py

@@ -3,8 +3,11 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.exceptions import DvcException, BadMetricError
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
+from dvc.exceptions import BadMetricError
+from dvc.exceptions import DvcException
 
 
 logger = logging.getLogger(__name__)

dvc/command/move.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/pipeline.py

@@ -1,13 +1,14 @@
 from __future__ import unicode_literals
 
-from dvc.utils import relpath
-from dvc.utils.compat import str
-
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.utils import relpath
+from dvc.utils.compat import str
 
 
 logger = logging.getLogger(__name__)

dvc/command/remote.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link, fix_subparsers
+from dvc.command.base import append_doc_link
+from dvc.command.base import fix_subparsers
 from dvc.command.config import CmdConfig
 from dvc.remote.config import RemoteConfig
 

dvc/command/remove.py

@@ -4,8 +4,9 @@ import argparse
 import logging
 
 import dvc.prompt as prompt
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)
@@ -28,7 +29,7 @@ class CmdRemove(CmdBase):
 
         raise DvcException(
             "Cannot purge without a confirmation from the user."
-            " Use '-f' to force."
+            " Use `-f` to force."
         )
 
     def run(self):

dvc/command/repro.py

@@ -1,10 +1,11 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
 import logging
+import os
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
 from dvc.exceptions import DvcException

dvc/command/root.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)

dvc/command/run.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 
@@ -27,9 +28,9 @@ class CmdRun(CmdBase):
             ]
         ):  # pragma: no cover
             logger.error(
-                "too few arguments. Specify at least one: '-d', '-o', '-O',"
-                " '-m', '-M', '--outs-persist', '--outs-persist-no-cache',"
-                " 'command'."
+                "too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
+                "`-m`, `-M`, `--outs-persist`, `--outs-persist-no-cache`, "
+                "`command`."
             )
             return 1
 

dvc/command/tag.py

@@ -1,8 +1,10 @@
 import logging
 
-from dvc.utils import to_yaml_string
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.utils import to_yaml_string
 
 
 logger = logging.getLogger(__name__)

dvc/command/unprotect.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/update.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, append_doc_link
 
 
 logger = logging.getLogger(__name__)

dvc/command/version.py

@@ -1,24 +1,24 @@
 from __future__ import unicode_literals
 
+import argparse
+import itertools
+import logging
 import os
 import platform
-import argparse
-import logging
 import uuid
-import itertools
 
 try:
     import psutil
 except ImportError:
     psutil = None
 
-from dvc.utils import is_binary
+from dvc.utils import is_binary, relpath
 from dvc.utils.compat import pathlib
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.system import System
-
+from dvc.utils.pkg import PKG
 
 logger = logging.getLogger(__name__)
 
@@ -31,17 +31,18 @@ class CmdVersion(CmdBaseNoRepo):
         python_version = platform.python_version()
         platform_type = platform.platform()
         binary = is_binary()
-
         info = (
             "DVC version: {dvc_version}\n"
             "Python version: {python_version}\n"
             "Platform: {platform_type}\n"
             "Binary: {binary}\n"
+            "Package: {package}\n"
         ).format(
             dvc_version=dvc_version,
             python_version=python_version,
             platform_type=platform_type,
             binary=binary,
+            package=PKG,
         )
 
         try:
@@ -60,8 +61,9 @@ class CmdVersion(CmdBaseNoRepo):
                 logger.warning(
                     "Unable to detect supported link types, as cache "
                     "directory '{}' doesn't exist. It is usually auto-created "
-                    "by commands such as `dvc add/fetch/pull/run/import`, but "
-                    "you could create it manually to enable this check."
+                    "by commands such as `dvc add/fetch/pull/run/import`, "
+                    "but you could create it manually to enable this "
+                    "check.".format(relpath(repo.cache.local.cache_dir))
                 )
 
             if psutil:

dvc/config.py

@@ -1,18 +1,24 @@
 """DVC config objects."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, open
-
-import os
-import re
 import copy
 import errno
-import configobj
 import logging
+import os
+import re
 
-from schema import Schema, Optional, And, Use, Regex, SchemaError
-from dvc.exceptions import DvcException, NotDvcRepoError
+import configobj
+from schema import And
+from schema import Optional
+from schema import Regex
+from schema import Schema
+from schema import SchemaError
+from schema import Use
+
+from dvc.exceptions import DvcException
+from dvc.exceptions import NotDvcRepoError
+from dvc.utils.compat import open
+from dvc.utils.compat import str
 
 logger = logging.getLogger(__name__)
 

dvc/daemon.py

@@ -1,15 +1,15 @@
 """Launch `dvc daemon` command in a separate detached process."""
-
 from __future__ import unicode_literals
 
-import os
-import sys
 import inspect
 import logging
+import os
+import sys
 from subprocess import Popen
 
 from dvc.env import DVC_DAEMON
-from dvc.utils import is_binary, fix_env
+from dvc.utils import fix_env
+from dvc.utils import is_binary
 from dvc.utils.compat import cast_bytes_py2
 
 

dvc/dagascii.py

@@ -1,14 +1,16 @@
 """Draws DAG in ASCII."""
-
-from __future__ import unicode_literals
 from __future__ import print_function
+from __future__ import unicode_literals
 
-import sys
 import math
+import sys
 
-from grandalf.graphs import Vertex, Edge, Graph
+from grandalf.graphs import Edge
+from grandalf.graphs import Graph
+from grandalf.graphs import Vertex
 from grandalf.layouts import SugiyamaLayout
-from grandalf.routing import route_with_lines, EdgeViewer
+from grandalf.routing import EdgeViewer
+from grandalf.routing import route_with_lines
 
 
 class VertexViewer(object):

dvc/data_cloud.py

@@ -1,10 +1,10 @@
 """Manages dvc remotes that user can use with push/pull/status comamnds."""
-
 from __future__ import unicode_literals
 
 import logging
 
-from dvc.config import Config, NoRemoteError
+from dvc.config import Config
+from dvc.config import NoRemoteError
 from dvc.remote import Remote
 
 
@@ -45,11 +45,11 @@ class DataCloud(object):
     def _init_remote(self, remote):
         return Remote(self.repo, name=remote)
 
-    def push(self, targets, jobs=None, remote=None, show_checksums=False):
+    def push(self, cache, jobs=None, remote=None, show_checksums=False):
         """Push data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to push to the cloud.
+            cache (NamedCache): named checksums to push to the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to push to.
                 By default remote from core.remote config option is used.
@@ -57,17 +57,17 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.push(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "push"),
             show_checksums=show_checksums,
         )
 
-    def pull(self, targets, jobs=None, remote=None, show_checksums=False):
+    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
         """Pull data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to pull from the cloud.
+            cache (NamedCache): named checksums to pull from the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
                 By default remote from core.remote config option is used.
@@ -75,25 +75,25 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.pull(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "pull"),
             show_checksums=show_checksums,
         )
 
-    def status(self, targets, jobs=None, remote=None, show_checksums=False):
+    def status(self, cache, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to check status for.
+            cache (NamedCache): named checksums to check status for.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to compare
-                targets to. By default remote from core.remote config option
+                cache to. By default remote from core.remote config option
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
         remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            targets, jobs=jobs, remote=remote, show_checksums=show_checksums
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )

dvc/dependency/__init__.py

@@ -2,20 +2,18 @@ from __future__ import unicode_literals
 
 import schema
 
-from dvc.scheme import Schemes
-
 import dvc.output as output
-from dvc.output.base import OutputBase
-from dvc.dependency.s3 import DependencyS3
+from .repo import DependencyREPO
 from dvc.dependency.gs import DependencyGS
-from dvc.dependency.local import DependencyLOCAL
 from dvc.dependency.hdfs import DependencyHDFS
-from dvc.dependency.ssh import DependencySSH
 from dvc.dependency.http import DependencyHTTP
 from dvc.dependency.https import DependencyHTTPS
-from .repo import DependencyREPO
-
+from dvc.dependency.local import DependencyLOCAL
+from dvc.dependency.s3 import DependencyS3
+from dvc.dependency.ssh import DependencySSH
+from dvc.output.base import OutputBase
 from dvc.remote import Remote
+from dvc.scheme import Schemes
 
 
 DEPS = [

dvc/dependency/gs.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.gs import OutputGS
 from dvc.dependency.base import DependencyBase
+from dvc.output.gs import OutputGS
 
 
 class DependencyGS(DependencyBase, OutputGS):

dvc/dependency/hdfs.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.hdfs import OutputHDFS
 from dvc.dependency.base import DependencyBase
+from dvc.output.hdfs import OutputHDFS
 
 
 class DependencyHDFS(DependencyBase, OutputHDFS):

dvc/dependency/http.py

@@ -1,8 +1,8 @@
 from __future__ import unicode_literals
 
+from dvc.dependency.base import DependencyBase
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP
-from dvc.dependency.base import DependencyBase
 
 
 class DependencyHTTP(DependencyBase, OutputBase):

dvc/dependency/https.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.remote.https import RemoteHTTPS
 from .http import DependencyHTTP
+from dvc.remote.https import RemoteHTTPS
 
 
 class DependencyHTTPS(DependencyHTTP):

dvc/dependency/repo.py

@@ -1,15 +1,14 @@
 from __future__ import unicode_literals
 
 import copy
+from contextlib import contextmanager
 
 from funcy import merge
 from schema import Optional
-from contextlib import contextmanager
-
-from dvc.external_repo import external_repo
-from dvc.utils.compat import str
 
 from .local import DependencyLOCAL
+from dvc.external_repo import external_repo
+from dvc.utils.compat import str
 
 
 class DependencyREPO(DependencyLOCAL):
@@ -35,6 +34,11 @@ class DependencyREPO(DependencyLOCAL):
     def is_in_repo(self):
         return False
 
+    @property
+    def repo_pair(self):
+        d = self.def_repo
+        return d[self.PARAM_URL], d[self.PARAM_REV_LOCK] or d[self.PARAM_REV]
+
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 

dvc/dependency/s3.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.s3 import OutputS3
 from dvc.dependency.base import DependencyBase
+from dvc.output.s3 import OutputS3
 
 
 class DependencyS3(DependencyBase, OutputS3):

dvc/dependency/ssh.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.output.ssh import OutputSSH
 from dvc.dependency.base import DependencyBase
+from dvc.output.ssh import OutputSSH
 
 
 class DependencySSH(DependencyBase, OutputSSH):

dvc/exceptions.py

@@ -1,12 +1,11 @@
 """Exceptions raised by the dvc."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, builtin_str
-
 import traceback
 
 from dvc.utils import relpath
+from dvc.utils.compat import builtin_str
+from dvc.utils.compat import str
 
 
 class DvcException(Exception):
@@ -179,7 +178,7 @@ class ConfirmRemoveError(DvcException):
     def __init__(self, path):
         super(ConfirmRemoveError, self).__init__(
             "unable to remove '{}' without a confirmation from the user. Use "
-            "'-f' to force.".format(path)
+            "`-f` to force.".format(path)
         )
 
 
@@ -209,7 +208,7 @@ class NoMetricsError(DvcException):
     def __init__(self):
         super(NoMetricsError, self).__init__(
             "no metric files in this repository. "
-            "Use 'dvc metrics add' to add a metric file to track."
+            "Use `dvc metrics add` to add a metric file to track."
         )
 
 
@@ -226,7 +225,7 @@ class StageFileCorruptedError(DvcException):
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
         super(RecursiveAddingWhileUsingFilename, self).__init__(
-            "using fname with recursive is not allowed."
+            "cannot use `fname` with multiple targets or `-R|--recursive`"
         )
 
 
@@ -247,7 +246,7 @@ class OverlappingOutputPathsError(DvcException):
 class CheckoutErrorSuggestGit(DvcException):
     def __init__(self, target, cause):
         super(CheckoutErrorSuggestGit, self).__init__(
-            "Did you mean 'git checkout {}'?".format(target), cause=cause
+            "Did you mean `git checkout {}`?".format(target), cause=cause
         )
 
 
@@ -325,3 +324,7 @@ class CheckoutError(DvcException):
             "forget to fetch?".format("\n".join(targets))
         )
         super(CheckoutError, self).__init__(m)
+
+
+class CollectCacheError(DvcException):
+    pass

dvc/external_repo.py

@@ -2,9 +2,9 @@ from __future__ import unicode_literals
 
 import os
 import tempfile
+from contextlib import contextmanager
 from distutils.dir_util import copy_tree
 
-from contextlib import contextmanager
 from funcy import retry
 
 from dvc.utils import remove

dvc/ignore.py

@@ -2,13 +2,12 @@ from __future__ import unicode_literals
 
 import logging
 import os
+
 from pathspec import PathSpec
 from pathspec.patterns import GitWildMatchPattern
 
 from dvc.utils import relpath
 
-from dvc.utils.compat import open
-
 logger = logging.getLogger(__name__)
 
 
@@ -20,13 +19,13 @@ class DvcIgnore(object):
 
 
 class DvcIgnorePatterns(DvcIgnore):
-    def __init__(self, ignore_file_path):
+    def __init__(self, ignore_file_path, tree):
         assert os.path.isabs(ignore_file_path)
 
         self.ignore_file_path = ignore_file_path
         self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
 
-        with open(ignore_file_path, encoding="utf-8") as fobj:
+        with tree.open(ignore_file_path, encoding="utf-8") as fobj:
             self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
 
     def __call__(self, root, dirs, files):
@@ -47,6 +46,9 @@ class DvcIgnorePatterns(DvcIgnore):
         return hash(self.ignore_file_path)
 
     def __eq__(self, other):
+        if not isinstance(other, DvcIgnorePatterns):
+            return NotImplemented
+
         return self.ignore_file_path == other.ignore_file_path
 
 
@@ -59,19 +61,29 @@ class DvcIgnoreDirs(DvcIgnore):
 
         return dirs, files
 
+    def __hash__(self):
+        return hash(tuple(self.basenames))
+
+    def __eq__(self, other):
+        if not isinstance(other, DvcIgnoreDirs):
+            return NotImplemented
+
+        return self.basenames == other.basenames
+
 
 class DvcIgnoreFilter(object):
-    def __init__(self, root_dir):
+    def __init__(self, root_dir, tree):
+        self.tree = tree
         self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
         self._update(root_dir)
-        for root, dirs, _ in os.walk(root_dir):
+        for root, dirs, _ in self.tree.walk(root_dir, dvcignore=self):
             for d in dirs:
                 self._update(os.path.join(root, d))
 
     def _update(self, dirname):
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if os.path.exists(ignore_file_path):
-            self.ignores.add(DvcIgnorePatterns(ignore_file_path))
+        if self.tree.exists(ignore_file_path):
+            self.ignores.add(DvcIgnorePatterns(ignore_file_path, self.tree))
 
     def __call__(self, root, dirs, files):
         for ignore in self.ignores:

dvc/istextfile.py

@@ -1,8 +1,8 @@
 """Use heuristics to guess if it is a text file or a binary file."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import is_py3, open
+from dvc.utils.compat import is_py3
+from dvc.utils.compat import open
 
 # Based on https://eli.thegreenplace.net/2011/10/19/
 # perls-guess-if-file-is-text-or-binary-implemented-in-python

dvc/lock.py

@@ -1,9 +1,8 @@
 """Manages dvc lock file."""
-
 from __future__ import unicode_literals
 
-import os
 import hashlib
+import os
 import time
 from datetime import timedelta
 
@@ -34,12 +33,31 @@ if is_py3:
         """
 
         def __init__(self, lockfile, tmp_dir=None):
-            lifetime = timedelta(days=365)  # Lock for good by default
+            import socket
+
             self._tmp_dir = tmp_dir
             if self._tmp_dir is not None:
                 makedirs(self._tmp_dir, exist_ok=True)
 
-            super(Lock, self).__init__(lockfile, lifetime=lifetime)
+            # NOTE: this is basically Lock.__init__ copy-paste, except that
+            # instead of using `socket.getfqdn()` we use `socket.gethostname()`
+            # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
+            # anything, which is way too slow. `gethostname()` is actually a
+            # fallback for `getfqdn()` when it is not able to resolve a
+            # canonical hostname through network. The claimfile that uses
+            # `self._hostname` is still usable, as it uses `pid` and random
+            # number to generate the resulting lock file name, which is unique
+            # enough for our application.
+            #
+            # [1] https://github.com/iterative/dvc/issues/2582
+            self._hostname = socket.gethostname()
+
+            self._lockfile = lockfile
+            self._lifetime = timedelta(days=365)  # Lock for good by default
+            self._separator = flufl.lock.SEP
+            self._set_claimfile()
+            self._owned = True
+            self._retry_errnos = []
 
         @property
         def lockfile(self):
@@ -68,6 +86,17 @@ if is_py3:
                     self._tmp_dir, filename + ".lock"
                 )
 
+        # Fix for __del__ bug in flufl.lock [1] which is causing errors on
+        # Python shutdown [2].
+        # [1] https://gitlab.com/warsaw/flufl.lock/issues/7
+        # [2] https://github.com/iterative/dvc/issues/2573
+        def __del__(self):
+            try:
+                if self._owned:
+                    self.finalize()
+            except ImportError:
+                pass
+
 
 else:
     import zc.lockfile

dvc/logger.py

@@ -1,15 +1,27 @@
 """Manages logging configuration for dvc repo."""
-
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, StringIO, RecursionError
-from dvc.progress import Tqdm
-
-import logging
-import logging.handlers
 import logging.config
+import logging.handlers
+
 import colorama
 
+from dvc.progress import Tqdm
+from dvc.utils.compat import RecursionError
+from dvc.utils.compat import str
+from dvc.utils.compat import StringIO
+
+
+FOOTER = (
+    "\n{yellow}Having any troubles?{nc}"
+    " Hit us up at {blue}https://dvc.org/support{nc},"
+    " we are always happy to help!"
+).format(
+    blue=colorama.Fore.BLUE,
+    nc=colorama.Fore.RESET,
+    yellow=colorama.Fore.YELLOW,
+)
+
 
 class LoggingException(Exception):
     def __init__(self, record):
@@ -22,6 +34,11 @@ class ExcludeErrorsFilter(logging.Filter):
         return record.levelno < logging.WARNING
 
 
+class ExcludeInfoFilter(logging.Filter):
+    def filter(self, record):
+        return record.levelno < logging.INFO
+
+
 class ColorFormatter(logging.Formatter):
     """Enable color support when logging to a terminal that supports it.
 
@@ -43,16 +60,6 @@ class ColorFormatter(logging.Formatter):
         "CRITICAL": colorama.Fore.RED,
     }
 
-    footer = (
-        "{yellow}Having any troubles?{nc}"
-        " Hit us up at {blue}https://dvc.org/support{nc},"
-        " we are always happy to help!"
-    ).format(
-        blue=colorama.Fore.BLUE,
-        nc=colorama.Fore.RESET,
-        yellow=colorama.Fore.YELLOW,
-    )
-
     def format(self, record):
         if record.levelname == "INFO":
             return record.msg
@@ -61,10 +68,7 @@ class ColorFormatter(logging.Formatter):
             exception, stack_trace = self._parse_exc(record.exc_info)
 
             return (
-                "{color}{levelname}{nc}: {description}"
-                "{stack_trace}\n"
-                "\n"
-                "{footer}"
+                "{color}{levelname}{nc}: {description}" "{stack_trace}\n"
             ).format(
                 color=self.color_code.get(record.levelname, ""),
                 nc=colorama.Fore.RESET,
@@ -72,7 +76,6 @@ class ColorFormatter(logging.Formatter):
                 description=self._description(record.msg, exception),
                 msg=record.msg,
                 stack_trace=stack_trace,
-                footer=self.footer,
             )
 
         return "{color}{levelname}{nc}: {msg}".format(
@@ -170,15 +173,25 @@ def setup(level=logging.INFO):
     logging.config.dictConfig(
         {
             "version": 1,
-            "filters": {"exclude_errors": {"()": ExcludeErrorsFilter}},
+            "filters": {
+                "exclude_errors": {"()": ExcludeErrorsFilter},
+                "exclude_info": {"()": ExcludeInfoFilter},
+            },
             "formatters": {"color": {"()": ColorFormatter}},
             "handlers": {
-                "console": {
+                "console_info": {
+                    "class": "dvc.logger.LoggerHandler",
+                    "level": "INFO",
+                    "formatter": "color",
+                    "stream": "ext://sys.stdout",
+                    "filters": ["exclude_errors"],
+                },
+                "console_debug": {
                     "class": "dvc.logger.LoggerHandler",
                     "level": "DEBUG",
                     "formatter": "color",
                     "stream": "ext://sys.stdout",
-                    "filters": ["exclude_errors"],
+                    "filters": ["exclude_info"],
                 },
                 "console_errors": {
                     "class": "dvc.logger.LoggerHandler",
@@ -190,15 +203,27 @@ def setup(level=logging.INFO):
             "loggers": {
                 "dvc": {
                     "level": level,
-                    "handlers": ["console", "console_errors"],
+                    "handlers": [
+                        "console_info",
+                        "console_debug",
+                        "console_errors",
+                    ],
                 },
                 "paramiko": {
                     "level": logging.CRITICAL,
-                    "handlers": ["console", "console_errors"],
+                    "handlers": [
+                        "console_info",
+                        "console_debug",
+                        "console_errors",
+                    ],
                 },
                 "flufl.lock": {
                     "level": logging.CRITICAL,
-                    "handlers": ["console", "console_errors"],
+                    "handlers": [
+                        "console_info",
+                        "console_debug",
+                        "console_errors",
+                    ],
                 },
             },
         }

dvc/main.py

@@ -1,19 +1,26 @@
 """Main entry point for dvc CLI."""
-
 from __future__ import unicode_literals
 
 import logging
 
-from dvc.cli import parse_args
-from dvc.lock import LockError
-from dvc.config import ConfigError
 from dvc.analytics import Analytics
-from dvc.exceptions import NotDvcRepoError, DvcParserError
-from dvc.remote.pool import close_pools
+from dvc.cli import parse_args
+from dvc.config import ConfigError
+from dvc.exceptions import DvcParserError
+from dvc.exceptions import NotDvcRepoError
 from dvc.external_repo import clean_repos
+from dvc.lock import LockError
+from dvc.logger import FOOTER
+from dvc.remote.pool import close_pools
 from dvc.utils.compat import is_py2
 
 
+# Workaround for CPython bug. See [1] and [2] for more info.
+# [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
+# [2] https://bugs.python.org/issue29288
+"".encode("idna")
+
+
 logger = logging.getLogger("dvc")
 
 
@@ -75,6 +82,9 @@ def main(argv=None):
         # so won't be reused by any other subsequent run anyway.
         clean_repos()
 
+    if ret != 0:
+        logger.info(FOOTER)
+
     Analytics().send_cmd(cmd, args, ret)
 
     return ret

dvc/output/__init__.py

@@ -2,20 +2,19 @@ from __future__ import unicode_literals
 
 import schema
 
-from dvc.scheme import Schemes
-from dvc.utils.compat import urlparse, str
-
 from dvc.output.base import OutputBase
-from dvc.output.s3 import OutputS3
 from dvc.output.gs import OutputGS
-from dvc.output.local import OutputLOCAL
 from dvc.output.hdfs import OutputHDFS
+from dvc.output.local import OutputLOCAL
+from dvc.output.s3 import OutputS3
 from dvc.output.ssh import OutputSSH
-
 from dvc.remote import Remote
-from dvc.remote.s3 import RemoteS3
 from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.local import RemoteLOCAL
+from dvc.remote.s3 import RemoteS3
+from dvc.scheme import Schemes
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
 
 OUTS = [
     OutputHDFS,

dvc/output/base.py

@@ -3,12 +3,16 @@ from __future__ import unicode_literals
 import logging
 from copy import copy
 
-from schema import Or, Optional
+from schema import Optional
+from schema import Or
 
 import dvc.prompt as prompt
+from dvc.cache import NamedCache
+from dvc.exceptions import CollectCacheError
 from dvc.exceptions import DvcException
-from dvc.utils.compat import str, urlparse
 from dvc.remote.base import RemoteBASE
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -286,7 +290,8 @@ class OutputBase(object):
 
     def checkout(self, force=False, progress_callback=None, tag=None):
         if not self.use_cache:
-            progress_callback(str(self.path_info), self.get_files_number())
+            if progress_callback:
+                progress_callback(str(self.path_info), self.get_files_number())
             return None
 
         if tag:
@@ -333,7 +338,7 @@ class OutputBase(object):
             self.remote.unprotect(self.path_info)
 
     def _collect_used_dir_cache(self, remote=None, force=False, jobs=None):
-        """Get a list of `info`s retaled to the given directory.
+        """Get a list of `info`s related to the given directory.
 
         - Pull the directory entry from the remote cache if it was changed.
 
@@ -345,25 +350,19 @@ class OutputBase(object):
             $ echo "bar" > directory/bar
             $ dvc add directory
 
-            It will return something similar to the following list:
+            It will return a NamedCache like:
 
-            [
-                { 'path': 'directory/foo', 'md5': 'c157a79031e1', ... },
-                { 'path': 'directory/bar', 'md5': 'd3b07384d113', ... },
-            ]
+            nc = NamedCache()
+            nc.add(self.scheme, 'c157a79031e1', 'directory/foo')
+            nc.add(self.scheme, 'd3b07384d113', 'directory/bar')
         """
 
-        ret = []
+        cache = NamedCache()
 
         if self.cache.changed_cache_file(self.checksum):
             try:
                 self.repo.cloud.pull(
-                    [
-                        {
-                            self.remote.PARAM_CHECKSUM: self.checksum,
-                            "name": str(self),
-                        }
-                    ],
+                    NamedCache.make("local", self.checksum, str(self)),
                     jobs=jobs,
                     remote=remote,
                     show_checksums=False,
@@ -378,20 +377,19 @@ class OutputBase(object):
                 "Would you like to continue? Use '-f' to force."
             )
             if not force and not prompt.confirm(msg.format(self.path_info)):
-                raise DvcException(
+                raise CollectCacheError(
                     "unable to fully collect used cache"
                     " without cache for directory '{}'".format(self)
                 )
             else:
-                return ret
+                return cache
 
         for entry in self.dir_cache:
-            info = copy(entry)
+            checksum = entry[self.remote.PARAM_CHECKSUM]
             path_info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            info["name"] = str(path_info)
-            ret.append(info)
+            cache.add(self.scheme, checksum, str(path_info))
 
-        return ret
+        return cache
 
     def get_used_cache(self, **kwargs):
         """Get a dumpd of the given `out`, with an entry including the branch.
@@ -403,7 +401,7 @@ class OutputBase(object):
         """
 
         if not self.use_cache:
-            return []
+            return NamedCache()
 
         if not self.info:
             logger.warning(
@@ -411,14 +409,14 @@ class OutputBase(object):
                 "not be collected. Use dvc repro to get your pipeline up to "
                 "date.".format(self, self.stage)
             )
-            return []
+            return NamedCache()
 
-        ret = [{self.remote.PARAM_CHECKSUM: self.checksum, "name": str(self)}]
+        ret = NamedCache.make(self.scheme, self.checksum, str(self))
 
         if not self.is_dir_checksum:
             return ret
 
-        ret.extend(self._collect_used_dir_cache(**kwargs))
+        ret.update(self._collect_used_dir_cache(**kwargs))
 
         return ret
 

dvc/output/gs.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.remote.gs import RemoteGS
 from dvc.output.s3 import OutputS3
+from dvc.remote.gs import RemoteGS
 
 
 class OutputGS(OutputS3):

dvc/output/local.py

@@ -1,13 +1,15 @@
 from __future__ import unicode_literals
 
-import os
 import logging
+import os
 
-from dvc.utils.compat import urlparse, fspath_py35, str
-from dvc.istextfile import istextfile
 from dvc.exceptions import DvcException
-from dvc.remote.local import RemoteLOCAL
+from dvc.istextfile import istextfile
 from dvc.output.base import OutputBase
+from dvc.remote.local import RemoteLOCAL
+from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)
@@ -26,7 +28,7 @@ class OutputLOCAL(OutputBase):
             # so we should expect both posix and windows style paths.
             # PathInfo accepts both, i.e. / works everywhere, \ only on win.
             #
-            # FIXME: if we have Windows path containig / or posix one with \
+            # FIXME: if we have Windows path containing / or posix one with \
             # then we have #2059 bug and can't really handle that.
             p = self.REMOTE.path_cls(path)
             if not p.is_absolute():

dvc/output/s3.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.remote.s3 import RemoteS3
 from dvc.output.base import OutputBase
+from dvc.remote.s3 import RemoteS3
 
 
 class OutputS3(OutputBase):

dvc/path_info.py

@@ -1,20 +1,24 @@
-from __future__ import unicode_literals, absolute_import
+from __future__ import absolute_import
+from __future__ import unicode_literals
 
-import sys
 import os
 import posixpath
+import sys
 
 from funcy import cached_property
 
-from dvc.utils.compat import str, builtin_str, basestring, is_py2
-from dvc.utils.compat import pathlib, urlparse
+from dvc.utils import relpath
+from dvc.utils.compat import basestring
+from dvc.utils.compat import builtin_str
+from dvc.utils.compat import is_py2
+from dvc.utils.compat import pathlib
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
 
 
 # On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
 # which is lossy, thus we can't use that,
 # see https://github.com/mcmtroffaes/pathlib2/issues/56.
-from dvc.utils import relpath
-
 if is_py2:
     fs_encoding = "utf-8"
 
@@ -112,52 +116,80 @@ class PosixPathInfo(PathInfo, pathlib.PurePosixPath):
     pass
 
 
+class _URLPathInfo(PosixPathInfo):
+    def __str__(self):
+        return self.__fspath__()
+
+    __unicode__ = __str__
+
+
 class _URLPathParents(object):
-    def __init__(self, pathcls, scheme, netloc, path):
-        self._scheme = scheme
-        self._netloc = netloc
-        self._parents = path.parents
-        self._pathcls = pathcls
+    def __init__(self, src):
+        self.src = src
+        self._parents = self.src._path.parents
 
     def __len__(self):
         return len(self._parents)
 
     def __getitem__(self, idx):
-        return self._pathcls.from_parts(
-            scheme=self._scheme,
-            netloc=self._netloc,
-            path=self._parents[idx].fspath,
-        )
+        return self.src.replace(path=self._parents[idx])
 
     def __repr__(self):
-        return "<{}.parents>".format(self._pathcls.__name__)
+        return "<{}.parents>".format(self.src)
 
 
 class URLInfo(object):
     DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22, "hdfs": 0}
 
     def __init__(self, url):
-        self.parsed = urlparse(url)
-        assert self.parsed.scheme != "remote"
+        p = urlparse(url)
+        assert not p.query and not p.params and not p.fragment
+        assert p.password is None
+
+        self.fill_parts(p.scheme, p.hostname, p.username, p.port, p.path)
 
     @classmethod
     def from_parts(
-        cls, scheme=None, netloc=None, host=None, user=None, port=None, path=""
+        cls, scheme=None, host=None, user=None, port=None, path="", netloc=None
     ):
-        assert scheme and (bool(host) ^ bool(netloc))
+        assert bool(host) ^ bool(netloc)
 
-        if netloc is None:
-            netloc = host
-            if user:
-                netloc = user + "@" + host
-            if port:
-                netloc += ":" + str(port)
-        return cls("{}://{}{}".format(scheme, netloc, path))
+        if netloc is not None:
+            return cls("{}://{}{}".format(scheme, netloc, path))
+
+        obj = cls.__new__(cls)
+        obj.fill_parts(scheme, host, user, port, path)
+        return obj
+
+    def fill_parts(self, scheme, host, user, port, path):
+        assert scheme != "remote"
+        assert isinstance(path, (basestring, _URLPathInfo))
+
+        self.scheme, self.host, self.user = scheme, host, user
+        self.port = int(port) if port else self.DEFAULT_PORTS.get(self.scheme)
+
+        if isinstance(path, _URLPathInfo):
+            self._spath = builtin_str(path)
+            self._path = path
+        else:
+            if path and path[0] != "/":
+                path = "/" + path
+            self._spath = path
+
+    @property
+    def _base_parts(self):
+        return (self.scheme, self.host, self.user, self.port)
+
+    @property
+    def parts(self):
+        return self._base_parts + self._path.parts
+
+    def replace(self, path=None):
+        return self.from_parts(*self._base_parts, path=path)
 
     @cached_property
     def url(self):
-        p = self.parsed
-        return "{}://{}{}".format(p.scheme, self.netloc, p.path)
+        return "{}://{}{}".format(self.scheme, self.netloc, self._spath)
 
     def __str__(self):
         return self.url
@@ -170,92 +202,60 @@ class URLInfo(object):
             other = self.__class__(other)
         return (
             self.__class__ == other.__class__
-            and self.scheme == other.scheme
-            and self.netloc == other.netloc
+            and self._base_parts == other._base_parts
             and self._path == other._path
         )
 
     def __hash__(self):
-        return hash(self.url)
+        return hash(self.parts)
 
     def __div__(self, other):
-        p = self.parsed
-        new_path = posixpath.join(p.path, str(other))
-        if not new_path.startswith("/"):
-            new_path = "/" + new_path
-        new_url = "{}://{}{}".format(p.scheme, p.netloc, new_path)
-        return self.__class__(new_url)
+        return self.replace(path=posixpath.join(self._spath, other))
 
     __truediv__ = __div__
 
-    def __getattr__(self, name):
-        # When deepcopy is called, it creates and object without __init__,
-        # self.parsed is not initialized and it causes infinite recursion.
-        # More on this special casing here:
-        # https://stackoverflow.com/a/47300262/298182
-        if name.startswith("__"):
-            raise AttributeError(name)
-        return getattr(self.parsed, name)
-
-    @cached_property
-    def netloc(self):
-        p = self.parsed
-        netloc = p.hostname
-        if p.username:
-            netloc = p.username + "@" + netloc
-        if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme):
-            netloc += ":" + str(p.port)
-        return netloc
-
     @property
-    def port(self):
-        return self.parsed.port or self.DEFAULT_PORTS.get(self.parsed.scheme)
-
-    @property
-    def host(self):
-        return self.parsed.hostname
-
-    @property
-    def user(self):
-        return self.parsed.username
+    def path(self):
+        return self._spath
 
     @cached_property
     def _path(self):
-        return PosixPathInfo(self.parsed.path)
+        return _URLPathInfo(self._spath)
 
     @property
     def name(self):
         return self._path.name
 
-    @property
-    def parts(self):
-        return (self.scheme, self.netloc) + self._path.parts
+    @cached_property
+    def netloc(self):
+        netloc = self.host
+        if self.user:
+            netloc = self.user + "@" + netloc
+        if self.port and int(self.port) != self.DEFAULT_PORTS.get(self.scheme):
+            netloc += ":" + str(self.port)
+        return netloc
 
     @property
     def bucket(self):
-        return self.parsed.netloc
+        return self.netloc
 
     @property
     def parent(self):
-        return self.from_parts(
-            scheme=self.scheme,
-            netloc=self.parsed.netloc,
-            path=self._path.parent.fspath,
-        )
+        return self.replace(path=self._path.parent)
 
     @property
     def parents(self):
-        return _URLPathParents(
-            type(self), self.scheme, self.parsed.netloc, self._path
-        )
+        return _URLPathParents(self)
 
     def relative_to(self, other):
-        if isinstance(other, str):
-            other = URLInfo(other)
-        if self.scheme != other.scheme or self.netloc != other.netloc:
-            raise ValueError(
-                "'{}' does not start with '{}'".format(self, other)
-            )
+        if isinstance(other, basestring):
+            other = self.__class__(other)
+        if self.__class__ != other.__class__:
+            msg = "'{}' has incompatible class with '{}'".format(self, other)
+            raise ValueError(msg)
+        if self._base_parts != other._base_parts:
+            msg = "'{}' does not start with '{}'".format(self, other)
+            raise ValueError(msg)
         return self._path.relative_to(other._path)
 
     def isin(self, other):
@@ -263,14 +263,12 @@ class URLInfo(object):
             other = self.__class__(other)
         elif self.__class__ != other.__class__:
             return False
-        return (
-            self.scheme == other.scheme
-            and self.netloc == other.netloc
-            and self._path.isin(other._path)
+        return self._base_parts == other._base_parts and self._path.isin(
+            other._path
         )
 
 
 class CloudURLInfo(URLInfo):
     @property
     def path(self):
-        return self.parsed.path.lstrip("/")
+        return self._spath.lstrip("/")

dvc/progress.py

@@ -1,33 +1,17 @@
 """Manages progress bars for dvc repo."""
 from __future__ import print_function
+
 import logging
-from tqdm import tqdm
-from concurrent.futures import ThreadPoolExecutor
+import sys
+from threading import RLock
+
 from funcy import merge
+from tqdm import tqdm
+
+from dvc.utils import env2bool
 
 logger = logging.getLogger(__name__)
-
-
-class TqdmThreadPoolExecutor(ThreadPoolExecutor):
-    """
-    Ensure worker progressbars are cleared away properly.
-    """
-
-    def __enter__(self):
-        """
-        Creates a blank initial dummy progress bar if needed so that workers
-        are forced to create "nested" bars.
-        """
-        blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False)
-        if blank_bar.pos > 0:
-            # already nested - don't need a placeholder bar
-            blank_bar.close()
-        self.bar = blank_bar
-        return super(TqdmThreadPoolExecutor, self).__enter__()
-
-    def __exit__(self, *a, **k):
-        super(TqdmThreadPoolExecutor, self).__exit__(*a, **k)
-        self.bar.close()
+tqdm.set_lock(RLock())
 
 
 class Tqdm(tqdm):
@@ -54,6 +38,7 @@ class Tqdm(tqdm):
         leave=False,
         bar_format=None,
         bytes=False,  # pylint: disable=W0622
+        file=None,
         **kwargs
     ):
         """
@@ -62,6 +47,9 @@ class Tqdm(tqdm):
         desc  : persists after `close()`
         level  : effective logging level for determining `disable`;
             used only if `disable` is unspecified
+        disable  : If (default: None), will be determined by logging level.
+            May be overridden to `True` due to non-TTY status.
+            Skip override by specifying env var `DVC_IGNORE_ISATTY`.
         kwargs  : anything accepted by `tqdm.tqdm()`
         """
         kwargs = kwargs.copy()
@@ -71,15 +59,26 @@ class Tqdm(tqdm):
                 unit="B", unit_scale=True, unit_divisor=1024, miniters=1
             )
             kwargs = merge(bytes_defaults, kwargs)
+        if file is None:
+            file = sys.stderr
         self.desc_persist = desc
+        # auto-disable based on `logger.level`
         if disable is None:
             disable = logger.getEffectiveLevel() > level
+        # auto-disable based on TTY
+        if (
+            not disable
+            and not env2bool("DVC_IGNORE_ISATTY")
+            and hasattr(file, "isatty")
+        ):
+            disable = not file.isatty()
         super(Tqdm, self).__init__(
             iterable=iterable,
             disable=disable,
             leave=leave,
             desc=desc,
             bar_format="!",
+            lock_args=(False,),
             **kwargs
         )
         if bar_format is None:
@@ -118,7 +117,7 @@ class Tqdm(tqdm):
         if ncols_desc:
             d["ncols_desc"] = ncols_desc
         else:
-            # work-around for zero-width desc
+            # work-around for zero-width description
             d["ncols_desc"] = 1
-            d["desc"] = 0
+            d["prefix"] = ""
         return d

dvc/prompt.py

@@ -1,10 +1,9 @@
 """Manages user prompts."""
-
-from __future__ import unicode_literals
 from __future__ import print_function
+from __future__ import unicode_literals
 
-import sys
 import logging
+import sys
 from getpass import getpass
 
 from dvc.utils.compat import input

dvc/remote/__init__.py

@@ -1,16 +1,15 @@
 from __future__ import unicode_literals
 
+from .config import RemoteConfig
 from dvc.remote.azure import RemoteAZURE
 from dvc.remote.gs import RemoteGS
 from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
-from dvc.remote.ssh import RemoteSSH
 from dvc.remote.http import RemoteHTTP
 from dvc.remote.https import RemoteHTTPS
+from dvc.remote.local import RemoteLOCAL
 from dvc.remote.oss import RemoteOSS
-
-from .config import RemoteConfig
+from dvc.remote.s3 import RemoteS3
+from dvc.remote.ssh import RemoteSSH
 
 
 REMOTES = [

dvc/remote/azure.py

@@ -1,20 +1,20 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+import logging
 import os
 import re
-import logging
-from datetime import datetime, timedelta
+from datetime import datetime
+from datetime import timedelta
 
 from funcy import cached_property
 
-from dvc.scheme import Schemes
-
-from dvc.utils.compat import urlparse
-from dvc.progress import Tqdm
 from dvc.config import Config
-from dvc.remote.base import RemoteBASE
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.scheme import Schemes
+from dvc.utils.compat import urlparse
 
 
 logger = logging.getLogger(__name__)

dvc/remote/base.py

@@ -1,30 +1,34 @@
 from __future__ import unicode_literals
 
-from operator import itemgetter
-from multiprocessing import cpu_count
+import itertools
 import json
 import logging
 import tempfile
-import itertools
-from functools import partial
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
-
-from dvc.ignore import DvcIgnore
-from dvc.utils.compat import str, basestring, urlparse, FileNotFoundError
-from dvc.remote.slow_link_detection import slow_link_guard
+from functools import partial
+from multiprocessing import cpu_count
+from operator import itemgetter
 
 import dvc.prompt as prompt
 from dvc.config import Config
-from dvc.exceptions import (
-    DvcException,
-    ConfirmRemoveError,
-    DvcIgnoreInCollectedDirError,
-)
+from dvc.exceptions import ConfirmRemoveError
+from dvc.exceptions import DvcException
+from dvc.exceptions import DvcIgnoreInCollectedDirError
+from dvc.ignore import DvcIgnore
+from dvc.path_info import PathInfo
+from dvc.path_info import URLInfo
 from dvc.progress import Tqdm
-from dvc.utils import LARGE_DIR_SIZE, tmp_fname, move, relpath, makedirs
+from dvc.remote.slow_link_detection import slow_link_guard
 from dvc.state import StateNoop
-from dvc.path_info import PathInfo, URLInfo
+from dvc.utils import makedirs
+from dvc.utils import relpath
+from dvc.utils import tmp_fname
+from dvc.utils.compat import basestring
+from dvc.utils.compat import FileNotFoundError
+from dvc.utils.compat import str
+from dvc.utils.compat import urlparse
+from dvc.utils.fs import move
 from dvc.utils.http import open_url
 
 logger = logging.getLogger(__name__)
@@ -177,27 +181,23 @@ class RemoteBASE(object):
         file_infos = list(file_infos)
         with ThreadPoolExecutor(max_workers=self.checksum_jobs) as executor:
             tasks = executor.map(self.get_file_checksum, file_infos)
-
-            if len(file_infos) > LARGE_DIR_SIZE:
-                logger.info(
-                    (
-                        "Computing md5 for a large number of files. "
-                        "This is only done once."
-                    )
-                )
-                tasks = Tqdm(tasks, total=len(file_infos), unit="md5")
-            checksums = dict(zip(file_infos, tasks))
+            with Tqdm(
+                tasks,
+                total=len(file_infos),
+                unit="md5",
+                desc="Computing hashes (only done once)",
+            ) as tasks:
+                checksums = dict(zip(file_infos, tasks))
         return checksums
 
     def _collect_dir(self, path_info):
-
         file_infos = set()
-        for root, _dirs, files in self.walk(path_info):
 
-            if DvcIgnore.DVCIGNORE_FILE in files:
-                raise DvcIgnoreInCollectedDirError(root)
+        for fname in self.walk_files(path_info):
+            if DvcIgnore.DVCIGNORE_FILE == fname.name:
+                raise DvcIgnoreInCollectedDirError(fname.parent)
 
-            file_infos.update(path_info / root / fname for fname in files)
+            file_infos.add(fname)
 
         checksums = {fi: self.state.get(fi) for fi in file_infos}
         not_in_state = {
@@ -466,7 +466,8 @@ class RemoteBASE(object):
         """
         return False
 
-    def walk(self, path_info):
+    def walk_files(self, path_info):
+        """Return a generator with `PathInfo`s to all the files"""
         raise NotImplementedError
 
     @staticmethod
@@ -620,14 +621,11 @@ class RemoteBASE(object):
                 # We ignore all the non-cache looking files
                 pass
 
-    def gc(self, cinfos):
-        used = self.extract_used_local_checksums(cinfos)
+    def gc(self, named_cache):
+        used = self.extract_used_local_checksums(named_cache)
 
         if self.scheme != "":
-            used |= {
-                info[self.PARAM_CHECKSUM]
-                for info in cinfos.get(self.scheme, [])
-            }
+            used.update(named_cache[self.scheme])
 
         removed = False
         for checksum in self.all():
@@ -834,11 +832,7 @@ class RemoteBASE(object):
         self.state.save(path_info, checksum)
 
     def _remove_redundant_files(self, path_info, dir_info, force):
-        existing_files = set(
-            path_info / root / fname
-            for root, _, files in self.walk(path_info)
-            for fname in files
-        )
+        existing_files = set(self.walk_files(path_info))
 
         needed_files = {
             path_info / entry[self.PARAM_RELPATH] for entry in dir_info
@@ -909,10 +903,8 @@ class RemoteBASE(object):
     def _get_unpacked_dir_names(self, checksums):
         return set()
 
-    def extract_used_local_checksums(self, cinfos):
-        from dvc.remote import RemoteLOCAL
-
-        used = {info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]}
+    def extract_used_local_checksums(self, named_cache):
+        used = set(named_cache["local"])
         unpacked = self._get_unpacked_dir_names(used)
         return used | unpacked
 

dvc/remote/config.py

@@ -1,10 +1,11 @@
-import os
 import logging
+import os
 import posixpath
 
+from dvc.config import Config
+from dvc.config import ConfigError
 from dvc.utils import relpath
 from dvc.utils.compat import urlparse
-from dvc.config import Config, ConfigError
 
 
 logger = logging.getLogger(__name__)

dvc/remote/gs.py

@@ -2,18 +2,55 @@ from __future__ import unicode_literals
 
 import logging
 from datetime import timedelta
+from functools import wraps
+
 from funcy import cached_property
-from dvc.utils.compat import FileNotFoundError
 
-from dvc.remote.base import RemoteBASE
 from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
+from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
+from dvc.utils.compat import FileNotFoundError  # skipcq: PYL-W0622
 
 logger = logging.getLogger(__name__)
 
 
+def dynamic_chunk_size(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        import requests
+        from google.cloud.storage.blob import Blob, _DEFAULT_CHUNKSIZE
+
+        # Default chunk size for gs is 100M, which might be too much for
+        # particular network (see [1]). So if we are getting ConnectionError,
+        # we should try lowering the chunk size until we reach the minimum
+        # allowed chunk size of 256K. Also note that `chunk_size` must be a
+        # multiple of 256K per the API specification.
+        #
+        # [1] https://github.com/iterative/dvc/issues/2572
+
+        # skipcq: PYL-W0212
+        multiplier = int(_DEFAULT_CHUNKSIZE / Blob._CHUNK_SIZE_MULTIPLE)
+        while True:
+            try:
+                # skipcq: PYL-W0212
+                chunk_size = Blob._CHUNK_SIZE_MULTIPLE * multiplier
+                return func(*args, chunk_size=chunk_size, **kwargs)
+            except requests.exceptions.ConnectionError:
+                multiplier = int(multiplier / 2)
+                if not multiplier:
+                    raise
+
+    return wrapper
+
+
+@dynamic_chunk_size
+def _upload_to_bucket(bucket, from_file, to_info, **kwargs):
+    blob = bucket.blob(to_info.path, **kwargs)
+    blob.upload_from_filename(from_file)
+
+
 class RemoteGS(RemoteBASE):
     scheme = Schemes.GS
     path_cls = CloudURLInfo
@@ -88,8 +125,7 @@ class RemoteGS(RemoteBASE):
 
     def _upload(self, from_file, to_info, **_kwargs):
         bucket = self.gs.bucket(to_info.bucket)
-        blob = bucket.blob(to_info.path)
-        blob.upload_from_filename(from_file)
+        _upload_to_bucket(bucket, from_file, to_info)
 
     def _download(self, from_info, to_file, **_kwargs):
         bucket = self.gs.bucket(from_info.bucket)

dvc/remote/hdfs.py

@@ -1,21 +1,24 @@
 from __future__ import unicode_literals
 
-import os
-import re
 import io
-import posixpath
 import logging
-from subprocess import Popen, PIPE
-from contextlib import contextmanager, closing
+import os
+import posixpath
+import re
+from contextlib import closing
+from contextlib import contextmanager
+from subprocess import PIPE
+from subprocess import Popen
 
+from .base import RemoteBASE
+from .base import RemoteCmdError
+from .pool import get_connection
 from dvc.config import Config
 from dvc.scheme import Schemes
-
-from dvc.utils.compat import urlparse, FileNotFoundError
-from dvc.utils import fix_env, tmp_fname
-
-from .pool import get_connection
-from .base import RemoteBASE, RemoteCmdError
+from dvc.utils import fix_env
+from dvc.utils import tmp_fname
+from dvc.utils.compat import FileNotFoundError
+from dvc.utils.compat import urlparse
 
 logger = logging.getLogger(__name__)
 

dvc/remote/http.py

@@ -1,19 +1,24 @@
 from __future__ import unicode_literals
 
 import logging
+
+from funcy import cached_property
+
+from dvc.config import Config
+from dvc.config import ConfigError
+from dvc.exceptions import DvcException
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
 from dvc.utils.compat import open
 
-from dvc.progress import Tqdm
-from dvc.exceptions import DvcException
-from dvc.config import Config, ConfigError
-from dvc.remote.base import RemoteBASE
-
 logger = logging.getLogger(__name__)
 
 
 class RemoteHTTP(RemoteBASE):
     scheme = Schemes.HTTP
+    SESSION_RETRIES = 5
+    SESSION_BACKOFF_FACTOR = 0.1
     REQUEST_TIMEOUT = 10
     CHUNK_SIZE = 2 ** 16
     PARAM_CHECKSUM = "etag"
@@ -76,6 +81,24 @@ class RemoteHTTP(RemoteBASE):
 
         return etag
 
+    @cached_property
+    def _session(self):
+        import requests
+        from requests.adapters import HTTPAdapter
+        from urllib3.util.retry import Retry
+
+        session = requests.Session()
+
+        retries = Retry(
+            total=self.SESSION_RETRIES,
+            backoff_factor=self.SESSION_BACKOFF_FACTOR,
+        )
+
+        session.mount("http://", HTTPAdapter(max_retries=retries))
+        session.mount("https://", HTTPAdapter(max_retries=retries))
+
+        return session
+
     def _request(self, method, url, **kwargs):
         import requests
 
@@ -83,7 +106,7 @@ class RemoteHTTP(RemoteBASE):
         kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
 
         try:
-            return requests.request(method, url, **kwargs)
+            return self._session.request(method, url, **kwargs)
         except requests.exceptions.RequestException:
             raise DvcException("could not perform a {} request".format(method))
 

dvc/remote/https.py

@@ -1,8 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.scheme import Schemes
-
 from .http import RemoteHTTP
+from dvc.scheme import Schemes
 
 
 class RemoteHTTPS(RemoteHTTP):

dvc/remote/local.py

@@ -1,39 +1,38 @@
 from __future__ import unicode_literals
 
-from dvc.scheme import Schemes
-from dvc.utils.compat import str, fspath_py35, open
-
+import errno
+import logging
 import os
 import stat
-import errno
-from shortuuid import uuid
-import logging
+from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 
-from dvc.system import System
-from dvc.remote.base import (
-    RemoteBASE,
-    STATUS_MAP,
-    STATUS_NEW,
-    STATUS_DELETED,
-    STATUS_MISSING,
-)
-from dvc.utils import (
-    remove,
-    move,
-    copyfile,
-    tmp_fname,
-    file_md5,
-    walk_files,
-    relpath,
-    dvc_walk,
-    makedirs,
-)
-from dvc.config import Config
-from dvc.exceptions import DvcException, DownloadError, UploadError
-from dvc.progress import Tqdm, TqdmThreadPoolExecutor
+from shortuuid import uuid
 
+from dvc.config import Config
+from dvc.exceptions import DownloadError
+from dvc.exceptions import DvcException
+from dvc.exceptions import UploadError
 from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.remote.base import STATUS_DELETED
+from dvc.remote.base import STATUS_MAP
+from dvc.remote.base import STATUS_MISSING
+from dvc.remote.base import STATUS_NEW
+from dvc.scheme import Schemes
+from dvc.system import System
+from dvc.utils import copyfile
+from dvc.utils import file_md5
+from dvc.utils import makedirs
+from dvc.utils import relpath
+from dvc.utils import remove
+from dvc.utils import tmp_fname
+from dvc.utils import walk_files
+from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import open
+from dvc.utils.compat import str
+from dvc.utils.fs import move
 
 logger = logging.getLogger(__name__)
 
@@ -145,11 +144,12 @@ class RemoteLOCAL(RemoteBASE):
     def getsize(path_info):
         return os.path.getsize(fspath_py35(path_info))
 
-    def walk(self, path_info):
-        return dvc_walk(path_info, self.repo.dvcignore)
+    def walk_files(self, path_info):
+        for fname in walk_files(path_info, self.repo.dvcignore):
+            yield PathInfo(fname)
 
     def get_file_checksum(self, path_info):
-        return file_md5(fspath_py35(path_info))[0]
+        return file_md5(path_info)[0]
 
     def remove(self, path_info):
         if path_info.scheme != "local":
@@ -224,7 +224,7 @@ class RemoteLOCAL(RemoteBASE):
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        makedirs(fspath_py35(to_info.parent), exist_ok=True)
+        makedirs(to_info.parent, exist_ok=True)
 
         tmp_file = tmp_fname(to_info)
         copyfile(
@@ -236,41 +236,16 @@ class RemoteLOCAL(RemoteBASE):
         self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
     ):
         copyfile(
-            fspath_py35(from_info),
-            to_file,
-            no_progress_bar=no_progress_bar,
-            name=name,
+            from_info, to_file, no_progress_bar=no_progress_bar, name=name
         )
 
     @staticmethod
     def open(path_info, mode="r", encoding=None):
         return open(fspath_py35(path_info), mode=mode, encoding=encoding)
 
-    def _group(self, checksum_infos, show_checksums=False):
-        by_md5 = {}
-
-        for info in checksum_infos:
-            md5 = info[self.PARAM_CHECKSUM]
-
-            if show_checksums:
-                by_md5[md5] = {"name": md5}
-                continue
-
-            name = info["name"]
-            branch = info.get("branch")
-            if branch:
-                name += "({})".format(branch)
-
-            if md5 not in by_md5.keys():
-                by_md5[md5] = {"name": name}
-            else:
-                by_md5[md5]["name"] += " " + name
-
-        return by_md5
-
     def status(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
@@ -279,15 +254,14 @@ class RemoteLOCAL(RemoteBASE):
         logger.debug(
             "Preparing to collect status from {}".format(remote.path_info)
         )
-        ret = self._group(checksum_infos, show_checksums=show_checksums) or {}
-        md5s = list(ret)
+        md5s = list(named_cache[self.scheme])
 
         logger.debug("Collecting information from local cache...")
         local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
 
         # This is a performance optimization. We can safely assume that,
         # if the resources that we want to fetch are already cached,
-        # there's no need to check the remote storage for the existance of
+        # there's no need to check the remote storage for the existence of
         # those files.
         if download and sorted(local_exists) == sorted(md5s):
             remote_exists = local_exists
@@ -299,6 +273,10 @@ class RemoteLOCAL(RemoteBASE):
                 )
             )
 
+        ret = {
+            checksum: {"name": checksum if show_checksums else " ".join(names)}
+            for checksum, names in named_cache[self.scheme].items()
+        }
         self._fill_statuses(ret, local_exists, remote_exists)
 
         self._log_missing_caches(ret)
@@ -338,7 +316,7 @@ class RemoteLOCAL(RemoteBASE):
 
     def _process(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
@@ -366,7 +344,7 @@ class RemoteLOCAL(RemoteBASE):
             jobs = remote.JOBS
 
         status_info = self.status(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
@@ -379,7 +357,7 @@ class RemoteLOCAL(RemoteBASE):
             return 0
 
         if jobs > 1:
-            with TqdmThreadPoolExecutor(max_workers=jobs) as executor:
+            with ThreadPoolExecutor(max_workers=jobs) as executor:
                 fails = sum(executor.map(func, *plans))
         else:
             fails = sum(map(func, *plans))
@@ -391,18 +369,18 @@ class RemoteLOCAL(RemoteBASE):
 
         return len(plans[0])
 
-    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def push(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
             download=False,
         )
 
-    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,

dvc/remote/oss.py

@@ -1,15 +1,14 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-import os
 import logging
-
-from dvc.scheme import Schemes
+import os
 
 from dvc.config import Config
-from dvc.remote.base import RemoteBASE
-from dvc.progress import Tqdm
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.scheme import Schemes
 
 
 logger = logging.getLogger(__name__)

dvc/remote/pool.py

@@ -1,5 +1,6 @@
 from collections import deque
 from contextlib import contextmanager
+
 from funcy import memoize
 
 

dvc/remote/s3.py

@@ -1,14 +1,17 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import os
 import logging
+import os
+
 from funcy import cached_property
 
-from dvc.progress import Tqdm
 from dvc.config import Config
-from dvc.remote.base import RemoteBASE
-from dvc.exceptions import DvcException, ETagMismatchError
+from dvc.exceptions import DvcException
+from dvc.exceptions import ETagMismatchError
 from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
@@ -36,7 +39,10 @@ class RemoteS3(RemoteBASE):
 
         self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
 
-        self.list_objects = config.get(Config.SECTION_AWS_LIST_OBJECTS)
+        if config.get(Config.SECTION_AWS_LIST_OBJECTS):
+            self.list_objects_api = "list_objects"
+        else:
+            self.list_objects_api = "list_objects_v2"
 
         self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
 
@@ -139,13 +145,13 @@ class RemoteS3(RemoteBASE):
         # multipart.
         #
         # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
-        # then it was transfered as a single part, which means that the chunk
+        # then it was transferred as a single part, which means that the chunk
         # size used to transfer it was greater or equal to the ContentLength
         # of that object. So to preserve that tag over the next transfer, we
         # could use any value >= ContentLength.
         #
         # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
-        # then it was transfered as a multipart, which means that the chunk
+        # then it was transferred as a multipart, which means that the chunk
         # size used to transfer it was less than ContentLength of that object.
         # Unfortunately, in general, it doesn't mean that the chunk size was
         # the same throughout the transfer, so it means that in order to
@@ -180,27 +186,57 @@ class RemoteS3(RemoteBASE):
         logger.debug("Removing {}".format(path_info))
         self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
 
-    def _list_paths(self, bucket, prefix):
+    def _list_objects(self, path_info, max_items=None):
         """ Read config for list object api, paginate through list objects."""
-        kwargs = {"Bucket": bucket, "Prefix": prefix}
-        if self.list_objects:
-            list_objects_api = "list_objects"
-        else:
-            list_objects_api = "list_objects_v2"
-        paginator = self.s3.get_paginator(list_objects_api)
+        kwargs = {
+            "Bucket": path_info.bucket,
+            "Prefix": path_info.path,
+            "PaginationConfig": {"MaxItems": max_items},
+        }
+        paginator = self.s3.get_paginator(self.list_objects_api)
         for page in paginator.paginate(**kwargs):
             contents = page.get("Contents", None)
             if not contents:
                 continue
             for item in contents:
-                yield item["Key"]
+                yield item
+
+    def _list_paths(self, path_info, max_items=None):
+        return (
+            item["Key"] for item in self._list_objects(path_info, max_items)
+        )
 
     def list_cache_paths(self):
-        return self._list_paths(self.path_info.bucket, self.path_info.path)
+        return self._list_paths(self.path_info)
 
     def exists(self, path_info):
-        paths = self._list_paths(path_info.bucket, path_info.path)
-        return any(path_info.path == path for path in paths)
+        dir_path = path_info / ""
+        fname = next(self._list_paths(path_info, max_items=1), "")
+        return path_info.path == fname or fname.startswith(dir_path.path)
+
+    def isdir(self, path_info):
+        # S3 doesn't have a concept for directories.
+        #
+        # Using `head_object` with a path pointing to a directory
+        # will throw a 404 error.
+        #
+        # A reliable way to know if a given path is a directory is by
+        # checking if there are more files sharing the same prefix
+        # with a `list_objects` call.
+        #
+        # We need to make sure that the path ends with a forward slash,
+        # since we can end with false-positives like the following example:
+        #
+        #       bucket
+        #       └── data
+        #          ├── alice
+        #          └── alpha
+        #
+        # Using `data/al` as prefix will return `[data/alice, data/alpha]`,
+        # While `data/al/` will return nothing.
+        #
+        dir_path = path_info / ""
+        return bool(list(self._list_paths(dir_path, max_items=1)))
 
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
         total = os.path.getsize(from_file)
@@ -234,3 +270,7 @@ class RemoteS3(RemoteBASE):
         return self.s3.generate_presigned_url(
             ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
         )
+
+    def walk_files(self, path_info, max_items=None):
+        for fname in self._list_paths(path_info, max_items):
+            yield path_info.replace(path=fname)

dvc/remote/slow_link_detection.py

@@ -1,9 +1,10 @@
 import logging
-import time
 import sys
+import time
 from functools import wraps
 
 import colorama
+
 from dvc.config import Config
 
 
@@ -18,7 +19,7 @@ this.message = (
     "See {blue}https://dvc.org/doc/commands-reference/config#cache{reset} "
     "for more information.\n"
     "To disable this message, run:\n"
-    "'dvc config cache.slow_link_warning false'".format(
+    "`dvc config cache.slow_link_warning false`".format(
         blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
     )
 )

dvc/remote/ssh/__init__.py

@@ -1,23 +1,25 @@
 from __future__ import unicode_literals
 
 import errno
-import itertools
-import io
-import os
 import getpass
+import io
+import itertools
 import logging
+import os
 import threading
 from concurrent.futures import ThreadPoolExecutor
-from contextlib import contextmanager, closing
+from contextlib import closing
+from contextlib import contextmanager
 
 import dvc.prompt as prompt
 from dvc.config import Config
-from dvc.utils import to_chunks
-from dvc.utils.compat import urlparse, StringIO
-from dvc.remote.base import RemoteBASE
-from dvc.scheme import Schemes
-from dvc.remote.pool import get_connection
 from dvc.progress import Tqdm
+from dvc.remote.base import RemoteBASE
+from dvc.remote.pool import get_connection
+from dvc.scheme import Schemes
+from dvc.utils import to_chunks
+from dvc.utils.compat import StringIO
+from dvc.utils.compat import urlparse
 
 logger = logging.getLogger(__name__)
 
@@ -115,9 +117,10 @@ class RemoteSSH(RemoteBASE):
             return identity_file[0]
         return None
 
-    def ssh(self, path_info):
+    def ensure_credentials(self, path_info=None):
+        if path_info is None:
+            path_info = self.path_info
         host, user, port = path_info.host, path_info.user, path_info.port
-
         # NOTE: we use the same password regardless of the server :(
         if self.ask_password and self.password is None:
             with saved_passwords_lock:
@@ -133,13 +136,16 @@ class RemoteSSH(RemoteBASE):
                     )
                 self.password = password
 
+    def ssh(self, path_info):
+        self.ensure_credentials(path_info)
+
         from .connection import SSHConnection
 
         return get_connection(
             SSHConnection,
-            host,
-            username=user,
-            port=port,
+            path_info.host,
+            username=path_info.user,
+            port=path_info.port,
             key_filename=self.keyfile,
             timeout=self.timeout,
             password=self.password,
@@ -260,10 +266,10 @@ class RemoteSSH(RemoteBASE):
         with self.ssh(self.path_info) as ssh:
             return list(ssh.walk_files(self.path_info.path))
 
-    def walk(self, path_info):
+    def walk_files(self, path_info):
         with self.ssh(path_info) as ssh:
-            for entry in ssh.walk(path_info.path):
-                yield entry
+            for fname in ssh.walk_files(path_info.path):
+                yield path_info.replace(path=fname)
 
     def makedirs(self, path_info):
         with self.ssh(path_info) as ssh:
@@ -306,6 +312,9 @@ class RemoteSSH(RemoteBASE):
         if not self.no_traverse:
             return list(set(checksums) & set(self.all()))
 
+        # possibly prompt for credentials before "Querying" progress output
+        self.ensure_credentials()
+
         with Tqdm(
             desc="Querying "
             + ("cache in " + name if name else "remote cache"),

dvc/remote/ssh/connection.py

@@ -1,8 +1,9 @@
+import errno
+import logging
 import os
 import posixpath
-import logging
-import errno
 import stat
+
 from funcy import cached_property
 
 try:
@@ -185,8 +186,26 @@ class SSHConnection:
             self.sftp.get(src, dest, callback=pbar.update_to)
 
     def move(self, src, dst):
+        """Rename src to dst, if it is not possible (in case src and dst are
+        on different filesystems) and actual physical copying of data is
+        happening.
+        """
         self.makedirs(posixpath.dirname(dst))
-        self.sftp.rename(src, dst)
+
+        try:
+            self.sftp.rename(src, dst)
+        except OSError:
+            self._atomic_copy(src, dst)
+            self.remove(src)
+
+    def _atomic_copy(self, src, dst):
+        tmp = tmp_fname(dst)
+
+        try:
+            self.copy(src, tmp)
+            self.sftp.rename(tmp, dst)
+        finally:
+            self.remove(tmp)
 
     def upload(self, src, dest, no_progress_bar=False, progress_title=None):
         self.makedirs(posixpath.dirname(dest))

dvc/repo/__init__.py

@@ -1,26 +1,28 @@
 from __future__ import unicode_literals
 
-import os
 import logging
+import os
 from contextlib import contextmanager
+from functools import wraps
 from itertools import chain
 
-from functools import wraps
 from funcy import cached_property
 
+from .graph import check_acyclic
+from .graph import get_pipeline
+from .graph import get_pipelines
+from .graph import get_stages
 from dvc.config import Config
-from dvc.exceptions import (
-    NotDvcRepoError,
-    OutputNotFoundError,
-    FileMissingError,
-)
+from dvc.exceptions import FileMissingError
+from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import OutputNotFoundError
 from dvc.ignore import DvcIgnoreFilter
 from dvc.path_info import PathInfo
 from dvc.remote.base import RemoteActionNotImplemented
-from dvc.utils.compat import open as _open, fspath_py35, FileNotFoundError
 from dvc.utils import relpath
-
-from .graph import check_acyclic, get_pipeline, get_pipelines, get_stages
+from dvc.utils.compat import FileNotFoundError
+from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import open as _open
 
 
 logger = logging.getLogger(__name__)
@@ -213,6 +215,7 @@ class Repo(object):
         all_branches=False,
         with_deps=False,
         all_tags=False,
+        all_commits=False,
         remote=None,
         force=False,
         jobs=None,
@@ -231,18 +234,14 @@ class Repo(object):
             A dictionary with Schemes (representing output's location) as keys,
             and a list with the outputs' `dumpd` as values.
         """
+        from dvc.cache import NamedCache
 
-        cache = {}
-        cache["local"] = []
-        cache["s3"] = []
-        cache["gs"] = []
-        cache["hdfs"] = []
-        cache["ssh"] = []
-        cache["azure"] = []
-        cache["repo"] = []
+        cache = NamedCache()
 
         for branch in self.brancher(
-            all_branches=all_branches, all_tags=all_tags
+            all_branches=all_branches,
+            all_tags=all_tags,
+            all_commits=all_commits,
         ):
             if targets:
                 stages = []
@@ -256,18 +255,16 @@ class Repo(object):
 
             for stage in stages:
                 if stage.is_repo_import:
-                    cache["repo"] += stage.deps
+                    dep, = stage.deps
+                    cache.external[dep.repo_pair].add(dep.def_path)
                     continue
 
                 for out in stage.outs:
-                    scheme = out.path_info.scheme
                     used_cache = out.get_used_cache(
                         remote=remote, force=force, jobs=jobs
                     )
-
-                    cache[scheme].extend(
-                        dict(entry, branch=branch) for entry in used_cache
-                    )
+                    suffix = "({})".format(branch) if branch else ""
+                    cache.update(used_cache, suffix=suffix)
 
         return cache
 
@@ -499,7 +496,7 @@ class Repo(object):
 
     @cached_property
     def dvcignore(self):
-        return DvcIgnoreFilter(self.root_dir)
+        return DvcIgnoreFilter(self.root_dir, self.tree)
 
     def close(self):
         self.scm.close()

dvc/repo/add.py

@@ -1,54 +1,67 @@
 from __future__ import unicode_literals
 
-import os
 import logging
-import colorama
+import os
 
-from dvc.repo.scm_context import scm_context
-from dvc.stage import Stage
-from dvc.utils import walk_files, LARGE_DIR_SIZE
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
+import colorama
+from six import string_types
 
 from . import locked
-
+from dvc.exceptions import RecursiveAddingWhileUsingFilename
+from dvc.progress import Tqdm
+from dvc.repo.scm_context import scm_context
+from dvc.stage import Stage
+from dvc.utils import LARGE_DIR_SIZE
+from dvc.utils import walk_files
 
 logger = logging.getLogger(__name__)
 
 
 @locked
 @scm_context
-def add(repo, target, recursive=False, no_commit=False, fname=None):
+def add(repo, targets, recursive=False, no_commit=False, fname=None):
     if recursive and fname:
         raise RecursiveAddingWhileUsingFilename()
 
-    targets = _find_all_targets(repo, target, recursive)
+    if isinstance(targets, string_types):
+        targets = [targets]
 
-    if os.path.isdir(target) and len(targets) > LARGE_DIR_SIZE:
-        logger.warning(
-            "You are adding a large directory '{target}' recursively,"
-            " consider tracking it as a whole instead.\n"
-            "{purple}HINT:{nc} Remove the generated DVC-file and then"
-            " run {cyan}dvc add {target}{nc}".format(
-                purple=colorama.Fore.MAGENTA,
-                cyan=colorama.Fore.CYAN,
-                nc=colorama.Style.RESET_ALL,
-                target=target,
-            )
-        )
+    stages_list = []
+    with Tqdm(total=len(targets), desc="Add", unit="file", leave=True) as pbar:
+        for target in targets:
+            sub_targets = _find_all_targets(repo, target, recursive)
+            pbar.total += len(sub_targets) - 1
 
-    stages = _create_stages(repo, targets, fname)
+            if os.path.isdir(target) and len(sub_targets) > LARGE_DIR_SIZE:
+                logger.warning(
+                    "You are adding a large directory '{target}' recursively,"
+                    " consider tracking it as a whole instead.\n"
+                    "{purple}HINT:{nc} Remove the generated DVC-file and then"
+                    " run {cyan}dvc add {target}{nc}".format(
+                        purple=colorama.Fore.MAGENTA,
+                        cyan=colorama.Fore.CYAN,
+                        nc=colorama.Style.RESET_ALL,
+                        target=target,
+                    )
+                )
 
-    repo.check_modified_graph(stages)
+            stages = _create_stages(repo, sub_targets, fname, pbar=pbar)
 
-    for stage in stages:
-        stage.save()
+            repo.check_modified_graph(stages)
 
-        if not no_commit:
-            stage.commit()
+            for stage in stages:
+                stage.save()
 
-        stage.dump()
+                if not no_commit:
+                    stage.commit()
 
-    return stages
+                stage.dump()
+
+            stages_list += stages
+        # remove filled bar bit of progress, leaving stats
+        pbar.bar_format = pbar.BAR_FMT_DEFAULT.replace("|{bar:10}|", " ")
+
+    return stages_list
 
 
 def _find_all_targets(repo, target, recursive):
@@ -64,15 +77,19 @@ def _find_all_targets(repo, target, recursive):
     return [target]
 
 
-def _create_stages(repo, targets, fname):
+def _create_stages(repo, targets, fname, pbar=None):
     stages = []
 
     for out in targets:
         stage = Stage.create(repo, outs=[out], add=True, fname=fname)
 
         if not stage:
+            if pbar is not None:
+                pbar.total -= 1
             continue
 
         stages.append(stage)
+        if pbar is not None:
+            pbar.update_desc(out)
 
     return stages

dvc/repo/brancher.py

@@ -1,8 +1,15 @@
+from funcy import group_by
+
 from dvc.scm.tree import WorkingTree
 
 
 def brancher(  # noqa: E302
-    self, branches=None, all_branches=False, tags=None, all_tags=False
+    self,
+    branches=None,
+    all_branches=False,
+    tags=None,
+    all_tags=False,
+    all_commits=False,
 ):
     """Generator that iterates over specified revisions.
 
@@ -18,7 +25,7 @@ def brancher(  # noqa: E302
             - empty string it there is no branches to iterate over
             - "Working Tree" if there are uncommited changes in the SCM repo
     """
-    if not any([branches, all_branches, tags, all_tags]):
+    if not any([branches, all_branches, tags, all_tags, all_commits]):
         yield ""
         return
 
@@ -30,25 +37,28 @@ def brancher(  # noqa: E302
     self.tree = WorkingTree(self.root_dir)
     yield "working tree"
 
-    if all_branches:
-        branches = scm.list_branches()
+    if all_commits:
+        revs = scm.list_all_commits()
+    else:
+        if all_branches:
+            branches = scm.list_branches()
 
-    if all_tags:
-        tags = scm.list_tags()
+        if all_tags:
+            tags = scm.list_tags()
 
-    if branches is not None:
-        revs.extend(branches)
+        if branches is not None:
+            revs.extend(branches)
 
-    if tags is not None:
-        revs.extend(tags)
+        if tags is not None:
+            revs.extend(tags)
 
     # NOTE: it might be a good idea to wrap this loop in try/finally block
     # to don't leave the tree on some unexpected branch after the
     # `brancher()`, but this could cause problems on exception handling
     # code which might expect the tree on which exception was raised to
     # stay in place. This behavior is a subject to change.
-    for rev in revs:
-        self.tree = scm.get_tree(rev)
-        yield rev
+    for sha, names in group_by(scm.resolve_rev, revs).items():
+        self.tree = scm.get_tree(sha)
+        yield ", ".join(names)
 
     self.tree = saved_tree

dvc/repo/checkout.py

@@ -2,21 +2,22 @@ from __future__ import unicode_literals
 
 import logging
 
-from dvc.exceptions import CheckoutErrorSuggestGit, CheckoutError
+from dvc.exceptions import CheckoutError
+from dvc.exceptions import CheckoutErrorSuggestGit
 from dvc.progress import Tqdm
 
 
 logger = logging.getLogger(__name__)
 
 
-def _cleanup_unused_links(self, all_stages):
+def _cleanup_unused_links(repo):
     used = [
         out.fspath
-        for stage in all_stages
+        for stage in repo.stages
         for out in stage.outs
         if out.scheme == "local"
     ]
-    self.state.remove_unused_links(used)
+    repo.state.remove_unused_links(used)
 
 
 def get_all_files_numbers(stages):
@@ -24,18 +25,27 @@ def get_all_files_numbers(stages):
 
 
 def _checkout(
-    self, target=None, with_deps=False, force=False, recursive=False
+    self, targets=None, with_deps=False, force=False, recursive=False
 ):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    try:
-        stages = self.collect(target, with_deps=with_deps, recursive=recursive)
-    except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
-        if not target:
-            raise
-        raise CheckoutErrorSuggestGit(target, exc)
+    stages = set()
+
+    if not targets:
+        targets = [None]
+        _cleanup_unused_links(self)
+
+    for target in targets:
+        try:
+            new = self.collect(
+                target, with_deps=with_deps, recursive=recursive
+            )
+            stages.update(new)
+        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+            if not target:
+                raise
+            raise CheckoutErrorSuggestGit(target, exc)
 
-    _cleanup_unused_links(self, self.stages)
     total = get_all_files_numbers(stages)
     if total == 0:
         logger.info("Nothing to do")
@@ -44,12 +54,6 @@ def _checkout(
         total=total, unit="file", desc="Checkout", disable=total == 0
     ) as pbar:
         for stage in stages:
-            if stage.locked:
-                logger.warning(
-                    "DVC-file '{path}' is locked. Its dependencies are"
-                    " not going to be checked out.".format(path=stage.relpath)
-                )
-
             failed.extend(
                 stage.checkout(force=force, progress_callback=pbar.update_desc)
             )

dvc/repo/diff.py

@@ -3,18 +3,14 @@ from __future__ import unicode_literals
 import os
 from errno import ENOENT
 
-
 import dvc.logger as logger
-from dvc.scm.base import FileNotInCommitError
-from dvc.scm.git import (
-    DIFF_A_TREE,
-    DIFF_B_TREE,
-    DIFF_A_REF,
-    DIFF_B_REF,
-    DIFF_EQUAL,
-)
-
 from . import locked
+from dvc.scm.base import FileNotInCommitError
+from dvc.scm.git import DIFF_A_REF
+from dvc.scm.git import DIFF_A_TREE
+from dvc.scm.git import DIFF_B_REF
+from dvc.scm.git import DIFF_B_TREE
+from dvc.scm.git import DIFF_EQUAL
 
 
 DIFF_TARGET = "target"

dvc/repo/fetch.py

@@ -2,8 +2,11 @@ from __future__ import unicode_literals
 
 import logging
 
+from dvc.cache import NamedCache
 from dvc.config import NoRemoteError
-from dvc.exceptions import DownloadError, OutputNotFoundError
+from dvc.exceptions import DownloadError
+from dvc.exceptions import OutputNotFoundError
+from dvc.external_repo import external_repo
 from dvc.scm.base import CloneError
 
 
@@ -24,7 +27,7 @@ def _fetch(
     """Download data items from a cloud and imported repositories
 
     Returns:
-        int: number of succesfully downloaded files
+        int: number of successfully downloaded files
 
     Raises:
         DownloadError: thrown when there are failed downloads, either
@@ -49,28 +52,53 @@ def _fetch(
 
     try:
         downloaded += self.cloud.pull(
-            used["local"], jobs, remote=remote, show_checksums=show_checksums
+            used, jobs, remote=remote, show_checksums=show_checksums
         )
     except NoRemoteError:
-        if not used["repo"] and used["local"]:
+        if not used.external and used["local"]:
             raise
-
     except DownloadError as exc:
         failed += exc.amount
 
-    for dep in used["repo"]:
-        try:
-            out = dep.fetch()
-            downloaded += out.get_files_number()
-        except DownloadError as exc:
-            failed += exc.amount
-        except (CloneError, OutputNotFoundError):
-            failed += 1
-            logger.exception(
-                "failed to fetch data for '{}'".format(dep.stage.outs[0])
-            )
+    for (repo_url, repo_rev), files in used.external.items():
+        d, f = _fetch_external(self, repo_url, repo_rev, files)
+        downloaded += d
+        failed += f
 
     if failed:
         raise DownloadError(failed)
 
     return downloaded
+
+
+def _fetch_external(self, repo_url, repo_rev, files):
+    failed = 0
+
+    cache_dir = self.cache.local.cache_dir
+    try:
+        with external_repo(repo_url, repo_rev, cache_dir=cache_dir) as repo:
+            with repo.state:
+                cache = NamedCache()
+                for name in files:
+                    try:
+                        out = repo.find_out_by_relpath(name)
+                    except OutputNotFoundError:
+                        failed += 1
+                        logger.exception(
+                            "failed to fetch data for '{}'".format(name)
+                        )
+                        continue
+                    else:
+                        cache.update(out.get_used_cache())
+
+                try:
+                    return repo.cloud.pull(cache), failed
+                except DownloadError as exc:
+                    failed += exc.amount
+    except CloneError:
+        failed += 1
+        logger.exception(
+            "failed to fetch data for '{}'".format(", ".join(files))
+        )
+
+    return 0, failed

dvc/repo/gc.py

@@ -1,53 +1,14 @@
 from __future__ import unicode_literals
 
-import collections
 import logging
 
 from . import locked
+from dvc.cache import NamedCache
 
 
 logger = logging.getLogger(__name__)
 
 
-def _merge_cache_lists(clists):
-    merged_cache = collections.defaultdict(list)
-
-    for cache_list in clists:
-        for scheme, cache in cache_list.items():
-            for item in cache:
-                if item not in merged_cache[scheme]:
-                    merged_cache[scheme].append(item)
-
-    return merged_cache
-
-
-def _load_all_used_cache(
-    repos,
-    all_branches=False,
-    with_deps=False,
-    all_tags=False,
-    remote=None,
-    force=False,
-    jobs=None,
-):
-    clists = []
-
-    for repo in repos:
-        repo_clist = repo.used_cache(
-            targets=None,
-            all_branches=all_branches,
-            with_deps=with_deps,
-            all_tags=all_tags,
-            remote=remote,
-            force=force,
-            jobs=jobs,
-        )
-
-        clists.append(repo_clist)
-
-    return clists
-
-
 def _do_gc(typ, func, clist):
     removed = func(clist)
     if not removed:
@@ -62,6 +23,7 @@ def gc(
     remote=None,
     with_deps=False,
     all_tags=False,
+    all_commits=False,
     force=False,
     jobs=None,
     repos=None,
@@ -79,37 +41,36 @@ def gc(
             stack.enter_context(repo.lock)
             stack.enter_context(repo.state)
 
-        all_clists = _load_all_used_cache(
-            all_repos + [self],
-            all_branches=all_branches,
-            with_deps=with_deps,
-            all_tags=all_tags,
-            remote=remote,
-            force=force,
-            jobs=jobs,
-        )
+        used = NamedCache()
+        for repo in all_repos + [self]:
+            used.update(
+                repo.used_cache(
+                    all_branches=all_branches,
+                    with_deps=with_deps,
+                    all_tags=all_tags,
+                    all_commits=all_commits,
+                    remote=remote,
+                    force=force,
+                    jobs=jobs,
+                )
+            )
 
-    if len(all_clists) > 1:
-        clist = _merge_cache_lists(all_clists)
-    else:
-        clist = all_clists[0]
-
-    _do_gc("local", self.cache.local.gc, clist)
+    _do_gc("local", self.cache.local.gc, used)
 
     if self.cache.s3:
-        _do_gc("s3", self.cache.s3.gc, clist)
+        _do_gc("s3", self.cache.s3.gc, used)
 
     if self.cache.gs:
-        _do_gc("gs", self.cache.gs.gc, clist)
+        _do_gc("gs", self.cache.gs.gc, used)
 
     if self.cache.ssh:
-        _do_gc("ssh", self.cache.ssh.gc, clist)
+        _do_gc("ssh", self.cache.ssh.gc, used)
 
     if self.cache.hdfs:
-        _do_gc("hdfs", self.cache.hdfs.gc, clist)
+        _do_gc("hdfs", self.cache.hdfs.gc, used)
 
     if self.cache.azure:
-        _do_gc("azure", self.cache.azure.gc, clist)
+        _do_gc("azure", self.cache.azure.gc, used)
 
     if cloud:
-        _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, clist)
+        _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, used)

dvc/repo/get.py

@@ -1,19 +1,18 @@
 import logging
 import os
+
 import shortuuid
 
-from dvc.stage import Stage
-from dvc.config import Config
-from dvc.exceptions import (
-    NotDvcRepoError,
-    UrlNotDvcRepoError,
-    GetDVCFileError,
-    OutputNotFoundError,
-)
-from dvc.path_info import PathInfo
+from dvc.exceptions import GetDVCFileError
+from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import OutputNotFoundError
+from dvc.exceptions import UrlNotDvcRepoError
 from dvc.external_repo import external_repo
+from dvc.path_info import PathInfo
+from dvc.stage import Stage
 from dvc.state import StateNoop
-from dvc.utils import remove, resolve_output
+from dvc.utils import remove
+from dvc.utils import resolve_output
 
 logger = logging.getLogger(__name__)
 
@@ -48,11 +47,7 @@ def get(url, path, out=None, rev=None):
             #
             # Also, we can't use theoretical "move" link type here, because
             # the same cache file might be used a few times in a directory.
-            repo.config.set(
-                Config.SECTION_CACHE,
-                Config.SECTION_CACHE_TYPE,
-                "reflink,hardlink,copy",
-            )
+            repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]
 
             o = repo.find_out_by_relpath(path)
             with repo.state:

dvc/repo/get_url.py

@@ -1,8 +1,7 @@
 import os
 
-import dvc.output as output
 import dvc.dependency as dependency
-
+import dvc.output as output
 from dvc.utils import resolve_output
 
 
@@ -12,7 +11,8 @@ def get_url(url, out=None):
 
     if os.path.exists(url):
         url = os.path.abspath(url)
-        out = os.path.abspath(out)
+
+    out = os.path.abspath(out)
 
     dep, = dependency.loads_from(None, [url])
     out, = output.loads_from(None, [out], use_cache=False)

dvc/repo/graph.py

@@ -5,10 +5,17 @@ def check_acyclic(graph):
     import networkx as nx
     from dvc.exceptions import CyclicGraphError
 
-    cycle = next(nx.simple_cycles(graph), None)
+    try:
+        edges = nx.find_cycle(graph, orientation="original")
+    except nx.NetworkXNoCycle:
+        return
 
-    if cycle:
-        raise CyclicGraphError(cycle)
+    stages = set()
+    for from_node, to_node, _ in edges:
+        stages.add(from_node)
+        stages.add(to_node)
+
+    raise CyclicGraphError(list(stages))
 
 
 def get_pipeline(pipelines, node):

dvc/repo/imp_url.py

@@ -1,7 +1,6 @@
-from dvc.utils import resolve_output
-from dvc.repo.scm_context import scm_context
-
 from . import locked as locked_repo
+from dvc.repo.scm_context import scm_context
+from dvc.utils import resolve_output
 
 
 @locked_repo

dvc/repo/init.py

@@ -1,13 +1,17 @@
-import os
-import colorama
 import logging
+import os
 
-from dvc.repo import Repo
-from dvc.scm import SCM, NoSCM
+import colorama
+
+from dvc.analytics import Analytics
 from dvc.config import Config
 from dvc.exceptions import InitError
-from dvc.utils import boxify, relpath, remove
-from dvc.analytics import Analytics
+from dvc.repo import Repo
+from dvc.scm import NoSCM
+from dvc.scm import SCM
+from dvc.utils import boxify
+from dvc.utils import relpath
+from dvc.utils import remove
 
 logger = logging.getLogger(__name__)
 
@@ -65,7 +69,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
     if isinstance(scm, NoSCM) and not no_scm:
         raise InitError(
             "{repo} is not tracked by any supported scm tool (e.g. git). "
-            "Use '--no-scm' if you don't want to use any scm.".format(
+            "Use `--no-scm` if you don't want to use any scm.".format(
                 repo=root_dir
             )
         )
@@ -73,7 +77,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
     if os.path.isdir(dvc_dir):
         if not force:
             raise InitError(
-                "'{repo}' exists. Use '-f' to force.".format(
+                "'{repo}' exists. Use `-f` to force.".format(
                     repo=relpath(dvc_dir)
                 )
             )

dvc/repo/metrics/modify.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from dvc.repo import locked
 from dvc.exceptions import DvcException
+from dvc.repo import locked
 
 
 @locked

dvc/repo/metrics/show.py

@@ -1,16 +1,20 @@
 from __future__ import unicode_literals
 
-import errno
-import os
 import csv
+import errno
 import json
 import logging
+import os
 
 from jsonpath_ng.ext import parse
 
+from dvc.exceptions import NoMetricsError
+from dvc.exceptions import OutputNotFoundError
 from dvc.repo import locked
-from dvc.exceptions import OutputNotFoundError, NoMetricsError
-from dvc.utils.compat import builtin_str, open, StringIO, csv_reader
+from dvc.utils.compat import builtin_str
+from dvc.utils.compat import csv_reader
+from dvc.utils.compat import open
+from dvc.utils.compat import StringIO
 
 NO_METRICS_FILE_AT_REFERENCE_WARNING = (
     "Metrics file '{}' does not exist at the reference '{}'."