dvc: bump to 0.63.4

This way we are able to properly utilize automatic connection pools and
not create new fds for each request, which overflows ulimit for max fds
very quickly on mac and windows. Kudos @pared for investigating 🎉

Fixes #2600

Signed-off-by: Ruslan Kuprieiev <[email protected]>

.pylintrc

@@ -139,7 +139,9 @@ disable=print-statement,
         deprecated-sys-function,
         exception-escape,
         comprehension-escape,
-        useless-object-inheritance
+        useless-object-inheritance,
+        missing-docstring,
+        bad-continuation
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -314,7 +316,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=100
+max-line-length=79
 
 # Maximum number of lines in a module.
 max-module-lines=1000

.travis.yml

@@ -19,7 +19,7 @@ stages:
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
-# Note that we specify language as ruby-2.4.0 to help us satisfy
+# Note that we specify language as ruby-2.4.3 to help us satisfy
 # fpm dependencies, as travis runs el capitan with an old ruby
 # version, that causes fpm to crash.
 jobs:
@@ -78,7 +78,7 @@ jobs:
     os: osx
     osx_image: xcode9.4
     language: ruby
-    rvm: 2.4.0
+    rvm: 2.4.3
     install:
     script: ./scripts/build_posix.sh
   - name: "Windows pkg"

README.rst

@@ -46,7 +46,7 @@ learning projects. Key features:
 
 #. it helps manage experiments with Git tags or branches and **metrics** tracking;
 
-It aims to replace tools like Excel and Docs that are being commonly used as a knowledge repo and
+**DVC** aims to replace tools like Excel and Google Docs that are being commonly used as a knowledge repo and
 a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
 data file suffixes and prefixes.
 

dvc/__init__.py

@@ -6,10 +6,12 @@ Make your data science projects reproducible and shareable.
 
 from __future__ import unicode_literals
 
-from dvc.version import __version__  # noqa: F401
 import warnings
+
 import dvc.logger
 
+from dvc.version import __version__  # noqa: F401
+
 
 dvc.logger.setup()
 

dvc/analytics.py

@@ -10,6 +10,7 @@ import errno
 import logging
 
 from dvc import __version__
+from dvc.utils import env2bool
 
 
 logger = logging.getLogger(__name__)
@@ -229,7 +230,7 @@ class Analytics(object):
     def is_enabled(cmd=None):
         from dvc.command.daemon import CmdDaemonBase
 
-        if os.getenv("DVC_TEST"):
+        if env2bool("DVC_TEST"):
             return False
 
         if isinstance(cmd, CmdDaemonBase):

dvc/cache.py

@@ -3,6 +3,7 @@
 from __future__ import unicode_literals
 
 import os
+from collections import defaultdict
 from funcy import cached_property
 
 from dvc.utils.compat import builtin_str
@@ -103,3 +104,29 @@ class Cache(object):
     ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
     hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
     azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
+
+
+class NamedCache(object):
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(set))
+        self.repo = []
+
+    @classmethod
+    def make(cls, scheme, checksum, name):
+        cache = cls()
+        cache.add(scheme, checksum, name)
+        return cache
+
+    def __getitem__(self, key):
+        return self._items[key]
+
+    def add(self, scheme, checksum, name):
+        self._items[scheme][checksum].add(name)
+
+    def update(self, cache, suffix=""):
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme]
+            for checksum, names in src.items():
+                dst[checksum].update(names)
+
+        self.repo.extend(cache.repo)

dvc/cli.py

@@ -144,6 +144,19 @@ def parse_args(argv=None):
         description=desc,
         parents=[parent_parser],
         formatter_class=argparse.RawTextHelpFormatter,
+        add_help=False,
+    )
+
+    # NOTE: We are doing this to capitalize help message.
+    # Unfortunately, there is no easier and clearer way to do it,
+    # as adding this argument in get_parent_parser() either in
+    # log_level_group or on parent_parser itself will cause unexpected error.
+    parser.add_argument(
+        "-h",
+        "--help",
+        action="help",
+        default=argparse.SUPPRESS,
+        help="Show this help message and exit.",
     )
 
     # NOTE: On some python versions action='version' prints to stderr

dvc/command/checkout.py

@@ -7,16 +7,12 @@ from dvc.command.base import CmdBase, append_doc_link
 
 class CmdCheckout(CmdBase):
     def run(self):
-        if not self.args.targets:
-            self.repo.checkout(force=self.args.force)
-        else:
-            for target in self.args.targets:
-                self.repo.checkout(
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    force=self.args.force,
-                    recursive=self.args.recursive,
-                )
+        self.repo.checkout(
+            targets=self.args.targets,
+            with_deps=self.args.with_deps,
+            force=self.args.force,
+            recursive=self.args.recursive,
+        )
         return 0
 
 

dvc/command/diff.py

@@ -9,7 +9,6 @@ from funcy import compact
 
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase, append_doc_link
-import dvc.repo.diff as diff
 
 
 logger = logging.getLogger(__name__)
@@ -37,6 +36,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_dir_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         engine = inflect.engine()
         changes_msg = (
             "{} {} untouched, {} {} modified, {} {} added, "
@@ -57,6 +58,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_file_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         if (
             dct.get(diff.DIFF_OLD_FILE)
             and dct.get(diff.DIFF_NEW_FILE)
@@ -79,6 +82,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_royal_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
             if dct.get("is_dir"):
                 return cls._get_dir_changes(dct)
@@ -88,6 +93,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _show(cls, diff_dct):
+        import dvc.repo.diff as diff
+
         msg = "dvc diff from {} to {}".format(
             diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
         )

dvc/command/get.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
 from dvc.exceptions import DvcException
 from .base import CmdBaseNoRepo, append_doc_link
 
@@ -13,6 +12,8 @@ logger = logging.getLogger(__name__)
 
 class CmdGet(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         try:
             Repo.get(
                 self.args.url,

dvc/command/get_url.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
 from dvc.exceptions import DvcException
 from .base import CmdBaseNoRepo, append_doc_link
 
@@ -13,6 +12,8 @@ logger = logging.getLogger(__name__)
 
 class CmdGetUrl(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         try:
             Repo.get_url(self.args.url, out=self.args.out)
             return 0

dvc/command/pipeline.py

@@ -9,8 +9,6 @@ import logging
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
 
-from dvc.repo.graph import get_pipeline
-
 
 logger = logging.getLogger(__name__)
 
@@ -43,6 +41,7 @@ class CmdPipelineShow(CmdBase):
     def __build_graph(self, target, commands, outs):
         import networkx
         from dvc.stage import Stage
+        from dvc.repo.graph import get_pipeline
 
         stage = Stage.load(self.repo, target)
         node = relpath(stage.path, self.repo.root_dir)

dvc/command/root.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.utils import relpath
 
@@ -12,6 +11,8 @@ logger = logging.getLogger(__name__)
 
 class CmdRoot(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         logger.info(relpath(Repo.find_root()))
         return 0
 

dvc/command/version.py

@@ -14,7 +14,6 @@ except ImportError:
 
 from dvc.utils import is_binary
 from dvc.utils.compat import pathlib
-from dvc.repo import Repo
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
 from dvc.exceptions import DvcException, NotDvcRepoError
@@ -26,6 +25,8 @@ logger = logging.getLogger(__name__)
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         dvc_version = __version__
         python_version = platform.python_version()
         platform_type = platform.platform()
@@ -47,9 +48,21 @@ class CmdVersion(CmdBaseNoRepo):
             repo = Repo()
             root_directory = repo.root_dir
 
-            info += "Cache: {cache}\n".format(
-                cache=self.get_linktype_support_info(repo)
-            )
+            # cache_dir might not exist yet (e.g. after `dvc init`), and we
+            # can't auto-create it, as it might cause issues if the user
+            # later decides to enable shared cache mode with
+            # `dvc config cache.shared group`.
+            if os.path.exists(repo.cache.local.cache_dir):
+                info += "Cache: {cache}\n".format(
+                    cache=self.get_linktype_support_info(repo)
+                )
+            else:
+                logger.warning(
+                    "Unable to detect supported link types, as cache "
+                    "directory '{}' doesn't exist. It is usually auto-created "
+                    "by commands such as `dvc add/fetch/pull/run/import`, but "
+                    "you could create it manually to enable this check."
+                )
 
             if psutil:
                 info += (

dvc/data_cloud.py

@@ -45,11 +45,11 @@ class DataCloud(object):
     def _init_remote(self, remote):
         return Remote(self.repo, name=remote)
 
-    def push(self, targets, jobs=None, remote=None, show_checksums=False):
+    def push(self, cache, jobs=None, remote=None, show_checksums=False):
         """Push data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to push to the cloud.
+            cache (NamedCache): named checksums to push to the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to push to.
                 By default remote from core.remote config option is used.
@@ -57,17 +57,17 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.push(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "push"),
             show_checksums=show_checksums,
         )
 
-    def pull(self, targets, jobs=None, remote=None, show_checksums=False):
+    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
         """Pull data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to pull from the cloud.
+            cache (NamedCache): named checksums to pull from the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
                 By default remote from core.remote config option is used.
@@ -75,25 +75,25 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.pull(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "pull"),
             show_checksums=show_checksums,
         )
 
-    def status(self, targets, jobs=None, remote=None, show_checksums=False):
+    def status(self, cache, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to check status for.
+            cache (NamedCache): named checksums to check status for.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to compare
-                targets to. By default remote from core.remote config option
+                cache to. By default remote from core.remote config option
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
         remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            targets, jobs=jobs, remote=remote, show_checksums=show_checksums
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )

dvc/exceptions.py

@@ -47,7 +47,7 @@ class OutputDuplicationError(DvcException):
             for stage in stages
         )
         msg = (
-            "file/directory '{}' is specified as an output in more than one"
+            "file/directory '{}' is specified as an output in more than one "
             "stage: {}"
         ).format(output, "\n    ".join(stages))
         super(OutputDuplicationError, self).__init__(msg)
@@ -82,8 +82,8 @@ class StagePathAsOutputError(DvcException):
         assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
         assert isinstance(fname, str) or isinstance(fname, builtin_str)
         msg = (
-            "current working directory '{cwd}' is specified as an output in"
-            " '{fname}'. Use another CWD to prevent any data removal.".format(
+            "current working directory '{cwd}' is specified as an output in "
+            "'{fname}'. Use another CWD to prevent any data removal.".format(
                 cwd=wdir, fname=fname
             )
         )
@@ -167,8 +167,8 @@ class CyclicGraphError(DvcException):
         assert isinstance(stages, list)
         stages = "\n".join("\t- {}".format(stage) for stage in stages)
         msg = (
-            "you've introduced a cycle in your pipeline that involves"
-            " the following stages:"
+            "you've introduced a cycle in your pipeline that involves "
+            "the following stages:"
             "\n"
             "{stages}".format(stages=stages)
         )
@@ -259,10 +259,11 @@ class ETagMismatchError(DvcException):
         )
 
 
-class OutputFileMissingError(DvcException):
-    def __init__(self, path):
-        super(OutputFileMissingError, self).__init__(
-            "Can't find {} neither locally nor on remote".format(path)
+class FileMissingError(DvcException):
+    def __init__(self, path, cause=None):
+        super(FileMissingError, self).__init__(
+            "Can't find '{}' neither locally nor on remote".format(path),
+            cause=cause,
         )
 
 

dvc/external_repo.py

@@ -1,50 +1,91 @@
 from __future__ import unicode_literals
 
 import os
-import logging
 import tempfile
+from distutils.dir_util import copy_tree
 
-from funcy import retry
 from contextlib import contextmanager
+from funcy import retry
 
-from dvc.config import Config
-from dvc.cache import CacheConfig
 from dvc.utils import remove
 
 
-logger = logging.getLogger(__name__)
-
-
-def _clone(cache_dir=None, url=None, rev=None, rev_lock=None):
-    from dvc.repo import Repo
-
-    _path = tempfile.mkdtemp("dvc-repo")
-
-    repo = Repo.clone(url, _path, rev=(rev_lock or rev))
-
-    try:
-        if cache_dir:
-            cache_config = CacheConfig(repo.config)
-            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
-    finally:
-        repo.close()
-
-    return Repo(_path)
-
-
-def _remove(repo):
-    repo.close()
-
-    if os.name == "nt":
-        # git.exe may hang for a while not permitting to remove temp dir
-        os_retry = retry(5, errors=OSError, timeout=0.1)
-        os_retry(remove)(repo.root_dir)
-    else:
-        remove(repo.root_dir)
+REPO_CACHE = {}
 
 
 @contextmanager
-def external_repo(**kwargs):
-    repo = _clone(**kwargs)
+def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
+    from dvc.repo import Repo
+
+    path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
+    repo = Repo(path)
     yield repo
-    _remove(repo)
+    repo.close()
+
+
+def _external_repo(url=None, rev=None, cache_dir=None):
+    from dvc.config import Config
+    from dvc.cache import CacheConfig
+    from dvc.repo import Repo
+
+    key = (url, rev, cache_dir)
+    if key in REPO_CACHE:
+        return REPO_CACHE[key]
+
+    new_path = tempfile.mkdtemp("dvc-erepo")
+
+    # Copy and adjust existing clone
+    if (url, None, None) in REPO_CACHE:
+        old_path = REPO_CACHE[url, None, None]
+
+        # This one unlike shutil.copytree() works with an existing dir
+        copy_tree(old_path, new_path)
+    else:
+        # Create a new clone
+        _clone_repo(url, new_path)
+
+        # Save clean clone dir so that we will have access to a default branch
+        clean_clone_path = tempfile.mkdtemp("dvc-erepo")
+        copy_tree(new_path, clean_clone_path)
+        REPO_CACHE[url, None, None] = clean_clone_path
+
+    # Adjust new clone/copy to fit rev and cache_dir
+    repo = Repo(new_path)
+    try:
+        if rev is not None:
+            repo.scm.checkout(rev)
+
+        if cache_dir is not None:
+            cache_config = CacheConfig(repo.config)
+            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
+    finally:
+        # Need to close/reopen repo to force config reread
+        repo.close()
+
+    REPO_CACHE[key] = new_path
+    return new_path
+
+
+def clean_repos():
+    # Outside code should not see cache while we are removing
+    repo_paths = list(REPO_CACHE.values())
+    REPO_CACHE.clear()
+
+    for path in repo_paths:
+        _remove(path)
+
+
+def _remove(path):
+    if os.name == "nt":
+        # git.exe may hang for a while not permitting to remove temp dir
+        os_retry = retry(5, errors=OSError, timeout=0.1)
+        os_retry(remove)(path)
+    else:
+        remove(path)
+
+
+def _clone_repo(url, path):
+    from dvc.scm.git import Git
+
+    git = Git.clone(url, path)
+    git.close()

dvc/lock.py

@@ -34,12 +34,31 @@ if is_py3:
         """
 
         def __init__(self, lockfile, tmp_dir=None):
-            lifetime = timedelta(days=365)  # Lock for good by default
+            import socket
+
             self._tmp_dir = tmp_dir
             if self._tmp_dir is not None:
                 makedirs(self._tmp_dir, exist_ok=True)
 
-            super(Lock, self).__init__(lockfile, lifetime=lifetime)
+            # NOTE: this is basically Lock.__init__ copy-paste, except that
+            # instead of using `socket.getfqdn()` we use `socket.gethostname()`
+            # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
+            # anything, which is way too slow. `gethostname()` is actually a
+            # fallback for `getfqdn()` when it is not able to resolve a
+            # canonical hostname through network. The claimfile that uses
+            # `self._hostname` is still usable, as it uses `pid` and random
+            # number to generate the resulting lock file name, which is unique
+            # enough for our application.
+            #
+            # [1] https://github.com/iterative/dvc/issues/2582
+            self._hostname = socket.gethostname()
+
+            self._lockfile = lockfile
+            self._lifetime = timedelta(days=365)  # Lock for good by default
+            self._separator = flufl.lock.SEP
+            self._set_claimfile()
+            self._owned = True
+            self._retry_errnos = []
 
         @property
         def lockfile(self):

dvc/logger.py

@@ -2,7 +2,7 @@
 
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, StringIO
+from dvc.utils.compat import str, StringIO, RecursionError
 from dvc.progress import Tqdm
 
 import logging
@@ -19,7 +19,7 @@ class LoggingException(Exception):
 
 class ExcludeErrorsFilter(logging.Filter):
     def filter(self, record):
-        return record.levelno < logging.ERROR
+        return record.levelno < logging.WARNING
 
 
 class ColorFormatter(logging.Formatter):
@@ -182,7 +182,7 @@ def setup(level=logging.INFO):
                 },
                 "console_errors": {
                     "class": "dvc.logger.LoggerHandler",
-                    "level": "ERROR",
+                    "level": "WARNING",
                     "formatter": "color",
                     "stream": "ext://sys.stderr",
                 },

dvc/main.py

@@ -10,6 +10,8 @@ from dvc.config import ConfigError
 from dvc.analytics import Analytics
 from dvc.exceptions import NotDvcRepoError, DvcParserError
 from dvc.remote.pool import close_pools
+from dvc.external_repo import clean_repos
+from dvc.utils.compat import is_py2
 
 
 logger = logging.getLogger("dvc")
@@ -53,15 +55,26 @@ def main(argv=None):
         ret = 253
     except DvcParserError:
         ret = 254
-    except Exception:  # pylint: disable=broad-except
-        logger.exception("unexpected error")
+    except Exception as exc:  # pylint: disable=broad-except
+        if isinstance(exc, UnicodeError) and is_py2:
+            logger.exception(
+                "unicode is not supported in DVC for Python 2 "
+                "(end-of-life January 1, 2020), please upgrade to Python 3"
+            )
+        else:
+            logger.exception("unexpected error")
         ret = 255
     finally:
         logger.setLevel(outerLogLevel)
+
         # Python 2 fails to close these clean occasionally and users see
         # weird error messages, so we do it manually
         close_pools()
 
+        # Remove cached repos in the end of the call, these are anonymous
+        # so won't be reused by any other subsequent run anyway.
+        clean_repos()
+
     Analytics().send_cmd(cmd, args, ret)
 
     return ret

dvc/output/base.py

@@ -5,6 +5,7 @@ from copy import copy
 
 from schema import Or, Optional
 
+from dvc.cache import NamedCache
 import dvc.prompt as prompt
 from dvc.exceptions import DvcException
 from dvc.utils.compat import str, urlparse
@@ -345,12 +346,11 @@ class OutputBase(object):
             $ echo "bar" > directory/bar
             $ dvc add directory
 
-            It will return something similar to the following list:
+            It will return a NamedCache like:
 
-            [
-                { 'path': 'directory/foo', 'md5': 'c157a79031e1', ... },
-                { 'path': 'directory/bar', 'md5': 'd3b07384d113', ... },
-            ]
+            nc = NamedCache()
+            nc.add(self.scheme, 'c157a79031e1', 'directory/foo')
+            nc.add(self.scheme, 'd3b07384d113', 'directory/bar')
         """
 
         ret = []
@@ -358,12 +358,7 @@ class OutputBase(object):
         if self.cache.changed_cache_file(self.checksum):
             try:
                 self.repo.cloud.pull(
-                    [
-                        {
-                            self.remote.PARAM_CHECKSUM: self.checksum,
-                            "name": str(self),
-                        }
-                    ],
+                    NamedCache.make("local", self.checksum, str(self)),
                     jobs=jobs,
                     remote=remote,
                     show_checksums=False,
@@ -385,13 +380,13 @@ class OutputBase(object):
             else:
                 return ret
 
+        cache = NamedCache()
         for entry in self.dir_cache:
-            info = copy(entry)
+            checksum = entry[self.remote.PARAM_CHECKSUM]
             path_info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            info["name"] = str(path_info)
-            ret.append(info)
+            cache.add(self.scheme, checksum, str(path_info))
 
-        return ret
+        return cache
 
     def get_used_cache(self, **kwargs):
         """Get a dumpd of the given `out`, with an entry including the branch.
@@ -403,7 +398,7 @@ class OutputBase(object):
         """
 
         if not self.use_cache:
-            return []
+            return NamedCache()
 
         if not self.info:
             logger.warning(
@@ -411,14 +406,14 @@ class OutputBase(object):
                 "not be collected. Use dvc repro to get your pipeline up to "
                 "date.".format(self, self.stage)
             )
-            return []
+            return NamedCache()
 
-        ret = [{self.remote.PARAM_CHECKSUM: self.checksum, "name": str(self)}]
+        ret = NamedCache.make(self.scheme, self.checksum, str(self))
 
         if not self.is_dir_checksum:
             return ret
 
-        ret.extend(self._collect_used_dir_cache(**kwargs))
+        ret.update(self._collect_used_dir_cache(**kwargs))
 
         return ret
 

dvc/progress.py

@@ -1,9 +1,11 @@
 """Manages progress bars for dvc repo."""
 from __future__ import print_function
 import logging
+import sys
 from tqdm import tqdm
 from concurrent.futures import ThreadPoolExecutor
 from funcy import merge
+from dvc.utils import env2bool
 
 logger = logging.getLogger(__name__)
 
@@ -54,6 +56,7 @@ class Tqdm(tqdm):
         leave=False,
         bar_format=None,
         bytes=False,  # pylint: disable=W0622
+        file=None,
         **kwargs
     ):
         """
@@ -62,6 +65,9 @@ class Tqdm(tqdm):
         desc  : persists after `close()`
         level  : effective logging level for determining `disable`;
             used only if `disable` is unspecified
+        disable  : If (default: None), will be determined by logging level.
+            May be overridden to `True` due to non-TTY status.
+            Skip override by specifying env var `DVC_IGNORE_ISATTY`.
         kwargs  : anything accepted by `tqdm.tqdm()`
         """
         kwargs = kwargs.copy()
@@ -71,9 +77,19 @@ class Tqdm(tqdm):
                 unit="B", unit_scale=True, unit_divisor=1024, miniters=1
             )
             kwargs = merge(bytes_defaults, kwargs)
+        if file is None:
+            file = sys.stderr
         self.desc_persist = desc
+        # auto-disable based on `logger.level`
         if disable is None:
             disable = logger.getEffectiveLevel() > level
+        # auto-disable based on TTY
+        if (
+            not disable
+            and not env2bool("DVC_IGNORE_ISATTY")
+            and hasattr(file, "isatty")
+        ):
+            disable = not file.isatty()
         super(Tqdm, self).__init__(
             iterable=iterable,
             disable=disable,
@@ -114,5 +130,11 @@ class Tqdm(tqdm):
         d = super(Tqdm, self).format_dict
         ncols = d["ncols"] or 80
         ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1
-        d["ncols_desc"] = max(ncols_desc, 0)
+        ncols_desc = max(ncols_desc, 0)
+        if ncols_desc:
+            d["ncols_desc"] = ncols_desc
+        else:
+            # work-around for zero-width desc
+            d["ncols_desc"] = 1
+            d["desc"] = 0
         return d

dvc/remote/base.py

@@ -1,17 +1,17 @@
 from __future__ import unicode_literals
 
-from dvc.ignore import DvcIgnore
-from dvc.utils.compat import str, basestring, urlparse, FileNotFoundError
-
+from operator import itemgetter
+from multiprocessing import cpu_count
 import json
 import logging
 import tempfile
 import itertools
-from operator import itemgetter
-from multiprocessing import cpu_count
 from functools import partial
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
+
+from dvc.ignore import DvcIgnore
+from dvc.utils.compat import str, basestring, urlparse, FileNotFoundError
 from dvc.remote.slow_link_detection import slow_link_guard
 
 import dvc.prompt as prompt
@@ -21,22 +21,19 @@ from dvc.exceptions import (
     ConfirmRemoveError,
     DvcIgnoreInCollectedDirError,
 )
-from dvc.progress import Tqdm, TqdmThreadPoolExecutor
+from dvc.progress import Tqdm
 from dvc.utils import LARGE_DIR_SIZE, tmp_fname, move, relpath, makedirs
 from dvc.state import StateNoop
 from dvc.path_info import PathInfo, URLInfo
 from dvc.utils.http import open_url
 
-
 logger = logging.getLogger(__name__)
 
-
 STATUS_OK = 1
 STATUS_MISSING = 2
 STATUS_NEW = 3
 STATUS_DELETED = 4
 
-
 STATUS_MAP = {
     # (local_exists, remote_exists)
     (True, True): STATUS_OK,
@@ -178,9 +175,7 @@ class RemoteBASE(object):
 
     def _calculate_checksums(self, file_infos):
         file_infos = list(file_infos)
-        with TqdmThreadPoolExecutor(
-            max_workers=self.checksum_jobs
-        ) as executor:
+        with ThreadPoolExecutor(max_workers=self.checksum_jobs) as executor:
             tasks = executor.map(self.get_file_checksum, file_infos)
 
             if len(file_infos) > LARGE_DIR_SIZE:
@@ -494,7 +489,7 @@ class RemoteBASE(object):
 
     def _save(self, path_info, checksum):
         to_info = self.checksum_to_path_info(checksum)
-        logger.info("Saving '{}' to '{}'.".format(path_info, to_info))
+        logger.debug("Saving '{}' to '{}'.".format(path_info, to_info))
         if self.isdir(path_info):
             self._save_dir(path_info, checksum)
             return
@@ -625,14 +620,11 @@ class RemoteBASE(object):
                 # We ignore all the non-cache looking files
                 pass
 
-    def gc(self, cinfos):
-        used = self.extract_used_local_checksums(cinfos)
+    def gc(self, named_cache):
+        used = self.extract_used_local_checksums(named_cache)
 
         if self.scheme != "":
-            used |= {
-                info[self.PARAM_CHECKSUM]
-                for info in cinfos.get(self.scheme, [])
-            }
+            used.update(named_cache[self.scheme])
 
         removed = False
         for checksum in self.all():
@@ -914,10 +906,8 @@ class RemoteBASE(object):
     def _get_unpacked_dir_names(self, checksums):
         return set()
 
-    def extract_used_local_checksums(self, cinfos):
-        from dvc.remote import RemoteLOCAL
-
-        used = {info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]}
+    def extract_used_local_checksums(self, named_cache):
+        used = set(named_cache["local"])
         unpacked = self._get_unpacked_dir_names(used)
         return used | unpacked
 

dvc/remote/hdfs.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import os
 import re
 import io
-import getpass
 import posixpath
 import logging
 from subprocess import Popen, PIPE
@@ -36,11 +35,7 @@ class RemoteHDFS(RemoteBASE):
 
         parsed = urlparse(url)
 
-        user = (
-            parsed.username
-            or config.get(Config.SECTION_REMOTE_USER)
-            or getpass.getuser()
-        )
+        user = parsed.username or config.get(Config.SECTION_REMOTE_USER)
 
         self.path_info = self.path_cls.from_parts(
             scheme=self.scheme,

dvc/remote/http.py

@@ -1,10 +1,11 @@
 from __future__ import unicode_literals
 
-from dvc.scheme import Schemes
-from dvc.utils.compat import open
-
 import logging
 
+from funcy import cached_property
+
+from dvc.scheme import Schemes
+from dvc.utils.compat import open
 from dvc.progress import Tqdm
 from dvc.exceptions import DvcException
 from dvc.config import Config, ConfigError
@@ -15,6 +16,8 @@ logger = logging.getLogger(__name__)
 
 class RemoteHTTP(RemoteBASE):
     scheme = Schemes.HTTP
+    SESSION_RETRIES = 5
+    SESSION_BACKOFF_FACTOR = 0.1
     REQUEST_TIMEOUT = 10
     CHUNK_SIZE = 2 ** 16
     PARAM_CHECKSUM = "etag"
@@ -77,6 +80,24 @@ class RemoteHTTP(RemoteBASE):
 
         return etag
 
+    @cached_property
+    def _session(self):
+        import requests
+        from requests.adapters import HTTPAdapter
+        from urllib3.util.retry import Retry
+
+        session = requests.Session()
+
+        retries = Retry(
+            total=self.SESSION_RETRIES,
+            backoff_factor=self.SESSION_BACKOFF_FACTOR,
+        )
+
+        session.mount("http://", HTTPAdapter(max_retries=retries))
+        session.mount("https://", HTTPAdapter(max_retries=retries))
+
+        return session
+
     def _request(self, method, url, **kwargs):
         import requests
 
@@ -84,7 +105,7 @@ class RemoteHTTP(RemoteBASE):
         kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
 
         try:
-            return requests.request(method, url, **kwargs)
+            return self._session.request(method, url, **kwargs)
         except requests.exceptions.RequestException:
             raise DvcException("could not perform a {} request".format(method))
 

dvc/remote/local.py

@@ -1,15 +1,14 @@
 from __future__ import unicode_literals
-
-from dvc.scheme import Schemes
-from dvc.utils.compat import str, fspath_py35, open
-
 import os
 import stat
 import errno
-from shortuuid import uuid
 import logging
 from functools import partial
 
+from shortuuid import uuid
+
+from dvc.scheme import Schemes
+from dvc.utils.compat import str, fspath_py35, open
 from dvc.system import System
 from dvc.remote.base import (
     RemoteBASE,
@@ -69,9 +68,6 @@ class RemoteLOCAL(RemoteBASE):
             cwd = config[Config.PRIVATE_CWD]
             cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
 
-        if cache_dir is not None and not os.path.exists(cache_dir):
-            os.mkdir(cache_dir)
-
         self.path_info = PathInfo(cache_dir) if cache_dir else None
         self._dir_info = {}
 
@@ -249,31 +245,9 @@ class RemoteLOCAL(RemoteBASE):
     def open(path_info, mode="r", encoding=None):
         return open(fspath_py35(path_info), mode=mode, encoding=encoding)
 
-    def _group(self, checksum_infos, show_checksums=False):
-        by_md5 = {}
-
-        for info in checksum_infos:
-            md5 = info[self.PARAM_CHECKSUM]
-
-            if show_checksums:
-                by_md5[md5] = {"name": md5}
-                continue
-
-            name = info["name"]
-            branch = info.get("branch")
-            if branch:
-                name += "({})".format(branch)
-
-            if md5 not in by_md5.keys():
-                by_md5[md5] = {"name": name}
-            else:
-                by_md5[md5]["name"] += " " + name
-
-        return by_md5
-
     def status(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
@@ -282,8 +256,7 @@ class RemoteLOCAL(RemoteBASE):
         logger.debug(
             "Preparing to collect status from {}".format(remote.path_info)
         )
-        ret = self._group(checksum_infos, show_checksums=show_checksums) or {}
-        md5s = list(ret)
+        md5s = list(named_cache[self.scheme])
 
         logger.debug("Collecting information from local cache...")
         local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
@@ -302,6 +275,10 @@ class RemoteLOCAL(RemoteBASE):
                 )
             )
 
+        ret = {
+            checksum: {"name": checksum if show_checksums else " ".join(names)}
+            for checksum, names in named_cache[self.scheme].items()
+        }
         self._fill_statuses(ret, local_exists, remote_exists)
 
         self._log_missing_caches(ret)
@@ -341,7 +318,7 @@ class RemoteLOCAL(RemoteBASE):
 
     def _process(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
@@ -369,7 +346,7 @@ class RemoteLOCAL(RemoteBASE):
             jobs = remote.JOBS
 
         status_info = self.status(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
@@ -394,18 +371,18 @@ class RemoteLOCAL(RemoteBASE):
 
         return len(plans[0])
 
-    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def push(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
             download=False,
         )
 
-    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,

dvc/remote/ssh/connection.py

@@ -106,7 +106,10 @@ class SSHConnection:
                 # Since paramiko errors are very vague we need to recheck
                 # whether it's because path already exists or something else
                 if e.errno == errno.EACCES or not self.exists(path):
-                    raise
+                    raise DvcException(
+                        "unable to create remote directory '{}'".format(path),
+                        cause=e,
+                    )
 
     def walk(self, directory, topdown=True):
         # NOTE: original os.walk() implementation [1] with default options was

dvc/repo/__init__.py

@@ -12,7 +12,7 @@ from dvc.config import Config
 from dvc.exceptions import (
     NotDvcRepoError,
     OutputNotFoundError,
-    OutputFileMissingError,
+    FileMissingError,
 )
 from dvc.ignore import DvcIgnoreFilter
 from dvc.path_info import PathInfo
@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
 def locked(f):
     @wraps(f)
     def wrapper(repo, *args, **kwargs):
-        with repo.lock:
+        with repo.lock, repo.state:
             ret = f(repo, *args, **kwargs)
             # Our graph cache is no longer valid after we release the repo.lock
             repo._reset()
@@ -164,7 +164,7 @@ class Repo(object):
             + updater.lock.files
         )
 
-        if self.cache.local.cache_dir.startswith(self.root_dir):
+        if self.cache.local.cache_dir.startswith(self.root_dir + os.sep):
             flist += [self.cache.local.cache_dir]
 
         self.scm.ignore_list(flist)
@@ -231,15 +231,9 @@ class Repo(object):
             A dictionary with Schemes (representing output's location) as keys,
             and a list with the outputs' `dumpd` as values.
         """
+        from dvc.cache import NamedCache
 
-        cache = {}
-        cache["local"] = []
-        cache["s3"] = []
-        cache["gs"] = []
-        cache["hdfs"] = []
-        cache["ssh"] = []
-        cache["azure"] = []
-        cache["repo"] = []
+        cache = NamedCache()
 
         for branch in self.brancher(
             all_branches=all_branches, all_tags=all_tags
@@ -256,18 +250,15 @@ class Repo(object):
 
             for stage in stages:
                 if stage.is_repo_import:
-                    cache["repo"] += stage.deps
+                    cache.repo += stage.deps
                     continue
 
                 for out in stage.outs:
-                    scheme = out.path_info.scheme
                     used_cache = out.get_used_cache(
                         remote=remote, force=force, jobs=jobs
                     )
-
-                    cache[scheme].extend(
-                        dict(entry, branch=branch) for entry in used_cache
-                    )
+                    suffix = "({})".format(branch) if branch else ""
+                    cache.update(used_cache, suffix=suffix)
 
         return cache
 
@@ -454,14 +445,29 @@ class Repo(object):
     @contextmanager
     def open(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
+        cause = None
         try:
-            with self._open(path, remote, mode, encoding) as fd:
-                yield fd
-        except FileNotFoundError:
-            raise OutputFileMissingError(relpath(path, self.root_dir))
+            out, = self.find_outs_by_path(path)
+        except OutputNotFoundError as e:
+            out = None
+            cause = e
 
-    def _open(self, path, remote=None, mode="r", encoding=None):
-        out, = self.find_outs_by_path(path)
+        if out and out.use_cache:
+            try:
+                with self._open_cached(out, remote, mode, encoding) as fd:
+                    yield fd
+                return
+            except FileNotFoundError as e:
+                raise FileMissingError(relpath(path, self.root_dir), cause=e)
+
+        if self.tree.exists(path):
+            with self.tree.open(path, mode, encoding) as fd:
+                yield fd
+            return
+
+        raise FileMissingError(relpath(path, self.root_dir), cause=cause)
+
+    def _open_cached(self, out, remote=None, mode="r", encoding=None):
         if out.isdir():
             raise ValueError("Can't open a dir")
 
@@ -489,15 +495,6 @@ class Repo(object):
     def close(self):
         self.scm.close()
 
-    @staticmethod
-    def clone(url, to_path, rev=None):
-        from dvc.scm.git import Git
-
-        git = Git.clone(url, to_path, rev=rev)
-        git.close()
-
-        return Repo(to_path)
-
     @locked
     def checkout(self, *args, **kwargs):
         return self._checkout(*args, **kwargs)
@@ -510,3 +507,4 @@ class Repo(object):
         self.__dict__.pop("graph", None)
         self.__dict__.pop("stages", None)
         self.__dict__.pop("pipelines", None)
+        self.__dict__.pop("dvcignore", None)

dvc/repo/add.py

@@ -36,18 +36,17 @@ def add(repo, target, recursive=False, no_commit=False, fname=None):
             )
         )
 
-    with repo.state:
-        stages = _create_stages(repo, targets, fname)
+    stages = _create_stages(repo, targets, fname)
 
-        repo.check_modified_graph(stages)
+    repo.check_modified_graph(stages)
 
-        for stage in stages:
-            stage.save()
+    for stage in stages:
+        stage.save()
 
-            if not no_commit:
-                stage.commit()
+        if not no_commit:
+            stage.commit()
 
-            stage.dump()
+        stage.dump()
 
     return stages
 

dvc/repo/brancher.py

@@ -1,3 +1,4 @@
+from funcy import group_by
 from dvc.scm.tree import WorkingTree
 
 
@@ -47,8 +48,8 @@ def brancher(  # noqa: E302
     # `brancher()`, but this could cause problems on exception handling
     # code which might expect the tree on which exception was raised to
     # stay in place. This behavior is a subject to change.
-    for rev in revs:
-        self.tree = scm.get_tree(rev)
-        yield rev
+    for sha, names in group_by(scm.resolve_rev, revs).items():
+        self.tree = scm.get_tree(sha)
+        yield ", ".join(names)
 
     self.tree = saved_tree

dvc/repo/checkout.py

@@ -24,39 +24,40 @@ def get_all_files_numbers(stages):
 
 
 def _checkout(
-    self, target=None, with_deps=False, force=False, recursive=False
+    self, targets=None, with_deps=False, force=False, recursive=False
 ):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    try:
-        stages = self.collect(target, with_deps=with_deps, recursive=recursive)
-    except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
-        if not target:
-            raise
-        raise CheckoutErrorSuggestGit(target, exc)
+    stages = set()
+    targets = targets or [None]
+    for target in targets:
+        try:
+            new = self.collect(
+                target, with_deps=with_deps, recursive=recursive
+            )
+            stages.update(new)
+        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+            if not target:
+                raise
+            raise CheckoutErrorSuggestGit(target, exc)
 
-    with self.state:
-        _cleanup_unused_links(self, self.stages)
-        total = get_all_files_numbers(stages)
-        if total == 0:
-            logger.info("Nothing to do")
-        failed = []
-        with Tqdm(
-            total=total, unit="file", desc="Checkout", disable=total == 0
-        ) as pbar:
-            for stage in stages:
-                if stage.locked:
-                    logger.warning(
-                        "DVC-file '{path}' is locked. Its dependencies are"
-                        " not going to be checked out.".format(
-                            path=stage.relpath
-                        )
-                    )
-
-                failed.extend(
-                    stage.checkout(
-                        force=force, progress_callback=pbar.update_desc
-                    )
+    _cleanup_unused_links(self, self.stages)
+    total = get_all_files_numbers(stages)
+    if total == 0:
+        logger.info("Nothing to do")
+    failed = []
+    with Tqdm(
+        total=total, unit="file", desc="Checkout", disable=total == 0
+    ) as pbar:
+        for stage in stages:
+            if stage.locked:
+                logger.warning(
+                    "DVC-file '{path}' is locked. Its dependencies are"
+                    " not going to be checked out.".format(path=stage.relpath)
                 )
-        if failed:
-            raise CheckoutError(failed)
+
+            failed.extend(
+                stage.checkout(force=force, progress_callback=pbar.update_desc)
+            )
+    if failed:
+        raise CheckoutError(failed)

dvc/repo/commit.py

@@ -4,8 +4,7 @@ from . import locked
 @locked
 def commit(self, target, with_deps=False, recursive=False, force=False):
     stages = self.collect(target, with_deps=with_deps, recursive=recursive)
-    with self.state:
-        for stage in stages:
-            stage.check_can_commit(force=force)
-            stage.commit()
-            stage.dump()
+    for stage in stages:
+        stage.check_can_commit(force=force)
+        stage.commit()
+        stage.dump()

dvc/repo/fetch.py

@@ -33,48 +33,44 @@ def _fetch(
         config.NoRemoteError: thrown when downloading only local files and no
             remote is configured
     """
-    with self.state:
-        used = self.used_cache(
-            targets,
-            all_branches=all_branches,
-            all_tags=all_tags,
-            with_deps=with_deps,
-            force=True,
-            remote=remote,
-            jobs=jobs,
-            recursive=recursive,
+    used = self.used_cache(
+        targets,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        with_deps=with_deps,
+        force=True,
+        remote=remote,
+        jobs=jobs,
+        recursive=recursive,
+    )
+
+    downloaded = 0
+    failed = 0
+
+    try:
+        downloaded += self.cloud.pull(
+            used, jobs, remote=remote, show_checksums=show_checksums
         )
+    except NoRemoteError:
+        if not used.repo and used["local"]:
+            raise
 
-        downloaded = 0
-        failed = 0
+    except DownloadError as exc:
+        failed += exc.amount
 
+    for dep in used.repo:
         try:
-            downloaded += self.cloud.pull(
-                used["local"],
-                jobs,
-                remote=remote,
-                show_checksums=show_checksums,
-            )
-        except NoRemoteError:
-            if not used["repo"] and used["local"]:
-                raise
-
+            out = dep.fetch()
+            downloaded += out.get_files_number()
         except DownloadError as exc:
             failed += exc.amount
+        except (CloneError, OutputNotFoundError):
+            failed += 1
+            logger.exception(
+                "failed to fetch data for '{}'".format(dep.stage.outs[0])
+            )
 
-        for dep in used["repo"]:
-            try:
-                out = dep.fetch()
-                downloaded += out.get_files_number()
-            except DownloadError as exc:
-                failed += exc.amount
-            except (CloneError, OutputNotFoundError):
-                failed += 1
-                logger.exception(
-                    "failed to fetch data for '{}'".format(dep.stage.outs[0])
-                )
+    if failed:
+        raise DownloadError(failed)
 
-        if failed:
-            raise DownloadError(failed)
-
-        return downloaded
+    return downloaded

dvc/repo/gc.py

@@ -1,54 +1,14 @@
 from __future__ import unicode_literals
 
-import collections
 import logging
 
+from dvc.cache import NamedCache
 from . import locked
 
 
 logger = logging.getLogger(__name__)
 
 
-def _merge_cache_lists(clists):
-    merged_cache = collections.defaultdict(list)
-
-    for cache_list in clists:
-        for scheme, cache in cache_list.items():
-            for item in cache:
-                if item not in merged_cache[scheme]:
-                    merged_cache[scheme].append(item)
-
-    return merged_cache
-
-
-def _load_all_used_cache(
-    repos,
-    all_branches=False,
-    with_deps=False,
-    all_tags=False,
-    remote=None,
-    force=False,
-    jobs=None,
-):
-    clists = []
-
-    for repo in repos:
-        with repo.state:
-            repo_clist = repo.used_cache(
-                targets=None,
-                all_branches=all_branches,
-                with_deps=with_deps,
-                all_tags=all_tags,
-                remote=remote,
-                force=force,
-                jobs=jobs,
-            )
-
-            clists.append(repo_clist)
-
-    return clists
-
-
 def _do_gc(typ, func, clist):
     removed = func(clist)
     if not removed:
@@ -67,45 +27,48 @@ def gc(
     jobs=None,
     repos=None,
 ):
+    from dvc.utils.compat import ExitStack
     from dvc.repo import Repo
 
-    all_repos = [self]
+    all_repos = []
 
     if repos:
-        all_repos.extend(Repo(path) for path in repos)
+        all_repos = [Repo(path) for path in repos]
 
-    all_clists = _load_all_used_cache(
-        all_repos,
-        all_branches=all_branches,
-        with_deps=with_deps,
-        all_tags=all_tags,
-        remote=remote,
-        force=force,
-        jobs=jobs,
-    )
+    with ExitStack() as stack:
+        for repo in all_repos:
+            stack.enter_context(repo.lock)
+            stack.enter_context(repo.state)
 
-    if len(all_clists) > 1:
-        clist = _merge_cache_lists(all_clists)
-    else:
-        clist = all_clists[0]
+        used = NamedCache()
+        for repo in all_repos + [self]:
+            used.update(
+                repo.used_cache(
+                    all_branches=all_branches,
+                    with_deps=with_deps,
+                    all_tags=all_tags,
+                    remote=remote,
+                    force=force,
+                    jobs=jobs,
+                )
+            )
 
-    with self.state:
-        _do_gc("local", self.cache.local.gc, clist)
+    _do_gc("local", self.cache.local.gc, used)
 
-        if self.cache.s3:
-            _do_gc("s3", self.cache.s3.gc, clist)
+    if self.cache.s3:
+        _do_gc("s3", self.cache.s3.gc, used)
 
-        if self.cache.gs:
-            _do_gc("gs", self.cache.gs.gc, clist)
+    if self.cache.gs:
+        _do_gc("gs", self.cache.gs.gc, used)
 
-        if self.cache.ssh:
-            _do_gc("ssh", self.cache.ssh.gc, clist)
+    if self.cache.ssh:
+        _do_gc("ssh", self.cache.ssh.gc, used)
 
-        if self.cache.hdfs:
-            _do_gc("hdfs", self.cache.hdfs.gc, clist)
+    if self.cache.hdfs:
+        _do_gc("hdfs", self.cache.hdfs.gc, used)
 
-        if self.cache.azure:
-            _do_gc("azure", self.cache.azure.gc, clist)
+    if self.cache.azure:
+        _do_gc("azure", self.cache.azure.gc, used)
 
-        if cloud:
-            _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, clist)
+    if cloud:
+        _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, used)

dvc/repo/get.py

@@ -13,15 +13,14 @@ from dvc.exceptions import (
 from dvc.path_info import PathInfo
 from dvc.external_repo import external_repo
 from dvc.state import StateNoop
-from dvc.utils import remove
-from dvc.utils.compat import urlparse
+from dvc.utils import remove, resolve_output
 
 logger = logging.getLogger(__name__)
 
 
 @staticmethod
 def get(url, path, out=None, rev=None):
-    out = out or os.path.basename(urlparse(path).path)
+    out = resolve_output(path, out)
 
     if Stage.is_valid_filename(out):
         raise GetDVCFileError()

dvc/repo/get_url.py

@@ -3,12 +3,12 @@ import os
 import dvc.output as output
 import dvc.dependency as dependency
 
-from dvc.utils.compat import urlparse
+from dvc.utils import resolve_output
 
 
 @staticmethod
 def get_url(url, out=None):
-    out = out or os.path.basename(urlparse(url).path)
+    out = resolve_output(url, out)
 
     if os.path.exists(url):
         url = os.path.abspath(url)

dvc/repo/imp_url.py

@@ -1,4 +1,4 @@
-from dvc.utils.compat import pathlib
+from dvc.utils import resolve_output
 from dvc.repo.scm_context import scm_context
 
 from . import locked as locked_repo
@@ -9,20 +9,18 @@ from . import locked as locked_repo
 def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
     from dvc.stage import Stage
 
-    out = out or pathlib.PurePath(url).name
+    out = resolve_output(url, out)
 
-    with self.state:
-        stage = Stage.create(
-            self, cmd=None, deps=[url], outs=[out], fname=fname, erepo=erepo
-        )
+    stage = Stage.create(
+        self, cmd=None, deps=[url], outs=[out], fname=fname, erepo=erepo
+    )
 
     if stage is None:
         return None
 
     self.check_modified_graph([stage])
 
-    with self.state:
-        stage.run()
+    stage.run()
 
     stage.locked = locked
 

dvc/repo/move.py

@@ -66,8 +66,7 @@ def move(self, from_path, to_path):
 
     to_out = Output.loads_from(stage, [to_path], out.use_cache, out.metric)[0]
 
-    with self.state:
-        out.move(to_out)
-        stage.save()
+    out.move(to_out)
+    stage.save()
 
     stage.dump()

dvc/repo/pull.py

@@ -24,11 +24,7 @@ def pull(
         with_deps=with_deps,
         recursive=recursive,
     )
-    for target in targets or [None]:
-        self._checkout(
-            target=target,
-            with_deps=with_deps,
-            force=force,
-            recursive=recursive,
-        )
+    self._checkout(
+        targets=targets, with_deps=with_deps, force=force, recursive=recursive
+    )
     return processed_files_count

dvc/repo/push.py

@@ -14,15 +14,14 @@ def push(
     all_tags=False,
     recursive=False,
 ):
-    with self.state:
-        used = self.used_cache(
-            targets,
-            all_branches=all_branches,
-            all_tags=all_tags,
-            with_deps=with_deps,
-            force=True,
-            remote=remote,
-            jobs=jobs,
-            recursive=recursive,
-        )["local"]
-        return self.cloud.push(used, jobs, remote=remote)
+    used = self.used_cache(
+        targets,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        with_deps=with_deps,
+        force=True,
+        remote=remote,
+        jobs=jobs,
+        recursive=recursive,
+    )
+    return self.cloud.push(used, jobs, remote=remote)

dvc/repo/reproduce.py

@@ -94,10 +94,9 @@ def reproduce(
         targets = self.collect(target, recursive=recursive, graph=active_graph)
 
     ret = []
-    with self.state:
-        for target in targets:
-            stages = _reproduce(self, active_graph, target, **kwargs)
-            ret.extend(stages)
+    for target in targets:
+        stages = _reproduce(self, active_graph, target, **kwargs)
+        ret.extend(stages)
 
     return ret
 

dvc/repo/run.py

@@ -9,17 +9,15 @@ from .scm_context import scm_context
 def run(self, no_exec=False, **kwargs):
     from dvc.stage import Stage
 
-    with self.state:
-        stage = Stage.create(self, **kwargs)
+    stage = Stage.create(self, **kwargs)
 
     if stage is None:
         return None
 
     self.check_modified_graph([stage])
 
-    with self.state:
-        if not no_exec:
-            stage.run(no_commit=kwargs.get("no_commit", False))
+    if not no_exec:
+        stage.run(no_commit=kwargs.get("no_commit", False))
 
     stage.dump()
 

dvc/repo/status.py

@@ -76,7 +76,7 @@ def _cloud_status(
         force=True,
         remote=remote,
         jobs=jobs,
-    )["local"]
+    )
 
     ret = {}
     status_info = self.cloud.status(used, jobs, remote=remote)
@@ -104,15 +104,14 @@ def status(
     with_deps=False,
     all_tags=False,
 ):
-    with self.state:
-        if cloud or remote:
-            return _cloud_status(
-                self,
-                targets,
-                jobs,
-                all_branches=all_branches,
-                with_deps=with_deps,
-                remote=remote,
-                all_tags=all_tags,
-            )
-        return _local_status(self, targets, with_deps=with_deps)
+    if cloud or remote:
+        return _cloud_status(
+            self,
+            targets,
+            jobs,
+            all_branches=all_branches,
+            with_deps=with_deps,
+            remote=remote,
+            all_tags=all_tags,
+        )
+    return _local_status(self, targets, with_deps=with_deps)

dvc/repo/update.py

@@ -6,7 +6,6 @@ def update(self, target):
     from dvc.stage import Stage
 
     stage = Stage.load(self, target)
-    with self.state:
-        stage.update()
+    stage.update()
 
     stage.dump()

dvc/scm/git/__init__.py

@@ -6,10 +6,11 @@ import os
 import logging
 
 from funcy import cached_property
+from pathspec.patterns import GitWildMatchPattern
 
 from dvc.exceptions import GitHookAlreadyExistsError
-from dvc.utils.compat import str, open
-from dvc.utils import fix_env, relpath
+from dvc.utils.compat import str, open, cast_bytes_py2
+from dvc.utils import fix_env, relpath, is_binary
 from dvc.scm.base import (
     Base,
     SCMError,
@@ -64,11 +65,24 @@ class Git(Base):
     def clone(url, to_path, rev=None):
         import git
 
+        ld_key = "LD_LIBRARY_PATH"
+
+        env = fix_env(None)
+        if is_binary() and ld_key not in env.keys():
+            # In fix_env, we delete LD_LIBRARY_PATH key if it was empty before
+            # PyInstaller modified it. GitPython, in git.Repo.clone_from, uses
+            # env to update its own internal state. When there is no key in
+            # env, this value is not updated and GitPython re-uses
+            # LD_LIBRARY_PATH that has been set by PyInstaller.
+            # See [1] for more info.
+            # [1] https://github.com/gitpython-developers/GitPython/issues/924
+            env[cast_bytes_py2(ld_key)] = ""
+
         try:
             tmp_repo = git.Repo.clone_from(
                 url,
                 to_path,
-                env=fix_env(None),  # needed before we can fix it in __init__
+                env=env,  # needed before we can fix it in __init__
                 no_single_branch=True,
             )
             tmp_repo.close()
@@ -141,7 +155,7 @@ class Git(Base):
             return
 
         msg = "Adding '{}' to '{}'.".format(relpath(path), relpath(gitignore))
-        logger.info(msg)
+        logger.debug(msg)
 
         self._add_entry_to_gitignore(entry, gitignore)
 
@@ -149,8 +163,9 @@ class Git(Base):
 
         self.ignored_paths.append(path)
 
-    @staticmethod
-    def _add_entry_to_gitignore(entry, gitignore):
+    def _add_entry_to_gitignore(self, entry, gitignore):
+        entry = GitWildMatchPattern.escape(entry)
+
         with open(gitignore, "a+", encoding="utf-8") as fobj:
             fobj.seek(0, os.SEEK_END)
             if fobj.tell() == 0:
@@ -345,6 +360,9 @@ class Git(Base):
     def get_rev(self):
         return self.repo.git.rev_parse("HEAD")
 
+    def resolve_rev(self, rev):
+        return self.repo.git.rev_parse(rev)
+
     def close(self):
         self.repo.close()
 

dvc/scm/git/tree.py

@@ -45,7 +45,8 @@ class GitTree(BaseTree):
     def tree_root(self):
         return self.git.working_dir
 
-    def open(self, path, binary=False):
+    def open(self, path, mode="r", encoding="utf-8"):
+        assert mode in {"r", "rb"}
 
         relative_path = relpath(path, self.git.working_dir)
 
@@ -61,9 +62,9 @@ class GitTree(BaseTree):
         # the `open()` behavior (since data_stream.read() returns bytes,
         # and `open` with default "r" mode returns str)
         data = obj.data_stream.read()
-        if binary:
+        if mode == "rb":
             return BytesIO(data)
-        return StringIO(data.decode("utf-8"))
+        return StringIO(data.decode(encoding))
 
     def exists(self, path):
         return self.git_object_by_path(path) is not None

dvc/scm/tree.py

@@ -11,7 +11,7 @@ class BaseTree(object):
     def tree_root(self):
         pass
 
-    def open(self, path, binary=False):
+    def open(self, path, mode="r", encoding="utf-8"):
         """Open file and return a stream."""
 
     def exists(self, path):
@@ -42,11 +42,9 @@ class WorkingTree(BaseTree):
     def tree_root(self):
         return self.repo_root
 
-    def open(self, path, binary=False):
+    def open(self, path, mode="r", encoding="utf-8"):
         """Open file and return a stream."""
-        if binary:
-            return open(path, "rb")
-        return open(path, encoding="utf-8")
+        return open(path, mode=mode, encoding=encoding)
 
     def exists(self, path):
         """Test whether a path exists."""

dvc/stage.py

@@ -293,7 +293,7 @@ class Stage(object):
         if ret:
             logger.warning("Stage '{}' changed.".format(self.relpath))
         else:
-            logger.info("Stage '{}' didn't change.".format(self.relpath))
+            logger.debug("Stage '{}' didn't change.".format(self.relpath))
 
         return ret
 
@@ -731,9 +731,8 @@ class Stage(object):
             msg += " and " if (changed_deps and changed_outs) else ""
             msg += "outputs {}".format(changed_outs) if changed_outs else ""
             msg += "md5" if not (changed_deps or changed_outs) else ""
-            msg += " of '{}' changed. Are you sure you commit it?".format(
-                self.relpath
-            )
+            msg += " of '{}' changed.".format(self.relpath)
+            msg += "Are you sure you want to commit it?"
             if not force and not prompt.confirm(msg):
                 raise StageCommitError(
                     "unable to commit changed '{}'. Use `-f|--force` to "
@@ -789,8 +788,34 @@ class Stage(object):
 
     def _run(self):
         self._check_missing_deps()
-        executable = os.getenv("SHELL") if os.name != "nt" else None
-        self._warn_if_fish(executable)
+
+        kwargs = {"cwd": self.wdir, "env": fix_env(None), "close_fds": True}
+
+        if os.name == "nt":
+            kwargs["shell"] = True
+            cmd = self.cmd
+        else:
+            # NOTE: when you specify `shell=True`, `Popen` [1] will default to
+            # `/bin/sh` on *nix and will add ["/bin/sh", "-c"] to your command.
+            # But we actually want to run the same shell that we are running
+            # from right now, which is usually determined by the `SHELL` env
+            # var. So instead, we compose our command on our own, making sure
+            # to include special flags to prevent shell from reading any
+            # configs and modifying env, which may change the behavior or the
+            # command we are running. See [2] for more info.
+            #
+            # [1] https://github.com/python/cpython/blob/3.7/Lib/subprocess.py
+            #                                                            #L1426
+            # [2] https://github.com/iterative/dvc/issues/2506
+            #                                           #issuecomment-535396799
+            kwargs["shell"] = False
+            executable = os.getenv("SHELL") or "/bin/sh"
+
+            self._warn_if_fish(executable)
+
+            opts = {"zsh": ["--no-rcs"], "bash": ["--noprofile", "--norc"]}
+            name = os.path.basename(executable).lower()
+            cmd = [executable] + opts.get(name, []) + ["-c", self.cmd]
 
         main_thread = isinstance(
             threading.current_thread(), threading._MainThread
@@ -799,14 +824,7 @@ class Stage(object):
         p = None
 
         try:
-            p = subprocess.Popen(
-                self.cmd,
-                cwd=self.wdir,
-                shell=True,
-                env=fix_env(os.environ),
-                executable=executable,
-                close_fds=True,
-            )
+            p = subprocess.Popen(cmd, **kwargs)
             if main_thread:
                 old_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
             p.communicate()

dvc/system.py

@@ -1,11 +1,10 @@
 from __future__ import unicode_literals
-
-from dvc.utils.compat import str, open, fspath
-
 import os
 import errno
 import shutil
 
+from dvc.utils.compat import str, open, fspath
+
 
 class System(object):
     @staticmethod

dvc/updater.py

@@ -9,7 +9,7 @@ from packaging import version
 
 from dvc import __version__
 from dvc.lock import Lock, LockError
-from dvc.utils import is_binary, boxify
+from dvc.utils import is_binary, boxify, env2bool
 
 
 logger = logging.getLogger(__name__)
@@ -45,7 +45,7 @@ class Updater(object):  # pragma: no cover
             logger.debug(msg.format(self.lock.lockfile, action))
 
     def check(self):
-        if os.getenv("CI") or os.getenv("DVC_TEST"):
+        if os.getenv("CI") or env2bool("DVC_TEST"):
             return
 
         self._with_lock(self._check, "checking")

dvc/utils/__init__.py

@@ -272,22 +272,45 @@ def fix_env(env=None):
     #
     # where
     #
-    #     PYENV_BIN_PATH - starts with $PYENV_ROOT, see pyenv-exec source code.
+    #     PYENV_BIN_PATH - might not start with $PYENV_ROOT if we are running
+    #         `system` version of the command, see pyenv-exec source code.
     #     bin_path - might not start with $PYENV_ROOT as it runs realpath on
-    #         it, see pyenv source code.
+    #         it, but always has `libexec` part in it, see pyenv source code.
     #     plugin_bin - might contain more than 1 entry, which start with
     #         $PYENV_ROOT, see pyenv source code.
     #
+    # Also, we know that whenever pyenv is running, it exports these env vars:
+    #
+    #     PYENV_DIR
+    #     PYENV_HOOK_PATH
+    #     PYENV_VERSION
+    #     PYENV_ROOT
+    #
     # So having this, we can make a rightful assumption about what parts of the
     # PATH we need to remove in order to get the original PATH.
-    path = env.get("PATH")
+    path = env.get("PATH", "")
+    parts = path.split(":")
+    bin_path = parts[1] if len(parts) > 2 else ""
+    pyenv_dir = env.get("PYENV_DIR")
+    pyenv_hook_path = env.get("PYENV_HOOK_PATH")
+    pyenv_version = env.get("PYENV_VERSION")
     pyenv_root = env.get("PYENV_ROOT")
-    if path and pyenv_root and path.startswith(pyenv_root):
+
+    env_matches = all([pyenv_dir, pyenv_hook_path, pyenv_version, pyenv_root])
+
+    bin_path_matches = os.path.basename(bin_path) == "libexec"
+
+    # NOTE: we don't support pyenv-win
+    if os.name != "nt" and env_matches and bin_path_matches:
         # removing PYENV_BIN_PATH and bin_path
-        parts = path.split(":")[2:]
-        # removing plugin_bin from the left
-        while pyenv_root in parts[0]:
-            del parts[0]
+        parts = parts[2:]
+
+        if parts:
+            # removing plugin_bin from the left
+            plugin_bin = os.path.join(pyenv_root, "plugins")
+            while parts[0].startswith(plugin_bin):
+                del parts[0]
+
         env["PATH"] = ":".join(parts)
 
     return env
@@ -437,3 +460,24 @@ def relpath(path, start=os.curdir):
     ):
         return path
     return os.path.relpath(path, start)
+
+
+def env2bool(var, undefined=False):
+    """
+    undefined: return value if env var is unset
+    """
+    var = os.getenv(var, None)
+    if var is None:
+        return undefined
+    return bool(re.search("1|y|yes|true", var, flags=re.I))
+
+
+def resolve_output(inp, out):
+    from dvc.utils.compat import urlparse
+
+    name = os.path.basename(urlparse(inp).path)
+    if not out:
+        return name
+    if os.path.isdir(out):
+        return os.path.join(out, name)
+    return out

dvc/utils/compat.py

@@ -18,6 +18,11 @@ is_py3 = _ver[0] == 3
 # simplified version of ipython_genutils/encoding.py
 DEFAULT_ENCODING = sys.getdefaultencoding()
 
+if _ver[:2] < (3, 5):
+    RecursionError = RuntimeError
+else:
+    RecursionError = RecursionError
+
 
 def no_code(x, encoding=None):
     return x
@@ -107,6 +112,7 @@ if is_py2:
     from io import open  # noqa: F401
     import pathlib2 as pathlib  # noqa: F401
     from collections import Mapping  # noqa: F401
+    from contextlib2 import ExitStack  # noqa: F401
 
     builtin_str = str  # noqa: F821
     bytes = str  # noqa: F821
@@ -150,6 +156,7 @@ elif is_py3:
     from io import StringIO, BytesIO  # noqa: F401
     import configparser as ConfigParser  # noqa: F401
     from collections.abc import Mapping  # noqa: F401
+    from contextlib import ExitStack  # noqa: F401
 
     builtin_str = str  # noqa: F821
     str = str  # noqa: F821

dvc/version.py

@@ -7,12 +7,12 @@ import os
 import subprocess
 
 
-_BASE_VERSION = "0.60.1"
+_BASE_VERSION = "0.63.4"
 
 
 def _generate_version(base_version):
     """Generate a version with information about the git repository"""
-    pkg_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+    pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
     if not _is_git_repo(pkg_dir) or not _have_git():
         return base_version

scripts/build_posix.sh

@@ -18,6 +18,15 @@ BIN_DIR=$BUILD_DIR/$INSTALL_DIR/bin
 DESC='Data Version Control - datasets, models, and experiments versioning for ML or data science projects'
 LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib
 
+FPM_PACKAGE_DIRS="usr"
+ZSH_CMPLT_DIR=usr/share/zsh/site-functions/_dvc
+if [[ "$(uname)" == 'Linux' ]]; then
+	BASH_CMPLT_DIR=etc/bash_completion.d
+	FPM_PACKAGE_DIRS="$FPM_PACKAGE_DIRS etc"
+else
+	BASH_CMPLT_DIR=usr/local/etc/bash_completion.d
+fi
+
 print_error()
 {
 	echo -e "\e[31m$1\e[0m"
@@ -51,7 +60,8 @@ fpm_build()
 	    $FPM_FLAGS \
 	    -n dvc \
 	    -v $VERSION \
-	    -C $BUILD_DIR usr
+	    -C $BUILD_DIR \
+	    $FPM_PACKAGE_DIRS
 }
 
 cleanup()
@@ -78,6 +88,9 @@ install_dependencies()
 
 	gem install --no-document fpm
 
+	print_info "Uprgading pip..."
+	pip install --upgrade pip
+
 	print_info "Installing requirements..."
 	pip install .[all] psutil
 
@@ -106,6 +119,16 @@ build_dvc()
 		popd
 		$BIN_DIR/dvc --help
 	fi
+
+        # NOTE: temporarily not adding scripts to mac package. See [1]
+        # [1] https://github.com/iterative/dvc/issues/2585
+        if [[ "$(uname)" == 'Linux' ]]; then
+            mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
+            cp scripts/completion/dvc.bash $BUILD_DIR/$BASH_CMPLT_DIR/dvc
+
+            mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
+            cp scripts/completion/dvc.zsh $BUILD_DIR/$ZSH_CMPLT_DIR
+        fi
 }
 
 cleanup

scripts/ci/before_install.sh

@@ -48,7 +48,7 @@ if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
         choco install python2
         echo 'PATH="/c/Python27:/c/Python27/Scripts:$PATH"' >> env.sh
     else
-        choco install python
+        choco install python --version 3.7.4
         echo 'PATH="/c/Python37:/c/Python37/Scripts:$PATH"' >> env.sh
     fi
 elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then

scripts/completion/dvc.bash

@@ -72,6 +72,10 @@ _dvc_version=''
 #       ${!x} ->  ${hello} ->  "world"
 #
 _dvc () {
+  replace_hyphen () {
+    echo $(echo $1 | sed 's/-/_/g')
+  }
+
   local word="${COMP_WORDS[COMP_CWORD]}"
 
   COMPREPLY=()
@@ -82,11 +86,11 @@ _dvc () {
       *)  COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
     esac
   elif [ "${COMP_CWORD}" -eq 2 ]; then
-    local options_list="_dvc_${COMP_WORDS[1]}"
+    local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})"
 
     COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
   elif [ "${COMP_CWORD}" -eq 3 ]; then
-    local options_list="_dvc_${COMP_WORDS[1]}_${COMP_WORDS[2]}"
+    local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})_$(replace_hyphen ${COMP_WORDS[2]})"
 
     COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
   fi

setup.py

@@ -10,13 +10,20 @@ import fastentrypoints  # noqa: F401
 
 
 # https://packaging.python.org/guides/single-sourcing-package-version/
-pkg_dir = os.path.dirname(__file__)
+pkg_dir = os.path.dirname(os.path.abspath(__file__))
+version_path = os.path.join(pkg_dir, "dvc", "version.py")
+if sys.version_info[0] == 2:
+    import imp
 
-# This will define __version__ implicitly
-with open(os.path.join(pkg_dir, "dvc", "version.py")) as fobj:
-    exec(fobj.read())
+    dvc_version = imp.load_source("dvc.version", version_path)
+else:
+    import importlib.util
 
-version = __version__  # noqa: F821
+    spec = importlib.util.spec_from_file_location("dvc.version", version_path)
+    dvc_version = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(dvc_version)
+
+version = dvc_version.__version__  # noqa: F821
 
 
 # To achieve consistency between the build version and the one provided
@@ -48,6 +55,7 @@ install_requires = [
     "colorama>=0.3.9",
     "configobj>=5.0.6",
     "gitpython>=2.1.8",
+    "gitdb2>=2.0.6",  # See https://github.com/iterative/dvc/issues/1880
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
@@ -63,7 +71,7 @@ install_requires = [
     "humanize>=0.5.1",
     "ruamel.yaml>=0.16.1",
     "funcy>=1.12",
-    "pathspec>=0.5.9",
+    "pathspec>=0.6.0",
     "shortuuid>=0.5.0",
     "tqdm>=4.35.0",
     "packaging>=19.0",
@@ -73,7 +81,7 @@ install_requires = [
 if sys.version_info[0] == 2:
     install_requires.append("networkx>=2.1,<2.3")
 else:
-    install_requires.append("networkx>=2.1")
+    install_requires.append("networkx>=2.1,<2.4")
 
 # Extra dependencies for remote integrations
 
@@ -96,7 +104,7 @@ if os.name != "nt" or sys.version_info[0] != 2:
 
 # Extra dependecies to run tests
 tests_requirements = [
-    "PyInstaller==3.4",
+    "PyInstaller==3.5",
     "wheel>=0.31.1",
     "pydot>=1.2.4",
     # Test requirements:

tests/__main__.py

@@ -7,7 +7,7 @@ REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 os.chdir(REPO_ROOT)
 
 os.putenv(
-    "PATH", "{}:{}".format(os.path.join(REPO_ROOT, "bin"), os.getenv("PATH"))
+    "PATH", ":".join([os.path.join(REPO_ROOT, "bin"), os.getenv("PATH")])
 )
 os.putenv("DVC_HOME", REPO_ROOT)
 

tests/conftest.py

@@ -13,6 +13,8 @@ from .basic_env import TestDirFixture, TestDvcGitFixture, TestGitFixture
 
 # Prevent updater and analytics from running their processes
 os.environ[cast_bytes_py2("DVC_TEST")] = cast_bytes_py2("true")
+# Ensure progress output even when not outputting to raw sys.stderr console
+os.environ[cast_bytes_py2("DVC_IGNORE_ISATTY")] = cast_bytes_py2("true")
 
 
 @pytest.fixture(autouse=True)

tests/func/test_add.py

@@ -325,7 +325,7 @@ class TestAddCommit(TestDvc):
         ret = main(["add", self.FOO, "--no-commit"])
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(self.FOO))
-        self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 0)
+        self.assertFalse(os.path.exists(self.dvc.cache.local.cache_dir))
 
         ret = main(["commit", self.FOO + ".dvc"])
         self.assertEqual(ret, 0)
@@ -634,3 +634,20 @@ def test_should_protect_on_repeated_add(link, dvc_repo, repo_dir):
     dvc_repo.add(repo_dir.FOO)
 
     assert not os.access(repo_dir.FOO, os.W_OK)
+
+
+def test_escape_gitignore_entries(git, dvc_repo, repo_dir):
+    fname = "file!with*weird#naming_[1].t?t"
+    ignored_fname = r"/file\!with\*weird\#naming_\[1\].t\?t"
+
+    if os.name == "nt":
+        # Some characters are not supported by Windows in the filename
+        # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+        fname = "file!with_weird#naming_[1].txt"
+        ignored_fname = r"/file\!with_weird\#naming_\[1\].txt"
+
+    os.rename(repo_dir.FOO, fname)
+
+    dvc_repo.add(fname)
+
+    assert ignored_fname in get_gitignore_content()

tests/func/test_api.py

@@ -1,8 +1,10 @@
+import os
+
 import pytest
 import shutil
 
 from dvc import api
-from dvc.exceptions import OutputFileMissingError
+from dvc.exceptions import FileMissingError
 from dvc.main import main
 from dvc.path_info import URLInfo
 from dvc.remote.config import RemoteConfig
@@ -134,7 +136,7 @@ def test_missing(repo_dir, dvc_repo, remote_url):
     # Remove cache to make foo missing
     shutil.rmtree(dvc_repo.cache.local.cache_dir)
 
-    with pytest.raises(OutputFileMissingError):
+    with pytest.raises(FileMissingError):
         api.read(repo_dir.FOO)
 
 
@@ -143,3 +145,30 @@ def _set_remote_url_and_commit(repo, remote_url):
     rconfig.modify("upstream", "url", remote_url)
     repo.scm.add([repo.config.config_file])
     repo.scm.commit("modify remote")
+
+
+def test_open_scm_controlled(dvc_repo, repo_dir):
+    stage, = dvc_repo.add(repo_dir.FOO)
+
+    stage_content = open(stage.path, "r").read()
+    with api.open(stage.path) as fd:
+        assert fd.read() == stage_content
+
+
+def test_open_not_cached(dvc_repo):
+    metric_file = "metric.txt"
+    metric_content = "0.6"
+    metric_code = "open('{}', 'w').write('{}')".format(
+        metric_file, metric_content
+    )
+    dvc_repo.run(
+        metrics_no_cache=[metric_file],
+        cmd=('python -c "{}"'.format(metric_code)),
+    )
+
+    with api.open(metric_file) as fd:
+        assert fd.read() == metric_content
+
+    os.remove(metric_file)
+    with pytest.raises(FileMissingError):
+        api.read(metric_file)

tests/func/test_cache.py

@@ -1,7 +1,6 @@
 import os
 import stat
 import pytest
-import shutil
 import configobj
 
 from dvc.cache import Cache
@@ -66,7 +65,7 @@ class TestExternalCacheDir(TestDvc):
         ret = main(["config", "cache.dir", cache_dir])
         self.assertEqual(ret, 0)
 
-        shutil.rmtree(".dvc/cache")
+        self.assertFalse(os.path.exists(self.dvc.cache.local.cache_dir))
 
         ret = main(["add", self.FOO])
         self.assertEqual(ret, 0)
@@ -101,7 +100,7 @@ class TestSharedCacheDir(TestDir):
             ret = main(["config", "cache.dir", cache_dir])
             self.assertEqual(ret, 0)
 
-            shutil.rmtree(os.path.join(".dvc", "cache"))
+            self.assertFalse(os.path.exists(os.path.join(".dvc", "cache")))
 
             with open("common", "w+") as fd:
                 fd.write("common")

tests/func/test_checkout.py

@@ -391,14 +391,14 @@ class TestCheckoutSuggestGit(TestRepro):
     def test(self):
 
         try:
-            self.dvc.checkout(target="gitbranch")
+            self.dvc.checkout(targets=["gitbranch"])
         except DvcException as exc:
             self.assertIsInstance(exc, CheckoutErrorSuggestGit)
             self.assertIsInstance(exc.cause, StageFileDoesNotExistError)
             self.assertIsNone(exc.cause.cause)
 
         try:
-            self.dvc.checkout(target=self.FOO)
+            self.dvc.checkout(targets=[self.FOO])
         except DvcException as exc:
             self.assertIsInstance(exc, CheckoutErrorSuggestGit)
             self.assertIsInstance(exc.cause, StageFileBadNameError)
@@ -422,7 +422,7 @@ class TestCheckoutRecursiveNotDirectory(TestDvc):
         ret = main(["add", self.FOO])
         self.assertEqual(0, ret)
 
-        self.dvc.checkout(target=self.FOO + ".dvc", recursive=True)
+        self.dvc.checkout(targets=[self.FOO + ".dvc"], recursive=True)
 
 
 class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
@@ -477,7 +477,7 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
 def test_checkout_no_checksum(repo_dir, dvc_repo):
     stage = dvc_repo.run(outs=[repo_dir.FOO], no_exec=True, cmd="somecmd")
     with pytest.raises(CheckoutError):
-        dvc_repo.checkout(stage.path, force=True)
+        dvc_repo.checkout([stage.path], force=True)
     assert not os.path.exists(repo_dir.FOO)
 
 
@@ -491,7 +491,7 @@ def test_should_relink_on_checkout(link, link_test_func, repo_dir, dvc_repo):
     dvc_repo.add(repo_dir.DATA_DIR)
     dvc_repo.unprotect(repo_dir.DATA_SUB)
 
-    dvc_repo.checkout(repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
+    dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX])
 
     assert link_test_func(repo_dir.DATA_SUB)
 
@@ -504,7 +504,7 @@ def test_should_protect_on_checkout(link, dvc_repo, repo_dir):
     dvc_repo.add(repo_dir.FOO)
     dvc_repo.unprotect(repo_dir.FOO)
 
-    dvc_repo.checkout(repo_dir.FOO + Stage.STAGE_FILE_SUFFIX)
+    dvc_repo.checkout([repo_dir.FOO + Stage.STAGE_FILE_SUFFIX])
 
     assert not os.access(repo_dir.FOO, os.W_OK)
 
@@ -517,7 +517,7 @@ def test_should_relink_only_one_file_in_dir(dvc_repo, repo_dir):
 
     link_spy = spy(System.symlink)
     with patch.object(dvc_repo.cache.local, "symlink", link_spy):
-        dvc_repo.checkout(repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
+        dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX])
 
     assert link_spy.mock.call_count == 1
 
@@ -529,6 +529,6 @@ def test_should_not_relink_on_unchanged_dependency(link, dvc_repo, repo_dir):
     dvc_repo.add(repo_dir.DATA_DIR)
 
     with patch.object(dvc_repo.cache.local, "link") as mock_link:
-        dvc_repo.checkout(repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
+        dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX])
 
     assert not mock_link.called

tests/func/test_data_cloud.py

@@ -13,8 +13,10 @@ import pytest
 from mock import patch
 
 from dvc.utils.compat import str
+from dvc.utils import env2bool
 from dvc.main import main
 from dvc.config import Config
+from dvc.cache import NamedCache
 from dvc.data_cloud import DataCloud
 from dvc.remote import (
     RemoteS3,
@@ -57,11 +59,9 @@ os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE
 
 
 def _should_test_aws():
-    dvc_test_aws = os.getenv("DVC_TEST_AWS")
-    if dvc_test_aws == "true":
-        return True
-    elif dvc_test_aws == "false":
-        return False
+    do_test = env2bool("DVC_TEST_AWS", undefined=None)
+    if do_test is not None:
+        return do_test
 
     if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"):
         return True
@@ -70,8 +70,9 @@ def _should_test_aws():
 
 
 def _should_test_gcp():
-    if os.getenv("DVC_TEST_GCP") == "true":
-        return True
+    do_test = env2bool("DVC_TEST_GCP", undefined=None)
+    if do_test is not None:
+        return do_test
 
     if not os.path.exists(TEST_GCP_CREDS_FILE):
         return False
@@ -92,10 +93,9 @@ def _should_test_gcp():
 
 
 def _should_test_azure():
-    if os.getenv("DVC_TEST_AZURE") == "true":
-        return True
-    elif os.getenv("DVC_TEST_AZURE") == "false":
-        return False
+    do_test = env2bool("DVC_TEST_AZURE", undefined=None)
+    if do_test is not None:
+        return do_test
 
     return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv(
         "AZURE_STORAGE_CONNECTION_STRING"
@@ -103,10 +103,9 @@ def _should_test_azure():
 
 
 def _should_test_oss():
-    if os.getenv("DVC_TEST_OSS") == "true":
-        return True
-    elif os.getenv("DVC_TEST_OSS") == "false":
-        return False
+    do_test = env2bool("DVC_TEST_OSS", undefined=None)
+    if do_test is not None:
+        return do_test
 
     return (
         os.getenv("OSS_ENDPOINT")
@@ -116,8 +115,9 @@ def _should_test_oss():
 
 
 def _should_test_ssh():
-    if os.getenv("DVC_TEST_SSH") == "true":
-        return True
+    do_test = env2bool("DVC_TEST_SSH", undefined=None)
+    if do_test is not None:
+        return do_test
 
     # FIXME: enable on windows
     if os.name == "nt":
@@ -290,72 +290,71 @@ class TestDataCloudBase(TestDvc):
         self.assertTrue(stage is not None)
         out = stage.outs[0]
         cache = out.cache_path
-        name = str(out)
         md5 = out.checksum
-        info = {"name": name, out.remote.PARAM_CHECKSUM: md5}
+        info = out.get_used_cache()
 
         stages = self.dvc.add(self.DATA_DIR)
         self.assertEqual(len(stages), 1)
         stage_dir = stages[0]
         self.assertTrue(stage_dir is not None)
         out_dir = stage_dir.outs[0]
-        cache_dir = out.cache_path
-        name_dir = str(out)
-        md5_dir = out.checksum
-        info_dir = {"name": name_dir, out_dir.remote.PARAM_CHECKSUM: md5_dir}
+        cache_dir = out_dir.cache_path
+        name_dir = str(out_dir)
+        md5_dir = out_dir.checksum
+        info_dir = NamedCache.make(out_dir.scheme, md5_dir, name_dir)
 
         with self.cloud.repo.state:
             # Check status
-            status = self.cloud.status([info], show_checksums=True)
+            status = self.cloud.status(info, show_checksums=True)
             expected = {md5: {"name": md5, "status": STATUS_NEW}}
             self.assertEqual(status, expected)
 
-            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            status_dir = self.cloud.status(info_dir, show_checksums=True)
             expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
             self.assertEqual(status_dir, expected)
 
             # Push and check status
-            self.cloud.push([info])
+            self.cloud.push(info)
             self.assertTrue(os.path.exists(cache))
             self.assertTrue(os.path.isfile(cache))
 
-            self.cloud.push([info_dir])
+            self.cloud.push(info_dir)
             self.assertTrue(os.path.isfile(cache_dir))
 
-            status = self.cloud.status([info], show_checksums=True)
+            status = self.cloud.status(info, show_checksums=True)
             expected = {md5: {"name": md5, "status": STATUS_OK}}
             self.assertEqual(status, expected)
 
-            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            status_dir = self.cloud.status(info_dir, show_checksums=True)
             expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
             self.assertEqual(status_dir, expected)
 
             # Remove and check status
             shutil.rmtree(self.dvc.cache.local.cache_dir)
 
-            status = self.cloud.status([info], show_checksums=True)
+            status = self.cloud.status(info, show_checksums=True)
             expected = {md5: {"name": md5, "status": STATUS_DELETED}}
             self.assertEqual(status, expected)
 
-            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            status_dir = self.cloud.status(info_dir, show_checksums=True)
             expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}}
             self.assertEqual(status_dir, expected)
 
             # Pull and check status
-            self.cloud.pull([info])
+            self.cloud.pull(info)
             self.assertTrue(os.path.exists(cache))
             self.assertTrue(os.path.isfile(cache))
             with open(cache, "r") as fd:
                 self.assertEqual(fd.read(), self.FOO_CONTENTS)
 
-            self.cloud.pull([info_dir])
+            self.cloud.pull(info_dir)
             self.assertTrue(os.path.isfile(cache_dir))
 
-            status = self.cloud.status([info], show_checksums=True)
+            status = self.cloud.status(info, show_checksums=True)
             expected = {md5: {"name": md5, "status": STATUS_OK}}
             self.assertEqual(status, expected)
 
-            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            status_dir = self.cloud.status(info_dir, show_checksums=True)
             expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
             self.assertTrue(status_dir, expected)
 

tests/func/test_external_repo.py

@@ -0,0 +1,28 @@
+import os
+from mock import patch
+
+from dvc.external_repo import external_repo
+from dvc.scm.git import Git
+
+
+def test_external_repo(erepo):
+    url = erepo.root_dir
+    # We will share cache dir, to fetch version file
+    cache_dir = erepo.dvc.cache.local.cache_dir
+
+    with patch.object(Git, "clone", wraps=Git.clone) as mock:
+        with external_repo(url, cache_dir=cache_dir) as repo:
+            with repo.open(os.path.join(repo.root_dir, "version")) as fd:
+                assert fd.read() == "master"
+
+        with external_repo(url, rev="branch", cache_dir=cache_dir) as repo:
+            with repo.open(os.path.join(repo.root_dir, "version")) as fd:
+                assert fd.read() == "branch"
+
+        # Check cache_dir is unset
+        with external_repo(url) as repo:
+            assert repo.cache.local.cache_dir.startswith(
+                repo.root_dir + os.sep
+            )
+
+        assert mock.call_count == 1

tests/func/test_get.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 import filecmp
 
@@ -5,6 +7,7 @@ import pytest
 
 from dvc.exceptions import UrlNotDvcRepoError, GetDVCFileError
 from dvc.repo import Repo
+from dvc.utils import makedirs
 
 from tests.utils import trees_equal
 
@@ -51,3 +54,17 @@ def test_get_from_non_dvc_repo(git_erepo):
 def test_get_a_dvc_file(repo_dir, erepo):
     with pytest.raises(GetDVCFileError):
         Repo.get(erepo.root_dir, "some_file.dvc")
+
+
+@pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
+def test_get_to_dir(dname, repo_dir, erepo):
+    src = erepo.FOO
+
+    makedirs(dname, exist_ok=True)
+
+    Repo.get(erepo.root_dir, src, dname)
+
+    dst = os.path.join(dname, os.path.basename(src))
+
+    assert os.path.isdir(dname)
+    assert filecmp.cmp(repo_dir.FOO, dst, shallow=False)

tests/func/test_get_url.py

@@ -1,7 +1,12 @@
+from __future__ import unicode_literals
+
 import os
 import filecmp
 
+import pytest
+
 from dvc.repo import Repo
+from dvc.utils import makedirs
 
 
 def test_get_file(repo_dir):
@@ -13,3 +18,17 @@ def test_get_file(repo_dir):
     assert os.path.exists(dst)
     assert os.path.isfile(dst)
     assert filecmp.cmp(repo_dir.FOO, dst, shallow=False)
+
+
+@pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
+def test_get_url_to_dir(dname, repo_dir):
+    src = repo_dir.DATA
+
+    makedirs(dname, exist_ok=True)
+
+    Repo.get_url(src, dname)
+
+    dst = os.path.join(dname, os.path.basename(src))
+
+    assert os.path.isdir(dname)
+    assert filecmp.cmp(repo_dir.DATA, dst, shallow=False)

tests/func/test_import.py

@@ -1,11 +1,14 @@
+from __future__ import unicode_literals
+
 import os
 import filecmp
-import shutil
 
 import pytest
+from mock import patch
 from tests.utils import trees_equal
 
 from dvc.stage import Stage
+from dvc.utils import makedirs
 from dvc.exceptions import DownloadError
 
 
@@ -73,9 +76,25 @@ def test_download_error_pulling_imported_stage(dvc_repo, erepo):
     dst_stage = Stage.load(dvc_repo, "foo_imported.dvc")
     dst_cache = dst_stage.outs[0].cache_path
 
-    shutil.rmtree(erepo.root_dir)
     os.remove(dst)
     os.remove(dst_cache)
 
-    with pytest.raises(DownloadError):
+    with patch(
+        "dvc.remote.RemoteLOCAL._download", side_effect=Exception
+    ), pytest.raises(DownloadError):
         dvc_repo.pull(["foo_imported.dvc"])
+
+
+@pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
+def test_import_to_dir(dname, repo_dir, dvc_repo, erepo):
+    src = erepo.FOO
+
+    makedirs(dname, exist_ok=True)
+
+    stage = dvc_repo.imp(erepo.root_dir, src, dname)
+
+    dst = os.path.join(dname, os.path.basename(src))
+
+    assert stage.outs[0].fspath == os.path.abspath(dst)
+    assert os.path.isdir(dname)
+    assert filecmp.cmp(repo_dir.FOO, dst, shallow=False)

tests/func/test_import_url.py

@@ -1,12 +1,17 @@
-import dvc
-
-from dvc.utils.compat import str
+from __future__ import unicode_literals
 
 import os
 from uuid import uuid4
 
-from dvc.main import main
+import pytest
+import filecmp
 from mock import patch
+
+import dvc
+from dvc.main import main
+from dvc.utils import makedirs
+from dvc.utils.compat import str
+
 from tests.basic_env import TestDvc
 from tests.utils import spy
 
@@ -86,3 +91,18 @@ class TestImportFilename(TestDvc):
         ret = main(["import-url", "--file", path, self.external_source])
         self.assertEqual(0, ret)
         self.assertTrue(os.path.exists(path))
+
+
+@pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
+def test_import_url_to_dir(dname, repo_dir, dvc_repo):
+    src = repo_dir.DATA
+
+    makedirs(dname, exist_ok=True)
+
+    stage = dvc_repo.imp_url(src, dname)
+
+    dst = os.path.join(dname, os.path.basename(src))
+
+    assert stage.outs[0].fspath == os.path.abspath(dst)
+    assert os.path.isdir(dname)
+    assert filecmp.cmp(repo_dir.DATA, dst, shallow=False)

tests/func/test_repro.py

@@ -943,7 +943,7 @@ class TestReproExternalBase(TestDvc):
         self.dvc.remove(cmd_stage.path, outs_only=True)
         self.assertNotEqual(self.dvc.status([cmd_stage.path]), {})
 
-        self.dvc.checkout(cmd_stage.path, force=True)
+        self.dvc.checkout([cmd_stage.path], force=True)
         self.assertEqual(self.dvc.status([cmd_stage.path]), {})
 
 
@@ -1280,7 +1280,7 @@ class TestReproNoCommit(TestRepro):
         shutil.rmtree(self.dvc.cache.local.cache_dir)
         ret = main(["repro", self.file1_stage, "--no-commit"])
         self.assertEqual(ret, 0)
-        self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 0)
+        self.assertFalse(os.path.exists(self.dvc.cache.local.cache_dir))
 
 
 class TestReproAlreadyCached(TestRepro):

tests/func/test_run.py

@@ -706,7 +706,7 @@ class TestRunCommit(TestDvc):
         )
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(fname))
-        self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 0)
+        self.assertFalse(os.path.exists(self.dvc.cache.local.cache_dir))
 
         ret = main(["commit", fname + ".dvc"])
         self.assertEqual(ret, 0)

tests/func/test_tag.py

@@ -47,47 +47,50 @@ class TestTag(TestDvc):
 
 class TestTagAll(TestDvc):
     def test(self):
-        ret = main(["add", self.FOO, self.BAR])
-        self.assertEqual(ret, 0)
+        with self._caplog.at_level(logging.INFO, logger="dvc"):
+            ret = main(["add", self.FOO, self.BAR])
+            self.assertEqual(ret, 0)
 
-        self._caplog.clear()
+            self._caplog.clear()
 
-        ret = main(["tag", "list"])
-        self.assertEqual(ret, 0)
+            ret = main(["tag", "list"])
+            self.assertEqual(ret, 0)
 
-        self.assertEqual("", self._caplog.text)
+            self.assertEqual("", self._caplog.text)
 
-        ret = main(["tag", "add", "v1"])
-        self.assertEqual(ret, 0)
+            ret = main(["tag", "add", "v1"])
+            self.assertEqual(ret, 0)
 
-        self._caplog.clear()
+            self._caplog.clear()
 
-        ret = main(["tag", "list"])
-        self.assertEqual(ret, 0)
+            ret = main(["tag", "list"])
+            self.assertEqual(ret, 0)
 
-        expected = {
-            "foo.dvc": {
-                "foo": {"v1": {"md5": "acbd18db4cc2f85cedef654fccc4a4d8"}}
-            },
-            "bar.dvc": {
-                "bar": {"v1": {"md5": "8978c98bb5a48c2fb5f2c4c905768afa"}}
-            },
-        }
+            expected = {
+                "foo.dvc": {
+                    "foo": {"v1": {"md5": "acbd18db4cc2f85cedef654fccc4a4d8"}}
+                },
+                "bar.dvc": {
+                    "bar": {"v1": {"md5": "8978c98bb5a48c2fb5f2c4c905768afa"}}
+                },
+            }
 
-        stdout = "\n".join(record.message for record in self._caplog.records)
-        received = from_yaml_string(stdout)
+            stdout = "\n".join(
+                record.message for record in self._caplog.records
+            )
+            received = from_yaml_string(stdout)
 
-        assert expected == received
+            assert expected == received
 
-        ret = main(["tag", "remove", "v1"])
-        self.assertEqual(ret, 0)
+            ret = main(["tag", "remove", "v1"])
+            self.assertEqual(ret, 0)
 
-        self._caplog.clear()
+            self._caplog.clear()
 
-        ret = main(["tag", "list"])
-        self.assertEqual(ret, 0)
+            ret = main(["tag", "list"])
+            self.assertEqual(ret, 0)
 
-        self.assertEqual("", self._caplog.text)
+            self.assertEqual("", self._caplog.text)
 
 
 class TestTagAddNoChecksumInfo(TestDvc):

tests/func/test_update.py

@@ -3,6 +3,7 @@ import shutil
 import filecmp
 
 from dvc.repo import Repo
+from dvc.external_repo import clean_repos
 
 
 def test_update_import(dvc_repo, erepo):
@@ -34,6 +35,10 @@ def test_update_import(dvc_repo, erepo):
 
     os.chdir(saved_dir)
 
+    # Caching in external repos doesn't see upstream updates within single
+    # cli call, so we need to clean the caches to see the changes.
+    clean_repos()
+
     assert dvc_repo.status([stage.path]) == {}
     dvc_repo.update(stage.path)
     assert dvc_repo.status([stage.path]) == {}

tests/func/test_version.py

@@ -5,7 +5,11 @@ from dvc.main import main
 from dvc.command.version import psutil
 
 
-def test_info_in_repo(dvc_repo, caplog):
+def test_info_in_repo(repo_dir, dvc_repo, caplog):
+    # adding a file so that dvc creates `.dvc/cache`, that is needed for proper
+    # supported link types check.
+    assert main(["add", repo_dir.FOO]) == 0
+
     assert main(["version"]) == 0
 
     assert re.search(re.compile(r"DVC version: \d+\.\d+\.\d+"), caplog.text)

tests/unit/remote/test_local.py

@@ -1,3 +1,4 @@
+from dvc.cache import NamedCache
 from dvc.remote.local import RemoteLOCAL
 
 
@@ -8,16 +9,11 @@ def test_status_download_optimization(mocker):
     """
     remote = RemoteLOCAL(None, {})
 
-    infos = [
-        {"name": "foo", "md5": "acbd18db4cc2f85cedef654fccc4a4d8"},
-        {"name": "bar", "md5": "37b51d194a7513e45b56f6524f2d51f2"},
-    ]
-
-    local_exists = [
-        "acbd18db4cc2f85cedef654fccc4a4d8",
-        "37b51d194a7513e45b56f6524f2d51f2",
-    ]
+    infos = NamedCache()
+    infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo")
+    infos.add("local", "37b51d194a7513e45b56f6524f2d51f2", "bar")
 
+    local_exists = list(infos["local"])
     mocker.patch.object(remote, "cache_exists", return_value=local_exists)
 
     other_remote = mocker.Mock()

tests/unit/test_logger.py

@@ -179,3 +179,10 @@ class TestColorFormatter:
                     logger.info("some info")
                     captured = capsys.readouterr()
                     assert captured.out == ""
+
+
+def test_handlers():
+    stdout, stderr = logger.handlers
+
+    assert stdout.level == logging.DEBUG
+    assert stderr.level == logging.WARNING

tests/unit/test_progress.py

@@ -1,17 +1,37 @@
 import logging
 from dvc.progress import Tqdm
+from dvc.utils import env2bool
+import sys
+import mock
 
 
-def test_quiet(caplog, capsys):
+def test_quiet_logging(caplog, capsys):
     with caplog.at_level(logging.CRITICAL, logger="dvc"):
         for _ in Tqdm(range(10)):
             pass
         out_err = capsys.readouterr()
         assert out_err.out == ""
         assert out_err.err == ""
+
+
+def test_quiet_notty(caplog, capsys):
     with caplog.at_level(logging.INFO, logger="dvc"):
         for _ in Tqdm(range(10)):
             pass
         out_err = capsys.readouterr()
         assert out_err.out == ""
-        assert "0/10" in out_err.err
+        if env2bool("DVC_IGNORE_ISATTY"):
+            assert "0/10" in out_err.err
+        else:
+            assert out_err.err == ""
+
+
+def test_default(caplog, capsys):
+    with caplog.at_level(logging.INFO, logger="dvc"):
+        # simulate interactive terminal
+        with mock.patch.object(sys.stderr, "isatty", return_value=True):
+            for _ in Tqdm(range(10)):
+                pass
+            out_err = capsys.readouterr()
+            assert out_err.out == ""
+            assert "0/10" in out_err.err

tests/unit/utils/test_utils.py

@@ -1,3 +1,5 @@
+import os
+
 import pytest
 
 from dvc.utils import to_chunks, fix_env
@@ -30,11 +32,15 @@ def test_to_chunks_num_chunks(num_chunks, expected_chunks):
     assert result == expected_chunks
 
 
+@pytest.mark.skipif(os.name == "nt", reason="pyenv-win is not supported")
 @pytest.mark.parametrize(
     "path, orig",
     [
         (
-            "/pyenv/bin:/pyenv/libexec:/pyenv/hook:/orig/path1:/orig/path2",
+            (
+                "/pyenv/bin:/pyenv/libexec:/pyenv/plugins/plugin:"
+                "/orig/path1:/orig/path2"
+            ),
             "/orig/path1:/orig/path2",
         ),
         (
@@ -42,7 +48,10 @@ def test_to_chunks_num_chunks(num_chunks, expected_chunks):
             "/orig/path1:/orig/path2",
         ),
         (
-            "/pyenv/bin:/some/libexec:/pyenv/hook:/orig/path1:/orig/path2",
+            (
+                "/pyenv/bin:/some/libexec:/pyenv/plugins/plugin:"
+                "/orig/path1:/orig/path2"
+            ),
             "/orig/path1:/orig/path2",
         ),
         ("/orig/path1:/orig/path2", "/orig/path1:/orig/path2"),
@@ -53,5 +62,11 @@ def test_to_chunks_num_chunks(num_chunks, expected_chunks):
     ],
 )
 def test_fix_env_pyenv(path, orig):
-    env = {"PATH": path, "PYENV_ROOT": "/pyenv"}
+    env = {
+        "PATH": path,
+        "PYENV_ROOT": "/pyenv",
+        "PYENV_VERSION": "3.7.2",
+        "PYENV_DIR": "/some/dir",
+        "PYENV_HOOK_PATH": "/some/hook/path",
+    }
     assert fix_env(env)["PATH"] == orig