dvc: bump to 0.64.1

* fix zero-width desc again

- fixes #2659

* correctly use tqdm API

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,5 @@
 * [ ] Have you followed the guidelines in our
-      [Contributing document](https://dvc.org/doc/user-guide/contributing)?
+      [Contributing document](https://dvc.org/doc/user-guide/contributing/core)?
 
 * [ ] Does your PR affect documented changes or does it add new functionality
       that should be documented? If yes, have you created a PR for

CONTRIBUTING.md

@@ -1 +1 @@
-### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing#contributing).
+### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing/core).

README.rst

@@ -202,7 +202,7 @@ Related technologies
 
 Contributing
 ============
-Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/>`_ for more
+Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0

dvc/analytics.py

@@ -10,6 +10,7 @@ import errno
 import logging
 
 from dvc import __version__
+from dvc.utils import env2bool
 
 
 logger = logging.getLogger(__name__)
@@ -229,7 +230,7 @@ class Analytics(object):
     def is_enabled(cmd=None):
         from dvc.command.daemon import CmdDaemonBase
 
-        if os.getenv("DVC_TEST"):
+        if env2bool("DVC_TEST"):
             return False
 
         if isinstance(cmd, CmdDaemonBase):

dvc/cache.py

@@ -3,6 +3,7 @@
 from __future__ import unicode_literals
 
 import os
+from collections import defaultdict
 from funcy import cached_property
 
 from dvc.utils.compat import builtin_str
@@ -103,3 +104,36 @@ class Cache(object):
     ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
     hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
     azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
+
+
+class NamedCache(object):
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(set))
+        self.external = defaultdict(set)
+
+    @classmethod
+    def make(cls, scheme, checksum, name):
+        cache = cls()
+        cache.add(scheme, checksum, name)
+        return cache
+
+    def __getitem__(self, key):
+        return self._items[key]
+
+    def add(self, scheme, checksum, name):
+        self._items[scheme][checksum].add(name)
+
+    def add_external(self, url, rev, path):
+        self.external[url, rev].add(path)
+
+    def update(self, cache, suffix=""):
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme]
+            for checksum, names in src.items():
+                if suffix:
+                    dst[checksum].update(n + suffix for n in names)
+                else:
+                    dst[checksum].update(names)
+
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)

dvc/command/checkout.py

@@ -7,16 +7,12 @@ from dvc.command.base import CmdBase, append_doc_link
 
 class CmdCheckout(CmdBase):
     def run(self):
-        if not self.args.targets:
-            self.repo.checkout(force=self.args.force)
-        else:
-            for target in self.args.targets:
-                self.repo.checkout(
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    force=self.args.force,
-                    recursive=self.args.recursive,
-                )
+        self.repo.checkout(
+            targets=self.args.targets,
+            with_deps=self.args.with_deps,
+            force=self.args.force,
+            recursive=self.args.recursive,
+        )
         return 0
 
 

dvc/command/gc.py

@@ -13,17 +13,19 @@ logger = logging.getLogger(__name__)
 
 class CmdGC(CmdBase):
     def run(self):
-        msg = "this will remove all cache except the cache that is used in "
-        if not self.args.all_branches and not self.args.all_tags:
-            msg += "the current git branch"
-        elif self.args.all_branches and not self.args.all_tags:
-            msg += "all git branches"
-        elif not self.args.all_branches and self.args.all_tags:
-            msg += "all git tags"
-        else:
-            msg += "all git branches and all git tags"
+        msg = "This will remove all cache except items used in "
 
-        if self.args.repos is not None and len(self.args.repos) > 0:
+        msg += "the working tree"
+        if self.args.all_commits:
+            msg += " and all git commits"
+        elif self.args.all_branches and self.args.all_tags:
+            msg += " and all git branches and tags"
+        elif self.args.all_branches:
+            msg += " and all git branches"
+        elif self.args.all_tags:
+            msg += " and all git tags"
+
+        if self.args.repos:
             msg += " of the current and the following repos:"
 
             for repo_path in self.args.repos:
@@ -40,6 +42,7 @@ class CmdGC(CmdBase):
         self.repo.gc(
             all_branches=self.args.all_branches,
             all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
             cloud=self.args.cloud,
             remote=self.args.remote,
             force=self.args.force,
@@ -76,6 +79,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Keep data files for all git tags.",
     )
+    gc_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Keep data files for all commits.",
+    )
     gc_parser.add_argument(
         "-c",
         "--cloud",

dvc/command/version.py

@@ -12,7 +12,7 @@ try:
 except ImportError:
     psutil = None
 
-from dvc.utils import is_binary
+from dvc.utils import is_binary, relpath
 from dvc.utils.compat import pathlib
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
@@ -60,8 +60,9 @@ class CmdVersion(CmdBaseNoRepo):
                 logger.warning(
                     "Unable to detect supported link types, as cache "
                     "directory '{}' doesn't exist. It is usually auto-created "
-                    "by commands such as `dvc add/fetch/pull/run/import`, but "
-                    "you could create it manually to enable this check."
+                    "by commands such as `dvc add/fetch/pull/run/import`, "
+                    "but you could create it manually to enable this "
+                    "check.".format(relpath(repo.cache.local.cache_dir))
                 )
 
             if psutil:

dvc/data_cloud.py

@@ -45,11 +45,11 @@ class DataCloud(object):
     def _init_remote(self, remote):
         return Remote(self.repo, name=remote)
 
-    def push(self, targets, jobs=None, remote=None, show_checksums=False):
+    def push(self, cache, jobs=None, remote=None, show_checksums=False):
         """Push data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to push to the cloud.
+            cache (NamedCache): named checksums to push to the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to push to.
                 By default remote from core.remote config option is used.
@@ -57,17 +57,17 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.push(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "push"),
             show_checksums=show_checksums,
         )
 
-    def pull(self, targets, jobs=None, remote=None, show_checksums=False):
+    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
         """Pull data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to pull from the cloud.
+            cache (NamedCache): named checksums to pull from the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
                 By default remote from core.remote config option is used.
@@ -75,25 +75,25 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.pull(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "pull"),
             show_checksums=show_checksums,
         )
 
-    def status(self, targets, jobs=None, remote=None, show_checksums=False):
+    def status(self, cache, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to check status for.
+            cache (NamedCache): named checksums to check status for.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to compare
-                targets to. By default remote from core.remote config option
+                cache to. By default remote from core.remote config option
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
         remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            targets, jobs=jobs, remote=remote, show_checksums=show_checksums
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )

dvc/dependency/repo.py

@@ -35,6 +35,11 @@ class DependencyREPO(DependencyLOCAL):
     def is_in_repo(self):
         return False
 
+    @property
+    def repo_pair(self):
+        d = self.def_repo
+        return d[self.PARAM_URL], d[self.PARAM_REV_LOCK] or d[self.PARAM_REV]
+
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 

dvc/exceptions.py

@@ -325,3 +325,7 @@ class CheckoutError(DvcException):
             "forget to fetch?".format("\n".join(targets))
         )
         super(CheckoutError, self).__init__(m)
+
+
+class CollectCacheError(DvcException):
+    pass

dvc/lock.py

@@ -34,12 +34,31 @@ if is_py3:
         """
 
         def __init__(self, lockfile, tmp_dir=None):
-            lifetime = timedelta(days=365)  # Lock for good by default
+            import socket
+
             self._tmp_dir = tmp_dir
             if self._tmp_dir is not None:
                 makedirs(self._tmp_dir, exist_ok=True)
 
-            super(Lock, self).__init__(lockfile, lifetime=lifetime)
+            # NOTE: this is basically Lock.__init__ copy-paste, except that
+            # instead of using `socket.getfqdn()` we use `socket.gethostname()`
+            # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
+            # anything, which is way too slow. `gethostname()` is actually a
+            # fallback for `getfqdn()` when it is not able to resolve a
+            # canonical hostname through network. The claimfile that uses
+            # `self._hostname` is still usable, as it uses `pid` and random
+            # number to generate the resulting lock file name, which is unique
+            # enough for our application.
+            #
+            # [1] https://github.com/iterative/dvc/issues/2582
+            self._hostname = socket.gethostname()
+
+            self._lockfile = lockfile
+            self._lifetime = timedelta(days=365)  # Lock for good by default
+            self._separator = flufl.lock.SEP
+            self._set_claimfile()
+            self._owned = True
+            self._retry_errnos = []
 
         @property
         def lockfile(self):

dvc/output/base.py

@@ -5,8 +5,9 @@ from copy import copy
 
 from schema import Or, Optional
 
+from dvc.cache import NamedCache
 import dvc.prompt as prompt
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, CollectCacheError
 from dvc.utils.compat import str, urlparse
 from dvc.remote.base import RemoteBASE
 
@@ -345,25 +346,19 @@ class OutputBase(object):
             $ echo "bar" > directory/bar
             $ dvc add directory
 
-            It will return something similar to the following list:
+            It will return a NamedCache like:
 
-            [
-                { 'path': 'directory/foo', 'md5': 'c157a79031e1', ... },
-                { 'path': 'directory/bar', 'md5': 'd3b07384d113', ... },
-            ]
+            nc = NamedCache()
+            nc.add(self.scheme, 'c157a79031e1', 'directory/foo')
+            nc.add(self.scheme, 'd3b07384d113', 'directory/bar')
         """
 
-        ret = []
+        cache = NamedCache()
 
         if self.cache.changed_cache_file(self.checksum):
             try:
                 self.repo.cloud.pull(
-                    [
-                        {
-                            self.remote.PARAM_CHECKSUM: self.checksum,
-                            "name": str(self),
-                        }
-                    ],
+                    NamedCache.make("local", self.checksum, str(self)),
                     jobs=jobs,
                     remote=remote,
                     show_checksums=False,
@@ -378,20 +373,19 @@ class OutputBase(object):
                 "Would you like to continue? Use '-f' to force."
             )
             if not force and not prompt.confirm(msg.format(self.path_info)):
-                raise DvcException(
+                raise CollectCacheError(
                     "unable to fully collect used cache"
                     " without cache for directory '{}'".format(self)
                 )
             else:
-                return ret
+                return cache
 
         for entry in self.dir_cache:
-            info = copy(entry)
+            checksum = entry[self.remote.PARAM_CHECKSUM]
             path_info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            info["name"] = str(path_info)
-            ret.append(info)
+            cache.add(self.scheme, checksum, str(path_info))
 
-        return ret
+        return cache
 
     def get_used_cache(self, **kwargs):
         """Get a dumpd of the given `out`, with an entry including the branch.
@@ -403,7 +397,7 @@ class OutputBase(object):
         """
 
         if not self.use_cache:
-            return []
+            return NamedCache()
 
         if not self.info:
             logger.warning(
@@ -411,14 +405,14 @@ class OutputBase(object):
                 "not be collected. Use dvc repro to get your pipeline up to "
                 "date.".format(self, self.stage)
             )
-            return []
+            return NamedCache()
 
-        ret = [{self.remote.PARAM_CHECKSUM: self.checksum, "name": str(self)}]
+        ret = NamedCache.make(self.scheme, self.checksum, str(self))
 
         if not self.is_dir_checksum:
             return ret
 
-        ret.extend(self._collect_used_dir_cache(**kwargs))
+        ret.update(self._collect_used_dir_cache(**kwargs))
 
         return ret
 

dvc/progress.py

@@ -1,9 +1,11 @@
 """Manages progress bars for dvc repo."""
 from __future__ import print_function
 import logging
+import sys
 from tqdm import tqdm
 from concurrent.futures import ThreadPoolExecutor
 from funcy import merge
+from dvc.utils import env2bool
 
 logger = logging.getLogger(__name__)
 
@@ -54,6 +56,7 @@ class Tqdm(tqdm):
         leave=False,
         bar_format=None,
         bytes=False,  # pylint: disable=W0622
+        file=None,
         **kwargs
     ):
         """
@@ -62,6 +65,9 @@ class Tqdm(tqdm):
         desc  : persists after `close()`
         level  : effective logging level for determining `disable`;
             used only if `disable` is unspecified
+        disable  : If (default: None), will be determined by logging level.
+            May be overridden to `True` due to non-TTY status.
+            Skip override by specifying env var `DVC_IGNORE_ISATTY`.
         kwargs  : anything accepted by `tqdm.tqdm()`
         """
         kwargs = kwargs.copy()
@@ -71,9 +77,19 @@ class Tqdm(tqdm):
                 unit="B", unit_scale=True, unit_divisor=1024, miniters=1
             )
             kwargs = merge(bytes_defaults, kwargs)
+        if file is None:
+            file = sys.stderr
         self.desc_persist = desc
+        # auto-disable based on `logger.level`
         if disable is None:
             disable = logger.getEffectiveLevel() > level
+        # auto-disable based on TTY
+        if (
+            not disable
+            and not env2bool("DVC_IGNORE_ISATTY")
+            and hasattr(file, "isatty")
+        ):
+            disable = not file.isatty()
         super(Tqdm, self).__init__(
             iterable=iterable,
             disable=disable,
@@ -118,7 +134,7 @@ class Tqdm(tqdm):
         if ncols_desc:
             d["ncols_desc"] = ncols_desc
         else:
-            # work-around for zero-width desc
+            # work-around for zero-width description
             d["ncols_desc"] = 1
-            d["desc"] = 0
+            d["prefix"] = ""
         return d

dvc/remote/base.py

@@ -620,14 +620,11 @@ class RemoteBASE(object):
                 # We ignore all the non-cache looking files
                 pass
 
-    def gc(self, cinfos):
-        used = self.extract_used_local_checksums(cinfos)
+    def gc(self, named_cache):
+        used = self.extract_used_local_checksums(named_cache)
 
         if self.scheme != "":
-            used |= {
-                info[self.PARAM_CHECKSUM]
-                for info in cinfos.get(self.scheme, [])
-            }
+            used.update(named_cache[self.scheme])
 
         removed = False
         for checksum in self.all():
@@ -909,10 +906,8 @@ class RemoteBASE(object):
     def _get_unpacked_dir_names(self, checksums):
         return set()
 
-    def extract_used_local_checksums(self, cinfos):
-        from dvc.remote import RemoteLOCAL
-
-        used = {info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]}
+    def extract_used_local_checksums(self, named_cache):
+        used = set(named_cache["local"])
         unpacked = self._get_unpacked_dir_names(used)
         return used | unpacked
 

dvc/remote/http.py

@@ -1,9 +1,11 @@
 from __future__ import unicode_literals
 
 import logging
+
+from funcy import cached_property
+
 from dvc.scheme import Schemes
 from dvc.utils.compat import open
-
 from dvc.progress import Tqdm
 from dvc.exceptions import DvcException
 from dvc.config import Config, ConfigError
@@ -14,6 +16,8 @@ logger = logging.getLogger(__name__)
 
 class RemoteHTTP(RemoteBASE):
     scheme = Schemes.HTTP
+    SESSION_RETRIES = 5
+    SESSION_BACKOFF_FACTOR = 0.1
     REQUEST_TIMEOUT = 10
     CHUNK_SIZE = 2 ** 16
     PARAM_CHECKSUM = "etag"
@@ -76,6 +80,24 @@ class RemoteHTTP(RemoteBASE):
 
         return etag
 
+    @cached_property
+    def _session(self):
+        import requests
+        from requests.adapters import HTTPAdapter
+        from urllib3.util.retry import Retry
+
+        session = requests.Session()
+
+        retries = Retry(
+            total=self.SESSION_RETRIES,
+            backoff_factor=self.SESSION_BACKOFF_FACTOR,
+        )
+
+        session.mount("http://", HTTPAdapter(max_retries=retries))
+        session.mount("https://", HTTPAdapter(max_retries=retries))
+
+        return session
+
     def _request(self, method, url, **kwargs):
         import requests
 
@@ -83,7 +105,7 @@ class RemoteHTTP(RemoteBASE):
         kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
 
         try:
-            return requests.request(method, url, **kwargs)
+            return self._session.request(method, url, **kwargs)
         except requests.exceptions.RequestException:
             raise DvcException("could not perform a {} request".format(method))
 

dvc/remote/local.py

@@ -1,15 +1,14 @@
 from __future__ import unicode_literals
-
-from dvc.scheme import Schemes
-from dvc.utils.compat import str, fspath_py35, open
-
 import os
 import stat
 import errno
-from shortuuid import uuid
 import logging
 from functools import partial
 
+from shortuuid import uuid
+
+from dvc.scheme import Schemes
+from dvc.utils.compat import str, fspath_py35, open
 from dvc.system import System
 from dvc.remote.base import (
     RemoteBASE,
@@ -246,31 +245,9 @@ class RemoteLOCAL(RemoteBASE):
     def open(path_info, mode="r", encoding=None):
         return open(fspath_py35(path_info), mode=mode, encoding=encoding)
 
-    def _group(self, checksum_infos, show_checksums=False):
-        by_md5 = {}
-
-        for info in checksum_infos:
-            md5 = info[self.PARAM_CHECKSUM]
-
-            if show_checksums:
-                by_md5[md5] = {"name": md5}
-                continue
-
-            name = info["name"]
-            branch = info.get("branch")
-            if branch:
-                name += "({})".format(branch)
-
-            if md5 not in by_md5.keys():
-                by_md5[md5] = {"name": name}
-            else:
-                by_md5[md5]["name"] += " " + name
-
-        return by_md5
-
     def status(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
@@ -279,8 +256,7 @@ class RemoteLOCAL(RemoteBASE):
         logger.debug(
             "Preparing to collect status from {}".format(remote.path_info)
         )
-        ret = self._group(checksum_infos, show_checksums=show_checksums) or {}
-        md5s = list(ret)
+        md5s = list(named_cache[self.scheme])
 
         logger.debug("Collecting information from local cache...")
         local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
@@ -299,6 +275,10 @@ class RemoteLOCAL(RemoteBASE):
                 )
             )
 
+        ret = {
+            checksum: {"name": checksum if show_checksums else " ".join(names)}
+            for checksum, names in named_cache[self.scheme].items()
+        }
         self._fill_statuses(ret, local_exists, remote_exists)
 
         self._log_missing_caches(ret)
@@ -338,7 +318,7 @@ class RemoteLOCAL(RemoteBASE):
 
     def _process(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
@@ -366,7 +346,7 @@ class RemoteLOCAL(RemoteBASE):
             jobs = remote.JOBS
 
         status_info = self.status(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
@@ -391,18 +371,18 @@ class RemoteLOCAL(RemoteBASE):
 
         return len(plans[0])
 
-    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def push(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
             download=False,
         )
 
-    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,

dvc/repo/__init__.py

@@ -213,6 +213,7 @@ class Repo(object):
         all_branches=False,
         with_deps=False,
         all_tags=False,
+        all_commits=False,
         remote=None,
         force=False,
         jobs=None,
@@ -231,18 +232,14 @@ class Repo(object):
             A dictionary with Schemes (representing output's location) as keys,
             and a list with the outputs' `dumpd` as values.
         """
+        from dvc.cache import NamedCache
 
-        cache = {}
-        cache["local"] = []
-        cache["s3"] = []
-        cache["gs"] = []
-        cache["hdfs"] = []
-        cache["ssh"] = []
-        cache["azure"] = []
-        cache["repo"] = []
+        cache = NamedCache()
 
         for branch in self.brancher(
-            all_branches=all_branches, all_tags=all_tags
+            all_branches=all_branches,
+            all_tags=all_tags,
+            all_commits=all_commits,
         ):
             if targets:
                 stages = []
@@ -256,18 +253,16 @@ class Repo(object):
 
             for stage in stages:
                 if stage.is_repo_import:
-                    cache["repo"] += stage.deps
+                    dep, = stage.deps
+                    cache.external[dep.repo_pair].add(dep.def_path)
                     continue
 
                 for out in stage.outs:
-                    scheme = out.path_info.scheme
                     used_cache = out.get_used_cache(
                         remote=remote, force=force, jobs=jobs
                     )
-
-                    cache[scheme].extend(
-                        dict(entry, branch=branch) for entry in used_cache
-                    )
+                    suffix = "({})".format(branch) if branch else ""
+                    cache.update(used_cache, suffix=suffix)
 
         return cache
 

dvc/repo/brancher.py

@@ -1,8 +1,14 @@
+from funcy import group_by
 from dvc.scm.tree import WorkingTree
 
 
 def brancher(  # noqa: E302
-    self, branches=None, all_branches=False, tags=None, all_tags=False
+    self,
+    branches=None,
+    all_branches=False,
+    tags=None,
+    all_tags=False,
+    all_commits=False,
 ):
     """Generator that iterates over specified revisions.
 
@@ -18,7 +24,7 @@ def brancher(  # noqa: E302
             - empty string it there is no branches to iterate over
             - "Working Tree" if there are uncommited changes in the SCM repo
     """
-    if not any([branches, all_branches, tags, all_tags]):
+    if not any([branches, all_branches, tags, all_tags, all_commits]):
         yield ""
         return
 
@@ -30,25 +36,28 @@ def brancher(  # noqa: E302
     self.tree = WorkingTree(self.root_dir)
     yield "working tree"
 
-    if all_branches:
-        branches = scm.list_branches()
+    if all_commits:
+        revs = scm.list_all_commits()
+    else:
+        if all_branches:
+            branches = scm.list_branches()
 
-    if all_tags:
-        tags = scm.list_tags()
+        if all_tags:
+            tags = scm.list_tags()
 
-    if branches is not None:
-        revs.extend(branches)
+        if branches is not None:
+            revs.extend(branches)
 
-    if tags is not None:
-        revs.extend(tags)
+        if tags is not None:
+            revs.extend(tags)
 
     # NOTE: it might be a good idea to wrap this loop in try/finally block
     # to don't leave the tree on some unexpected branch after the
     # `brancher()`, but this could cause problems on exception handling
     # code which might expect the tree on which exception was raised to
     # stay in place. This behavior is a subject to change.
-    for rev in revs:
-        self.tree = scm.get_tree(rev)
-        yield rev
+    for sha, names in group_by(scm.resolve_rev, revs).items():
+        self.tree = scm.get_tree(sha)
+        yield ", ".join(names)
 
     self.tree = saved_tree

dvc/repo/checkout.py

@@ -24,16 +24,22 @@ def get_all_files_numbers(stages):
 
 
 def _checkout(
-    self, target=None, with_deps=False, force=False, recursive=False
+    self, targets=None, with_deps=False, force=False, recursive=False
 ):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    try:
-        stages = self.collect(target, with_deps=with_deps, recursive=recursive)
-    except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
-        if not target:
-            raise
-        raise CheckoutErrorSuggestGit(target, exc)
+    stages = set()
+    targets = targets or [None]
+    for target in targets:
+        try:
+            new = self.collect(
+                target, with_deps=with_deps, recursive=recursive
+            )
+            stages.update(new)
+        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+            if not target:
+                raise
+            raise CheckoutErrorSuggestGit(target, exc)
 
     _cleanup_unused_links(self, self.stages)
     total = get_all_files_numbers(stages)

dvc/repo/fetch.py

@@ -5,6 +5,8 @@ import logging
 from dvc.config import NoRemoteError
 from dvc.exceptions import DownloadError, OutputNotFoundError
 from dvc.scm.base import CloneError
+from dvc.external_repo import external_repo
+from dvc.cache import NamedCache
 
 
 logger = logging.getLogger(__name__)
@@ -49,28 +51,53 @@ def _fetch(
 
     try:
         downloaded += self.cloud.pull(
-            used["local"], jobs, remote=remote, show_checksums=show_checksums
+            used, jobs, remote=remote, show_checksums=show_checksums
         )
     except NoRemoteError:
-        if not used["repo"] and used["local"]:
+        if not used.external and used["local"]:
             raise
-
     except DownloadError as exc:
         failed += exc.amount
 
-    for dep in used["repo"]:
-        try:
-            out = dep.fetch()
-            downloaded += out.get_files_number()
-        except DownloadError as exc:
-            failed += exc.amount
-        except (CloneError, OutputNotFoundError):
-            failed += 1
-            logger.exception(
-                "failed to fetch data for '{}'".format(dep.stage.outs[0])
-            )
+    for (repo_url, repo_rev), files in used.external.items():
+        d, f = _fetch_external(self, repo_url, repo_rev, files)
+        downloaded += d
+        failed += f
 
     if failed:
         raise DownloadError(failed)
 
     return downloaded
+
+
+def _fetch_external(self, repo_url, repo_rev, files):
+    failed = 0
+
+    cache_dir = self.cache.local.cache_dir
+    try:
+        with external_repo(repo_url, repo_rev, cache_dir=cache_dir) as repo:
+            cache = NamedCache()
+            for name in files:
+                try:
+                    out = repo.find_out_by_relpath(name)
+                except OutputNotFoundError:
+                    failed += 1
+                    logger.exception(
+                        "failed to fetch data for '{}'".format(name)
+                    )
+                    continue
+                else:
+                    cache.update(out.get_used_cache())
+
+            with repo.state:
+                try:
+                    return repo.cloud.pull(cache), failed
+                except DownloadError as exc:
+                    failed += exc.amount
+    except CloneError:
+        failed += 1
+        logger.exception(
+            "failed to fetch data for '{}'".format(", ".join(files))
+        )
+
+    return 0, failed

dvc/repo/gc.py

@@ -1,53 +1,14 @@
 from __future__ import unicode_literals
 
-import collections
 import logging
 
+from dvc.cache import NamedCache
 from . import locked
 
 
 logger = logging.getLogger(__name__)
 
 
-def _merge_cache_lists(clists):
-    merged_cache = collections.defaultdict(list)
-
-    for cache_list in clists:
-        for scheme, cache in cache_list.items():
-            for item in cache:
-                if item not in merged_cache[scheme]:
-                    merged_cache[scheme].append(item)
-
-    return merged_cache
-
-
-def _load_all_used_cache(
-    repos,
-    all_branches=False,
-    with_deps=False,
-    all_tags=False,
-    remote=None,
-    force=False,
-    jobs=None,
-):
-    clists = []
-
-    for repo in repos:
-        repo_clist = repo.used_cache(
-            targets=None,
-            all_branches=all_branches,
-            with_deps=with_deps,
-            all_tags=all_tags,
-            remote=remote,
-            force=force,
-            jobs=jobs,
-        )
-
-        clists.append(repo_clist)
-
-    return clists
-
-
 def _do_gc(typ, func, clist):
     removed = func(clist)
     if not removed:
@@ -62,6 +23,7 @@ def gc(
     remote=None,
     with_deps=False,
     all_tags=False,
+    all_commits=False,
     force=False,
     jobs=None,
     repos=None,
@@ -79,37 +41,36 @@ def gc(
             stack.enter_context(repo.lock)
             stack.enter_context(repo.state)
 
-        all_clists = _load_all_used_cache(
-            all_repos + [self],
-            all_branches=all_branches,
-            with_deps=with_deps,
-            all_tags=all_tags,
-            remote=remote,
-            force=force,
-            jobs=jobs,
-        )
+        used = NamedCache()
+        for repo in all_repos + [self]:
+            used.update(
+                repo.used_cache(
+                    all_branches=all_branches,
+                    with_deps=with_deps,
+                    all_tags=all_tags,
+                    all_commits=all_commits,
+                    remote=remote,
+                    force=force,
+                    jobs=jobs,
+                )
+            )
 
-    if len(all_clists) > 1:
-        clist = _merge_cache_lists(all_clists)
-    else:
-        clist = all_clists[0]
-
-    _do_gc("local", self.cache.local.gc, clist)
+    _do_gc("local", self.cache.local.gc, used)
 
     if self.cache.s3:
-        _do_gc("s3", self.cache.s3.gc, clist)
+        _do_gc("s3", self.cache.s3.gc, used)
 
     if self.cache.gs:
-        _do_gc("gs", self.cache.gs.gc, clist)
+        _do_gc("gs", self.cache.gs.gc, used)
 
     if self.cache.ssh:
-        _do_gc("ssh", self.cache.ssh.gc, clist)
+        _do_gc("ssh", self.cache.ssh.gc, used)
 
     if self.cache.hdfs:
-        _do_gc("hdfs", self.cache.hdfs.gc, clist)
+        _do_gc("hdfs", self.cache.hdfs.gc, used)
 
     if self.cache.azure:
-        _do_gc("azure", self.cache.azure.gc, clist)
+        _do_gc("azure", self.cache.azure.gc, used)
 
     if cloud:
-        _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, clist)
+        _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, used)

dvc/repo/get.py

@@ -21,6 +21,7 @@ logger = logging.getLogger(__name__)
 @staticmethod
 def get(url, path, out=None, rev=None):
     out = resolve_output(path, out)
+    path = path.lstrip("/")
 
     if Stage.is_valid_filename(out):
         raise GetDVCFileError()

dvc/repo/imp.py

@@ -2,5 +2,6 @@ def imp(self, url, path, out=None, rev=None):
     erepo = {"url": url}
     if rev is not None:
         erepo["rev"] = rev
+        path = path.lstrip("/")
 
     return self.imp_url(path, out=out, erepo=erepo, locked=True)

dvc/repo/pull.py

@@ -24,11 +24,7 @@ def pull(
         with_deps=with_deps,
         recursive=recursive,
     )
-    for target in targets or [None]:
-        self._checkout(
-            target=target,
-            with_deps=with_deps,
-            force=force,
-            recursive=recursive,
-        )
+    self._checkout(
+        targets=targets, with_deps=with_deps, force=force, recursive=recursive
+    )
     return processed_files_count

dvc/repo/push.py

@@ -23,5 +23,5 @@ def push(
         remote=remote,
         jobs=jobs,
         recursive=recursive,
-    )["local"]
+    )
     return self.cloud.push(used, jobs, remote=remote)

dvc/repo/status.py

@@ -76,7 +76,7 @@ def _cloud_status(
         force=True,
         remote=remote,
         jobs=jobs,
-    )["local"]
+    )
 
     ret = {}
     status_info = self.cloud.status(used, jobs, remote=remote)

dvc/scm/base.py

@@ -120,6 +120,10 @@ class Base(object):
         """Returns a list of available tags in the repo."""
         return []
 
+    def list_all_commits(self):  # pylint: disable=no-self-use
+        """Returns a list of commits in the repo."""
+        return []
+
     def install(self):
         """Adds dvc commands to SCM hooks for the repo."""
 

dvc/scm/git/__init__.py

@@ -245,6 +245,9 @@ class Git(Base):
     def list_tags(self):
         return [t.name for t in self.repo.tags]
 
+    def list_all_commits(self):
+        return [c.hexsha for c in self.repo.iter_commits("--all")]
+
     def _install_hook(self, name, cmd):
         command = (
             '[ "$3" = "0" ]'
@@ -360,6 +363,9 @@ class Git(Base):
     def get_rev(self):
         return self.repo.git.rev_parse("HEAD")
 
+    def resolve_rev(self, rev):
+        return self.repo.git.rev_parse(rev)
+
     def close(self):
         self.repo.close()
 

dvc/system.py

@@ -1,11 +1,10 @@
 from __future__ import unicode_literals
-
-from dvc.utils.compat import str, open, fspath
-
 import os
 import errno
 import shutil
 
+from dvc.utils.compat import str, open, fspath
+
 
 class System(object):
     @staticmethod

dvc/updater.py

@@ -9,7 +9,7 @@ from packaging import version
 
 from dvc import __version__
 from dvc.lock import Lock, LockError
-from dvc.utils import is_binary, boxify
+from dvc.utils import is_binary, boxify, env2bool
 
 
 logger = logging.getLogger(__name__)
@@ -45,7 +45,7 @@ class Updater(object):  # pragma: no cover
             logger.debug(msg.format(self.lock.lockfile, action))
 
     def check(self):
-        if os.getenv("CI") or os.getenv("DVC_TEST"):
+        if os.getenv("CI") or env2bool("DVC_TEST"):
             return
 
         self._with_lock(self._check, "checking")

dvc/utils/__init__.py

@@ -462,6 +462,16 @@ def relpath(path, start=os.curdir):
     return os.path.relpath(path, start)
 
 
+def env2bool(var, undefined=False):
+    """
+    undefined: return value if env var is unset
+    """
+    var = os.getenv(var, None)
+    if var is None:
+        return undefined
+    return bool(re.search("1|y|yes|true", var, flags=re.I))
+
+
 def resolve_output(inp, out):
     from dvc.utils.compat import urlparse
 

dvc/version.py

@@ -7,7 +7,7 @@ import os
 import subprocess
 
 
-_BASE_VERSION = "0.63.2"
+_BASE_VERSION = "0.64.1"
 
 
 def _generate_version(base_version):

setup.py

@@ -175,7 +175,7 @@ setup(
     ],
     packages=find_packages(exclude=["tests"]),
     include_package_data=True,
-    url="http://dataversioncontrol.com",
+    url="http://dvc.org",
     entry_points={"console_scripts": ["dvc = dvc.main:main"]},
     cmdclass={"build_py": build_py},
     zip_safe=False,

tests/__main__.py

@@ -7,7 +7,7 @@ REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 os.chdir(REPO_ROOT)
 
 os.putenv(
-    "PATH", "{}:{}".format(os.path.join(REPO_ROOT, "bin"), os.getenv("PATH"))
+    "PATH", ":".join([os.path.join(REPO_ROOT, "bin"), os.getenv("PATH")])
 )
 os.putenv("DVC_HOME", REPO_ROOT)
 

tests/conftest.py

@@ -13,6 +13,8 @@ from .basic_env import TestDirFixture, TestDvcGitFixture, TestGitFixture
 
 # Prevent updater and analytics from running their processes
 os.environ[cast_bytes_py2("DVC_TEST")] = cast_bytes_py2("true")
+# Ensure progress output even when not outputting to raw sys.stderr console
+os.environ[cast_bytes_py2("DVC_IGNORE_ISATTY")] = cast_bytes_py2("true")
 
 
 @pytest.fixture(autouse=True)

tests/func/test_checkout.py

@@ -391,14 +391,14 @@ class TestCheckoutSuggestGit(TestRepro):
     def test(self):
 
         try:
-            self.dvc.checkout(target="gitbranch")
+            self.dvc.checkout(targets=["gitbranch"])
         except DvcException as exc:
             self.assertIsInstance(exc, CheckoutErrorSuggestGit)
             self.assertIsInstance(exc.cause, StageFileDoesNotExistError)
             self.assertIsNone(exc.cause.cause)
 
         try:
-            self.dvc.checkout(target=self.FOO)
+            self.dvc.checkout(targets=[self.FOO])
         except DvcException as exc:
             self.assertIsInstance(exc, CheckoutErrorSuggestGit)
             self.assertIsInstance(exc.cause, StageFileBadNameError)
@@ -422,7 +422,7 @@ class TestCheckoutRecursiveNotDirectory(TestDvc):
         ret = main(["add", self.FOO])
         self.assertEqual(0, ret)
 
-        self.dvc.checkout(target=self.FOO + ".dvc", recursive=True)
+        self.dvc.checkout(targets=[self.FOO + ".dvc"], recursive=True)
 
 
 class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
@@ -477,7 +477,7 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
 def test_checkout_no_checksum(repo_dir, dvc_repo):
     stage = dvc_repo.run(outs=[repo_dir.FOO], no_exec=True, cmd="somecmd")
     with pytest.raises(CheckoutError):
-        dvc_repo.checkout(stage.path, force=True)
+        dvc_repo.checkout([stage.path], force=True)
     assert not os.path.exists(repo_dir.FOO)
 
 
@@ -491,7 +491,7 @@ def test_should_relink_on_checkout(link, link_test_func, repo_dir, dvc_repo):
     dvc_repo.add(repo_dir.DATA_DIR)
     dvc_repo.unprotect(repo_dir.DATA_SUB)
 
-    dvc_repo.checkout(repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
+    dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX])
 
     assert link_test_func(repo_dir.DATA_SUB)
 
@@ -504,7 +504,7 @@ def test_should_protect_on_checkout(link, dvc_repo, repo_dir):
     dvc_repo.add(repo_dir.FOO)
     dvc_repo.unprotect(repo_dir.FOO)
 
-    dvc_repo.checkout(repo_dir.FOO + Stage.STAGE_FILE_SUFFIX)
+    dvc_repo.checkout([repo_dir.FOO + Stage.STAGE_FILE_SUFFIX])
 
     assert not os.access(repo_dir.FOO, os.W_OK)
 
@@ -517,7 +517,7 @@ def test_should_relink_only_one_file_in_dir(dvc_repo, repo_dir):
 
     link_spy = spy(System.symlink)
     with patch.object(dvc_repo.cache.local, "symlink", link_spy):
-        dvc_repo.checkout(repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
+        dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX])
 
     assert link_spy.mock.call_count == 1
 
@@ -529,6 +529,6 @@ def test_should_not_relink_on_unchanged_dependency(link, dvc_repo, repo_dir):
     dvc_repo.add(repo_dir.DATA_DIR)
 
     with patch.object(dvc_repo.cache.local, "link") as mock_link:
-        dvc_repo.checkout(repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
+        dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX])
 
     assert not mock_link.called

tests/func/test_data_cloud.py

@@ -13,8 +13,10 @@ import pytest
 from mock import patch
 
 from dvc.utils.compat import str
+from dvc.utils import env2bool
 from dvc.main import main
 from dvc.config import Config
+from dvc.cache import NamedCache
 from dvc.data_cloud import DataCloud
 from dvc.remote import (
     RemoteS3,
@@ -57,11 +59,9 @@ os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE
 
 
 def _should_test_aws():
-    dvc_test_aws = os.getenv("DVC_TEST_AWS")
-    if dvc_test_aws == "true":
-        return True
-    elif dvc_test_aws == "false":
-        return False
+    do_test = env2bool("DVC_TEST_AWS", undefined=None)
+    if do_test is not None:
+        return do_test
 
     if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"):
         return True
@@ -70,8 +70,9 @@ def _should_test_aws():
 
 
 def _should_test_gcp():
-    if os.getenv("DVC_TEST_GCP") == "true":
-        return True
+    do_test = env2bool("DVC_TEST_GCP", undefined=None)
+    if do_test is not None:
+        return do_test
 
     if not os.path.exists(TEST_GCP_CREDS_FILE):
         return False
@@ -92,10 +93,9 @@ def _should_test_gcp():
 
 
 def _should_test_azure():
-    if os.getenv("DVC_TEST_AZURE") == "true":
-        return True
-    elif os.getenv("DVC_TEST_AZURE") == "false":
-        return False
+    do_test = env2bool("DVC_TEST_AZURE", undefined=None)
+    if do_test is not None:
+        return do_test
 
     return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv(
         "AZURE_STORAGE_CONNECTION_STRING"
@@ -103,10 +103,9 @@ def _should_test_azure():
 
 
 def _should_test_oss():
-    if os.getenv("DVC_TEST_OSS") == "true":
-        return True
-    elif os.getenv("DVC_TEST_OSS") == "false":
-        return False
+    do_test = env2bool("DVC_TEST_OSS", undefined=None)
+    if do_test is not None:
+        return do_test
 
     return (
         os.getenv("OSS_ENDPOINT")
@@ -116,8 +115,9 @@ def _should_test_oss():
 
 
 def _should_test_ssh():
-    if os.getenv("DVC_TEST_SSH") == "true":
-        return True
+    do_test = env2bool("DVC_TEST_SSH", undefined=None)
+    if do_test is not None:
+        return do_test
 
     # FIXME: enable on windows
     if os.name == "nt":
@@ -290,72 +290,71 @@ class TestDataCloudBase(TestDvc):
         self.assertTrue(stage is not None)
         out = stage.outs[0]
         cache = out.cache_path
-        name = str(out)
         md5 = out.checksum
-        info = {"name": name, out.remote.PARAM_CHECKSUM: md5}
+        info = out.get_used_cache()
 
         stages = self.dvc.add(self.DATA_DIR)
         self.assertEqual(len(stages), 1)
         stage_dir = stages[0]
         self.assertTrue(stage_dir is not None)
         out_dir = stage_dir.outs[0]
-        cache_dir = out.cache_path
-        name_dir = str(out)
-        md5_dir = out.checksum
-        info_dir = {"name": name_dir, out_dir.remote.PARAM_CHECKSUM: md5_dir}
+        cache_dir = out_dir.cache_path
+        name_dir = str(out_dir)
+        md5_dir = out_dir.checksum
+        info_dir = NamedCache.make(out_dir.scheme, md5_dir, name_dir)
 
         with self.cloud.repo.state:
             # Check status
-            status = self.cloud.status([info], show_checksums=True)
+            status = self.cloud.status(info, show_checksums=True)
             expected = {md5: {"name": md5, "status": STATUS_NEW}}
             self.assertEqual(status, expected)
 
-            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            status_dir = self.cloud.status(info_dir, show_checksums=True)
             expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
             self.assertEqual(status_dir, expected)
 
             # Push and check status
-            self.cloud.push([info])
+            self.cloud.push(info)
             self.assertTrue(os.path.exists(cache))
             self.assertTrue(os.path.isfile(cache))
 
-            self.cloud.push([info_dir])
+            self.cloud.push(info_dir)
             self.assertTrue(os.path.isfile(cache_dir))
 
-            status = self.cloud.status([info], show_checksums=True)
+            status = self.cloud.status(info, show_checksums=True)
             expected = {md5: {"name": md5, "status": STATUS_OK}}
             self.assertEqual(status, expected)
 
-            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            status_dir = self.cloud.status(info_dir, show_checksums=True)
             expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
             self.assertEqual(status_dir, expected)
 
             # Remove and check status
             shutil.rmtree(self.dvc.cache.local.cache_dir)
 
-            status = self.cloud.status([info], show_checksums=True)
+            status = self.cloud.status(info, show_checksums=True)
             expected = {md5: {"name": md5, "status": STATUS_DELETED}}
             self.assertEqual(status, expected)
 
-            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            status_dir = self.cloud.status(info_dir, show_checksums=True)
             expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}}
             self.assertEqual(status_dir, expected)
 
             # Pull and check status
-            self.cloud.pull([info])
+            self.cloud.pull(info)
             self.assertTrue(os.path.exists(cache))
             self.assertTrue(os.path.isfile(cache))
             with open(cache, "r") as fd:
                 self.assertEqual(fd.read(), self.FOO_CONTENTS)
 
-            self.cloud.pull([info_dir])
+            self.cloud.pull(info_dir)
             self.assertTrue(os.path.isfile(cache_dir))
 
-            status = self.cloud.status([info], show_checksums=True)
+            status = self.cloud.status(info, show_checksums=True)
             expected = {md5: {"name": md5, "status": STATUS_OK}}
             self.assertEqual(status, expected)
 
-            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            status_dir = self.cloud.status(info_dir, show_checksums=True)
             expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
             self.assertTrue(status_dir, expected)
 

tests/func/test_gc.py

@@ -1,10 +1,15 @@
+from __future__ import unicode_literals
 import os
+
+import pytest
 import configobj
 
 from git import Repo
 
+from dvc.utils.compat import pathlib
 from dvc.main import main
 from dvc.repo import Repo as DvcRepo
+from dvc.exceptions import CollectCacheError
 
 from tests.basic_env import TestDvcGit
 from tests.basic_env import TestDir
@@ -173,3 +178,53 @@ class TestGCMultipleDvcRepos(TestDvcGit):
 
         self.dvc.gc()
         self._check_cache(2)
+
+
+def test_all_commits(git, dvc_repo):
+    def add_and_commit():
+        stages = dvc_repo.add(str(testfile))
+        dvc_repo.scm.add([s.relpath for s in stages])
+        dvc_repo.scm.commit("message")
+
+    cache_dir = os.path.join(dvc_repo.root_dir, ".dvc", "cache")
+    testfile = pathlib.Path("testfile")
+
+    testfile.write_text("uncommited")
+    dvc_repo.add(str(testfile))
+
+    testfile.write_text("commited")
+    add_and_commit()
+
+    testfile.write_text("modified")
+    add_and_commit()
+
+    testfile.write_text("workspace")
+    dvc_repo.add(str(testfile))
+
+    N = _count_files(cache_dir)
+
+    dvc_repo.gc(all_commits=True)
+
+    # Only one uncommited file should go away
+    assert _count_files(cache_dir) == N - 1
+
+
+def _count_files(path):
+    return sum(len(files) for _, _, files in os.walk(path))
+
+
+def test_gc_no_dir_cache(repo_dir, dvc_repo):
+    dvc_repo.add(repo_dir.FOO)
+    dvc_repo.add(repo_dir.BAR)
+    dir_stage, = dvc_repo.add(repo_dir.DATA_DIR)
+
+    os.unlink(dir_stage.outs[0].cache_path)
+
+    with pytest.raises(CollectCacheError):
+        dvc_repo.gc()
+
+    assert _count_files(dvc_repo.cache.local.cache_dir) == 4
+
+    dvc_repo.gc(force=True)
+
+    assert _count_files(dvc_repo.cache.local.cache_dir) == 2

tests/func/test_import.py

@@ -98,3 +98,20 @@ def test_import_to_dir(dname, repo_dir, dvc_repo, erepo):
     assert stage.outs[0].fspath == os.path.abspath(dst)
     assert os.path.isdir(dname)
     assert filecmp.cmp(repo_dir.FOO, dst, shallow=False)
+
+
+def test_pull_non_workspace(git, dvc_repo, erepo):
+    src = "version"
+    dst = src
+
+    stage = dvc_repo.imp(erepo.root_dir, src, dst, rev="branch")
+    dvc_repo.scm.add([stage.relpath])
+    dvc_repo.scm.commit("imported branch")
+    dvc_repo.scm.tag("ref-to-branch")
+
+    # Ovewrite via import
+    dvc_repo.imp(erepo.root_dir, src, dst, rev="master")
+
+    os.remove(stage.outs[0].cache_path)
+    dvc_repo.fetch(all_tags=True)
+    assert os.path.exists(stage.outs[0].cache_path)

tests/func/test_repro.py

@@ -943,7 +943,7 @@ class TestReproExternalBase(TestDvc):
         self.dvc.remove(cmd_stage.path, outs_only=True)
         self.assertNotEqual(self.dvc.status([cmd_stage.path]), {})
 
-        self.dvc.checkout(cmd_stage.path, force=True)
+        self.dvc.checkout([cmd_stage.path], force=True)
         self.assertEqual(self.dvc.status([cmd_stage.path]), {})
 
 

tests/unit/remote/test_local.py

@@ -1,3 +1,4 @@
+from dvc.cache import NamedCache
 from dvc.remote.local import RemoteLOCAL
 
 
@@ -8,16 +9,11 @@ def test_status_download_optimization(mocker):
     """
     remote = RemoteLOCAL(None, {})
 
-    infos = [
-        {"name": "foo", "md5": "acbd18db4cc2f85cedef654fccc4a4d8"},
-        {"name": "bar", "md5": "37b51d194a7513e45b56f6524f2d51f2"},
-    ]
-
-    local_exists = [
-        "acbd18db4cc2f85cedef654fccc4a4d8",
-        "37b51d194a7513e45b56f6524f2d51f2",
-    ]
+    infos = NamedCache()
+    infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo")
+    infos.add("local", "37b51d194a7513e45b56f6524f2d51f2", "bar")
 
+    local_exists = list(infos["local"])
     mocker.patch.object(remote, "cache_exists", return_value=local_exists)
 
     other_remote = mocker.Mock()

tests/unit/test_progress.py

@@ -1,17 +1,37 @@
 import logging
 from dvc.progress import Tqdm
+from dvc.utils import env2bool
+import sys
+import mock
 
 
-def test_quiet(caplog, capsys):
+def test_quiet_logging(caplog, capsys):
     with caplog.at_level(logging.CRITICAL, logger="dvc"):
         for _ in Tqdm(range(10)):
             pass
         out_err = capsys.readouterr()
         assert out_err.out == ""
         assert out_err.err == ""
+
+
+def test_quiet_notty(caplog, capsys):
     with caplog.at_level(logging.INFO, logger="dvc"):
         for _ in Tqdm(range(10)):
             pass
         out_err = capsys.readouterr()
         assert out_err.out == ""
-        assert "0/10" in out_err.err
+        if env2bool("DVC_IGNORE_ISATTY"):
+            assert "0/10" in out_err.err
+        else:
+            assert out_err.err == ""
+
+
+def test_default(caplog, capsys):
+    with caplog.at_level(logging.INFO, logger="dvc"):
+        # simulate interactive terminal
+        with mock.patch.object(sys.stderr, "isatty", return_value=True):
+            for _ in Tqdm(range(10)):
+                pass
+            out_err = capsys.readouterr()
+            assert out_err.out == ""
+            assert "0/10" in out_err.err