dvc: bump to 0.63.4

This way we are able to properly utilize automatic connection pools and
not create new fds for each request, which overflows ulimit for max fds
very quickly on mac and windows. Kudos @pared for investigating 🎉

Fixes #2600

Signed-off-by: Ruslan Kuprieiev <[email protected]>

.travis.yml

@@ -19,7 +19,7 @@ stages:
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
-# Note that we specify language as ruby-2.4.0 to help us satisfy
+# Note that we specify language as ruby-2.4.3 to help us satisfy
 # fpm dependencies, as travis runs el capitan with an old ruby
 # version, that causes fpm to crash.
 jobs:
@@ -78,7 +78,7 @@ jobs:
     os: osx
     osx_image: xcode9.4
     language: ruby
-    rvm: 2.4.0
+    rvm: 2.4.3
     install:
     script: ./scripts/build_posix.sh
   - name: "Windows pkg"

dvc/__init__.py

@@ -6,10 +6,12 @@ Make your data science projects reproducible and shareable.
 
 from __future__ import unicode_literals
 
-from dvc.version import __version__  # noqa: F401
 import warnings
+
 import dvc.logger
 
+from dvc.version import __version__  # noqa: F401
+
 
 dvc.logger.setup()
 

dvc/analytics.py

@@ -10,6 +10,7 @@ import errno
 import logging
 
 from dvc import __version__
+from dvc.utils import env2bool
 
 
 logger = logging.getLogger(__name__)
@@ -229,7 +230,7 @@ class Analytics(object):
     def is_enabled(cmd=None):
         from dvc.command.daemon import CmdDaemonBase
 
-        if os.getenv("DVC_TEST"):
+        if env2bool("DVC_TEST"):
             return False
 
         if isinstance(cmd, CmdDaemonBase):

dvc/cache.py

@@ -3,6 +3,7 @@
 from __future__ import unicode_literals
 
 import os
+from collections import defaultdict
 from funcy import cached_property
 
 from dvc.utils.compat import builtin_str
@@ -103,3 +104,29 @@ class Cache(object):
     ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
     hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
     azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
+
+
+class NamedCache(object):
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(set))
+        self.repo = []
+
+    @classmethod
+    def make(cls, scheme, checksum, name):
+        cache = cls()
+        cache.add(scheme, checksum, name)
+        return cache
+
+    def __getitem__(self, key):
+        return self._items[key]
+
+    def add(self, scheme, checksum, name):
+        self._items[scheme][checksum].add(name)
+
+    def update(self, cache, suffix=""):
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme]
+            for checksum, names in src.items():
+                dst[checksum].update(names)
+
+        self.repo.extend(cache.repo)

dvc/command/checkout.py

@@ -7,16 +7,12 @@ from dvc.command.base import CmdBase, append_doc_link
 
 class CmdCheckout(CmdBase):
     def run(self):
-        if not self.args.targets:
-            self.repo.checkout(force=self.args.force)
-        else:
-            for target in self.args.targets:
-                self.repo.checkout(
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    force=self.args.force,
-                    recursive=self.args.recursive,
-                )
+        self.repo.checkout(
+            targets=self.args.targets,
+            with_deps=self.args.with_deps,
+            force=self.args.force,
+            recursive=self.args.recursive,
+        )
         return 0
 
 

dvc/data_cloud.py

@@ -45,11 +45,11 @@ class DataCloud(object):
     def _init_remote(self, remote):
         return Remote(self.repo, name=remote)
 
-    def push(self, targets, jobs=None, remote=None, show_checksums=False):
+    def push(self, cache, jobs=None, remote=None, show_checksums=False):
         """Push data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to push to the cloud.
+            cache (NamedCache): named checksums to push to the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to push to.
                 By default remote from core.remote config option is used.
@@ -57,17 +57,17 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.push(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "push"),
             show_checksums=show_checksums,
         )
 
-    def pull(self, targets, jobs=None, remote=None, show_checksums=False):
+    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
         """Pull data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to pull from the cloud.
+            cache (NamedCache): named checksums to pull from the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
                 By default remote from core.remote config option is used.
@@ -75,25 +75,25 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.pull(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "pull"),
             show_checksums=show_checksums,
         )
 
-    def status(self, targets, jobs=None, remote=None, show_checksums=False):
+    def status(self, cache, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to check status for.
+            cache (NamedCache): named checksums to check status for.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to compare
-                targets to. By default remote from core.remote config option
+                cache to. By default remote from core.remote config option
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
         remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            targets, jobs=jobs, remote=remote, show_checksums=show_checksums
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )

dvc/external_repo.py

@@ -4,8 +4,8 @@ import os
 import tempfile
 from distutils.dir_util import copy_tree
 
-from funcy import retry
 from contextlib import contextmanager
+from funcy import retry
 
 from dvc.utils import remove
 

dvc/lock.py

@@ -34,12 +34,31 @@ if is_py3:
         """
 
         def __init__(self, lockfile, tmp_dir=None):
-            lifetime = timedelta(days=365)  # Lock for good by default
+            import socket
+
             self._tmp_dir = tmp_dir
             if self._tmp_dir is not None:
                 makedirs(self._tmp_dir, exist_ok=True)
 
-            super(Lock, self).__init__(lockfile, lifetime=lifetime)
+            # NOTE: this is basically Lock.__init__ copy-paste, except that
+            # instead of using `socket.getfqdn()` we use `socket.gethostname()`
+            # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
+            # anything, which is way too slow. `gethostname()` is actually a
+            # fallback for `getfqdn()` when it is not able to resolve a
+            # canonical hostname through network. The claimfile that uses
+            # `self._hostname` is still usable, as it uses `pid` and random
+            # number to generate the resulting lock file name, which is unique
+            # enough for our application.
+            #
+            # [1] https://github.com/iterative/dvc/issues/2582
+            self._hostname = socket.gethostname()
+
+            self._lockfile = lockfile
+            self._lifetime = timedelta(days=365)  # Lock for good by default
+            self._separator = flufl.lock.SEP
+            self._set_claimfile()
+            self._owned = True
+            self._retry_errnos = []
 
         @property
         def lockfile(self):

dvc/output/base.py

@@ -5,6 +5,7 @@ from copy import copy
 
 from schema import Or, Optional
 
+from dvc.cache import NamedCache
 import dvc.prompt as prompt
 from dvc.exceptions import DvcException
 from dvc.utils.compat import str, urlparse
@@ -345,12 +346,11 @@ class OutputBase(object):
             $ echo "bar" > directory/bar
             $ dvc add directory
 
-            It will return something similar to the following list:
+            It will return a NamedCache like:
 
-            [
-                { 'path': 'directory/foo', 'md5': 'c157a79031e1', ... },
-                { 'path': 'directory/bar', 'md5': 'd3b07384d113', ... },
-            ]
+            nc = NamedCache()
+            nc.add(self.scheme, 'c157a79031e1', 'directory/foo')
+            nc.add(self.scheme, 'd3b07384d113', 'directory/bar')
         """
 
         ret = []
@@ -358,12 +358,7 @@ class OutputBase(object):
         if self.cache.changed_cache_file(self.checksum):
             try:
                 self.repo.cloud.pull(
-                    [
-                        {
-                            self.remote.PARAM_CHECKSUM: self.checksum,
-                            "name": str(self),
-                        }
-                    ],
+                    NamedCache.make("local", self.checksum, str(self)),
                     jobs=jobs,
                     remote=remote,
                     show_checksums=False,
@@ -385,13 +380,13 @@ class OutputBase(object):
             else:
                 return ret
 
+        cache = NamedCache()
         for entry in self.dir_cache:
-            info = copy(entry)
+            checksum = entry[self.remote.PARAM_CHECKSUM]
             path_info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            info["name"] = str(path_info)
-            ret.append(info)
+            cache.add(self.scheme, checksum, str(path_info))
 
-        return ret
+        return cache
 
     def get_used_cache(self, **kwargs):
         """Get a dumpd of the given `out`, with an entry including the branch.
@@ -403,7 +398,7 @@ class OutputBase(object):
         """
 
         if not self.use_cache:
-            return []
+            return NamedCache()
 
         if not self.info:
             logger.warning(
@@ -411,14 +406,14 @@ class OutputBase(object):
                 "not be collected. Use dvc repro to get your pipeline up to "
                 "date.".format(self, self.stage)
             )
-            return []
+            return NamedCache()
 
-        ret = [{self.remote.PARAM_CHECKSUM: self.checksum, "name": str(self)}]
+        ret = NamedCache.make(self.scheme, self.checksum, str(self))
 
         if not self.is_dir_checksum:
             return ret
 
-        ret.extend(self._collect_used_dir_cache(**kwargs))
+        ret.update(self._collect_used_dir_cache(**kwargs))
 
         return ret
 

dvc/progress.py

@@ -1,9 +1,11 @@
 """Manages progress bars for dvc repo."""
 from __future__ import print_function
 import logging
+import sys
 from tqdm import tqdm
 from concurrent.futures import ThreadPoolExecutor
 from funcy import merge
+from dvc.utils import env2bool
 
 logger = logging.getLogger(__name__)
 
@@ -54,6 +56,7 @@ class Tqdm(tqdm):
         leave=False,
         bar_format=None,
         bytes=False,  # pylint: disable=W0622
+        file=None,
         **kwargs
     ):
         """
@@ -62,6 +65,9 @@ class Tqdm(tqdm):
         desc  : persists after `close()`
         level  : effective logging level for determining `disable`;
             used only if `disable` is unspecified
+        disable  : If (default: None), will be determined by logging level.
+            May be overridden to `True` due to non-TTY status.
+            Skip override by specifying env var `DVC_IGNORE_ISATTY`.
         kwargs  : anything accepted by `tqdm.tqdm()`
         """
         kwargs = kwargs.copy()
@@ -71,9 +77,19 @@ class Tqdm(tqdm):
                 unit="B", unit_scale=True, unit_divisor=1024, miniters=1
             )
             kwargs = merge(bytes_defaults, kwargs)
+        if file is None:
+            file = sys.stderr
         self.desc_persist = desc
+        # auto-disable based on `logger.level`
         if disable is None:
             disable = logger.getEffectiveLevel() > level
+        # auto-disable based on TTY
+        if (
+            not disable
+            and not env2bool("DVC_IGNORE_ISATTY")
+            and hasattr(file, "isatty")
+        ):
+            disable = not file.isatty()
         super(Tqdm, self).__init__(
             iterable=iterable,
             disable=disable,
@@ -114,5 +130,11 @@ class Tqdm(tqdm):
         d = super(Tqdm, self).format_dict
         ncols = d["ncols"] or 80
         ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1
-        d["ncols_desc"] = max(ncols_desc, 0)
+        ncols_desc = max(ncols_desc, 0)
+        if ncols_desc:
+            d["ncols_desc"] = ncols_desc
+        else:
+            # work-around for zero-width desc
+            d["ncols_desc"] = 1
+            d["desc"] = 0
         return d

dvc/remote/base.py

@@ -1,17 +1,17 @@
 from __future__ import unicode_literals
 
-from dvc.ignore import DvcIgnore
-from dvc.utils.compat import str, basestring, urlparse, FileNotFoundError
-
+from operator import itemgetter
+from multiprocessing import cpu_count
 import json
 import logging
 import tempfile
 import itertools
-from operator import itemgetter
-from multiprocessing import cpu_count
 from functools import partial
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
+
+from dvc.ignore import DvcIgnore
+from dvc.utils.compat import str, basestring, urlparse, FileNotFoundError
 from dvc.remote.slow_link_detection import slow_link_guard
 
 import dvc.prompt as prompt
@@ -27,16 +27,13 @@ from dvc.state import StateNoop
 from dvc.path_info import PathInfo, URLInfo
 from dvc.utils.http import open_url
 
-
 logger = logging.getLogger(__name__)
 
-
 STATUS_OK = 1
 STATUS_MISSING = 2
 STATUS_NEW = 3
 STATUS_DELETED = 4
 
-
 STATUS_MAP = {
     # (local_exists, remote_exists)
     (True, True): STATUS_OK,
@@ -623,14 +620,11 @@ class RemoteBASE(object):
                 # We ignore all the non-cache looking files
                 pass
 
-    def gc(self, cinfos):
-        used = self.extract_used_local_checksums(cinfos)
+    def gc(self, named_cache):
+        used = self.extract_used_local_checksums(named_cache)
 
         if self.scheme != "":
-            used |= {
-                info[self.PARAM_CHECKSUM]
-                for info in cinfos.get(self.scheme, [])
-            }
+            used.update(named_cache[self.scheme])
 
         removed = False
         for checksum in self.all():
@@ -912,10 +906,8 @@ class RemoteBASE(object):
     def _get_unpacked_dir_names(self, checksums):
         return set()
 
-    def extract_used_local_checksums(self, cinfos):
-        from dvc.remote import RemoteLOCAL
-
-        used = {info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]}
+    def extract_used_local_checksums(self, named_cache):
+        used = set(named_cache["local"])
         unpacked = self._get_unpacked_dir_names(used)
         return used | unpacked
 

dvc/remote/hdfs.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import os
 import re
 import io
-import getpass
 import posixpath
 import logging
 from subprocess import Popen, PIPE
@@ -36,11 +35,7 @@ class RemoteHDFS(RemoteBASE):
 
         parsed = urlparse(url)
 
-        user = (
-            parsed.username
-            or config.get(Config.SECTION_REMOTE_USER)
-            or getpass.getuser()
-        )
+        user = parsed.username or config.get(Config.SECTION_REMOTE_USER)
 
         self.path_info = self.path_cls.from_parts(
             scheme=self.scheme,

dvc/remote/http.py

@@ -1,10 +1,11 @@
 from __future__ import unicode_literals
 
-from dvc.scheme import Schemes
-from dvc.utils.compat import open
-
 import logging
 
+from funcy import cached_property
+
+from dvc.scheme import Schemes
+from dvc.utils.compat import open
 from dvc.progress import Tqdm
 from dvc.exceptions import DvcException
 from dvc.config import Config, ConfigError
@@ -15,6 +16,8 @@ logger = logging.getLogger(__name__)
 
 class RemoteHTTP(RemoteBASE):
     scheme = Schemes.HTTP
+    SESSION_RETRIES = 5
+    SESSION_BACKOFF_FACTOR = 0.1
     REQUEST_TIMEOUT = 10
     CHUNK_SIZE = 2 ** 16
     PARAM_CHECKSUM = "etag"
@@ -77,6 +80,24 @@ class RemoteHTTP(RemoteBASE):
 
         return etag
 
+    @cached_property
+    def _session(self):
+        import requests
+        from requests.adapters import HTTPAdapter
+        from urllib3.util.retry import Retry
+
+        session = requests.Session()
+
+        retries = Retry(
+            total=self.SESSION_RETRIES,
+            backoff_factor=self.SESSION_BACKOFF_FACTOR,
+        )
+
+        session.mount("http://", HTTPAdapter(max_retries=retries))
+        session.mount("https://", HTTPAdapter(max_retries=retries))
+
+        return session
+
     def _request(self, method, url, **kwargs):
         import requests
 
@@ -84,7 +105,7 @@ class RemoteHTTP(RemoteBASE):
         kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
 
         try:
-            return requests.request(method, url, **kwargs)
+            return self._session.request(method, url, **kwargs)
         except requests.exceptions.RequestException:
             raise DvcException("could not perform a {} request".format(method))
 

dvc/remote/local.py

@@ -1,15 +1,14 @@
 from __future__ import unicode_literals
-
-from dvc.scheme import Schemes
-from dvc.utils.compat import str, fspath_py35, open
-
 import os
 import stat
 import errno
-from shortuuid import uuid
 import logging
 from functools import partial
 
+from shortuuid import uuid
+
+from dvc.scheme import Schemes
+from dvc.utils.compat import str, fspath_py35, open
 from dvc.system import System
 from dvc.remote.base import (
     RemoteBASE,
@@ -246,31 +245,9 @@ class RemoteLOCAL(RemoteBASE):
     def open(path_info, mode="r", encoding=None):
         return open(fspath_py35(path_info), mode=mode, encoding=encoding)
 
-    def _group(self, checksum_infos, show_checksums=False):
-        by_md5 = {}
-
-        for info in checksum_infos:
-            md5 = info[self.PARAM_CHECKSUM]
-
-            if show_checksums:
-                by_md5[md5] = {"name": md5}
-                continue
-
-            name = info["name"]
-            branch = info.get("branch")
-            if branch:
-                name += "({})".format(branch)
-
-            if md5 not in by_md5.keys():
-                by_md5[md5] = {"name": name}
-            else:
-                by_md5[md5]["name"] += " " + name
-
-        return by_md5
-
     def status(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
@@ -279,8 +256,7 @@ class RemoteLOCAL(RemoteBASE):
         logger.debug(
             "Preparing to collect status from {}".format(remote.path_info)
         )
-        ret = self._group(checksum_infos, show_checksums=show_checksums) or {}
-        md5s = list(ret)
+        md5s = list(named_cache[self.scheme])
 
         logger.debug("Collecting information from local cache...")
         local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
@@ -299,6 +275,10 @@ class RemoteLOCAL(RemoteBASE):
                 )
             )
 
+        ret = {
+            checksum: {"name": checksum if show_checksums else " ".join(names)}
+            for checksum, names in named_cache[self.scheme].items()
+        }
         self._fill_statuses(ret, local_exists, remote_exists)
 
         self._log_missing_caches(ret)
@@ -338,7 +318,7 @@ class RemoteLOCAL(RemoteBASE):
 
     def _process(
         self,
-        checksum_infos,
+        named_cache,
         remote,
         jobs=None,
         show_checksums=False,
@@ -366,7 +346,7 @@ class RemoteLOCAL(RemoteBASE):
             jobs = remote.JOBS
 
         status_info = self.status(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
@@ -391,18 +371,18 @@ class RemoteLOCAL(RemoteBASE):
 
         return len(plans[0])
 
-    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def push(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,
             download=False,
         )
 
-    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
-            checksum_infos,
+            named_cache,
             remote,
             jobs=jobs,
             show_checksums=show_checksums,

dvc/repo/__init__.py

@@ -231,15 +231,9 @@ class Repo(object):
             A dictionary with Schemes (representing output's location) as keys,
             and a list with the outputs' `dumpd` as values.
         """
+        from dvc.cache import NamedCache
 
-        cache = {}
-        cache["local"] = []
-        cache["s3"] = []
-        cache["gs"] = []
-        cache["hdfs"] = []
-        cache["ssh"] = []
-        cache["azure"] = []
-        cache["repo"] = []
+        cache = NamedCache()
 
         for branch in self.brancher(
             all_branches=all_branches, all_tags=all_tags
@@ -256,18 +250,15 @@ class Repo(object):
 
             for stage in stages:
                 if stage.is_repo_import:
-                    cache["repo"] += stage.deps
+                    cache.repo += stage.deps
                     continue
 
                 for out in stage.outs:
-                    scheme = out.path_info.scheme
                     used_cache = out.get_used_cache(
                         remote=remote, force=force, jobs=jobs
                     )
-
-                    cache[scheme].extend(
-                        dict(entry, branch=branch) for entry in used_cache
-                    )
+                    suffix = "({})".format(branch) if branch else ""
+                    cache.update(used_cache, suffix=suffix)
 
         return cache
 

dvc/repo/brancher.py

@@ -1,3 +1,4 @@
+from funcy import group_by
 from dvc.scm.tree import WorkingTree
 
 
@@ -47,8 +48,8 @@ def brancher(  # noqa: E302
     # `brancher()`, but this could cause problems on exception handling
     # code which might expect the tree on which exception was raised to
     # stay in place. This behavior is a subject to change.
-    for rev in revs:
-        self.tree = scm.get_tree(rev)
-        yield rev
+    for sha, names in group_by(scm.resolve_rev, revs).items():
+        self.tree = scm.get_tree(sha)
+        yield ", ".join(names)
 
     self.tree = saved_tree

dvc/repo/checkout.py

@@ -24,16 +24,22 @@ def get_all_files_numbers(stages):
 
 
 def _checkout(
-    self, target=None, with_deps=False, force=False, recursive=False
+    self, targets=None, with_deps=False, force=False, recursive=False
 ):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    try:
-        stages = self.collect(target, with_deps=with_deps, recursive=recursive)
-    except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
-        if not target:
-            raise
-        raise CheckoutErrorSuggestGit(target, exc)
+    stages = set()
+    targets = targets or [None]
+    for target in targets:
+        try:
+            new = self.collect(
+                target, with_deps=with_deps, recursive=recursive
+            )
+            stages.update(new)
+        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+            if not target:
+                raise
+            raise CheckoutErrorSuggestGit(target, exc)
 
     _cleanup_unused_links(self, self.stages)
     total = get_all_files_numbers(stages)

dvc/repo/fetch.py

@@ -49,16 +49,16 @@ def _fetch(
 
     try:
         downloaded += self.cloud.pull(
-            used["local"], jobs, remote=remote, show_checksums=show_checksums
+            used, jobs, remote=remote, show_checksums=show_checksums
         )
     except NoRemoteError:
-        if not used["repo"] and used["local"]:
+        if not used.repo and used["local"]:
             raise
 
     except DownloadError as exc:
         failed += exc.amount
 
-    for dep in used["repo"]:
+    for dep in used.repo:
         try:
             out = dep.fetch()
             downloaded += out.get_files_number()

dvc/repo/gc.py

@@ -1,53 +1,14 @@
 from __future__ import unicode_literals
 
-import collections
 import logging
 
+from dvc.cache import NamedCache
 from . import locked
 
 
 logger = logging.getLogger(__name__)
 
 
-def _merge_cache_lists(clists):
-    merged_cache = collections.defaultdict(list)
-
-    for cache_list in clists:
-        for scheme, cache in cache_list.items():
-            for item in cache:
-                if item not in merged_cache[scheme]:
-                    merged_cache[scheme].append(item)
-
-    return merged_cache
-
-
-def _load_all_used_cache(
-    repos,
-    all_branches=False,
-    with_deps=False,
-    all_tags=False,
-    remote=None,
-    force=False,
-    jobs=None,
-):
-    clists = []
-
-    for repo in repos:
-        repo_clist = repo.used_cache(
-            targets=None,
-            all_branches=all_branches,
-            with_deps=with_deps,
-            all_tags=all_tags,
-            remote=remote,
-            force=force,
-            jobs=jobs,
-        )
-
-        clists.append(repo_clist)
-
-    return clists
-
-
 def _do_gc(typ, func, clist):
     removed = func(clist)
     if not removed:
@@ -79,37 +40,35 @@ def gc(
             stack.enter_context(repo.lock)
             stack.enter_context(repo.state)
 
-        all_clists = _load_all_used_cache(
-            all_repos + [self],
-            all_branches=all_branches,
-            with_deps=with_deps,
-            all_tags=all_tags,
-            remote=remote,
-            force=force,
-            jobs=jobs,
-        )
+        used = NamedCache()
+        for repo in all_repos + [self]:
+            used.update(
+                repo.used_cache(
+                    all_branches=all_branches,
+                    with_deps=with_deps,
+                    all_tags=all_tags,
+                    remote=remote,
+                    force=force,
+                    jobs=jobs,
+                )
+            )
 
-    if len(all_clists) > 1:
-        clist = _merge_cache_lists(all_clists)
-    else:
-        clist = all_clists[0]
-
-    _do_gc("local", self.cache.local.gc, clist)
+    _do_gc("local", self.cache.local.gc, used)
 
     if self.cache.s3:
-        _do_gc("s3", self.cache.s3.gc, clist)
+        _do_gc("s3", self.cache.s3.gc, used)
 
     if self.cache.gs:
-        _do_gc("gs", self.cache.gs.gc, clist)
+        _do_gc("gs", self.cache.gs.gc, used)
 
     if self.cache.ssh:
-        _do_gc("ssh", self.cache.ssh.gc, clist)
+        _do_gc("ssh", self.cache.ssh.gc, used)
 
     if self.cache.hdfs:
-        _do_gc("hdfs", self.cache.hdfs.gc, clist)
+        _do_gc("hdfs", self.cache.hdfs.gc, used)
 
     if self.cache.azure:
-        _do_gc("azure", self.cache.azure.gc, clist)
+        _do_gc("azure", self.cache.azure.gc, used)
 
     if cloud:
-        _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, clist)
+        _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, used)

dvc/repo/pull.py

@@ -24,11 +24,7 @@ def pull(
         with_deps=with_deps,
         recursive=recursive,
     )
-    for target in targets or [None]:
-        self._checkout(
-            target=target,
-            with_deps=with_deps,
-            force=force,
-            recursive=recursive,
-        )
+    self._checkout(
+        targets=targets, with_deps=with_deps, force=force, recursive=recursive
+    )
     return processed_files_count

dvc/repo/push.py

@@ -23,5 +23,5 @@ def push(
         remote=remote,
         jobs=jobs,
         recursive=recursive,
-    )["local"]
+    )
     return self.cloud.push(used, jobs, remote=remote)

dvc/repo/status.py

@@ -76,7 +76,7 @@ def _cloud_status(
         force=True,
         remote=remote,
         jobs=jobs,
-    )["local"]
+    )
 
     ret = {}
     status_info = self.cloud.status(used, jobs, remote=remote)

dvc/scm/git/__init__.py

@@ -360,6 +360,9 @@ class Git(Base):
     def get_rev(self):
         return self.repo.git.rev_parse("HEAD")
 
+    def resolve_rev(self, rev):
+        return self.repo.git.rev_parse(rev)
+
     def close(self):
         self.repo.close()
 

dvc/system.py

@@ -1,11 +1,10 @@
 from __future__ import unicode_literals
-
-from dvc.utils.compat import str, open, fspath
-
 import os
 import errno
 import shutil
 
+from dvc.utils.compat import str, open, fspath
+
 
 class System(object):
     @staticmethod

dvc/updater.py

@@ -9,7 +9,7 @@ from packaging import version
 
 from dvc import __version__
 from dvc.lock import Lock, LockError
-from dvc.utils import is_binary, boxify
+from dvc.utils import is_binary, boxify, env2bool
 
 
 logger = logging.getLogger(__name__)
@@ -45,7 +45,7 @@ class Updater(object):  # pragma: no cover
             logger.debug(msg.format(self.lock.lockfile, action))
 
     def check(self):
-        if os.getenv("CI") or os.getenv("DVC_TEST"):
+        if os.getenv("CI") or env2bool("DVC_TEST"):
             return
 
         self._with_lock(self._check, "checking")

dvc/utils/__init__.py

@@ -462,6 +462,16 @@ def relpath(path, start=os.curdir):
     return os.path.relpath(path, start)
 
 
+def env2bool(var, undefined=False):
+    """
+    undefined: return value if env var is unset
+    """
+    var = os.getenv(var, None)
+    if var is None:
+        return undefined
+    return bool(re.search("1|y|yes|true", var, flags=re.I))
+
+
 def resolve_output(inp, out):
     from dvc.utils.compat import urlparse
 

dvc/version.py

@@ -7,7 +7,7 @@ import os
 import subprocess
 
 
-_BASE_VERSION = "0.63.0"
+_BASE_VERSION = "0.63.4"
 
 
 def _generate_version(base_version):

tests/__main__.py

@@ -7,7 +7,7 @@ REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 os.chdir(REPO_ROOT)
 
 os.putenv(
-    "PATH", "{}:{}".format(os.path.join(REPO_ROOT, "bin"), os.getenv("PATH"))
+    "PATH", ":".join([os.path.join(REPO_ROOT, "bin"), os.getenv("PATH")])
 )
 os.putenv("DVC_HOME", REPO_ROOT)
 

tests/conftest.py

@@ -13,6 +13,8 @@ from .basic_env import TestDirFixture, TestDvcGitFixture, TestGitFixture
 
 # Prevent updater and analytics from running their processes
 os.environ[cast_bytes_py2("DVC_TEST")] = cast_bytes_py2("true")
+# Ensure progress output even when not outputting to raw sys.stderr console
+os.environ[cast_bytes_py2("DVC_IGNORE_ISATTY")] = cast_bytes_py2("true")
 
 
 @pytest.fixture(autouse=True)

tests/func/test_checkout.py

@@ -391,14 +391,14 @@ class TestCheckoutSuggestGit(TestRepro):
     def test(self):
 
         try:
-            self.dvc.checkout(target="gitbranch")
+            self.dvc.checkout(targets=["gitbranch"])
         except DvcException as exc:
             self.assertIsInstance(exc, CheckoutErrorSuggestGit)
             self.assertIsInstance(exc.cause, StageFileDoesNotExistError)
             self.assertIsNone(exc.cause.cause)
 
         try:
-            self.dvc.checkout(target=self.FOO)
+            self.dvc.checkout(targets=[self.FOO])
         except DvcException as exc:
             self.assertIsInstance(exc, CheckoutErrorSuggestGit)
             self.assertIsInstance(exc.cause, StageFileBadNameError)
@@ -422,7 +422,7 @@ class TestCheckoutRecursiveNotDirectory(TestDvc):
         ret = main(["add", self.FOO])
         self.assertEqual(0, ret)
 
-        self.dvc.checkout(target=self.FOO + ".dvc", recursive=True)
+        self.dvc.checkout(targets=[self.FOO + ".dvc"], recursive=True)
 
 
 class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
@@ -477,7 +477,7 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
 def test_checkout_no_checksum(repo_dir, dvc_repo):
     stage = dvc_repo.run(outs=[repo_dir.FOO], no_exec=True, cmd="somecmd")
     with pytest.raises(CheckoutError):
-        dvc_repo.checkout(stage.path, force=True)
+        dvc_repo.checkout([stage.path], force=True)
     assert not os.path.exists(repo_dir.FOO)
 
 
@@ -491,7 +491,7 @@ def test_should_relink_on_checkout(link, link_test_func, repo_dir, dvc_repo):
     dvc_repo.add(repo_dir.DATA_DIR)
     dvc_repo.unprotect(repo_dir.DATA_SUB)
 
-    dvc_repo.checkout(repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
+    dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX])
 
     assert link_test_func(repo_dir.DATA_SUB)
 
@@ -504,7 +504,7 @@ def test_should_protect_on_checkout(link, dvc_repo, repo_dir):
     dvc_repo.add(repo_dir.FOO)
     dvc_repo.unprotect(repo_dir.FOO)
 
-    dvc_repo.checkout(repo_dir.FOO + Stage.STAGE_FILE_SUFFIX)
+    dvc_repo.checkout([repo_dir.FOO + Stage.STAGE_FILE_SUFFIX])
 
     assert not os.access(repo_dir.FOO, os.W_OK)
 
@@ -517,7 +517,7 @@ def test_should_relink_only_one_file_in_dir(dvc_repo, repo_dir):
 
     link_spy = spy(System.symlink)
     with patch.object(dvc_repo.cache.local, "symlink", link_spy):
-        dvc_repo.checkout(repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
+        dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX])
 
     assert link_spy.mock.call_count == 1
 
@@ -529,6 +529,6 @@ def test_should_not_relink_on_unchanged_dependency(link, dvc_repo, repo_dir):
     dvc_repo.add(repo_dir.DATA_DIR)
 
     with patch.object(dvc_repo.cache.local, "link") as mock_link:
-        dvc_repo.checkout(repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
+        dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX])
 
     assert not mock_link.called

tests/func/test_data_cloud.py

@@ -13,8 +13,10 @@ import pytest
 from mock import patch
 
 from dvc.utils.compat import str
+from dvc.utils import env2bool
 from dvc.main import main
 from dvc.config import Config
+from dvc.cache import NamedCache
 from dvc.data_cloud import DataCloud
 from dvc.remote import (
     RemoteS3,
@@ -57,11 +59,9 @@ os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE
 
 
 def _should_test_aws():
-    dvc_test_aws = os.getenv("DVC_TEST_AWS")
-    if dvc_test_aws == "true":
-        return True
-    elif dvc_test_aws == "false":
-        return False
+    do_test = env2bool("DVC_TEST_AWS", undefined=None)
+    if do_test is not None:
+        return do_test
 
     if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"):
         return True
@@ -70,8 +70,9 @@ def _should_test_aws():
 
 
 def _should_test_gcp():
-    if os.getenv("DVC_TEST_GCP") == "true":
-        return True
+    do_test = env2bool("DVC_TEST_GCP", undefined=None)
+    if do_test is not None:
+        return do_test
 
     if not os.path.exists(TEST_GCP_CREDS_FILE):
         return False
@@ -92,10 +93,9 @@ def _should_test_gcp():
 
 
 def _should_test_azure():
-    if os.getenv("DVC_TEST_AZURE") == "true":
-        return True
-    elif os.getenv("DVC_TEST_AZURE") == "false":
-        return False
+    do_test = env2bool("DVC_TEST_AZURE", undefined=None)
+    if do_test is not None:
+        return do_test
 
     return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv(
         "AZURE_STORAGE_CONNECTION_STRING"
@@ -103,10 +103,9 @@ def _should_test_azure():
 
 
 def _should_test_oss():
-    if os.getenv("DVC_TEST_OSS") == "true":
-        return True
-    elif os.getenv("DVC_TEST_OSS") == "false":
-        return False
+    do_test = env2bool("DVC_TEST_OSS", undefined=None)
+    if do_test is not None:
+        return do_test
 
     return (
         os.getenv("OSS_ENDPOINT")
@@ -116,8 +115,9 @@ def _should_test_oss():
 
 
 def _should_test_ssh():
-    if os.getenv("DVC_TEST_SSH") == "true":
-        return True
+    do_test = env2bool("DVC_TEST_SSH", undefined=None)
+    if do_test is not None:
+        return do_test
 
     # FIXME: enable on windows
     if os.name == "nt":
@@ -290,72 +290,71 @@ class TestDataCloudBase(TestDvc):
         self.assertTrue(stage is not None)
         out = stage.outs[0]
         cache = out.cache_path
-        name = str(out)
         md5 = out.checksum
-        info = {"name": name, out.remote.PARAM_CHECKSUM: md5}
+        info = out.get_used_cache()
 
         stages = self.dvc.add(self.DATA_DIR)
         self.assertEqual(len(stages), 1)
         stage_dir = stages[0]
         self.assertTrue(stage_dir is not None)
         out_dir = stage_dir.outs[0]
-        cache_dir = out.cache_path
-        name_dir = str(out)
-        md5_dir = out.checksum
-        info_dir = {"name": name_dir, out_dir.remote.PARAM_CHECKSUM: md5_dir}
+        cache_dir = out_dir.cache_path
+        name_dir = str(out_dir)
+        md5_dir = out_dir.checksum
+        info_dir = NamedCache.make(out_dir.scheme, md5_dir, name_dir)
 
         with self.cloud.repo.state:
             # Check status
-            status = self.cloud.status([info], show_checksums=True)
+            status = self.cloud.status(info, show_checksums=True)
             expected = {md5: {"name": md5, "status": STATUS_NEW}}
             self.assertEqual(status, expected)
 
-            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            status_dir = self.cloud.status(info_dir, show_checksums=True)
             expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
             self.assertEqual(status_dir, expected)
 
             # Push and check status
-            self.cloud.push([info])
+            self.cloud.push(info)
             self.assertTrue(os.path.exists(cache))
             self.assertTrue(os.path.isfile(cache))
 
-            self.cloud.push([info_dir])
+            self.cloud.push(info_dir)
             self.assertTrue(os.path.isfile(cache_dir))
 
-            status = self.cloud.status([info], show_checksums=True)
+            status = self.cloud.status(info, show_checksums=True)
             expected = {md5: {"name": md5, "status": STATUS_OK}}
             self.assertEqual(status, expected)
 
-            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            status_dir = self.cloud.status(info_dir, show_checksums=True)
             expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
             self.assertEqual(status_dir, expected)
 
             # Remove and check status
             shutil.rmtree(self.dvc.cache.local.cache_dir)
 
-            status = self.cloud.status([info], show_checksums=True)
+            status = self.cloud.status(info, show_checksums=True)
             expected = {md5: {"name": md5, "status": STATUS_DELETED}}
             self.assertEqual(status, expected)
 
-            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            status_dir = self.cloud.status(info_dir, show_checksums=True)
             expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}}
             self.assertEqual(status_dir, expected)
 
             # Pull and check status
-            self.cloud.pull([info])
+            self.cloud.pull(info)
             self.assertTrue(os.path.exists(cache))
             self.assertTrue(os.path.isfile(cache))
             with open(cache, "r") as fd:
                 self.assertEqual(fd.read(), self.FOO_CONTENTS)
 
-            self.cloud.pull([info_dir])
+            self.cloud.pull(info_dir)
             self.assertTrue(os.path.isfile(cache_dir))
 
-            status = self.cloud.status([info], show_checksums=True)
+            status = self.cloud.status(info, show_checksums=True)
             expected = {md5: {"name": md5, "status": STATUS_OK}}
             self.assertEqual(status, expected)
 
-            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            status_dir = self.cloud.status(info_dir, show_checksums=True)
             expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
             self.assertTrue(status_dir, expected)
 

tests/func/test_repro.py

@@ -943,7 +943,7 @@ class TestReproExternalBase(TestDvc):
         self.dvc.remove(cmd_stage.path, outs_only=True)
         self.assertNotEqual(self.dvc.status([cmd_stage.path]), {})
 
-        self.dvc.checkout(cmd_stage.path, force=True)
+        self.dvc.checkout([cmd_stage.path], force=True)
         self.assertEqual(self.dvc.status([cmd_stage.path]), {})
 
 

tests/unit/remote/test_local.py

@@ -1,3 +1,4 @@
+from dvc.cache import NamedCache
 from dvc.remote.local import RemoteLOCAL
 
 
@@ -8,16 +9,11 @@ def test_status_download_optimization(mocker):
     """
     remote = RemoteLOCAL(None, {})
 
-    infos = [
-        {"name": "foo", "md5": "acbd18db4cc2f85cedef654fccc4a4d8"},
-        {"name": "bar", "md5": "37b51d194a7513e45b56f6524f2d51f2"},
-    ]
-
-    local_exists = [
-        "acbd18db4cc2f85cedef654fccc4a4d8",
-        "37b51d194a7513e45b56f6524f2d51f2",
-    ]
+    infos = NamedCache()
+    infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo")
+    infos.add("local", "37b51d194a7513e45b56f6524f2d51f2", "bar")
 
+    local_exists = list(infos["local"])
     mocker.patch.object(remote, "cache_exists", return_value=local_exists)
 
     other_remote = mocker.Mock()

tests/unit/test_progress.py

@@ -1,17 +1,37 @@
 import logging
 from dvc.progress import Tqdm
+from dvc.utils import env2bool
+import sys
+import mock
 
 
-def test_quiet(caplog, capsys):
+def test_quiet_logging(caplog, capsys):
     with caplog.at_level(logging.CRITICAL, logger="dvc"):
         for _ in Tqdm(range(10)):
             pass
         out_err = capsys.readouterr()
         assert out_err.out == ""
         assert out_err.err == ""
+
+
+def test_quiet_notty(caplog, capsys):
     with caplog.at_level(logging.INFO, logger="dvc"):
         for _ in Tqdm(range(10)):
             pass
         out_err = capsys.readouterr()
         assert out_err.out == ""
-        assert "0/10" in out_err.err
+        if env2bool("DVC_IGNORE_ISATTY"):
+            assert "0/10" in out_err.err
+        else:
+            assert out_err.err == ""
+
+
+def test_default(caplog, capsys):
+    with caplog.at_level(logging.INFO, logger="dvc"):
+        # simulate interactive terminal
+        with mock.patch.object(sys.stderr, "isatty", return_value=True):
+            for _ in Tqdm(range(10)):
+                pass
+            out_err = capsys.readouterr()
+            assert out_err.out == ""
+            assert "0/10" in out_err.err