dvc: bump to 0.86.2

* get/import: update cmd and url arg descs with correct behavior

* get/import: update path arg desc as well (See previous commit)

* get/imp: udpate help output with simpler terms
per https://github.com/iterative/dvc/pull/3277#pullrequestreview-352650016

* api: url->URL in a docstring

* metrics show: update help output per
per https://github.com/iterative/dvc.org/pull/933#pullrequestreview-351862579

* diff: update `diff` and `metrics diff` help desc to match docs
per https://github.com/iterative/dvc.org/pull/933#pullrequestreview-356890960

* diff: update autocomplete scripts
per https://github.com/iterative/dvc/pull/3308#issuecomment-585220300

* diff: update help output again
per https://github.com/iterative/dvc.org/pull/933#pullrequestreview-356890960

* diff/metrics diff: update help output yet again
per https://github.com/iterative/dvc.org/pull/933#discussion_r378654316

Co-authored-by: Ruslan Kuprieiev <[email protected]>

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,4 +5,3 @@
 * [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect everything to be addressed. Please review them carefully and fix those that actually improve code or fix bugs.
 
 Thank you for the contribution - we'll try to review it as soon as possible. 🙏
-

.gitignore

@@ -26,7 +26,7 @@ innosetup/config.ini
 .coverage
 .coverage.*
 
-*.swp
+*.sw?
 
 pip-wheel-metadata/
 .vscode/

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: 19.3b0
+    rev: 19.10b0
     hooks:
     - id: black
       language_version: python3
@@ -9,3 +9,9 @@ repos:
     hooks:
     - id: flake8
       language_version: python3
+-   repo: https://github.com/lovesegfault/beautysh
+    rev: master
+    hooks:
+    - id: beautysh
+      language_version: python3
+      args: [-i, '2']  # 2-space indentaion

.restyled.yaml

@@ -1,7 +1,7 @@
 ---
 restylers:
   - name: black
-    image: restyled/restyler-black:v19.3b0
+    image: restyled/restyler-black:v19.10b0
     command:
     - black
     arguments: []

.travis.yml

@@ -81,6 +81,8 @@ jobs:
      - name: snapcraft
        channel: stable
        confinement: classic
+     - name: review-tools
+       channel: stable
      - name: lxd
        channel: stable
     env:

debian/changelog

@@ -2,4 +2,4 @@ dvc (0.75.b1) experimental; urgency=low
 
   * First native Debian package version.
 
- -- Dmitry Petrov <dmitry@dataversioncontrol.com>  Sat, 07 Dec 2019 09:49:28 +0300
+ -- Dmitry Petrov <dmitry@dvc.org>  Sat, 07 Dec 2019 09:49:28 +0300

debian/control

@@ -1,5 +1,5 @@
 Source: dvc
-Maintainer: Dmitry Petrov <dmitry@dataversioncontrol.com>
+Maintainer: Dmitry Petrov <dmitry@dvc.org>
 Section: python
 Priority: optional
 Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)

debian/copyright

@@ -1,10 +1,10 @@
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: DVC
-Upstream-Contact: Dmitry Petrov <dmitry@dataversioncontrol.com>
+Upstream-Contact: Dmitry Petrov <dmitry@dvc.org>
 Source: https://github.com/iterative/dvc
 
 Files: *
-Copyright: Dmitry Petrov <dmitry@dataversioncontrol.com>
+Copyright: Dmitry Petrov <dmitry@dvc.org>
 License: Apache 2.0
  On Debian systems, the full text of the Apache 2.0
  License can be found in the file

dvc/analytics.py

@@ -54,11 +54,8 @@ def is_enabled():
         return False
 
     enabled = to_bool(
-        Config(validate=False)
-        .config.get(Config.SECTION_CORE, {})
-        .get(Config.SECTION_CORE_ANALYTICS, "true")
+        Config(validate=False).get("core", {}).get("analytics", "true")
     )
-
     logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
 
     return enabled
@@ -76,7 +73,10 @@ def send(report):
     headers = {"content-type": "application/json"}
 
     with open(report, "rb") as fobj:
-        requests.post(url, data=fobj, headers=headers, timeout=5)
+        try:
+            requests.post(url, data=fobj, headers=headers, timeout=5)
+        except requests.exceptions.RequestException:
+            logger.debug("failed to send analytics report", exc_info=True)
 
     os.remove(report)
 
@@ -141,7 +141,7 @@ def _find_or_create_user_id():
 
     IDs are generated randomly with UUID.
     """
-    config_dir = Config.get_global_config_dir()
+    config_dir = Config.get_dir("global")
     fname = os.path.join(config_dir, "user_id")
     lockfile = os.path.join(config_dir, "user_id.lock")
 

dvc/api.py

@@ -1,50 +1,13 @@
-from builtins import open as builtin_open
-import importlib
 import os
-import sys
 from contextlib import contextmanager, _GeneratorContextManager as GCM
-import threading
-
-from funcy import wrap_with
-import ruamel.yaml
-from voluptuous import Schema, Required, Invalid
 
 from dvc.repo import Repo
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.external_repo import external_repo
 
 
-SUMMON_FILE_SCHEMA = Schema(
-    {
-        Required("objects"): [
-            {
-                Required("name"): str,
-                "meta": dict,
-                Required("summon"): {
-                    Required("type"): str,
-                    "deps": [str],
-                    str: object,
-                },
-            }
-        ]
-    }
-)
-SUMMON_PYTHON_SCHEMA = Schema(
-    {
-        Required("type"): "python",
-        Required("call"): str,
-        "args": dict,
-        "deps": [str],
-    }
-)
-
-
-class SummonError(DvcException):
-    pass
-
-
 class UrlNotDvcRepoError(DvcException):
-    """Thrown if given url is not a DVC repository.
+    """Thrown if given URL is not a DVC repository.
 
     Args:
         url (str): URL to the repository
@@ -120,128 +83,6 @@ def _make_repo(repo_url=None, rev=None):
         yield repo
 
 
-def summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml", args=None):
-    """Instantiate an object described in the `summon_file`."""
-    with prepare_summon(
-        name, repo=repo, rev=rev, summon_file=summon_file
-    ) as desc:
-        try:
-            summon_dict = SUMMON_PYTHON_SCHEMA(desc.obj["summon"])
-        except Invalid as exc:
-            raise SummonError(str(exc)) from exc
-
-        _args = {**summon_dict.get("args", {}), **(args or {})}
-        return _invoke_method(summon_dict["call"], _args, desc.repo.root_dir)
-
-
-@contextmanager
-def prepare_summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml"):
-    """Does a couple of things every summon needs as a prerequisite:
-    clones the repo, parses the summon file and pulls the deps.
-
-    Calling code is expected to complete the summon logic following
-    instructions stated in "summon" dict of the object spec.
-
-    Returns a SummonDesc instance, which contains references to a Repo object,
-    named object specification and resolved paths to deps.
-    """
-    with _make_repo(repo, rev=rev) as _repo:
-        _require_dvc(_repo)
-        try:
-            path = os.path.join(_repo.root_dir, summon_file)
-            obj = _get_object_spec(name, path)
-            yield SummonDesc(_repo, obj)
-        except SummonError as exc:
-            raise SummonError(
-                str(exc) + " at '{}' in '{}'".format(summon_file, repo)
-            ) from exc.__cause__
-
-
-class SummonDesc:
-    def __init__(self, repo, obj):
-        self.repo = repo
-        self.obj = obj
-        self._pull_deps()
-
-    @property
-    def deps(self):
-        return [os.path.join(self.repo.root_dir, d) for d in self._deps]
-
-    @property
-    def _deps(self):
-        return self.obj["summon"].get("deps", [])
-
-    def _pull_deps(self):
-        if not self._deps:
-            return
-
-        outs = [self.repo.find_out_by_relpath(d) for d in self._deps]
-
-        with self.repo.state:
-            for out in outs:
-                self.repo.cloud.pull(out.get_used_cache())
-                out.checkout()
-
-
-def _get_object_spec(name, path):
-    """
-    Given a summonable object's name, search for it on the given file
-    and return its description.
-    """
-    try:
-        with builtin_open(path, "r") as fobj:
-            content = SUMMON_FILE_SCHEMA(ruamel.yaml.safe_load(fobj.read()))
-            objects = [x for x in content["objects"] if x["name"] == name]
-
-        if not objects:
-            raise SummonError("No object with name '{}'".format(name))
-        elif len(objects) >= 2:
-            raise SummonError(
-                "More than one object with name '{}'".format(name)
-            )
-
-        return objects[0]
-
-    except FileNotFoundError as exc:
-        raise SummonError("Summon file not found") from exc
-    except ruamel.yaml.YAMLError as exc:
-        raise SummonError("Failed to parse summon file") from exc
-    except Invalid as exc:
-        raise SummonError(str(exc)) from exc
-
-
-@wrap_with(threading.Lock())
-def _invoke_method(call, args, path):
-    # XXX: Some issues with this approach:
-    #   * Import will pollute sys.modules
-    #   * sys.path manipulation is "theoretically" not needed,
-    #     but tests are failing for an unknown reason.
-    cwd = os.getcwd()
-
-    try:
-        os.chdir(path)
-        sys.path.insert(0, path)
-        method = _import_string(call)
-        return method(**args)
-    finally:
-        os.chdir(cwd)
-        sys.path.pop(0)
-
-
-def _import_string(import_name):
-    """Imports an object based on a string.
-    Useful to delay import to not load everything on startup.
-    Use dotted notaion in `import_name`, e.g. 'dvc.remote.gs.RemoteGS'.
-
-    :return: imported object
-    """
-    if "." in import_name:
-        module, obj = import_name.rsplit(".", 1)
-    else:
-        return importlib.import_module(import_name)
-    return getattr(importlib.import_module(module), obj)
-
-
 def _require_dvc(repo):
     if not isinstance(repo, Repo):
         raise UrlNotDvcRepoError(repo.url)

dvc/cache.py

@@ -1,25 +1,8 @@
 """Manages cache of a DVC repo."""
-import os
 from collections import defaultdict
 
 from funcy import cached_property
 
-from dvc.config import Config
-
-
-class CacheConfig(object):
-    def __init__(self, config):
-        self.config = config
-
-    def set_dir(self, dname, level=None):
-        from dvc.remote.config import RemoteConfig
-
-        configobj = self.config.get_configobj(level)
-        path = RemoteConfig.resolve_path(dname, configobj.filename)
-        self.config.set(
-            Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level
-        )
-
 
 def _make_remote_property(name):
     """
@@ -70,37 +53,22 @@ class Cache(object):
         from dvc.remote import Remote
 
         self.repo = repo
+        self.config = config = repo.config["cache"]
 
-        self.config = config = repo.config.config[Config.SECTION_CACHE]
-        local = config.get(Config.SECTION_CACHE_LOCAL)
+        local = config.get("local")
 
         if local:
-            name = Config.SECTION_REMOTE_FMT.format(local)
-            settings = repo.config.config[name]
+            settings = {"name": local}
         else:
-            default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
-            cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
-            cache_type = config.get(Config.SECTION_CACHE_TYPE)
-            protected = config.get(Config.SECTION_CACHE_PROTECTED)
-            shared = config.get(Config.SECTION_CACHE_SHARED)
-
-            settings = {
-                Config.PRIVATE_CWD: config.get(
-                    Config.PRIVATE_CWD, repo.dvc_dir
-                ),
-                Config.SECTION_REMOTE_URL: cache_dir,
-                Config.SECTION_CACHE_TYPE: cache_type,
-                Config.SECTION_CACHE_PROTECTED: protected,
-                Config.SECTION_CACHE_SHARED: shared,
-            }
+            settings = {**config, "url": config["dir"]}
 
         self.local = Remote(repo, **settings)
 
-    s3 = _make_remote_property(Config.SECTION_CACHE_S3)
-    gs = _make_remote_property(Config.SECTION_CACHE_GS)
-    ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
-    hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
-    azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
+    s3 = _make_remote_property("s3")
+    gs = _make_remote_property("gs")
+    ssh = _make_remote_property("ssh")
+    hdfs = _make_remote_property("hdfs")
+    azure = _make_remote_property("azure")
 
 
 class NamedCache(object):

dvc/cli.py

@@ -3,37 +3,40 @@ import argparse
 import logging
 import sys
 
-import dvc.command.add as add
-import dvc.command.cache as cache
-import dvc.command.checkout as checkout
-import dvc.command.commit as commit
-import dvc.command.config as config
-import dvc.command.daemon as daemon
-import dvc.command.data_sync as data_sync
-import dvc.command.destroy as destroy
-import dvc.command.diff as diff
-import dvc.command.gc as gc
-import dvc.command.get as get
-import dvc.command.get_url as get_url
-import dvc.command.imp as imp
-import dvc.command.imp_url as imp_url
-import dvc.command.init as init
-import dvc.command.install as install
-import dvc.command.lock as lock
-import dvc.command.metrics as metrics
-import dvc.command.move as move
-import dvc.command.pipeline as pipeline
-import dvc.command.remote as remote
-import dvc.command.remove as remove
-import dvc.command.repro as repro
-import dvc.command.root as root
-import dvc.command.run as run
-import dvc.command.tag as tag
-import dvc.command.unprotect as unprotect
-import dvc.command.update as update
-import dvc.command.version as version
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import DvcParserError
+from .command import (
+    add,
+    cache,
+    checkout,
+    commit,
+    config,
+    daemon,
+    data_sync,
+    destroy,
+    diff,
+    gc,
+    get,
+    get_url,
+    imp,
+    imp_url,
+    init,
+    install,
+    ls,
+    lock,
+    metrics,
+    move,
+    pipeline,
+    remote,
+    remove,
+    repro,
+    root,
+    run,
+    tag,
+    unprotect,
+    update,
+    version,
+)
+from .command.base import fix_subparsers
+from .exceptions import DvcParserError
 
 
 logger = logging.getLogger(__name__)
@@ -60,6 +63,7 @@ COMMANDS = [
     metrics,
     install,
     root,
+    ls,
     lock,
     pipeline,
     daemon,
@@ -89,7 +93,7 @@ class DvcParser(argparse.ArgumentParser):
 
 class VersionAction(argparse.Action):  # pragma: no cover
     # pylint: disable=too-few-public-methods
-    """Shows dvc version and exits."""
+    """Shows DVC version and exits."""
 
     def __call__(self, parser, namespace, values, option_string=None):
         from dvc import __version__

dvc/command/add.py

@@ -1,11 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.exceptions import DvcException
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
-
+from dvc.command.base import append_doc_link, CmdBase
+from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
 logger = logging.getLogger(__name__)
 

dvc/command/base.py

@@ -35,7 +35,7 @@ class CmdBase(object):
         self.repo = Repo()
         self.config = self.repo.config
         self.args = args
-        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        hardlink_lock = self.config["core"].get("hardlink_lock", False)
         updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
         updater.check()
 

dvc/command/cache.py

@@ -1,18 +1,13 @@
 import argparse
 
-from dvc.cache import CacheConfig
-from dvc.command.base import append_doc_link
-from dvc.command.base import fix_subparsers
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
 
 
 class CmdCacheDir(CmdConfig):
-    def __init__(self, args):
-        super().__init__(args)
-        self.cache_config = CacheConfig(self.config)
-
     def run(self):
-        self.cache_config.set_dir(self.args.value, level=self.args.level)
+        with self.config.edit(level=self.args.level) as edit:
+            edit["cache"]["dir"] = self.args.value
         return 0
 
 

dvc/command/commit.py

@@ -47,7 +47,7 @@ def add_parser(subparsers, parent_parser):
         "--force",
         action="store_true",
         default=False,
-        help="Commit even if checksums for dependencies/outputs changed.",
+        help="Commit even if hash value for dependencies/outputs changed.",
     )
     commit_parser.add_argument(
         "-d",

dvc/command/config.py

@@ -1,10 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
-from dvc.config import Config
-
+from dvc.command.base import append_doc_link, CmdBaseNoRepo
+from dvc.config import Config, ConfigError
 
 logger = logging.getLogger(__name__)
 
@@ -18,17 +16,21 @@ class CmdConfig(CmdBaseNoRepo):
     def run(self):
         section, opt = self.args.name.lower().strip().split(".", 1)
 
-        if self.args.unset:
-            self.config.unset(section, opt, level=self.args.level)
-        elif self.args.value is None:
-            logger.info(self.config.get(section, opt, level=self.args.level))
-        else:
-            self.config.set(
-                section, opt, self.args.value, level=self.args.level
-            )
+        if self.args.value is None and not self.args.unset:
+            conf = self.config.load_one(self.args.level)
+            self._check(conf, section, opt)
+            logger.info(conf[section][opt])
+            return 0
 
-        is_write = self.args.unset or self.args.value is not None
-        if is_write and self.args.name == "cache.type":
+        with self.config.edit(self.args.level) as conf:
+            if self.args.unset:
+                self._check(conf, section, opt)
+                del conf[section][opt]
+            else:
+                self._check(conf, section)
+                conf[section][opt] = self.args.value
+
+        if self.args.name == "cache.type":
             logger.warning(
                 "You have changed the 'cache.type' option. This doesn't update"
                 " any existing workspace file links, but it can be done with:"
@@ -37,30 +39,39 @@ class CmdConfig(CmdBaseNoRepo):
 
         return 0
 
+    def _check(self, conf, section, opt=None):
+        if section not in conf:
+            msg = "section {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+
+        if opt and opt not in conf[section]:
+            msg = "option {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+
 
 parent_config_parser = argparse.ArgumentParser(add_help=False)
 parent_config_parser.add_argument(
     "--global",
     dest="level",
     action="store_const",
-    const=Config.LEVEL_GLOBAL,
+    const="global",
     help="Use global config.",
 )
 parent_config_parser.add_argument(
     "--system",
     dest="level",
     action="store_const",
-    const=Config.LEVEL_SYSTEM,
+    const="system",
     help="Use system config.",
 )
 parent_config_parser.add_argument(
     "--local",
     dest="level",
     action="store_const",
-    const=Config.LEVEL_LOCAL,
+    const="local",
     help="Use local config.",
 )
-parent_config_parser.set_defaults(level=Config.LEVEL_REPO)
+parent_config_parser.set_defaults(level="repo")
 
 
 def add_parser(subparsers, parent_parser):

dvc/command/daemon.py

@@ -16,9 +16,7 @@ class CmdDaemonUpdater(CmdDaemonBase):
         root_dir = Repo.find_root()
         dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
         config = Config(dvc_dir, validate=False)
-        hardlink_lock = config.config.get("core", {}).get(
-            "hardlink_lock", False
-        )
+        hardlink_lock = config.get("core", {}).get("hardlink_lock", False)
         updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
         updater.fetch(detach=False)
 

dvc/command/data_sync.py

@@ -148,7 +148,8 @@ def add_parser(subparsers, _parent_parser):
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
-    PUSH_HELP = "Push data files to a DVC remote storage."
+    PUSH_HELP = "Uploads files or directories tracked by DVC"
+    " to remote storage."
 
     push_parser = subparsers.add_parser(
         "push",
@@ -191,7 +192,8 @@ def add_parser(subparsers, _parent_parser):
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
-    FETCH_HELP = "Fetch data files from a DVC remote storage."
+    FETCH_HELP = "Get files or directories tracked by DVC from remote storage"
+    " into the cache."
 
     fetch_parser = subparsers.add_parser(
         "fetch",

dvc/command/diff.py

@@ -1,12 +1,11 @@
 import argparse
+import json
 import logging
+import os
 
-import humanize
-import inflect
-from funcy import compact
+import colorama
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 
@@ -15,156 +14,144 @@ logger = logging.getLogger(__name__)
 
 class CmdDiff(CmdBase):
     @staticmethod
-    def _print_size(size):
-        if size < 0:
-            change = "decreased by {}"
-        elif size > 0:
-            change = "increased by {}"
-        else:
-            change = "not changed"
-        natur_size = humanize.naturalsize(abs(size))
-        return change.format(natur_size)
+    def _format(diff):
+        """
+        Given a diff structure, generate a string of paths separated
+        by new lines and grouped together by their state.
 
-    @staticmethod
-    def _get_md5_string(sign, file_name, checksum):
-        sample_msg = ""
-        if file_name:
-            sample_msg = "{}{} with md5 {}\n"
-            sample_msg = sample_msg.format(sign, file_name, checksum)
-        return sample_msg
+        A group's header is colored and its entries are sorted to enhance
+        readability, for example:
 
-    @classmethod
-    def _get_dir_changes(cls, dct):
-        import dvc.repo.diff as diff
+            Added:
+                another_file.txt
+                backup.tar
+                dir/
+                dir/1
 
-        engine = inflect.engine()
-        changes_msg = (
-            "{} {} untouched, {} {} modified, {} {} added, "
-            "{} {} deleted, size was {}"
+        An example of a diff formatted when entries contain checksums:
+
+            Added:
+                d3b07384 foo
+
+            Modified:
+                c157a790..f98bf6f1 bar
+
+        If a group has no entries, it won't be included in the result.
+
+        At the bottom, include a summary with the number of files per state.
+        """
+
+        def _digest(checksum):
+            if type(checksum) is str:
+                return checksum[0:8]
+            return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
+
+        colors = {
+            "added": colorama.Fore.GREEN,
+            "modified": colorama.Fore.YELLOW,
+            "deleted": colorama.Fore.RED,
+        }
+
+        summary = {}
+        groups = []
+
+        for state in ["added", "deleted", "modified"]:
+            summary[state] = 0
+            entries = diff[state]
+
+            if not entries:
+                continue
+
+            content = []
+
+            for entry in entries:
+                path = entry["path"]
+                checksum = entry.get("checksum")
+                summary[state] += 1 if not path.endswith(os.sep) else 0
+                content.append(
+                    "{space}{checksum}{separator}{path}".format(
+                        space="    ",
+                        checksum=_digest(checksum) if checksum else "",
+                        separator="  " if checksum else "",
+                        path=entry["path"],
+                    )
+                )
+
+            groups.append(
+                "{color}{header}{nc}:\n{content}".format(
+                    color=colors[state],
+                    header=state.capitalize(),
+                    nc=colorama.Fore.RESET,
+                    content="\n".join(content),
+                )
+            )
+
+        groups.append(
+            "files summary: {added} added, {deleted} deleted,"
+            " {modified} modified".format_map(summary)
         )
-        changes_msg = changes_msg.format(
-            dct[diff.DIFF_IDENT],
-            engine.plural("file", dct[diff.DIFF_IDENT]),
-            dct[diff.DIFF_CHANGE],
-            engine.plural("file", dct[diff.DIFF_CHANGE]),
-            dct[diff.DIFF_NEW],
-            engine.plural("file", dct[diff.DIFF_NEW]),
-            dct[diff.DIFF_DEL],
-            engine.plural("file", dct[diff.DIFF_DEL]),
-            cls._print_size(dct[diff.DIFF_SIZE]),
-        )
-        return changes_msg
 
-    @classmethod
-    def _get_file_changes(cls, dct):
-        import dvc.repo.diff as diff
-
-        if (
-            dct.get(diff.DIFF_OLD_FILE)
-            and dct.get(diff.DIFF_NEW_FILE)
-            and dct[diff.DIFF_SIZE] == 0
-        ):
-            msg = "file size was not changed"
-        elif dct.get(diff.DIFF_NEW_FILE):
-            msg = "added file with size {}".format(
-                humanize.naturalsize(dct[diff.DIFF_SIZE])
-            )
-        elif dct.get(diff.DIFF_OLD_FILE):
-            msg = "deleted file with size {}".format(
-                humanize.naturalsize(abs(dct[diff.DIFF_SIZE]))
-            )
-        else:
-            msg = "file was modified, file size {}".format(
-                cls._print_size(dct[diff.DIFF_SIZE])
-            )
-        return msg
-
-    @classmethod
-    def _get_royal_changes(cls, dct):
-        import dvc.repo.diff as diff
-
-        if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
-            if dct.get("is_dir"):
-                return cls._get_dir_changes(dct)
-            else:
-                return cls._get_file_changes(dct)
-        return "size is ?"
-
-    @classmethod
-    def _show(cls, diff_dct):
-        import dvc.repo.diff as diff
-
-        msg = "dvc diff from {} to {}".format(
-            diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
-        )
-        if diff_dct.get(diff.DIFF_EQUAL):
-            logger.info(msg)
-            return
-        for dct in diff_dct[diff.DIFF_LIST]:
-            msg += "\n\ndiff for '{}'\n".format(dct[diff.DIFF_TARGET])
-            msg += cls._get_md5_string(
-                "-",
-                dct.get(diff.DIFF_OLD_FILE),
-                dct.get(diff.DIFF_OLD_CHECKSUM),
-            )
-            msg += cls._get_md5_string(
-                "+",
-                dct.get(diff.DIFF_NEW_FILE),
-                dct.get(diff.DIFF_NEW_CHECKSUM),
-            )
-            msg += "\n"
-            msg += cls._get_royal_changes(dct)
-        logger.info(msg)
-        return msg
+        return "\n\n".join(groups)
 
     def run(self):
         try:
-            msg = self.repo.diff(
-                self.args.a_ref, target=self.args.target, b_ref=self.args.b_ref
-            )
-            self._show(msg)
+            diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
+
+            if not any(diff.values()):
+                return 0
+
+            if not self.args.checksums:
+                for _, entries in diff.items():
+                    for entry in entries:
+                        del entry["checksum"]
+
+            if self.args.show_json:
+                res = json.dumps(diff)
+            else:
+                res = self._format(diff)
+
+            logger.info(res)
+
         except DvcException:
-            msg = "failed to get 'diff {}'"
-            args = " ".join(
-                compact([self.args.target, self.args.a_ref, self.args.b_ref])
-            )
-            msg = msg.format(args)
-            logger.exception(msg)
+            logger.exception("failed to get diff")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     DIFF_DESCRIPTION = (
-        "Show diff of a data file or a directory that is under DVC control.\n"
-        "Some basic statistics summary, how many files were deleted/changed."
+        "Show changes between commits in the DVC repository,"
+        " or between a commit and the workspace."
+        "Prints the list of paths added, modified, or deleted."
     )
-    DIFF_HELP = "Show a diff of a DVC controlled data file or a directory."
     diff_parser = subparsers.add_parser(
         "diff",
         parents=[parent_parser],
         description=append_doc_link(DIFF_DESCRIPTION, "diff"),
-        help=DIFF_HELP,
+        help=DIFF_DESCRIPTION,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     diff_parser.add_argument(
-        "-t",
-        "--target",
-        help=(
-            "Source path to a data file or directory. Default None. "
-            "If not specified, compares all files and directories "
-            "that are under DVC control in the current working space."
-        ),
+        "a_rev",
+        help="Old Git commit to compare (defaults to HEAD)",
+        nargs="?",
+        default="HEAD",
     )
     diff_parser.add_argument(
-        "a_ref", help="Git reference from which diff calculates"
-    )
-    diff_parser.add_argument(
-        "b_ref",
-        help=(
-            "Git reference until which diff calculates, if omitted "
-            "diff shows the difference between current HEAD and a_ref"
-        ),
+        "b_rev",
+        help=("New Git commit to compare (defaults to the current workspace)"),
         nargs="?",
     )
+    diff_parser.add_argument(
+        "--show-json",
+        help="Format the output into a JSON",
+        action="store_true",
+        default=False,
+    )
+    diff_parser.add_argument(
+        "--checksums",
+        help="Display hash value for each entry",
+        action="store_true",
+        default=False,
+    )
     diff_parser.set_defaults(func=CmdDiff)

dvc/command/gc.py

@@ -52,10 +52,10 @@ class CmdGC(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    GC_HELP = "Collect unused data from DVC cache or a remote storage."
+    GC_HELP = "Garbage collect unused objects from cache or remote storage."
     GC_DESCRIPTION = (
-        "Deletes all files in the cache or a remote which are not in\n"
-        "use by the specified git references (defaults to just HEAD)."
+        "Removes all files in the cache or a remote which are not in\n"
+        "use by the specified Git revisions (defaults to just HEAD)."
     )
     gc_parser = subparsers.add_parser(
         "gc",

dvc/command/get.py

@@ -51,9 +51,7 @@ class CmdGet(CmdBaseNoRepo):
 
 
 def add_parser(subparsers, parent_parser):
-    GET_HELP = (
-        "Download file or directory from any DVC project or Git repository."
-    )
+    GET_HELP = "Download file or directory tracked by DVC or by Git."
     get_parser = subparsers.add_parser(
         "get",
         parents=[parent_parser],
@@ -62,18 +60,16 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     get_parser.add_argument(
-        "url",
-        help="Location of DVC project or Git repository to download from",
+        "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the project or repository",
+        "path", help="Path to a file or directory within the repository"
     )
     get_parser.add_argument(
         "-o", "--out", nargs="?", help="Destination path to download files to"
     )
     get_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
+        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
     )
     get_parser.add_argument(
         "--show-url",

dvc/command/imp.py

@@ -30,7 +30,7 @@ class CmdImport(CmdBase):
 
 def add_parser(subparsers, parent_parser):
     IMPORT_HELP = (
-        "Download file or directory from any DVC project or Git repository "
+        "Download file or directory tracked by DVC or by Git "
         "into the workspace, and track it."
     )
 
@@ -42,17 +42,15 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawTextHelpFormatter,
     )
     import_parser.add_argument(
-        "url",
-        help="Location of DVC project or Git repository to download from",
+        "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the project or repository",
+        "path", help="Path to a file or directory within the repository"
     )
     import_parser.add_argument(
         "-o", "--out", nargs="?", help="Destination path to download files to"
     )
     import_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
+        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
     )
     import_parser.set_defaults(func=CmdImport)

dvc/command/ls.py

@@ -0,0 +1,68 @@
+import argparse
+import logging
+
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
+from dvc.exceptions import DvcException
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdList(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            nodes = Repo.ls(
+                self.args.url,
+                self.args.target,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+                outs_only=self.args.outs_only,
+            )
+            if nodes:
+                logger.info("\n".join(nodes))
+            return 0
+        except DvcException:
+            logger.exception("failed to list '{}'".format(self.args.url))
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    LIST_HELP = "List files and DVC outputs in the repo."
+    list_parser = subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(LIST_HELP, "list"),
+        help=LIST_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    list_parser.add_argument(
+        "url",
+        help="Supported urls:\n"
+        "/path/to/file\n"
+        "/path/to/directory\n"
+        "C:\\\\path\\to\\file\n"
+        "C:\\\\path\\to\\directory\n"
+        "https://github.com/path/to/repo\n"
+        "[email protected]:path/to/repo.git\n",
+    )
+    list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        help="Recursively list files.",
+    )
+    list_parser.add_argument(
+        "--outs-only", action="store_true", help="Show only DVC outputs."
+    )
+    list_parser.add_argument(
+        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
+    )
+    list_parser.add_argument(
+        "target",
+        nargs="?",
+        help="Path to directory within the repository to list outputs for",
+    )
+    list_parser.set_defaults(func=CmdList)

dvc/command/metrics.py

@@ -136,8 +136,8 @@ class CmdMetricsDiff(CmdBase):
     def run(self):
         try:
             diff = self.repo.metrics.diff(
-                a_ref=self.args.a_ref,
-                b_ref=self.args.b_ref,
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
                 targets=self.args.targets,
                 typ=self.args.type,
                 xpath=self.args.xpath,
@@ -273,7 +273,8 @@ def add_parser(subparsers, parent_parser):
     metrics_remove_parser.add_argument("path", help="Path to a metric file.")
     metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
 
-    METRICS_DIFF_HELP = "Output metric values."
+    METRICS_DIFF_HELP = "Show changes in metrics between commits"
+    " in the DVC repository, or between a commit and the workspace."
     metrics_diff_parser = metrics_subparsers.add_parser(
         "diff",
         parents=[parent_parser],
@@ -282,20 +283,12 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_diff_parser.add_argument(
-        "a_ref",
-        nargs="?",
-        help=(
-            "Git reference from which diff is calculated. "
-            "If omitted `HEAD`(latest commit) is used."
-        ),
+        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
     )
     metrics_diff_parser.add_argument(
-        "b_ref",
+        "b_rev",
         nargs="?",
-        help=(
-            "Git reference to which diff is calculated. "
-            "If omitted current working tree is used."
-        ),
+        help=("New Git commit to compare (defaults to the current workspace)"),
     )
     metrics_diff_parser.add_argument(
         "--targets",

dvc/command/pipeline.py

@@ -31,45 +31,50 @@ class CmdPipelineShow(CmdBase):
             else:
                 logger.info(stage.path_in_repo)
 
-    def __build_graph(self, target, commands, outs):
+    def _build_graph(self, target, commands, outs):
         import networkx
         from dvc.stage import Stage
         from dvc.repo.graph import get_pipeline
 
-        stage = Stage.load(self.repo, target)
-        G = get_pipeline(self.repo.pipelines, stage)
+        target_stage = Stage.load(self.repo, target)
+        G = get_pipeline(self.repo.pipelines, target_stage)
 
-        nodes = []
-        for stage in G:
+        nodes = set()
+        for stage in networkx.dfs_preorder_nodes(G, target_stage):
             if commands:
                 if stage.cmd is None:
                     continue
-                nodes.append(stage.cmd)
+                nodes.add(stage.cmd)
             elif outs:
                 for out in stage.outs:
-                    nodes.append(str(out))
+                    nodes.add(str(out))
+                for dep in stage.deps:
+                    nodes.add(str(dep))
             else:
-                nodes.append(stage.relpath)
+                nodes.add(stage.relpath)
 
         edges = []
-        for from_stage, to_stage in G.edges():
-            if commands:
-                if to_stage.cmd is None:
-                    continue
-                edges.append((from_stage.cmd, to_stage.cmd))
-            elif outs:
-                for from_out in from_stage.outs:
-                    for to_out in to_stage.outs:
-                        edges.append((str(from_out), str(to_out)))
-            else:
-                edges.append((from_stage.relpath, to_stage.relpath))
 
-        return nodes, edges, networkx.is_tree(G)
+        if outs:
+            for stage in networkx.dfs_preorder_nodes(G, target_stage):
+                for dep in stage.deps:
+                    for out in stage.outs:
+                        edges.append((str(out), str(dep)))
+        else:
+            for from_stage, to_stage in networkx.dfs_edges(G, target_stage):
+                if commands:
+                    if to_stage.cmd is None:
+                        continue
+                    edges.append((from_stage.cmd, to_stage.cmd))
+                else:
+                    edges.append((from_stage.relpath, to_stage.relpath))
+
+        return list(nodes), edges, networkx.is_tree(G)
 
     def _show_ascii(self, target, commands, outs):
         from dvc.dagascii import draw
 
-        nodes, edges, _ = self.__build_graph(target, commands, outs)
+        nodes, edges, _ = self._build_graph(target, commands, outs)
 
         if not nodes:
             return
@@ -79,7 +84,7 @@ class CmdPipelineShow(CmdBase):
     def _show_dependencies_tree(self, target, commands, outs):
         from treelib import Tree
 
-        nodes, edges, is_tree = self.__build_graph(target, commands, outs)
+        nodes, edges, is_tree = self._build_graph(target, commands, outs)
         if not nodes:
             return
         if not is_tree:
@@ -100,12 +105,12 @@ class CmdPipelineShow(CmdBase):
             observe_list.pop(0)
         tree.show()
 
-    def __write_dot(self, target, commands, outs):
+    def _write_dot(self, target, commands, outs):
         import io
         import networkx
         from networkx.drawing.nx_pydot import write_dot
 
-        _, edges, _ = self.__build_graph(target, commands, outs)
+        _, edges, _ = self._build_graph(target, commands, outs)
         edges = [edge[::-1] for edge in edges]
 
         simple_g = networkx.DiGraph()
@@ -126,9 +131,7 @@ class CmdPipelineShow(CmdBase):
                         target, self.args.commands, self.args.outs
                     )
                 elif self.args.dot:
-                    self.__write_dot(
-                        target, self.args.commands, self.args.outs
-                    )
+                    self._write_dot(target, self.args.commands, self.args.outs)
                 elif self.args.tree:
                     self._show_dependencies_tree(
                         target, self.args.commands, self.args.outs

dvc/command/remote.py

@@ -1,71 +1,99 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import fix_subparsers
+from dvc.config import ConfigError
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.remote.config import RemoteConfig
+from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
 
-class CmdRemoteConfig(CmdConfig):
+class CmdRemote(CmdConfig):
     def __init__(self, args):
         super().__init__(args)
-        self.remote_config = RemoteConfig(self.config)
+
+        if getattr(self.args, "name", None):
+            self.args.name = self.args.name.lower()
+
+    def _check_exists(self, conf):
+        if self.args.name not in conf["remote"]:
+            raise ConfigError(
+                "remote '{}' doesn't exists.".format(self.args.name)
+            )
 
 
-class CmdRemoteAdd(CmdRemoteConfig):
+class CmdRemoteAdd(CmdRemote):
     def run(self):
         if self.args.default:
             logger.info(
                 "Setting '{}' as a default remote.".format(self.args.name)
             )
-        self.remote_config.add(
-            self.args.name,
-            self.args.url,
-            force=self.args.force,
-            default=self.args.default,
-            level=self.args.level,
-        )
+
+        with self.config.edit(self.args.level) as conf:
+            if self.args.name in conf["remote"] and not self.args.force:
+                raise ConfigError(
+                    "remote '{}' already exists. Use `-f|--force` to "
+                    "overwrite it.".format(self.args.name)
+                )
+
+            conf["remote"][self.args.name] = {"url": self.args.url}
+            if self.args.default:
+                conf["core"]["remote"] = self.args.name
+
         return 0
 
 
-class CmdRemoteRemove(CmdRemoteConfig):
+class CmdRemoteRemove(CmdRemote):
     def run(self):
-        self.remote_config.remove(self.args.name, level=self.args.level)
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            del conf["remote"][self.args.name]
+
+        # Remove core.remote refs to this remote in any shadowing configs
+        for level in reversed(self.config.LEVELS):
+            with self.config.edit(level) as conf:
+                if conf["core"].get("remote") == self.args.name:
+                    del conf["core"]["remote"]
+
+            if level == self.args.level:
+                break
+
         return 0
 
 
-class CmdRemoteModify(CmdRemoteConfig):
+class CmdRemoteModify(CmdRemote):
     def run(self):
-        self.remote_config.modify(
-            self.args.name,
-            self.args.option,
-            self.args.value,
-            level=self.args.level,
-        )
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            conf["remote"][self.args.name][self.args.option] = self.args.value
         return 0
 
 
-class CmdRemoteDefault(CmdRemoteConfig):
+class CmdRemoteDefault(CmdRemote):
     def run(self):
+
         if self.args.name is None and not self.args.unset:
-            name = self.remote_config.get_default(level=self.args.level)
-            print(name)
+            conf = self.config.load_one(self.args.level)
+            try:
+                print(conf["core"]["remote"])
+            except KeyError:
+                logger.info("No default remote set")
+                return 1
         else:
-            self.remote_config.set_default(
-                self.args.name, unset=self.args.unset, level=self.args.level
-            )
+            with self.config.edit(self.args.level) as conf:
+                if self.args.unset:
+                    conf["core"].pop("remote", None)
+                else:
+                    conf["core"]["remote"] = self.args.name
         return 0
 
 
-class CmdRemoteList(CmdRemoteConfig):
+class CmdRemoteList(CmdRemote):
     def run(self):
-        for name, url in self.remote_config.list(
-            level=self.args.level
-        ).items():
-            logger.info("{}\t{}".format(name, url))
+        conf = self.config.load_one(self.args.level)
+        for name, conf in conf["remote"].items():
+            logger.info("{}\t{}".format(name, conf["url"]))
         return 0
 
 
@@ -99,7 +127,9 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser.add_argument("name", help="Name of the remote")
     remote_add_parser.add_argument(
         "url",
-        help="URL. See full list of supported urls at " "man.dvc.org/remote",
+        help="URL. See full list of supported urls at {}".format(
+            format_link("https://man.dvc.org/remote")
+        ),
     )
     remote_add_parser.add_argument(
         "-d",

dvc/command/update.py

@@ -14,7 +14,7 @@ class CmdUpdate(CmdBase):
         ret = 0
         for target in self.args.targets:
             try:
-                self.repo.update(target)
+                self.repo.update(target, self.args.rev)
             except DvcException:
                 logger.exception("failed to update '{}'.".format(target))
                 ret = 1
@@ -33,4 +33,7 @@ def add_parser(subparsers, parent_parser):
     update_parser.add_argument(
         "targets", nargs="+", help="DVC-files to update."
     )
+    update_parser.add_argument(
+        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+    )
     update_parser.set_defaults(func=CmdUpdate)

dvc/config.py

@@ -1,17 +1,17 @@
 """DVC config objects."""
-
-import copy
-import errno
+from contextlib import contextmanager
 import logging
 import os
 import re
+from urllib.parse import urlparse
 
+from funcy import cached_property, re_find, walk_values, compact
 import configobj
-from voluptuous import Schema, Required, Optional, Invalid
-from voluptuous import All, Any, Lower, Range, Coerce, Match
+from voluptuous import Schema, Optional, Invalid, ALLOW_EXTRA
+from voluptuous import All, Any, Lower, Range, Coerce
 
-from dvc.exceptions import DvcException
-from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -24,15 +24,7 @@ class ConfigError(DvcException):
 
 
 class NoRemoteError(ConfigError):
-    def __init__(self, command):
-        msg = (
-            "no remote specified. Setup default remote with\n"
-            "    dvc remote default <name>\n"
-            "or use:\n"
-            "    dvc {} -r <name>\n".format(command)
-        )
-
-        super().__init__(msg)
+    pass
 
 
 def supported_cache_type(types):
@@ -56,13 +48,12 @@ def supported_cache_type(types):
 
 
 # Checks that value is either true or false and converts it to bool
-Bool = All(
+to_bool = Bool = All(
     Lower,
     Any("true", "false"),
     lambda v: v == "true",
     msg="expected true or false",
 )
-to_bool = Schema(Bool)
 
 
 def Choices(*choices):
@@ -75,7 +66,124 @@ def Choices(*choices):
     return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
 
 
-class Config(object):  # pylint: disable=too-many-instance-attributes
+def ByUrl(mapping):
+    schemas = walk_values(Schema, mapping)
+
+    def validate(data):
+        if "url" not in data:
+            raise Invalid("expected 'url'")
+
+        parsed = urlparse(data["url"])
+        # Windows absolute paths should really have scheme == "" (local)
+        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+            return schemas[""](data)
+        if parsed.scheme not in schemas:
+            raise Invalid("Unsupported URL type {}://".format(parsed.scheme))
+
+        return schemas[parsed.scheme](data)
+
+    return validate
+
+
+class RelPath(str):
+    pass
+
+
+REMOTE_COMMON = {
+    "url": str,
+    "checksum_jobs": All(Coerce(int), Range(1)),
+    "no_traverse": Bool,
+    "verify": Bool,
+}
+LOCAL_COMMON = {
+    "type": supported_cache_type,
+    Optional("protected", default=False): Bool,
+    "shared": All(Lower, Choices("group")),
+    Optional("slow_link_warning", default=True): Bool,
+}
+SCHEMA = {
+    "core": {
+        "remote": Lower,
+        "checksum_jobs": All(Coerce(int), Range(1)),
+        Optional("interactive", default=False): Bool,
+        Optional("analytics", default=True): Bool,
+        Optional("hardlink_lock", default=False): Bool,
+    },
+    "cache": {
+        "local": str,
+        "s3": str,
+        "gs": str,
+        "hdfs": str,
+        "ssh": str,
+        "azure": str,
+        # This is for default local cache
+        "dir": str,
+        **LOCAL_COMMON,
+    },
+    "remote": {
+        str: ByUrl(
+            {
+                "": {**LOCAL_COMMON, **REMOTE_COMMON},
+                "s3": {
+                    "region": str,
+                    "profile": str,
+                    "credentialpath": str,
+                    "endpointurl": str,
+                    Optional("listobjects", default=False): Bool,
+                    Optional("use_ssl", default=True): Bool,
+                    "sse": str,
+                    "acl": str,
+                    "grant_read": str,
+                    "grant_read_acp": str,
+                    "grant_write_acp": str,
+                    "grant_full_control": str,
+                    **REMOTE_COMMON,
+                },
+                "gs": {
+                    "projectname": str,
+                    "credentialpath": str,
+                    **REMOTE_COMMON,
+                },
+                "ssh": {
+                    "type": supported_cache_type,
+                    "port": Coerce(int),
+                    "user": str,
+                    "password": str,
+                    "ask_password": Bool,
+                    "keyfile": str,
+                    "timeout": Coerce(int),
+                    "gss_auth": Bool,
+                    **REMOTE_COMMON,
+                },
+                "hdfs": {"user": str, **REMOTE_COMMON},
+                "azure": {"connection_string": str, **REMOTE_COMMON},
+                "oss": {
+                    "oss_key_id": str,
+                    "oss_key_secret": str,
+                    "oss_endpoint": str,
+                    **REMOTE_COMMON,
+                },
+                "gdrive": {
+                    "gdrive_client_id": str,
+                    "gdrive_client_secret": str,
+                    "gdrive_user_credentials_file": str,
+                    **REMOTE_COMMON,
+                },
+                "http": REMOTE_COMMON,
+                "https": REMOTE_COMMON,
+                "remote": {str: object},  # Any of the above options are valid
+            }
+        )
+    },
+    "state": {
+        "row_limit": All(Coerce(int), Range(1)),
+        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
+    },
+}
+COMPILED_SCHEMA = Schema(SCHEMA)
+
+
+class Config(dict):
     """Class that manages configuration files for a DVC repo.
 
     Args:
@@ -93,168 +201,14 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     APPNAME = "dvc"
     APPAUTHOR = "iterative"
 
-    # NOTE: used internally in RemoteLOCAL to know config
-    # location, that url should resolved relative to.
-    PRIVATE_CWD = "_cwd"
+    # In the order they shadow each other
+    LEVELS = ("system", "global", "repo", "local")
 
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
-    CREDENTIALPATH = "credentialpath"
-
-    LEVEL_LOCAL = 0
-    LEVEL_REPO = 1
-    LEVEL_GLOBAL = 2
-    LEVEL_SYSTEM = 3
-
-    SECTION_CORE = "core"
-    SECTION_CORE_LOGLEVEL = "loglevel"
-    SECTION_CORE_LOGLEVEL_SCHEMA = All(
-        Lower, Choices("info", "debug", "warning", "error")
-    )
-    SECTION_CORE_REMOTE = "remote"
-    SECTION_CORE_INTERACTIVE = "interactive"
-    SECTION_CORE_ANALYTICS = "analytics"
-    SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_CORE_HARDLINK_LOCK = "hardlink_lock"
-
-    SECTION_CACHE = "cache"
-    SECTION_CACHE_DIR = "dir"
-    SECTION_CACHE_TYPE = "type"
-    SECTION_CACHE_PROTECTED = "protected"
-    SECTION_CACHE_SHARED = "shared"
-    SECTION_CACHE_SHARED_SCHEMA = All(Lower, Choices("group"))
-    SECTION_CACHE_LOCAL = "local"
-    SECTION_CACHE_S3 = "s3"
-    SECTION_CACHE_GS = "gs"
-    SECTION_CACHE_SSH = "ssh"
-    SECTION_CACHE_HDFS = "hdfs"
-    SECTION_CACHE_AZURE = "azure"
-    SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
-    SECTION_CACHE_SCHEMA = {
-        SECTION_CACHE_LOCAL: str,
-        SECTION_CACHE_S3: str,
-        SECTION_CACHE_GS: str,
-        SECTION_CACHE_HDFS: str,
-        SECTION_CACHE_SSH: str,
-        SECTION_CACHE_AZURE: str,
-        SECTION_CACHE_DIR: str,
-        SECTION_CACHE_TYPE: supported_cache_type,
-        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
-        SECTION_CACHE_SHARED: SECTION_CACHE_SHARED_SCHEMA,
-        PRIVATE_CWD: str,
-        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): Bool,
-    }
-
-    SECTION_CORE_SCHEMA = {
-        SECTION_CORE_LOGLEVEL: SECTION_CORE_LOGLEVEL_SCHEMA,
-        SECTION_CORE_REMOTE: Lower,
-        Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
-        Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
-        SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
-        Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool,
-    }
-
-    # aws specific options
-    SECTION_AWS_CREDENTIALPATH = CREDENTIALPATH
-    SECTION_AWS_ENDPOINT_URL = "endpointurl"
-    SECTION_AWS_LIST_OBJECTS = "listobjects"
-    SECTION_AWS_REGION = "region"
-    SECTION_AWS_PROFILE = "profile"
-    SECTION_AWS_USE_SSL = "use_ssl"
-    SECTION_AWS_SSE = "sse"
-    SECTION_AWS_ACL = "acl"
-    SECTION_AWS_GRANT_READ = "grant_read"
-    SECTION_AWS_GRANT_READ_ACP = "grant_read_acp"
-    SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp"
-    SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control"
-
-    # gcp specific options
-    SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH
-    SECTION_GCP_PROJECTNAME = "projectname"
-
-    # azure specific option
-    SECTION_AZURE_CONNECTION_STRING = "connection_string"
-
-    # Alibabacloud oss options
-    SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
-    SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
-    SECTION_OSS_ENDPOINT = "oss_endpoint"
-
-    # GDrive options
-    SECTION_GDRIVE_CLIENT_ID = "gdrive_client_id"
-    SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret"
-    SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file"
-
-    SECTION_REMOTE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
-    SECTION_REMOTE_FMT = 'remote "{}"'
-    SECTION_REMOTE_URL = "url"
-    SECTION_REMOTE_USER = "user"
-    SECTION_REMOTE_PORT = "port"
-    SECTION_REMOTE_KEY_FILE = "keyfile"
-    SECTION_REMOTE_TIMEOUT = "timeout"
-    SECTION_REMOTE_PASSWORD = "password"
-    SECTION_REMOTE_ASK_PASSWORD = "ask_password"
-    SECTION_REMOTE_GSS_AUTH = "gss_auth"
-    SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
-    SECTION_REMOTE_VERIFY = "verify"
-    SECTION_REMOTE_SCHEMA = {
-        Required(SECTION_REMOTE_URL): str,
-        SECTION_AWS_REGION: str,
-        SECTION_AWS_PROFILE: str,
-        SECTION_AWS_CREDENTIALPATH: str,
-        SECTION_AWS_ENDPOINT_URL: str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): Bool,
-        Optional(SECTION_AWS_USE_SSL, default=True): Bool,
-        SECTION_AWS_SSE: str,
-        SECTION_AWS_ACL: str,
-        SECTION_AWS_GRANT_READ: str,
-        SECTION_AWS_GRANT_READ_ACP: str,
-        SECTION_AWS_GRANT_WRITE_ACP: str,
-        SECTION_AWS_GRANT_FULL_CONTROL: str,
-        SECTION_GCP_PROJECTNAME: str,
-        SECTION_CACHE_TYPE: supported_cache_type,
-        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
-        SECTION_REMOTE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
-        SECTION_REMOTE_USER: str,
-        SECTION_REMOTE_PORT: Coerce(int),
-        SECTION_REMOTE_KEY_FILE: str,
-        SECTION_REMOTE_TIMEOUT: Coerce(int),
-        SECTION_REMOTE_PASSWORD: str,
-        SECTION_REMOTE_ASK_PASSWORD: Bool,
-        SECTION_REMOTE_GSS_AUTH: Bool,
-        SECTION_AZURE_CONNECTION_STRING: str,
-        SECTION_OSS_ACCESS_KEY_ID: str,
-        SECTION_OSS_ACCESS_KEY_SECRET: str,
-        SECTION_OSS_ENDPOINT: str,
-        SECTION_GDRIVE_CLIENT_ID: str,
-        SECTION_GDRIVE_CLIENT_SECRET: str,
-        SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
-        PRIVATE_CWD: str,
-        SECTION_REMOTE_NO_TRAVERSE: Bool,
-        SECTION_REMOTE_VERIFY: Bool,
-    }
-
-    SECTION_STATE = "state"
-    SECTION_STATE_ROW_LIMIT = "row_limit"
-    SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
-    SECTION_STATE_SCHEMA = {
-        SECTION_STATE_ROW_LIMIT: All(Coerce(int), Range(1)),
-        SECTION_STATE_ROW_CLEANUP_QUOTA: All(Coerce(int), Range(0, 100)),
-    }
-
-    SCHEMA = {
-        Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
-        Match(SECTION_REMOTE_REGEX): SECTION_REMOTE_SCHEMA,
-        Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
-        Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
-    }
-    COMPILED_SCHEMA = Schema(SCHEMA)
-
     def __init__(self, dvc_dir=None, validate=True):
         self.dvc_dir = dvc_dir
-        self.validate = validate
 
         if not dvc_dir:
             try:
@@ -266,33 +220,31 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
-        self.load()
+        self.load(validate=validate)
 
-    @staticmethod
-    def get_global_config_dir():
-        """Returns global config location. E.g. ~/.config/dvc/config.
+    @classmethod
+    def get_dir(cls, level):
+        from appdirs import user_config_dir, site_config_dir
 
-        Returns:
-            str: path to the global config directory.
-        """
-        from appdirs import user_config_dir
+        assert level in ("global", "system")
 
-        return user_config_dir(
-            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
-        )
+        if level == "global":
+            return user_config_dir(cls.APPNAME, cls.APPAUTHOR)
+        if level == "system":
+            return site_config_dir(cls.APPNAME, cls.APPAUTHOR)
 
-    @staticmethod
-    def get_system_config_dir():
-        """Returns system config location. E.g. /etc/dvc.conf.
+    @cached_property
+    def files(self):
+        files = {
+            level: os.path.join(self.get_dir(level), self.CONFIG)
+            for level in ("system", "global")
+        }
 
-        Returns:
-            str: path to the system config directory.
-        """
-        from appdirs import site_config_dir
+        if self.dvc_dir is not None:
+            files["repo"] = os.path.join(self.dvc_dir, self.CONFIG)
+            files["local"] = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
 
-        return site_config_dir(
-            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
-        )
+        return files
 
     @staticmethod
     def init(dvc_dir):
@@ -308,243 +260,137 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         open(config_file, "w+").close()
         return Config(dvc_dir)
 
-    def _resolve_cache_path(self, config):
-        cache = config.get(self.SECTION_CACHE)
-        if cache is None:
-            return
-
-        cache_dir = cache.get(self.SECTION_CACHE_DIR)
-        if cache_dir is None:
-            return
-
-        cache[self.PRIVATE_CWD] = os.path.dirname(config.filename)
-
-    def _resolve_paths(self, config):
-        if config.filename is None:
-            return config
-
-        ret = copy.deepcopy(config)
-        self._resolve_cache_path(ret)
-
-        for section in ret.values():
-            if self.SECTION_REMOTE_URL not in section.keys():
-                continue
-
-            section[self.PRIVATE_CWD] = os.path.dirname(ret.filename)
-
-        return ret
-
-    def _load_configs(self):
-        system_config_file = os.path.join(
-            self.get_system_config_dir(), self.CONFIG
-        )
-
-        global_config_file = os.path.join(
-            self.get_global_config_dir(), self.CONFIG
-        )
-
-        self._system_config = configobj.ConfigObj(system_config_file)
-        self._global_config = configobj.ConfigObj(global_config_file)
-        self._repo_config = configobj.ConfigObj()
-        self._local_config = configobj.ConfigObj()
-
-        if not self.dvc_dir:
-            return
-
-        config_file = os.path.join(self.dvc_dir, self.CONFIG)
-        config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
-
-        self._repo_config = configobj.ConfigObj(config_file)
-        self._local_config = configobj.ConfigObj(config_local_file)
-
-    @property
-    def config_local_file(self):
-        return self._local_config.filename
-
-    @property
-    def config_file(self):
-        return self._repo_config.filename
-
-    def load(self):
+    def load(self, validate=True):
         """Loads config from all the config files.
 
         Raises:
             dvc.config.ConfigError: thrown if config has invalid format.
         """
-        self._load_configs()
+        conf = {}
+        for level in self.LEVELS:
+            if level in self.files:
+                _merge(conf, self.load_one(level))
 
-        self.config = configobj.ConfigObj()
-        for c in [
-            self._system_config,
-            self._global_config,
-            self._repo_config,
-            self._local_config,
-        ]:
-            c = self._resolve_paths(c)
-            c = self._lower(c)
-            self.config.merge(c)
+        if validate:
+            conf = self.validate(conf)
 
-        if not self.validate:
-            return
+        self.clear()
+        self.update(conf)
 
-        d = self.config.dict()
-        try:
-            d = self.COMPILED_SCHEMA(d)
-        except Invalid as exc:
-            raise ConfigError(str(exc)) from exc
-        self.config = configobj.ConfigObj(d, write_empty_values=True)
+        # Add resolved default cache.dir
+        if not self["cache"].get("dir") and self.dvc_dir:
+            self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
-    def save(self, config=None):
-        """Saves config to config files.
+    def load_one(self, level):
+        conf = _load_config(self.files[level])
+        conf = self._load_paths(conf, self.files[level])
 
-        Raises:
-            dvc.config.ConfigError: thrown if failed to write config file.
-        """
-        if config is not None:
-            clist = [config]
-        else:
-            clist = [
-                self._system_config,
-                self._global_config,
-                self._repo_config,
-                self._local_config,
-            ]
+        # Autovivify sections
+        for key in COMPILED_SCHEMA.schema:
+            conf.setdefault(key, {})
 
-        for conf in clist:
-            self._save(conf)
+        return conf
 
+    @staticmethod
+    def _load_paths(conf, filename):
+        abs_conf_dir = os.path.abspath(os.path.dirname(filename))
+
+        def resolve(path):
+            if os.path.isabs(path) or re.match(r"\w+://", path):
+                return path
+            return RelPath(os.path.join(abs_conf_dir, path))
+
+        return Config._map_dirs(conf, resolve)
+
+    @staticmethod
+    def _save_paths(conf, filename):
+        conf_dir = os.path.dirname(filename)
+
+        def rel(path):
+            if re.match(r"\w+://", path):
+                return path
+
+            if isinstance(path, RelPath) or not os.path.isabs(path):
+                return relpath(path, conf_dir)
+            return path
+
+        return Config._map_dirs(conf, rel)
+
+    @staticmethod
+    def _map_dirs(conf, func):
+        dirs_schema = {"cache": {"dir": func}, "remote": {str: {"url": func}}}
+        return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
+
+    @contextmanager
+    def edit(self, level="repo"):
+        if level in {"repo", "local"} and self.dvc_dir is None:
+            raise ConfigError("Not inside a dvc repo")
+
+        conf = self.load_one(level)
+        yield conf
+
+        conf = self._save_paths(conf, self.files[level])
+        _save_config(self.files[level], conf)
         self.load()
 
     @staticmethod
-    def _save(config):
-        if config.filename is None:
-            return
-
-        logger.debug("Writing '{}'.".format(config.filename))
-        dname = os.path.dirname(os.path.abspath(config.filename))
+    def validate(data):
         try:
-            os.makedirs(dname)
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                raise
-        config.write()
+            return COMPILED_SCHEMA(data)
+        except Invalid as exc:
+            raise ConfigError(str(exc)) from None
 
-    def unset(self, section, opt=None, level=None, force=False):
-        """Unsets specified option and/or section in the config.
 
-        Args:
-            section (str): section name.
-            opt (str): optional option name.
-            level (int): config level to use.
-            force (bool): don't error-out even if section doesn't exist. False
-                by default.
+def _load_config(filename):
+    conf_obj = configobj.ConfigObj(filename)
+    return _parse_remotes(_lower_keys(conf_obj.dict()))
 
-        Raises:
-            dvc.config.ConfigError: thrown if section doesn't exist and
-                `force != True`.
-        """
-        config = self.get_configobj(level)
 
-        if section not in config.keys():
-            if force:
-                return
-            raise ConfigError("section '{}' doesn't exist".format(section))
+def _save_config(filename, conf_dict):
+    logger.debug("Writing '{}'.".format(filename))
+    os.makedirs(os.path.dirname(filename), exist_ok=True)
 
-        if opt:
-            if opt not in config[section].keys():
-                if force:
-                    return
-                raise ConfigError(
-                    "option '{}.{}' doesn't exist".format(section, opt)
-                )
-            del config[section][opt]
+    config = configobj.ConfigObj(_pack_remotes(conf_dict))
+    config.filename = filename
+    config.write()
 
-            if not config[section]:
-                del config[section]
+
+def _parse_remotes(conf):
+    result = {"remote": {}}
+
+    for section, val in conf.items():
+        name = re_find(r'^\s*remote\s*"(.*)"\s*$', section)
+        if name:
+            result["remote"][name] = val
         else:
-            del config[section]
+            result[section] = val
 
-        self.save(config)
+    return result
 
-    def set(self, section, opt, value, level=None, force=True):
-        """Sets specified option in the config.
 
-        Args:
-            section (str): section name.
-            opt (str): option name.
-            value: value to set option to.
-            level (int): config level to use.
-            force (bool): set option even if section already exists. True by
-                default.
+def _pack_remotes(conf):
+    # Drop empty sections
+    result = compact(conf)
 
-        Raises:
-            dvc.config.ConfigError: thrown if section already exists and
-                `force != True`.
+    # Transform remote.name -> 'remote "name"'
+    for name, val in conf["remote"].items():
+        result['remote "{}"'.format(name)] = val
+    result.pop("remote", None)
 
-        """
-        config = self.get_configobj(level)
+    return result
 
-        if section not in config.keys():
-            config[section] = {}
-        elif not force:
-            raise ConfigError(
-                "Section '{}' already exists. Use `-f|--force` to overwrite "
-                "section with new value.".format(section)
-            )
 
-        config[section][opt] = value
-        self.save(config)
+def _merge(into, update):
+    """Merges second dict into first recursively"""
+    for key, val in update.items():
+        if isinstance(into.get(key), dict) and isinstance(val, dict):
+            _merge(into[key], val)
+        else:
+            into[key] = val
 
-    def get(self, section, opt=None, level=None):
-        """Return option value from the config.
 
-        Args:
-            section (str): section name.
-            opt (str): option name.
-            level (int): config level to use.
-
-        Returns:
-            value (str, int): option value.
-        """
-        config = self.get_configobj(level)
-
-        if section not in config.keys():
-            raise ConfigError("section '{}' doesn't exist".format(section))
-
-        if opt not in config[section].keys():
-            raise ConfigError(
-                "option '{}.{}' doesn't exist".format(section, opt)
-            )
-
-        return config[section][opt]
-
-    @staticmethod
-    def _lower(config):
-        new_config = configobj.ConfigObj()
-        for s_key, s_value in config.items():
-            new_s = {}
-            for key, value in s_value.items():
-                new_s[key.lower()] = str(value)
-            new_config[s_key.lower()] = new_s
-        return new_config
-
-    def get_configobj(self, level):
-        configs = {
-            self.LEVEL_LOCAL: self._local_config,
-            self.LEVEL_REPO: self._repo_config,
-            self.LEVEL_GLOBAL: self._global_config,
-            self.LEVEL_SYSTEM: self._system_config,
-        }
-
-        return configs.get(level, self._repo_config)
-
-    def list_options(self, section_regex, option, level=None):
-        ret = {}
-        config = self.get_configobj(level)
-        for section in config.keys():
-            r = re.match(section_regex, section)
-            if r:
-                name = r.group("name")
-                value = config[section].get(option, "")
-                ret[name] = value
-        return ret
+def _lower_keys(data):
+    return {
+        k.lower(): _lower_keys(v) if isinstance(v, dict) else v
+        for k, v in data.items()
+    }

dvc/dagascii.py

@@ -14,7 +14,7 @@ from grandalf.routing import EdgeViewer
 from grandalf.routing import route_with_lines
 
 from dvc.env import DVC_PAGER
-
+from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
@@ -45,7 +45,9 @@ def find_pager():
 
     logger.warning(
         "Unable to find `less` in the PATH. Check out "
-        "man.dvc.org/doc/command-reference/pipeline/show for more info."
+        "{} for more info.".format(
+            format_link("https://man.dvc.org/pipeline/show")
+        )
     )
     return pydoc.plainpager
 

dvc/data_cloud.py

@@ -2,7 +2,6 @@
 
 import logging
 
-from dvc.config import Config
 from dvc.config import NoRemoteError
 from dvc.remote import Remote
 
@@ -24,22 +23,27 @@ class DataCloud(object):
     def __init__(self, repo):
         self.repo = repo
 
-    @property
-    def _config(self):
-        return self.repo.config.config
-
-    @property
-    def _core(self):
-        return self._config.get(Config.SECTION_CORE, {})
-
     def get_remote(self, remote=None, command="<command>"):
         if not remote:
-            remote = self._core.get(Config.SECTION_CORE_REMOTE)
+            remote = self.repo.config["core"].get("remote")
 
         if remote:
             return self._init_remote(remote)
 
-        raise NoRemoteError(command)
+        if bool(self.repo.config["remote"]):
+            error_msg = (
+                "no remote specified. Setup default remote with\n"
+                "    dvc remote default <remote name>\n"
+                "or use:\n"
+                "    dvc {} -r <remote name>".format(command)
+            )
+        else:
+            error_msg = (
+                "no remote specified. Create a default remote with\n"
+                "    dvc remote add -d <remote name> <remote url>"
+            )
+
+        raise NoRemoteError(error_msg)
 
     def _init_remote(self, remote):
         return Remote(self.repo, name=remote)

dvc/dependency/base.py

@@ -27,5 +27,5 @@ class DependencyBase(object):
     IsNotFileOrDirError = DependencyIsNotFileOrDirError
     IsStageFileError = DependencyIsStageFileError
 
-    def update(self):
+    def update(self, rev=None):
         pass

dvc/dependency/repo.py

@@ -80,6 +80,9 @@ class DependencyREPO(DependencyLOCAL):
 
             repo.pull_to(self.def_path, to.path_info)
 
-    def update(self):
+    def update(self, rev=None):
+        if rev:
+            self.def_repo[self.PARAM_REV] = rev
+
         with self._make_repo(locked=False) as repo:
             self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()

dvc/exceptions.py

@@ -1,6 +1,6 @@
 """Exceptions raised by the dvc."""
 
-from dvc.utils import relpath
+from dvc.utils import relpath, format_link
 
 
 class DvcException(Exception):
@@ -112,18 +112,7 @@ class MoveNotDataSourceError(DvcException):
 
 
 class NotDvcRepoError(DvcException):
-    """Thrown if a directory is not a DVC repo.
-
-    Args:
-        root (str): path to the directory.
-    """
-
-    def __init__(self, root):
-        msg = (
-            "you are not inside of a DVC repository "
-            "(checked up to mount point '{}')"
-        )
-        super().__init__(msg.format(root))
+    """Thrown if a directory is not a DVC repo"""
 
 
 class DvcParserError(DvcException):
@@ -199,17 +188,10 @@ class RecursiveAddingWhileUsingFilename(DvcException):
 
 
 class OverlappingOutputPathsError(DvcException):
-    def __init__(self, out_1, out_2):
-        super().__init__(
-            "Paths for outs:\n'{}'('{}')\n'{}'('{}')\noverlap. To avoid "
-            "unpredictable behaviour, rerun command with non overlapping outs "
-            "paths.".format(
-                str(out_1),
-                out_1.stage.relpath,
-                str(out_2),
-                out_2.stage.relpath,
-            )
-        )
+    def __init__(self, parent, overlapping_out, message):
+        self.parent = parent
+        self.overlapping_out = overlapping_out
+        super().__init__(message)
 
 
 class CheckoutErrorSuggestGit(DvcException):
@@ -244,9 +226,10 @@ class DvcIgnoreInCollectedDirError(DvcException):
 class GitHookAlreadyExistsError(DvcException):
     def __init__(self, hook_name):
         super().__init__(
-            "Hook '{}' already exists. Please refer to "
-            "https://man.dvc.org/install "
-            "for more info.".format(hook_name)
+            "Hook '{}' already exists. Please refer to {} for more "
+            "info.".format(
+                hook_name, format_link("https://man.dvc.org/install")
+            )
         )
 
 
@@ -304,9 +287,35 @@ class HTTPError(DvcException):
 
 
 class PathMissingError(DvcException):
-    def __init__(self, path, repo):
+    default_msg = (
+        "The path '{}' does not exist in the target repository '{}'"
+        " neither as an output nor a git-handled file."
+    )
+    default_msg_output_only = (
+        "The path '{}' does not exist in the target repository '{}'"
+        " as an output."
+    )
+
+    def __init__(self, path, repo, output_only=False):
         msg = (
-            "The path '{}' does not exist in the target repository '{}'"
-            " neither as an output nor a git-handled file."
+            self.default_msg
+            if not output_only
+            else self.default_msg_output_only
         )
         super().__init__(msg.format(path, repo))
+        self.output_only = output_only
+
+
+class RemoteCacheRequiredError(DvcException):
+    def __init__(self, path_info):
+        super().__init__(
+            (
+                "Current operation was unsuccessful because '{}' requires "
+                "existing cache on '{}' remote. See {} for information on how "
+                "to set up remote cache."
+            ).format(
+                path_info,
+                path_info.scheme,
+                format_link("https://man.dvc.org/config#cache"),
+            )
+        )

dvc/external_repo.py

@@ -1,24 +1,31 @@
+import logging
 import os
 import tempfile
 from contextlib import contextmanager
 from distutils.dir_util import copy_tree
+import threading
 
-from funcy import retry, suppress, memoize, cached_property
+from funcy import retry, suppress, wrap_with, cached_property
 
+from dvc.path_info import PathInfo
 from dvc.compat import fspath
 from dvc.repo import Repo
-from dvc.config import Config, NoRemoteError, NotDvcRepoError
+from dvc.config import NoRemoteError, NotDvcRepoError
 from dvc.exceptions import NoRemoteInExternalRepoError
 from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
 from dvc.exceptions import FileMissingError, PathMissingError
-from dvc.remote import RemoteConfig
-from dvc.utils.fs import remove, fs_copy
-from dvc.scm import SCM
+from dvc.utils.fs import remove, fs_copy, move
+from dvc.utils import tmp_fname
+from dvc.scm.git import Git
+
+
+logger = logging.getLogger(__name__)
 
 
 @contextmanager
-def external_repo(url, rev=None):
-    path = _cached_clone(url, rev)
+def external_repo(url, rev=None, for_write=False):
+    logger.debug("Creating external repo {}@{}", url, rev)
+    path = _cached_clone(url, rev, for_write=for_write)
     try:
         repo = ExternalRepo(path, url)
     except NotDvcRepoError:
@@ -36,14 +43,21 @@ def external_repo(url, rev=None):
         raise PathMissingError(exc.path, url)
     finally:
         repo.close()
+        if for_write:
+            _remove(path)
+
+
+CLONES = {}
+CACHE_DIRS = {}
 
 
 def clean_repos():
     # Outside code should not see cache while we are removing
-    repo_paths = list(_cached_clone.memory.values())
-    _cached_clone.memory.clear()
+    paths = list(CLONES.values()) + list(CACHE_DIRS.values())
+    CLONES.clear()
+    CACHE_DIRS.clear()
 
-    for path in repo_paths:
+    for path in paths:
         _remove(path)
 
 
@@ -51,55 +65,80 @@ class ExternalRepo(Repo):
     def __init__(self, root_dir, url):
         super().__init__(root_dir)
         self.url = url
+        self._set_cache_dir()
         self._set_upstream()
 
     def pull_to(self, path, to_info):
-        try:
-            out = None
-            with suppress(OutputNotFoundError):
-                out = self.find_out_by_relpath(path)
+        """
+        Pull the corresponding file or directory specified by `path` and
+        checkout it into `to_info`.
 
+        It works with files tracked by Git and DVC, and also local files
+        outside the repository.
+        """
+        out = None
+        path_info = PathInfo(self.root_dir) / path
+
+        with suppress(OutputNotFoundError):
+            (out,) = self.find_outs_by_path(fspath(path_info), strict=False)
+
+        try:
             if out and out.use_cache:
-                self._pull_cached(out, to_info)
+                self._pull_cached(out, path_info, to_info)
                 return
 
-            # Git handled files can't have absolute path
+            # Check if it is handled by Git (it can't have an absolute path)
             if os.path.isabs(path):
                 raise FileNotFoundError
 
-            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
+            fs_copy(fspath(path_info), fspath(to_info))
         except FileNotFoundError:
             raise PathMissingError(path, self.url)
 
-    def _pull_cached(self, out, to_info):
+    def _pull_cached(self, out, path_info, dest):
         with self.state:
+            tmp = PathInfo(tmp_fname(dest))
+            src = tmp / path_info.relative_to(out.path_info)
+
+            out.path_info = tmp
+
             # Only pull unless all needed cache is present
-            if out.changed_cache():
-                self.cloud.pull(out.get_used_cache())
+            if out.changed_cache(filter_info=src):
+                self.cloud.pull(out.get_used_cache(filter_info=src))
+
+            failed = out.checkout(filter_info=src)
+
+            move(src, dest)
+            remove(tmp)
 
-            out.path_info = to_info
-            failed = out.checkout()
-            # This might happen when pull haven't really pulled all the files
             if failed:
                 raise FileNotFoundError
 
+    @wrap_with(threading.Lock())
+    def _set_cache_dir(self):
+        try:
+            cache_dir = CACHE_DIRS[self.url]
+        except KeyError:
+            cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
+
+        self.cache.local.cache_dir = cache_dir
+
     def _set_upstream(self):
         # check if the URL is local and no default remote is present
         # add default remote pointing to the original repo's cache location
         if os.path.isdir(self.url):
-            rconfig = RemoteConfig(self.config)
-            if not rconfig.has_default():
+            if not self.config["core"].get("remote"):
                 src_repo = Repo(self.url)
                 try:
-                    rconfig.add(
-                        "auto-generated-upstream",
-                        src_repo.cache.local.cache_dir,
-                        default=True,
-                        level=Config.LEVEL_LOCAL,
-                    )
+                    cache_dir = src_repo.cache.local.cache_dir
                 finally:
                     src_repo.close()
 
+                self.config["remote"]["auto-generated-upstream"] = {
+                    "url": cache_dir
+                }
+                self.config["core"]["remote"] = "auto-generated-upstream"
+
 
 class ExternalGitRepo:
     def __init__(self, root_dir, url):
@@ -108,7 +147,7 @@ class ExternalGitRepo:
 
     @cached_property
     def scm(self):
-        return SCM(self.root_dir)
+        return Git(self.root_dir)
 
     def close(self):
         if "scm" in self.__dict__:
@@ -137,41 +176,65 @@ class ExternalGitRepo:
             raise PathMissingError(path, self.url)
 
 
-@memoize
-def _cached_clone(url, rev):
+def _cached_clone(url, rev, for_write=False):
     """Clone an external git repo to a temporary directory.
 
     Returns the path to a local temporary directory with the specified
-    revision checked out.
+    revision checked out. If for_write is set prevents reusing this dir via
+    cache.
     """
-    new_path = tempfile.mkdtemp("dvc-erepo")
+    if not for_write and Git.is_sha(rev) and (url, rev) in CLONES:
+        return CLONES[url, rev]
 
-    if url in _cached_clone.memory:
-        # Copy and an existing clean clone
-        # This one unlike shutil.copytree() works with an existing dir
-        copy_tree(_cached_clone.memory[url], new_path)
-    else:
-        # Create a new clone
-        _clone_repo(url, new_path)
+    clone_path = _clone_default_branch(url, rev)
+    rev_sha = Git(clone_path).resolve_rev(rev or "HEAD")
 
-        # Save clean clone dir so that we will have access to a default branch
-        clean_clone_path = tempfile.mkdtemp("dvc-erepo")
-        copy_tree(new_path, clean_clone_path)
-        _cached_clone.memory[url] = clean_clone_path
+    if not for_write and (url, rev_sha) in CLONES:
+        return CLONES[url, rev_sha]
+
+    # Copy to a new dir to keep the clone clean
+    repo_path = tempfile.mkdtemp("dvc-erepo")
+    copy_tree(clone_path, repo_path)
 
     # Check out the specified revision
     if rev is not None:
-        _git_checkout(new_path, rev)
+        _git_checkout(repo_path, rev)
 
-    return new_path
+    if not for_write:
+        CLONES[url, rev_sha] = repo_path
+    return repo_path
 
 
-def _git_checkout(repo_path, revision):
-    from dvc.scm import Git
+@wrap_with(threading.Lock())
+def _clone_default_branch(url, rev):
+    """Get or create a clean clone of the url.
 
+    The cloned is reactualized with git pull unless rev is a known sha.
+    """
+    clone_path = CLONES.get(url)
+
+    git = None
+    try:
+        if clone_path:
+            git = Git(clone_path)
+            # Do not pull for known shas, branches and tags might move
+            if not Git.is_sha(rev) or not git.has_rev(rev):
+                git.pull()
+        else:
+            clone_path = tempfile.mkdtemp("dvc-clone")
+            git = Git.clone(url, clone_path)
+            CLONES[url] = clone_path
+    finally:
+        if git:
+            git.close()
+
+    return clone_path
+
+
+def _git_checkout(repo_path, rev):
     git = Git(repo_path)
     try:
-        git.checkout(revision)
+        git.checkout(rev)
     finally:
         git.close()
 
@@ -183,10 +246,3 @@ def _remove(path):
         os_retry(remove)(path)
     else:
         remove(path)
-
-
-def _clone_repo(url, path):
-    from dvc.scm.git import Git
-
-    git = Git.clone(url, path)
-    git.close()

dvc/logger.py

@@ -1,6 +1,5 @@
 """Manages logging configuration for DVC repo."""
 
-import traceback
 import logging.config
 import logging.handlers
 
@@ -37,21 +36,19 @@ class ExcludeInfoFilter(logging.Filter):
 
 
 class ColorFormatter(logging.Formatter):
-    """Enable color support when logging to a terminal that supports it.
+    """Spit out colored text in supported terminals.
 
-    Color support on Windows versions that do not support ANSI color codes is
-    enabled by use of the colorama__ library.
+    colorama__ makes ANSI escape character sequences work under Windows.
     See the colorama documentation for details.
 
     __ https://pypi.python.org/pypi/colorama
 
-    For records containing `exc_info`, it will use a custom `_walk_exc` to
-    retrieve the whole traceback.
+    If record has an extra `tb_only` attribute, it will not show the
+    exception cause, just the message and the traceback.
     """
 
     color_code = {
         "DEBUG": colorama.Fore.BLUE,
-        "INFO": "",
         "WARNING": colorama.Fore.YELLOW,
         "ERROR": colorama.Fore.RED,
         "CRITICAL": colorama.Fore.RED,
@@ -63,80 +60,40 @@ class ColorFormatter(logging.Formatter):
         if record.levelname == "INFO":
             return msg
 
-        if record.levelname == "ERROR" or record.levelname == "CRITICAL":
-            exception, stack_trace = self._parse_exc(record)
+        if record.exc_info:
+            if getattr(record, "tb_only", False):
+                cause = ""
+            else:
+                cause = ": ".join(_iter_causes(record.exc_info[1]))
 
-            return (
-                "{color}{levelname}{nc}: {description}" "{stack_trace}\n"
-            ).format(
-                color=self.color_code.get(record.levelname, ""),
-                nc=colorama.Fore.RESET,
-                levelname=record.levelname,
-                description=self._description(msg, exception),
-                msg=msg,
-                stack_trace=stack_trace,
+            msg = "{message}{separator}{cause}".format(
+                message=msg or "",
+                separator=" - " if msg and cause else "",
+                cause=cause,
             )
 
-        return "{color}{levelname}{nc}: {msg}".format(
-            color=self.color_code.get(record.levelname, ""),
+            if _is_verbose():
+                msg += _stack_trace(record.exc_info)
+
+        return "{asctime}{color}{levelname}{nc}: {msg}".format(
+            asctime=self.formatTime(record, self.datefmt),
+            color=self.color_code[record.levelname],
             nc=colorama.Fore.RESET,
             levelname=record.levelname,
             msg=msg,
         )
 
-    def _current_level(self):
-        return logging.getLogger("dvc").getEffectiveLevel()
+    def formatTime(self, record, datefmt=None):
+        # only show if current level is set to DEBUG
+        # also, skip INFO as it is used for UI
+        if not _is_verbose() or record.levelno == logging.INFO:
+            return ""
 
-    def _is_visible(self, record):
-        return record.levelno >= self._current_level()
-
-    def _description(self, message, exception):
-        description = ""
-
-        if exception and message:
-            description = "{message} - {exception}"
-        elif exception:
-            description = "{exception}"
-        elif message:
-            description = "{message}"
-
-        return description.format(message=message, exception=exception)
-
-    def _walk_exc(self, exc_info):
-        exc = exc_info[1]
-
-        exc_list = [str(exc)]
-
-        while hasattr(exc, "__cause__") and exc.__cause__:
-            exc_list.append(str(exc.__cause__))
-            exc = exc.__cause__
-
-        return exc_list
-
-    def _parse_exc(self, record):
-        tb_only = getattr(record, "tb_only", False)
-
-        if not record.exc_info:
-            return (None, "")
-
-        exc_list = self._walk_exc(record.exc_info)
-        tb = traceback.format_exception(*record.exc_info)
-
-        exception = None if tb_only else ": ".join(exc_list)
-
-        if self._current_level() == logging.DEBUG:
-            stack_trace = (
-                "\n" "{red}{line}{nc}\n" "{stack_trace}" "{red}{line}{nc}"
-            ).format(
-                red=colorama.Fore.RED,
-                nc=colorama.Fore.RESET,
-                line="-" * 60,
-                stack_trace="".join(tb),
-            )
-        else:
-            stack_trace = ""
-
-        return (exception, stack_trace)
+        return "{green}{date}{nc} ".format(
+            green=colorama.Fore.GREEN,
+            date=super().formatTime(record, datefmt),
+            nc=colorama.Fore.RESET,
+        )
 
 
 class LoggerHandler(logging.StreamHandler):
@@ -158,6 +115,32 @@ class LoggerHandler(logging.StreamHandler):
             self.handleError(record)
 
 
+def _is_verbose():
+    return logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG
+
+
+def _iter_causes(exc):
+    while exc:
+        yield str(exc)
+        exc = exc.__cause__
+
+
+def _stack_trace(exc_info):
+    import traceback
+
+    return (
+        "\n"
+        "{red}{line}{nc}\n"
+        "{trace}"
+        "{red}{line}{nc}".format(
+            red=colorama.Fore.RED,
+            line="-" * 60,
+            trace="".join(traceback.format_exception(*exc_info)),
+            nc=colorama.Fore.RESET,
+        )
+    )
+
+
 def setup(level=logging.INFO):
     colorama.init()
 

dvc/main.py

@@ -11,6 +11,7 @@ from dvc.exceptions import NotDvcRepoError
 from dvc.external_repo import clean_repos
 from dvc.logger import FOOTER
 from dvc.utils import format_link
+from dvc.remote.pool import close_pools
 
 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
@@ -73,6 +74,11 @@ def main(argv=None):
     finally:
         logger.setLevel(outerLogLevel)
 
+        # Closing pools by-hand to prevent wierd messages when closing SSH
+        # connections. See https://github.com/iterative/dvc/issues/3248 for
+        # more info.
+        close_pools()
+
         # Remove cached repos in the end of the call, these are anonymous
         # so won't be reused by any other subsequent run anyway.
         clean_repos()

dvc/output/__init__.py

@@ -1,5 +1,5 @@
 from urllib.parse import urlparse
-from voluptuous import Any, Required
+from voluptuous import Any, Required, Lower, Length, Coerce, And, SetTo
 
 from dvc.output.base import OutputBase
 from dvc.output.gs import OutputGS
@@ -29,6 +29,12 @@ OUTS_MAP = {
     Schemes.LOCAL: OutputLOCAL,
 }
 
+CHECKSUM_SCHEMA = Any(
+    None,
+    And(str, Length(max=0), SetTo(None)),
+    And(Any(str, And(int, Coerce(str))), Length(min=3), Lower),
+)
+
 # NOTE: currently there are only 3 possible checksum names:
 #
 #    1) md5 (LOCAL, SSH, GS);
@@ -37,15 +43,15 @@ OUTS_MAP = {
 #
 # so when a few types of outputs share the same name, we only need
 # specify it once.
-CHECKSUM_SCHEMA = {
-    RemoteLOCAL.PARAM_CHECKSUM: Any(str, None),
-    RemoteS3.PARAM_CHECKSUM: Any(str, None),
-    RemoteHDFS.PARAM_CHECKSUM: Any(str, None),
+CHECKSUMS_SCHEMA = {
+    RemoteLOCAL.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    RemoteS3.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    RemoteHDFS.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
 }
 
-TAGS_SCHEMA = {str: CHECKSUM_SCHEMA}
+TAGS_SCHEMA = {str: CHECKSUMS_SCHEMA}
 
-SCHEMA = CHECKSUM_SCHEMA.copy()
+SCHEMA = CHECKSUMS_SCHEMA.copy()
 SCHEMA[Required(OutputBase.PARAM_PATH)] = str
 SCHEMA[OutputBase.PARAM_CACHE] = bool
 SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA

dvc/output/base.py

@@ -6,7 +6,7 @@ from voluptuous import Any
 
 import dvc.prompt as prompt
 from dvc.cache import NamedCache
-from dvc.exceptions import CollectCacheError
+from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError
 from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteBASE
 
@@ -98,14 +98,9 @@ class OutputBase(object):
         self.persist = persist
         self.tags = None if self.IS_DEPENDENCY else (tags or {})
 
-        if self.use_cache and self.cache is None:
-            raise DvcException(
-                "no cache location setup for '{}' outputs.".format(
-                    self.REMOTE.scheme
-                )
-            )
-
         self.path_info = self._parse_path(remote, path)
+        if self.use_cache and self.cache is None:
+            raise RemoteCacheRequiredError(self.path_info)
 
     def _parse_path(self, remote, path):
         if remote:
@@ -167,11 +162,11 @@ class OutputBase(object):
     def changed_checksum(self):
         return self.checksum != self.remote.get_checksum(self.path_info)
 
-    def changed_cache(self):
+    def changed_cache(self, filter_info=None):
         if not self.use_cache or not self.checksum:
             return True
 
-        return self.cache.changed_cache(self.checksum)
+        return self.cache.changed_cache(self.checksum, filter_info=filter_info)
 
     def status(self):
         if self.checksum and self.use_cache and self.changed_cache():
@@ -410,14 +405,14 @@ class OutputBase(object):
 
         if self.stage.is_repo_import:
             cache = NamedCache()
-            dep, = self.stage.deps
+            (dep,) = self.stage.deps
             cache.external[dep.repo_pair].add(dep.def_path)
             return cache
 
         if not self.info:
             logger.warning(
                 "Output '{}'({}) is missing version info. Cache for it will "
-                "not be collected. Use dvc repro to get your pipeline up to "
+                "not be collected. Use `dvc repro` to get your pipeline up to "
                 "date.".format(self, self.stage)
             )
             return NamedCache()

dvc/output/local.py

@@ -18,6 +18,12 @@ class OutputLOCAL(OutputBase):
     REMOTE = RemoteLOCAL
     sep = os.sep
 
+    def __init__(self, stage, path, *args, **kwargs):
+        if stage and path_isin(path, stage.repo.root_dir):
+            path = relpath(path, stage.wdir)
+
+        super().__init__(stage, path, *args, **kwargs)
+
     def _parse_path(self, remote, path):
         parsed = urlparse(path)
         if parsed.scheme == "remote":

dvc/progress.py

@@ -4,7 +4,6 @@ import logging
 import sys
 from threading import RLock
 
-from funcy import merge
 from tqdm import tqdm
 
 from dvc.utils import env2bool
@@ -31,7 +30,10 @@ class Tqdm(tqdm):
     )
     BAR_FMT_NOTOTAL = (
         "{desc:{ncols_desc}.{ncols_desc}}{n_fmt}"
-        " [{elapsed}<??:??, {rate_fmt:>11}{postfix}]"
+        " [{elapsed}, {rate_fmt:>11}{postfix}]"
+    )
+    BYTES_DEFAULTS = dict(
+        unit="B", unit_scale=True, unit_divisor=1024, miniters=1
     )
 
     def __init__(
@@ -44,6 +46,7 @@ class Tqdm(tqdm):
         bar_format=None,
         bytes=False,  # pylint: disable=W0622
         file=None,
+        total=None,
         **kwargs
     ):
         """
@@ -52,23 +55,22 @@ class Tqdm(tqdm):
         desc  : persists after `close()`
         level  : effective logging level for determining `disable`;
             used only if `disable` is unspecified
-        disable  : If (default: None), will be determined by logging level.
+        disable  : If (default: None) or False,
+            will be determined by logging level.
             May be overridden to `True` due to non-TTY status.
             Skip override by specifying env var `DVC_IGNORE_ISATTY`.
         kwargs  : anything accepted by `tqdm.tqdm()`
         """
         kwargs = kwargs.copy()
-        kwargs.setdefault("unit_scale", True)
         if bytes:
-            bytes_defaults = dict(
-                unit="B", unit_scale=True, unit_divisor=1024, miniters=1
-            )
-            kwargs = merge(bytes_defaults, kwargs)
+            kwargs = {**self.BYTES_DEFAULTS, **kwargs}
+        else:
+            kwargs.setdefault("unit_scale", total > 999 if total else True)
         if file is None:
             file = sys.stderr
         self.desc_persist = desc
         # auto-disable based on `logger.level`
-        if disable is None:
+        if not disable:
             disable = logger.getEffectiveLevel() > level
         # auto-disable based on TTY
         if (
@@ -84,6 +86,7 @@ class Tqdm(tqdm):
             desc=desc,
             bar_format="!",
             lock_args=(False,),
+            total=total,
             **kwargs
         )
         if bar_format is None:
@@ -114,6 +117,10 @@ class Tqdm(tqdm):
     def close(self):
         if self.desc_persist is not None:
             self.set_description_str(self.desc_persist, refresh=False)
+        # unknown/zero ETA
+        self.bar_format = self.bar_format.replace("<{remaining}", "")
+        # remove completed bar
+        self.bar_format = self.bar_format.replace("|{bar:10}|", " ")
         super().close()
 
     @property

dvc/remote/__init__.py

@@ -1,4 +1,6 @@
-from .config import RemoteConfig
+import posixpath
+from urllib.parse import urlparse
+
 from dvc.remote.azure import RemoteAZURE
 from dvc.remote.gdrive import RemoteGDrive
 from dvc.remote.gs import RemoteGS
@@ -25,9 +27,9 @@ REMOTES = [
 ]
 
 
-def _get(config):
+def _get(remote_conf):
     for remote in REMOTES:
-        if remote.supported(config):
+        if remote.supported(remote_conf):
             return remote
     return RemoteLOCAL
 
@@ -35,8 +37,39 @@ def _get(config):
 def Remote(repo, **kwargs):
     name = kwargs.get("name")
     if name:
-        remote_config = RemoteConfig(repo.config)
-        settings = remote_config.get_settings(name)
+        remote_conf = repo.config["remote"][name.lower()]
     else:
-        settings = kwargs
-    return _get(settings)(repo, settings)
+        remote_conf = kwargs
+    remote_conf = _resolve_remote_refs(repo.config, remote_conf)
+    return _get(remote_conf)(repo, remote_conf)
+
+
+def _resolve_remote_refs(config, remote_conf):
+    # Support for cross referenced remotes.
+    # This will merge the settings, shadowing base ref with remote_conf.
+    # For example, having:
+    #
+    #       dvc remote add server ssh://localhost
+    #       dvc remote modify server user root
+    #       dvc remote modify server ask_password true
+    #
+    #       dvc remote add images remote://server/tmp/pictures
+    #       dvc remote modify images user alice
+    #       dvc remote modify images ask_password false
+    #       dvc remote modify images password asdf1234
+    #
+    # Results on a config dictionary like:
+    #
+    #       {
+    #           "url": "ssh://localhost/tmp/pictures",
+    #           "user": "alice",
+    #           "password": "asdf1234",
+    #           "ask_password": False,
+    #       }
+    parsed = urlparse(remote_conf["url"])
+    if parsed.scheme != "remote":
+        return remote_conf
+
+    base = config["remote"][parsed.netloc]
+    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
+    return {**base, **remote_conf, "url": url}

dvc/remote/azure.py

@@ -7,7 +7,6 @@ import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.config import Config
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
@@ -33,7 +32,7 @@ class RemoteAZURE(RemoteBASE):
     def __init__(self, repo, config):
         super().__init__(repo, config)
 
-        url = config.get(Config.SECTION_REMOTE_URL, "azure://")
+        url = config.get("url", "azure://")
 
         match = re.match(self.REGEX, url)  # backward compatibility
         path = match.group("path")
@@ -44,7 +43,7 @@ class RemoteAZURE(RemoteBASE):
         )
 
         self.connection_string = (
-            config.get(Config.SECTION_AZURE_CONNECTION_STRING)
+            config.get("connection_string")
             or match.group("connection_string")  # backward compatibility
             or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
         )

dvc/remote/base.py

@@ -13,11 +13,11 @@ from operator import itemgetter
 from shortuuid import uuid
 
 import dvc.prompt as prompt
-from dvc.config import Config
 from dvc.exceptions import (
     DvcException,
     ConfirmRemoveError,
     DvcIgnoreInCollectedDirError,
+    RemoteCacheRequiredError,
 )
 from dvc.ignore import DvcIgnore
 from dvc.path_info import PathInfo, URLInfo
@@ -65,7 +65,7 @@ class RemoteMissingDepsError(DvcException):
 class DirCacheError(DvcException):
     def __init__(self, checksum):
         super().__init__(
-            "Failed to load dir cache for checksum: '{}'.".format(checksum)
+            "Failed to load dir cache for hash value: '{}'.".format(checksum)
         )
 
 
@@ -88,24 +88,18 @@ class RemoteBASE(object):
         self.repo = repo
 
         self._check_requires(config)
-        self.checksum_jobs = self._get_checksum_jobs(config)
 
+        self.checksum_jobs = (
+            config.get("checksum_jobs")
+            or (self.repo and self.repo.config["core"].get("checksum_jobs"))
+            or self.CHECKSUM_JOBS
+        )
         self.protected = False
-        self.no_traverse = config.get(
-            Config.SECTION_REMOTE_NO_TRAVERSE, self.DEFAULT_NO_TRAVERSE
-        )
-        self.verify = config.get(
-            Config.SECTION_REMOTE_VERIFY, self.DEFAULT_VERIFY
-        )
+        self.no_traverse = config.get("no_traverse", self.DEFAULT_NO_TRAVERSE)
+        self.verify = config.get("verify", self.DEFAULT_VERIFY)
         self._dir_info = {}
 
-        types = config.get(Config.SECTION_CACHE_TYPE, None)
-        if types:
-            if isinstance(types, str):
-                types = [t.strip() for t in types.split(",")]
-            self.cache_types = types
-        else:
-            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
+        self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES)
         self.cache_type_confirmed = False
 
     def _check_requires(self, config):
@@ -122,9 +116,7 @@ class RemoteBASE(object):
         if not missing:
             return
 
-        url = config.get(
-            Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
-        )
+        url = config.get("url", "{}://".format(self.scheme))
         msg = (
             "URL '{}' is supported but requires these missing "
             "dependencies: {}. If you have installed dvc using pip, "
@@ -145,19 +137,6 @@ class RemoteBASE(object):
         ).format(url, missing, " ".join(missing), self.scheme)
         raise RemoteMissingDepsError(msg)
 
-    def _get_checksum_jobs(self, config):
-        checksum_jobs = config.get(Config.SECTION_REMOTE_CHECKSUM_JOBS)
-        if checksum_jobs:
-            return checksum_jobs
-
-        if self.repo:
-            core = self.repo.config.config.get(Config.SECTION_CORE, {})
-            return core.get(
-                Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
-            )
-
-        return self.CHECKSUM_JOBS
-
     def __repr__(self):
         return "{class_name}: '{path_info}'".format(
             class_name=type(self).__name__,
@@ -169,7 +148,7 @@ class RemoteBASE(object):
         if isinstance(config, (str, bytes)):
             url = config
         else:
-            url = config[Config.SECTION_REMOTE_URL]
+            url = config["url"]
 
         # NOTE: silently skipping remote, calling code should handle that
         parsed = urlparse(url)
@@ -190,7 +169,7 @@ class RemoteBASE(object):
                 tasks,
                 total=len(file_infos),
                 unit="md5",
-                desc="Computing hashes (only done once)",
+                desc="Computing file/dir hashes (only done once)",
             ) as tasks:
                 checksums = dict(zip(file_infos, tasks))
         return checksums
@@ -233,6 +212,9 @@ class RemoteBASE(object):
         return sorted(result, key=itemgetter(self.PARAM_RELPATH))
 
     def get_dir_checksum(self, path_info):
+        if not self.cache:
+            raise RemoteCacheRequiredError(path_info)
+
         dir_info = self._collect_dir(path_info)
         checksum, tmp_info = self._get_dir_info_checksum(dir_info)
         new_info = self.cache.checksum_to_path_info(checksum)
@@ -338,13 +320,13 @@ class RemoteBASE(object):
 
         A file is considered changed if:
             - It doesn't exist on the working directory (was unlinked)
-            - Checksum is not computed (saving a new file)
-            - The checkusm stored in the State is different from the given one
+            - Hash value is not computed (saving a new file)
+            - The hash value stored is different from the given one
             - There's no file in the cache
 
         Args:
             path_info: dict with path information.
-            checksum: expected checksum for this data.
+            checksum: expected hash value for this data.
 
         Returns:
             bool: True if data has changed, False otherwise.
@@ -360,7 +342,7 @@ class RemoteBASE(object):
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
         if checksum is None:
-            logger.debug("checksum for '{}' is missing.", path_info)
+            logger.debug("hash value for '{}' is missing.", path_info)
             return True
 
         if self.changed_cache(checksum):
@@ -372,7 +354,7 @@ class RemoteBASE(object):
         actual = self.get_checksum(path_info)
         if checksum != actual:
             logger.debug(
-                "checksum '{}'(actual '{}') for '{}' has changed.",
+                "hash value '{}' for '{}' has changed (actual '{}').",
                 checksum,
                 actual,
                 path_info,
@@ -489,7 +471,9 @@ class RemoteBASE(object):
         cache_info = self.checksum_to_path_info(checksum)
         dir_info = self.get_dir_cache(checksum)
 
-        for entry in dir_info:
+        for entry in Tqdm(
+            dir_info, desc="Saving " + path_info.name, unit="file"
+        ):
             entry_info = path_info / entry[self.PARAM_RELPATH]
             entry_checksum = entry[self.PARAM_CHECKSUM]
             self._save_file(entry_info, entry_checksum, save_link=False)
@@ -872,9 +856,7 @@ class RemoteBASE(object):
         """The file is changed we need to checkout a new copy"""
         cache_info = self.checksum_to_path_info(checksum)
         if self.exists(path_info):
-            logger.warning(
-                "data '{}' exists. Removing before checkout.", path_info
-            )
+            logger.debug("data '{}' will be replaced.", path_info)
             self.safe_remove(path_info, force=force)
 
         self.link(cache_info, path_info)
@@ -887,7 +869,6 @@ class RemoteBASE(object):
         """Optional: Implement only if the remote needs to create
         directories before copying/linking/moving data
         """
-        pass
 
     def _checkout_dir(
         self,
@@ -956,7 +937,7 @@ class RemoteBASE(object):
         skip = False
         if not checksum:
             logger.warning(
-                "No checksum info found for '{}'. " "It won't be created.",
+                "No file hash info found for '{}'. " "It won't be created.",
                 path_info,
             )
             self.safe_remove(path_info, force=force)

dvc/remote/config.py

@@ -1,163 +0,0 @@
-import logging
-import os
-import posixpath
-from urllib.parse import urlparse
-
-from dvc.config import Config, ConfigError
-from dvc.utils import relpath
-
-
-logger = logging.getLogger(__name__)
-
-
-class RemoteConfig(object):
-    def __init__(self, config):
-        self.config = config
-
-    def get_settings(self, name):
-        """
-        Args:
-            name (str): The name of the remote that we want to retrieve
-
-        Returns:
-            dict: The content beneath the given remote name.
-
-        Example:
-            >>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
-            >>> get_settings("server")
-            {'url': 'ssh://localhost/'}
-        """
-        settings = self.config.config.get(
-            Config.SECTION_REMOTE_FMT.format(name.lower())
-        )
-
-        if settings is None:
-            raise ConfigError(
-                "Unable to find remote section '{}'".format(name)
-            )
-
-        parsed = urlparse(settings["url"])
-
-        # Support for cross referenced remotes.
-        # This will merge the settings, giving priority to the outer reference.
-        # For example, having:
-        #
-        #       dvc remote add server ssh://localhost
-        #       dvc remote modify server user root
-        #       dvc remote modify server ask_password true
-        #
-        #       dvc remote add images remote://server/tmp/pictures
-        #       dvc remote modify images user alice
-        #       dvc remote modify images ask_password false
-        #       dvc remote modify images password asdf1234
-        #
-        # Results on a config dictionary like:
-        #
-        #       {
-        #           "url": "ssh://localhost/tmp/pictures",
-        #           "user": "alice",
-        #           "password": "asdf1234",
-        #           "ask_password": False,
-        #       }
-        #
-        if parsed.scheme == "remote":
-            reference = self.get_settings(parsed.netloc)
-            url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
-            merged = reference.copy()
-            merged.update(settings)
-            merged["url"] = url
-            return merged
-
-        return settings
-
-    @staticmethod
-    def resolve_path(path, config_file):
-        """Resolve path relative to config file location.
-
-        Args:
-            path: Path to be resolved.
-            config_file: Path to config file, which `path` is specified
-                relative to.
-
-        Returns:
-            Path relative to the `config_file` location. If `path` is an
-            absolute path then it will be returned without change.
-
-        """
-        if os.path.isabs(path):
-            return path
-        return relpath(path, os.path.dirname(config_file))
-
-    def add(self, name, url, default=False, force=False, level=None):
-        from dvc.remote import _get, RemoteLOCAL
-
-        configobj = self.config.get_configobj(level)
-        remote = _get({Config.SECTION_REMOTE_URL: url})
-        if remote == RemoteLOCAL and not url.startswith("remote://"):
-            url = self.resolve_path(url, configobj.filename)
-
-        self.config.set(
-            Config.SECTION_REMOTE_FMT.format(name),
-            Config.SECTION_REMOTE_URL,
-            url,
-            force=force,
-            level=level,
-        )
-        if default:
-            self.config.set(
-                Config.SECTION_CORE,
-                Config.SECTION_CORE_REMOTE,
-                name,
-                level=level,
-            )
-
-    def remove(self, name, level=None):
-        self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level)
-
-        if level is None:
-            level = Config.LEVEL_REPO
-
-        for lev in [
-            Config.LEVEL_LOCAL,
-            Config.LEVEL_REPO,
-            Config.LEVEL_GLOBAL,
-            Config.LEVEL_SYSTEM,
-        ]:
-            self.config.unset(
-                Config.SECTION_CORE,
-                Config.SECTION_CORE_REMOTE,
-                level=lev,
-                force=True,
-            )
-            if lev == level:
-                break
-
-    def modify(self, name, option, value, level=None):
-        # try to get remote settings to verify that section exists
-        self.get_settings(name)
-
-        self.config.set(
-            Config.SECTION_REMOTE_FMT.format(name), option, value, level=level
-        )
-
-    def list(self, level=None):
-        return self.config.list_options(
-            Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level
-        )
-
-    def set_default(self, name, unset=False, level=None):
-        if unset:
-            self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE)
-            return
-        self.config.set(
-            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level
-        )
-
-    def get_default(self, level=None):
-        return self.config.get(
-            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level
-        )
-
-    def has_default(self):
-        core = self.config.config[Config.SECTION_CORE]
-        return bool(core.get(Config.SECTION_CORE_REMOTE))

dvc/remote/gdrive.py

@@ -1,8 +1,9 @@
 import os
 import posixpath
 import logging
-import threading
 import re
+import threading
+from urllib.parse import urlparse
 
 from funcy import retry, compose, decorator, wrap_with
 from funcy.py3 import cat
@@ -11,9 +12,8 @@ from dvc.progress import Tqdm
 from dvc.scheme import Schemes
 from dvc.path_info import CloudURLInfo
 from dvc.remote.base import RemoteBASE
-from dvc.config import Config
 from dvc.exceptions import DvcException
-from dvc.utils import tmp_fname
+from dvc.utils import tmp_fname, format_link
 
 logger = logging.getLogger(__name__)
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
@@ -23,22 +23,31 @@ class GDriveRetriableError(DvcException):
     pass
 
 
+class GDrivePathNotFound(DvcException):
+    def __init__(self, path_info):
+        super().__init__("Google Drive path '{}' not found.".format(path_info))
+
+
 class GDriveAccessTokenRefreshError(DvcException):
-    pass
+    def __init__(self):
+        super().__init__("Google Drive access token refreshment is failed.")
 
 
 class GDriveMissedCredentialKeyError(DvcException):
-    pass
+    def __init__(self, path):
+        super().__init__(
+            "Google Drive user credentials file '{}' "
+            "misses value for key.".format(path)
+        )
 
 
 @decorator
 def _wrap_pydrive_retriable(call):
-    from apiclient import errors
     from pydrive2.files import ApiRequestError
 
     try:
         result = call()
-    except (ApiRequestError, errors.HttpError) as exception:
+    except ApiRequestError as exception:
         retry_codes = ["403", "500", "502", "503", "504"]
         if any(
             "HttpError {}".format(code) in str(exception)
@@ -50,17 +59,32 @@ def _wrap_pydrive_retriable(call):
 
 
 gdrive_retry = compose(
-    # 8 tries, start at 0.5s, multiply by golden ratio, cap at 10s
+    # 15 tries, start at 0.5s, multiply by golden ratio, cap at 20s
     retry(
-        8, GDriveRetriableError, timeout=lambda a: min(0.5 * 1.618 ** a, 10)
+        15, GDriveRetriableError, timeout=lambda a: min(0.5 * 1.618 ** a, 20)
     ),
     _wrap_pydrive_retriable,
 )
 
 
+class GDriveURLInfo(CloudURLInfo):
+    def __init__(self, url):
+        super().__init__(url)
+
+        # GDrive URL host part is case sensitive,
+        # we are restoring it here.
+        p = urlparse(url)
+        self.host = p.netloc
+        assert self.netloc == self.host
+
+        # Normalize path. Important since we have a cache (path to ID)
+        # and don't want to deal with different variations of path in it.
+        self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
+
+
 class RemoteGDrive(RemoteBASE):
     scheme = Schemes.GDRIVE
-    path_cls = CloudURLInfo
+    path_cls = GDriveURLInfo
     REQUIRES = {"pydrive2": "pydrive2"}
     DEFAULT_NO_TRAVERSE = False
     DEFAULT_VERIFY = True
@@ -70,48 +94,164 @@ class RemoteGDrive(RemoteBASE):
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
-        self.path_info = self.path_cls(config[Config.SECTION_REMOTE_URL])
+        self.path_info = self.path_cls(config["url"])
 
-        bucket = re.search(
-            "{}://(.*)".format(self.scheme),
-            config[Config.SECTION_REMOTE_URL],
-            re.IGNORECASE,
-        )
-        self.bucket = (
-            bucket.group(1).split("/")[0] if bucket else self.path_info.bucket
-        )
+        if not self.path_info.bucket:
+            raise DvcException(
+                "Empty Google Drive URL '{}'. Learn more at "
+                "{}.".format(
+                    config["url"],
+                    format_link("https://man.dvc.org/remote/add"),
+                )
+            )
 
-        self.config = config
-        self.init_drive()
-
-    def init_drive(self):
-        self.client_id = self.config.get(Config.SECTION_GDRIVE_CLIENT_ID, None)
-        self.client_secret = self.config.get(
-            Config.SECTION_GDRIVE_CLIENT_SECRET, None
-        )
-        if not self.client_id or not self.client_secret:
+        self._bucket = self.path_info.bucket
+        self._client_id = config.get("gdrive_client_id")
+        self._client_secret = config.get("gdrive_client_secret")
+        if not self._client_id or not self._client_secret:
             raise DvcException(
                 "Please specify Google Drive's client id and "
-                "secret in DVC's config. Learn more at "
-                "https://man.dvc.org/remote/add."
+                "secret in DVC config. Learn more at "
+                "{}.".format(format_link("https://man.dvc.org/remote/add"))
             )
-        self.gdrive_user_credentials_path = (
+        self._gdrive_user_credentials_path = (
             tmp_fname(os.path.join(self.repo.tmp_dir, ""))
             if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
-            else self.config.get(
-                Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE,
+            else config.get(
+                "gdrive_user_credentials_file",
                 os.path.join(
                     self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE
                 ),
             )
         )
 
+        self._list_params = None
+        self._gdrive = None
+
+        self._cache_initialized = False
+        self._remote_root_id = None
+        self._cached_dirs = None
+        self._cached_ids = None
+
+    @property
+    @wrap_with(threading.RLock())
+    def drive(self):
+        from pydrive2.auth import RefreshError
+
+        if not self._gdrive:
+            from pydrive2.auth import GoogleAuth
+            from pydrive2.drive import GoogleDrive
+
+            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                with open(
+                    self._gdrive_user_credentials_path, "w"
+                ) as credentials_file:
+                    credentials_file.write(
+                        os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
+                    )
+
+            GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
+            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
+                "client_id": self._client_id,
+                "client_secret": self._client_secret,
+                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                "token_uri": "https://oauth2.googleapis.com/token",
+                "revoke_uri": "https://oauth2.googleapis.com/revoke",
+                "redirect_uri": "",
+            }
+            GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
+            GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
+            GoogleAuth.DEFAULT_SETTINGS[
+                "save_credentials_file"
+            ] = self._gdrive_user_credentials_path
+            GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
+            GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
+                "https://www.googleapis.com/auth/drive",
+                "https://www.googleapis.com/auth/drive.appdata",
+            ]
+
+            # Pass non existent settings path to force DEFAULT_SETTINGS loading
+            gauth = GoogleAuth(settings_file="")
+
+            try:
+                gauth.CommandLineAuth()
+            except RefreshError as exc:
+                raise GDriveAccessTokenRefreshError from exc
+            except KeyError as exc:
+                raise GDriveMissedCredentialKeyError(
+                    self._gdrive_user_credentials_path
+                ) from exc
+            # Handle pydrive2.auth.AuthenticationError and other auth failures
+            except Exception as exc:
+                raise DvcException(
+                    "Google Drive authentication failed"
+                ) from exc
+            finally:
+                if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                    os.remove(self._gdrive_user_credentials_path)
+
+            self._gdrive = GoogleDrive(gauth)
+
+        return self._gdrive
+
+    @wrap_with(threading.RLock())
+    def _initialize_cache(self):
+        if self._cache_initialized:
+            return
+
+        cached_dirs = {}
+        cached_ids = {}
+        self._remote_root_id = self._get_remote_id(self.path_info)
+        for dir1 in self.gdrive_list_item(
+            "'{}' in parents and trashed=false".format(self._remote_root_id)
+        ):
+            remote_path = posixpath.join(self.path_info.path, dir1["title"])
+            cached_dirs.setdefault(remote_path, []).append(dir1["id"])
+            cached_ids[dir1["id"]] = dir1["title"]
+
+        self._cached_dirs = cached_dirs
+        self._cached_ids = cached_ids
+        self._cache_initialized = True
+
+    @property
+    def cached_dirs(self):
+        if not self._cache_initialized:
+            self._initialize_cache()
+        return self._cached_dirs
+
+    @property
+    def cached_ids(self):
+        if not self._cache_initialized:
+            self._initialize_cache()
+        return self._cached_ids
+
+    @property
+    def remote_root_id(self):
+        if not self._cache_initialized:
+            self._initialize_cache()
+        return self._remote_root_id
+
+    @property
+    def list_params(self):
+        if not self._list_params:
+            params = {"corpora": "default"}
+            if self._bucket != "root" and self._bucket != "appDataFolder":
+                params["driveId"] = self._get_remote_drive_id(self._bucket)
+                params["corpora"] = "drive"
+            self._list_params = params
+        return self._list_params
+
     @gdrive_retry
     def gdrive_upload_file(
-        self, args, no_progress_bar=True, from_file="", progress_name=""
+        self,
+        parent_id,
+        title,
+        no_progress_bar=True,
+        from_file="",
+        progress_name="",
     ):
         item = self.drive.CreateFile(
-            {"title": args["title"], "parents": [{"id": args["parent_id"]}]}
+            {"title": title, "parents": [{"id": parent_id}]}
         )
 
         with open(from_file, "rb") as fobj:
@@ -134,9 +274,11 @@ class RemoteGDrive(RemoteBASE):
     def gdrive_download_file(
         self, file_id, to_file, progress_name, no_progress_bar
     ):
-        gdrive_file = self.drive.CreateFile({"id": file_id})
+        param = {"id": file_id}
+        # it does not create a file on the remote
+        gdrive_file = self.drive.CreateFile(param)
         bar_format = (
-            "Donwloading {desc:{ncols_desc}.{ncols_desc}}... "
+            "Downloading {desc:{ncols_desc}.{ncols_desc}}... "
             + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
         )
         with Tqdm(
@@ -145,7 +287,10 @@ class RemoteGDrive(RemoteBASE):
             gdrive_file.GetContentFile(to_file)
 
     def gdrive_list_item(self, query):
-        file_list = self.drive.ListFile({"q": query, "maxResults": 1000})
+        param = {"q": query, "maxResults": 1000}
+        param.update(self.list_params)
+
+        file_list = self.drive.ListFile(param)
 
         # Isolate and decorate fetching of remote drive items in pages
         get_list = gdrive_retry(lambda: next(file_list, None))
@@ -153,114 +298,39 @@ class RemoteGDrive(RemoteBASE):
         # Fetch pages until None is received, lazily flatten the thing
         return cat(iter(get_list, None))
 
-    def cache_root_dirs(self):
-        cached_dirs = {}
-        cached_ids = {}
-        for dir1 in self.gdrive_list_item(
-            "'{}' in parents and trashed=false".format(self.remote_root_id)
-        ):
-            remote_path = posixpath.join(self.path_info.path, dir1["title"])
-            cached_dirs.setdefault(remote_path, []).append(dir1["id"])
-            cached_ids[dir1["id"]] = dir1["title"]
-        return cached_dirs, cached_ids
-
-    @property
-    def cached_dirs(self):
-        if not hasattr(self, "_cached_dirs"):
-            self.drive
-        return self._cached_dirs
-
-    @property
-    def cached_ids(self):
-        if not hasattr(self, "_cached_ids"):
-            self.drive
-        return self._cached_ids
-
-    @property
     @wrap_with(threading.RLock())
-    def drive(self):
-        from pydrive2.auth import RefreshError
+    def gdrive_create_dir(self, parent_id, title, remote_path):
+        if parent_id == self.remote_root_id:
+            cached = self.cached_dirs.get(remote_path, [])
+            if cached:
+                return cached[0]
 
-        if not hasattr(self, "_gdrive"):
-            from pydrive2.auth import GoogleAuth
-            from pydrive2.drive import GoogleDrive
+        item = self._create_remote_dir(parent_id, title)
 
-            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
-                with open(
-                    self.gdrive_user_credentials_path, "w"
-                ) as credentials_file:
-                    credentials_file.write(
-                        os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
-                    )
+        if parent_id == self.remote_root_id:
+            self.cached_dirs.setdefault(remote_path, []).append(item["id"])
+            self.cached_ids[item["id"]] = item["title"]
 
-            GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
-            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
-                "client_id": self.client_id,
-                "client_secret": self.client_secret,
-                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-                "token_uri": "https://oauth2.googleapis.com/token",
-                "revoke_uri": "https://oauth2.googleapis.com/revoke",
-                "redirect_uri": "",
-            }
-            GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
-            GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
-            GoogleAuth.DEFAULT_SETTINGS[
-                "save_credentials_file"
-            ] = self.gdrive_user_credentials_path
-            GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
-            GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
-                "https://www.googleapis.com/auth/drive",
-                "https://www.googleapis.com/auth/drive.appdata",
-            ]
-
-            # Pass non existent settings path to force DEFAULT_SETTINGS loading
-            gauth = GoogleAuth(settings_file="")
-
-            try:
-                gauth.CommandLineAuth()
-            except RefreshError as exc:
-                raise GDriveAccessTokenRefreshError(
-                    "Google Drive's access token refreshment is failed"
-                ) from exc
-            except KeyError as exc:
-                raise GDriveMissedCredentialKeyError(
-                    "Google Drive's user credentials file '{}' "
-                    "misses value for key '{}'".format(
-                        self.gdrive_user_credentials_path, str(exc)
-                    )
-                )
-            # Handle pydrive2.auth.AuthenticationError and others auth failures
-            except Exception as exc:
-                raise DvcException(
-                    "Google Drive authentication failed"
-                ) from exc
-            finally:
-                if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
-                    os.remove(self.gdrive_user_credentials_path)
-
-            self._gdrive = GoogleDrive(gauth)
-
-            self.remote_root_id = self.get_remote_id(
-                self.path_info, create=True
-            )
-            self._cached_dirs, self._cached_ids = self.cache_root_dirs()
-
-        return self._gdrive
+        return item["id"]
 
     @gdrive_retry
-    def create_remote_dir(self, parent_id, title):
+    def _create_remote_dir(self, parent_id, title):
+        parent = {"id": parent_id}
         item = self.drive.CreateFile(
-            {
-                "title": title,
-                "parents": [{"id": parent_id}],
-                "mimeType": FOLDER_MIME_TYPE,
-            }
+            {"title": title, "parents": [parent], "mimeType": FOLDER_MIME_TYPE}
         )
         item.Upload()
         return item
 
     @gdrive_retry
-    def get_remote_item(self, name, parents_ids):
+    def _delete_remote_file(self, remote_id):
+        param = {"id": remote_id}
+        # it does not create a file on the remote
+        item = self.drive.CreateFile(param)
+        item.Delete()
+
+    @gdrive_retry
+    def _get_remote_item(self, name, parents_ids):
         if not parents_ids:
             return None
         query = "({})".format(
@@ -272,64 +342,78 @@ class RemoteGDrive(RemoteBASE):
 
         query += " and trashed=false and title='{}'".format(name)
 
+        # Remote might contain items with duplicated path (titles).
+        # We thus limit number of items.
+        param = {"q": query, "maxResults": 1}
+        param.update(self.list_params)
+
         # Limit found remote items count to 1 in response
-        item_list = self.drive.ListFile(
-            {"q": query, "maxResults": 1}
-        ).GetList()
+        item_list = self.drive.ListFile(param).GetList()
         return next(iter(item_list), None)
 
-    def resolve_remote_item_from_path(self, path_parts, create):
-        parents_ids = [self.bucket]
-        current_path = ""
-        for path_part in path_parts:
-            current_path = posixpath.join(current_path, path_part)
-            remote_ids = self.get_remote_id_from_cache(current_path)
-            if remote_ids:
-                parents_ids = remote_ids
-                continue
-            item = self.get_remote_item(path_part, parents_ids)
-            if not item and create:
-                item = self.create_remote_dir(parents_ids[0], path_part)
-            elif not item:
-                return None
-            parents_ids = [item["id"]]
-        return item
+    @gdrive_retry
+    def _get_remote_drive_id(self, remote_id):
+        param = {"id": remote_id}
+        # it does not create a file on the remote
+        item = self.drive.CreateFile(param)
+        item.FetchMetadata("driveId")
+        return item.get("driveId", None)
 
-    def get_remote_id_from_cache(self, remote_path):
-        if hasattr(self, "_cached_dirs"):
-            return self.cached_dirs.get(remote_path, [])
+    def _get_cached_remote_ids(self, path):
+        if not path:
+            return [self._bucket]
+        if self._cache_initialized:
+            if path == self.path_info.path:
+                return [self.remote_root_id]
+            return self.cached_dirs.get(path, [])
         return []
 
-    def get_remote_id(self, path_info, create=False):
-        remote_ids = self.get_remote_id_from_cache(path_info.path)
-
+    def _path_to_remote_ids(self, path, create):
+        remote_ids = self._get_cached_remote_ids(path)
         if remote_ids:
-            return remote_ids[0]
+            return remote_ids
 
-        file1 = self.resolve_remote_item_from_path(
-            path_info.path.split("/"), create
-        )
-        return file1["id"] if file1 else ""
+        parent_path, part = posixpath.split(path)
+        parent_ids = self._path_to_remote_ids(parent_path, create)
+        item = self._get_remote_item(part, parent_ids)
+
+        if not item:
+            return (
+                [self.gdrive_create_dir(parent_ids[0], part, path)]
+                if create
+                else []
+            )
+
+        return [item["id"]]
+
+    def _get_remote_id(self, path_info, create=False):
+        assert path_info.bucket == self._bucket
+
+        remote_ids = self._path_to_remote_ids(path_info.path, create)
+        if not remote_ids:
+            raise GDrivePathNotFound(path_info)
+
+        return remote_ids[0]
 
     def exists(self, path_info):
-        return self.get_remote_id(path_info) != ""
+        try:
+            self._get_remote_id(path_info)
+        except GDrivePathNotFound:
+            return False
+        else:
+            return True
 
     def _upload(self, from_file, to_info, name, no_progress_bar):
         dirname = to_info.parent
-        if dirname:
-            parent_id = self.get_remote_id(dirname, True)
-        else:
-            parent_id = to_info.bucket
+        assert dirname
+        parent_id = self._get_remote_id(dirname, True)
 
         self.gdrive_upload_file(
-            {"title": to_info.name, "parent_id": parent_id},
-            no_progress_bar,
-            from_file,
-            name,
+            parent_id, to_info.name, no_progress_bar, from_file, name
         )
 
     def _download(self, from_info, to_file, name, no_progress_bar):
-        file_id = self.get_remote_id(from_info)
+        file_id = self._get_remote_id(from_info)
         self.gdrive_download_file(file_id, to_file, name, no_progress_bar)
 
     def all(self):
@@ -351,3 +435,7 @@ class RemoteGDrive(RemoteBASE):
             except ValueError:
                 # We ignore all the non-cache looking files
                 logger.debug('Ignoring path as "non-cache looking"')
+
+    def remove(self, path_info):
+        remote_id = self._get_remote_id(path_info)
+        self._delete_remote_file(remote_id)

dvc/remote/gs.py

@@ -7,7 +7,6 @@ import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
@@ -75,11 +74,11 @@ class RemoteGS(RemoteBASE):
     def __init__(self, repo, config):
         super().__init__(repo, config)
 
-        url = config.get(Config.SECTION_REMOTE_URL, "gs:///")
+        url = config.get("url", "gs:///")
         self.path_info = self.path_cls(url)
 
-        self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
-        self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
+        self.projectname = config.get("projectname", None)
+        self.credentialpath = config.get("credentialpath")
 
     @wrap_prop(threading.Lock())
     @cached_property

dvc/remote/hdfs.py

@@ -10,7 +10,6 @@ from urllib.parse import urlparse
 
 from .base import RemoteBASE, RemoteCmdError
 from .pool import get_connection
-from dvc.config import Config
 from dvc.scheme import Schemes
 from dvc.utils import fix_env, tmp_fname
 
@@ -26,13 +25,12 @@ class RemoteHDFS(RemoteBASE):
     def __init__(self, repo, config):
         super().__init__(repo, config)
         self.path_info = None
-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
         if not url:
             return
 
         parsed = urlparse(url)
-
-        user = parsed.username or config.get(Config.SECTION_REMOTE_USER)
+        user = parsed.username or config.get("user")
 
         self.path_info = self.path_cls.from_parts(
             scheme=self.scheme,

dvc/remote/http.py

@@ -3,7 +3,6 @@ import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.config import Config
 from dvc.config import ConfigError
 from dvc.exceptions import DvcException, HTTPError
 from dvc.progress import Tqdm
@@ -24,7 +23,7 @@ class RemoteHTTP(RemoteBASE):
     def __init__(self, repo, config):
         super().__init__(repo, config)
 
-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
         self.path_info = self.path_cls(url) if url else None
 
         if not self.no_traverse:

dvc/remote/local.py

@@ -7,27 +7,17 @@ from functools import partial
 
 from shortuuid import uuid
 
-from dvc.config import Config
-from dvc.exceptions import DownloadError
-from dvc.exceptions import DvcException
-from dvc.exceptions import UploadError
+from dvc.compat import fspath_py35
+from dvc.exceptions import DvcException, DownloadError, UploadError
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
-from dvc.remote.base import STATUS_DELETED
-from dvc.remote.base import STATUS_MAP
-from dvc.remote.base import STATUS_MISSING
-from dvc.remote.base import STATUS_NEW
+from dvc.remote.base import RemoteBASE, STATUS_MAP
+from dvc.remote.base import STATUS_DELETED, STATUS_MISSING, STATUS_NEW
 from dvc.scheme import Schemes
 from dvc.scm.tree import is_working_tree
 from dvc.system import System
-from dvc.utils.fs import copyfile
-from dvc.utils import file_md5
-from dvc.utils import relpath
-from dvc.utils import tmp_fname
-from dvc.compat import fspath_py35
-from dvc.utils.fs import move, makedirs, remove
-from dvc.utils.fs import walk_files
+from dvc.utils import file_md5, relpath, tmp_fname
+from dvc.utils.fs import copyfile, move, makedirs, remove, walk_files
 
 logger = logging.getLogger(__name__)
 
@@ -46,22 +36,16 @@ class RemoteLOCAL(RemoteBASE):
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
-        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
+        self.protected = config.get("protected", False)
 
-        shared = config.get(Config.SECTION_CACHE_SHARED)
+        shared = config.get("shared")
         self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
 
         if self.protected:
             # cache files are set to be read-only for everyone
             self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
 
-        cache_dir = config.get(Config.SECTION_REMOTE_URL)
-
-        if cache_dir is not None and not os.path.isabs(cache_dir):
-            cwd = config[Config.PRIVATE_CWD]
-            cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
-
-        self.cache_dir = cache_dir
+        self.cache_dir = config.get("url")
         self._dir_info = {}
 
     @property

dvc/remote/oss.py

@@ -4,7 +4,6 @@ import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.config import Config
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
@@ -42,21 +41,19 @@ class RemoteOSS(RemoteBASE):
     def __init__(self, repo, config):
         super().__init__(repo, config)
 
-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
         self.path_info = self.path_cls(url) if url else None
 
-        self.endpoint = config.get(Config.SECTION_OSS_ENDPOINT) or os.getenv(
-            "OSS_ENDPOINT"
-        )
+        self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")
 
         self.key_id = (
-            config.get(Config.SECTION_OSS_ACCESS_KEY_ID)
+            config.get("oss_key_id")
             or os.getenv("OSS_ACCESS_KEY_ID")
             or "defaultId"
         )
 
         self.key_secret = (
-            config.get(Config.SECTION_OSS_ACCESS_KEY_SECRET)
+            config.get("oss_key_secret")
             or os.getenv("OSS_ACCESS_KEY_SECRET")
             or "defaultSecret"
         )

dvc/remote/s3.py

@@ -6,7 +6,7 @@ import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.config import Config, ConfigError
+from dvc.config import ConfigError
 from dvc.exceptions import DvcException
 from dvc.exceptions import ETagMismatchError
 from dvc.path_info import CloudURLInfo
@@ -26,35 +26,33 @@ class RemoteS3(RemoteBASE):
     def __init__(self, repo, config):
         super().__init__(repo, config)
 
-        url = config.get(Config.SECTION_REMOTE_URL, "s3://")
+        url = config.get("url", "s3://")
         self.path_info = self.path_cls(url)
 
-        self.region = config.get(Config.SECTION_AWS_REGION)
+        self.region = config.get("region")
+        self.profile = config.get("profile")
+        self.endpoint_url = config.get("endpointurl")
 
-        self.profile = config.get(Config.SECTION_AWS_PROFILE)
-
-        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
-
-        if config.get(Config.SECTION_AWS_LIST_OBJECTS):
+        if config.get("listobjects"):
             self.list_objects_api = "list_objects"
         else:
             self.list_objects_api = "list_objects_v2"
 
-        self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
+        self.use_ssl = config.get("use_ssl", True)
 
         self.extra_args = {}
 
-        self.sse = config.get(Config.SECTION_AWS_SSE, "")
+        self.sse = config.get("sse")
         if self.sse:
             self.extra_args["ServerSideEncryption"] = self.sse
 
-        self.acl = config.get(Config.SECTION_AWS_ACL, "")
+        self.acl = config.get("acl")
         if self.acl:
             self.extra_args["ACL"] = self.acl
 
         self._append_aws_grants_to_extra_args(config)
 
-        shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
+        shared_creds = config.get("credentialpath")
         if shared_creds:
             os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
 
@@ -322,14 +320,14 @@ class RemoteS3(RemoteBASE):
         """
 
         grants = {
-            Config.SECTION_AWS_GRANT_FULL_CONTROL: "GrantFullControl",
-            Config.SECTION_AWS_GRANT_READ: "GrantRead",
-            Config.SECTION_AWS_GRANT_READ_ACP: "GrantReadACP",
-            Config.SECTION_AWS_GRANT_WRITE_ACP: "GrantWriteACP",
+            "grant_full_control": "GrantFullControl",
+            "grant_read": "GrantRead",
+            "grant_read_acp": "GrantReadACP",
+            "grant_write_acp": "GrantWriteACP",
         }
 
         for grant_option, extra_args_key in grants.items():
-            if config.get(grant_option, ""):
+            if config.get(grant_option):
                 if self.acl:
                     raise ConfigError(
                         "`acl` and `grant_*` AWS S3 config options "

dvc/remote/slow_link_detection.py

@@ -1,12 +1,10 @@
 import logging
 import sys
 import time
+
 from functools import wraps
-
 import colorama
 
-from dvc.config import Config
-
 
 logger = logging.getLogger(__name__)
 this = sys.modules[__name__]
@@ -31,11 +29,9 @@ def slow_link_guard(f):
         if this.already_displayed:
             return f(remote, *args, **kwargs)
 
-        config = remote.repo.config.config.get(Config.SECTION_CACHE, {})
-        cache_type = config.get(Config.SECTION_CACHE_TYPE)
-        should_warn = config.get(Config.SECTION_CACHE_SLOW_LINK_WARNING, True)
-
-        if not should_warn or cache_type:
+        cache_conf = remote.repo.config["cache"]
+        slow_link_warning = cache_conf.get("slow_link_warning", True)
+        if not slow_link_warning or cache_conf.get("type"):
             return f(remote, *args, **kwargs)
 
         start = time.time()

dvc/remote/ssh/__init__.py

@@ -9,10 +9,9 @@ from concurrent.futures import ThreadPoolExecutor
 from contextlib import closing, contextmanager
 from urllib.parse import urlparse
 
-from funcy import memoize, wrap_with
+from funcy import memoize, wrap_with, silent, first
 
 import dvc.prompt as prompt
-from dvc.config import Config
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.remote.pool import get_connection
@@ -50,20 +49,20 @@ class RemoteSSH(RemoteBASE):
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
         if url:
             parsed = urlparse(url)
             user_ssh_config = self._load_user_ssh_config(parsed.hostname)
 
             host = user_ssh_config.get("hostname", parsed.hostname)
             user = (
-                config.get(Config.SECTION_REMOTE_USER)
+                config.get("user")
                 or parsed.username
                 or user_ssh_config.get("user")
                 or getpass.getuser()
             )
             port = (
-                config.get(Config.SECTION_REMOTE_PORT)
+                config.get("port")
                 or parsed.port
                 or self._try_get_ssh_config_port(user_ssh_config)
                 or self.DEFAULT_PORT
@@ -80,14 +79,12 @@ class RemoteSSH(RemoteBASE):
             user_ssh_config = {}
 
         self.keyfile = config.get(
-            Config.SECTION_REMOTE_KEY_FILE
+            "keyfile"
         ) or self._try_get_ssh_config_keyfile(user_ssh_config)
-        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
-        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
-        self.ask_password = config.get(
-            Config.SECTION_REMOTE_ASK_PASSWORD, False
-        )
-        self.gss_auth = config.get(Config.SECTION_REMOTE_GSS_AUTH, False)
+        self.timeout = config.get("timeout", self.TIMEOUT)
+        self.password = config.get("password", None)
+        self.ask_password = config.get("ask_password", False)
+        self.gss_auth = config.get("gss_auth", False)
 
     @staticmethod
     def ssh_config_filename():
@@ -110,17 +107,11 @@ class RemoteSSH(RemoteBASE):
 
     @staticmethod
     def _try_get_ssh_config_port(user_ssh_config):
-        try:
-            return int(user_ssh_config.get("port"))
-        except (ValueError, TypeError):
-            return None
+        return silent(int)(user_ssh_config.get("port"))
 
     @staticmethod
     def _try_get_ssh_config_keyfile(user_ssh_config):
-        identity_file = user_ssh_config.get("identityfile")
-        if identity_file and len(identity_file) > 0:
-            return identity_file[0]
-        return None
+        return first(user_ssh_config.get("identityfile") or ())
 
     def ensure_credentials(self, path_info=None):
         if path_info is None:

dvc/repo/__init__.py

@@ -1,4 +1,3 @@
-import logging
 import os
 from contextlib import contextmanager
 from functools import wraps
@@ -21,9 +20,6 @@ from dvc.utils.fs import path_isin
 from .graph import check_acyclic, get_pipeline, get_pipelines
 
 
-logger = logging.getLogger(__name__)
-
-
 def locked(f):
     @wraps(f)
     def wrapper(repo, *args, **kwargs):
@@ -43,6 +39,7 @@ class Repo(object):
     from dvc.repo.install import install
     from dvc.repo.add import add
     from dvc.repo.remove import remove
+    from dvc.repo.ls import ls
     from dvc.repo.lock import lock as lock_stage
     from dvc.repo.move import move
     from dvc.repo.run import run
@@ -87,7 +84,7 @@ class Repo(object):
         self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
         makedirs(self.tmp_dir, exist_ok=True)
 
-        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        hardlink_lock = self.config["core"].get("hardlink_lock", False)
         self.lock = make_lock(
             os.path.join(self.dvc_dir, "lock"),
             tmp_dir=os.path.join(self.dvc_dir, "tmp"),
@@ -97,13 +94,7 @@ class Repo(object):
 
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
-        self.state = State(self, self.config.config)
-
-        core = self.config.config[Config.SECTION_CORE]
-
-        level = core.get(Config.SECTION_CORE_LOGLEVEL)
-        if level:
-            logger.setLevel(level.upper())
+        self.state = State(self)
 
         self.cache = Cache(self)
         self.cloud = DataCloud(self)
@@ -125,23 +116,28 @@ class Repo(object):
         self._reset()
 
     def __repr__(self):
-        return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
+        return "{}: '{}'".format(self.__class__.__name__, self.root_dir)
 
     @classmethod
     def find_root(cls, root=None):
-        if root is None:
-            root = os.getcwd()
-        else:
-            root = os.path.abspath(os.path.realpath(root))
+        root_dir = os.path.realpath(root or os.curdir)
+
+        if not os.path.isdir(root_dir):
+            raise NotDvcRepoError("directory '{}' does not exist".format(root))
 
         while True:
-            dvc_dir = os.path.join(root, cls.DVC_DIR)
+            dvc_dir = os.path.join(root_dir, cls.DVC_DIR)
             if os.path.isdir(dvc_dir):
-                return root
-            if os.path.ismount(root):
+                return root_dir
+            if os.path.ismount(root_dir):
                 break
-            root = os.path.dirname(root)
-        raise NotDvcRepoError(root)
+            root_dir = os.path.dirname(root_dir)
+
+        message = (
+            "you are not inside of a DVC repository "
+            "(checked up to mount point '{}')"
+        ).format(root_dir)
+        raise NotDvcRepoError(message)
 
     @classmethod
     def find_dvc_dir(cls, root=None):
@@ -164,7 +160,7 @@ class Repo(object):
         updater = Updater(self.dvc_dir)
 
         flist = (
-            [self.config.config_local_file, updater.updater_file]
+            [self.config.files["local"], updater.updater_file]
             + [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir]
             + self.state.files
         )
@@ -205,7 +201,7 @@ class Repo(object):
             return [(stage, None) for stage in self.stages]
 
         try:
-            out, = self.find_outs_by_path(target, strict=False)
+            (out,) = self.find_outs_by_path(target, strict=False)
             filter_info = PathInfo(os.path.abspath(target))
             return [(out.stage, filter_info)]
         except OutputNotFoundError:
@@ -329,7 +325,17 @@ class Repo(object):
             for out in stage.outs:
                 for p in out.path_info.parents:
                     if p in outs:
-                        raise OverlappingOutputPathsError(outs[p], out)
+                        msg = (
+                            "Paths for outs:\n'{}'('{}')\n'{}'('{}')\n"
+                            "overlap. To avoid unpredictable behaviour, "
+                            "rerun command with non overlapping outs paths."
+                        ).format(
+                            str(outs[p]),
+                            outs[p].stage.relpath,
+                            str(out),
+                            out.stage.relpath,
+                        )
+                        raise OverlappingOutputPathsError(outs[p], out, msg)
 
         for stage in stages:
             stage_path_info = PathInfo(stage.path)
@@ -399,14 +405,13 @@ class Repo(object):
 
         abs_path = os.path.abspath(path)
         path_info = PathInfo(abs_path)
-        is_dir = self.tree.isdir(abs_path)
         match = path_info.__eq__ if strict else path_info.isin_or_eq
 
         def func(out):
             if out.scheme == "local" and match(out.path_info):
                 return True
 
-            if is_dir and recursive and out.path_info.isin(path_info):
+            if recursive and out.path_info.isin(path_info):
                 return True
 
             return False
@@ -419,7 +424,7 @@ class Repo(object):
 
     def find_out_by_relpath(self, relpath):
         path = os.path.join(self.root_dir, relpath)
-        out, = self.find_outs_by_path(path)
+        (out,) = self.find_outs_by_path(path)
         return out
 
     def is_dvc_internal(self, path):

dvc/repo/add.py

@@ -4,11 +4,15 @@ import os
 import colorama
 
 from . import locked
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
-from dvc.progress import Tqdm
-from dvc.repo.scm_context import scm_context
-from dvc.stage import Stage
-from dvc.utils import LARGE_DIR_SIZE
+from ..exceptions import (
+    RecursiveAddingWhileUsingFilename,
+    OverlappingOutputPathsError,
+)
+from ..output.base import OutputDoesNotExistError
+from ..progress import Tqdm
+from ..repo.scm_context import scm_context
+from ..stage import Stage
+from ..utils import LARGE_DIR_SIZE
 
 logger = logging.getLogger(__name__)
 
@@ -23,7 +27,12 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
         targets = [targets]
 
     stages_list = []
-    with Tqdm(total=len(targets), desc="Add", unit="file", leave=True) as pbar:
+    num_targets = len(targets)
+    with Tqdm(total=num_targets, desc="Add", unit="file", leave=True) as pbar:
+        if num_targets == 1:
+            # clear unneeded top-level progress bar for single target
+            pbar.bar_format = "Adding..."
+            pbar.refresh()
         for target in targets:
             sub_targets = _find_all_targets(repo, target, recursive)
             pbar.total += len(sub_targets) - 1
@@ -43,19 +52,46 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
 
             stages = _create_stages(repo, sub_targets, fname, pbar=pbar)
 
-            repo.check_modified_graph(stages)
+            try:
+                repo.check_modified_graph(stages)
+            except OverlappingOutputPathsError as exc:
+                msg = (
+                    "Cannot add '{out}', because it is overlapping with other "
+                    "DVC tracked output: '{parent}'.\n"
+                    "To include '{out}' in '{parent}', run "
+                    "'dvc commit {parent_stage}'"
+                ).format(
+                    out=exc.overlapping_out.path_info,
+                    parent=exc.parent.path_info,
+                    parent_stage=exc.parent.stage.relpath,
+                )
+                raise OverlappingOutputPathsError(
+                    exc.parent, exc.overlapping_out, msg
+                )
 
-            for stage in stages:
-                stage.save()
+            with Tqdm(
+                total=len(stages),
+                desc="Processing",
+                unit="file",
+                disable=len(stages) == 1,
+            ) as pbar_stages:
+                for stage in stages:
+                    try:
+                        stage.save()
+                    except OutputDoesNotExistError:
+                        pbar.n -= 1
+                        raise
 
-                if not no_commit:
-                    stage.commit()
+                    if not no_commit:
+                        stage.commit()
 
-                stage.dump()
+                    stage.dump()
+                    pbar_stages.update()
 
             stages_list += stages
-        # remove filled bar bit of progress, leaving stats
-        pbar.bar_format = pbar.BAR_FMT_DEFAULT.replace("|{bar:10}|", " ")
+
+        if num_targets == 1:  # restore bar format for stats
+            pbar.bar_format = pbar.BAR_FMT_DEFAULT
 
     return stages_list
 
@@ -64,7 +100,12 @@ def _find_all_targets(repo, target, recursive):
     if os.path.isdir(target) and recursive:
         return [
             fname
-            for fname in repo.tree.walk_files(target)
+            for fname in Tqdm(
+                repo.tree.walk_files(target),
+                desc="Searching " + target,
+                bar_format=Tqdm.BAR_FMT_NOTOTAL,
+                unit="file",
+            )
             if not repo.is_dvc_internal(fname)
             if not Stage.is_stage_file(fname)
             if not repo.scm.belongs_to_scm(fname)
@@ -76,8 +117,15 @@ def _find_all_targets(repo, target, recursive):
 def _create_stages(repo, targets, fname, pbar=None):
     stages = []
 
-    for out in targets:
-        stage = Stage.create(repo, outs=[out], add=True, fname=fname)
+    for out in Tqdm(
+        targets,
+        desc="Creating DVC-files",
+        disable=len(targets) < LARGE_DIR_SIZE,
+        unit="file",
+    ):
+        stage = Stage.create(
+            repo, outs=[out], accompany_outs=True, fname=fname
+        )
 
         if not stage:
             if pbar is not None:

dvc/repo/diff.py

@@ -1,246 +1,76 @@
 import os
-from errno import ENOENT
 
-from dvc import logger
-from dvc.scm.base import FileNotInCommitError
-from dvc.scm.git import DIFF_A_REF, DIFF_B_REF, DIFF_A_TREE, DIFF_B_TREE
-from dvc.scm.git import DIFF_EQUAL
-from . import locked
-
-
-DIFF_TARGET = "target"
-DIFF_IS_DIR = "is_dir"
-DIFF_OLD_FILE = "old_file"
-DIFF_OLD_CHECKSUM = "old_checksum"
-DIFF_NEW_FILE = "new_file"
-DIFF_NEW_CHECKSUM = "new_checksum"
-DIFF_SIZE = "size_diff"
-DIFF_DEL = "del"
-DIFF_IDENT = "ident"
-DIFF_CHANGE = "changes"
-DIFF_NEW = "new"
-DIFF_MOVE = "moves"
-DIFF_LIST = "diffs"
-DIFF_SIZE_UNKNOWN = "?"
-DIFF_A_OUTPUT = "a_output"
-DIFF_B_OUTPUT = "b_output"
-DIFF_DELETED = "deleted_file"
-DIFF_IS_NEW = "created_file"
-
-
-def _file_not_exists(error, result):
-    if error.errno == ENOENT:
-        result.update({DIFF_SIZE: DIFF_SIZE_UNKNOWN})
-    else:
-        raise error
-
-
-def _extract_dir(self, dir_not_exists, output):
-    """Extract the content of dvc tree file
-    Args:
-        self(object) - Repo class instance
-        dir_not_exists(bool) - flag for directory existence
-        output(object) - OutputLOCAL class instance
-    Returns:
-        dict - dictionary with keys - paths to file in .dvc/cache
-                               values -checksums for that files
-    """
-    if not dir_not_exists:
-        lst = output.dir_cache
-        return {i["relpath"]: i["md5"] for i in lst}
-    return {}
-
-
-def _get_dir_info(dir_not_exists, a_output):
-    if not dir_not_exists:
-        return str(a_output), a_output.checksum
-    return "", ""
-
-
-def _ident_files(a_entries, b_entries):
-    keys = [
-        key for key in a_entries.keys() if b_entries.get(key) == a_entries[key]
-    ]
-    return len(keys)
-
-
-def _modified_files(self, a_entries, b_entries):
-    keys = [key for key in a_entries.keys() if key in b_entries]
-    diff_size = 0
-    modified_count = 0
-    for key in keys:
-        if a_entries[key] != b_entries[key]:
-            modified_count += 1
-            diff_size += os.path.getsize(
-                self.cache.local.get(b_entries[key])
-            ) - os.path.getsize(self.cache.local.get(a_entries[key]))
-    return modified_count, diff_size
-
-
-def _deleted_files(self, a_entries, b_entries):
-    diff_size = 0
-    deleted_count = 0
-    for key, value in a_entries.items():
-        if key not in b_entries:
-            deleted_count += 1
-            diff_size -= os.path.getsize(self.cache.local.get(a_entries[key]))
-    return deleted_count, diff_size
-
-
-def _new_files(self, a_entries, b_entries):
-    diff_size = 0
-    new_count = 0
-    for key, value in b_entries.items():
-        if key not in a_entries:
-            new_count += 1
-            diff_size += os.path.getsize(self.cache.local.get(b_entries[key]))
-    return new_count, diff_size
-
-
-def _get_tree_changes(self, a_entries, b_entries):
-    result = {
-        DIFF_DEL: 0,
-        DIFF_IDENT: 0,
-        DIFF_CHANGE: 0,
-        DIFF_NEW: 0,
-        DIFF_MOVE: 0,
-    }
-    result[DIFF_IDENT] = _ident_files(a_entries, b_entries)
-    result[DIFF_CHANGE], diff_size = _modified_files(
-        self, a_entries, b_entries
-    )
-    result[DIFF_SIZE] = diff_size
-    result[DIFF_DEL], diff_size = _deleted_files(self, a_entries, b_entries)
-    result[DIFF_SIZE] += diff_size
-    result[DIFF_NEW], diff_size = _new_files(self, a_entries, b_entries)
-    result[DIFF_SIZE] += diff_size
-    return result
-
-
-def _check_local_cache(a_out, is_checked):
-    if a_out is not None and a_out.scheme != "local":
-        is_checked.append(str(a_out))
-        return True
-    return False
-
-
-def _is_dir(path, a_outs, b_outs):
-    if a_outs.get(path):
-        return a_outs[path].is_dir_checksum
-    else:
-        return b_outs[path].is_dir_checksum
-
-
-def _get_diff_outs(self, diff_dct):
-    self.tree = diff_dct[DIFF_A_TREE]
-    a_outs = {str(out): out for st in self.stages for out in st.outs}
-    self.tree = diff_dct[DIFF_B_TREE]
-    b_outs = {str(out): out for st in self.stages for out in st.outs}
-    outs_paths = set(a_outs.keys())
-    outs_paths.update(b_outs.keys())
-    results = {}
-    non_local_cache = []
-    for path in outs_paths:
-        check1 = _check_local_cache(a_outs.get(path), non_local_cache)
-        check2 = _check_local_cache(b_outs.get(path), non_local_cache)
-        # skip files/directories with non-local cache for now
-        if check1 or check2:
-            continue
-        results[path] = {}
-        results[path][DIFF_A_OUTPUT] = a_outs.get(path)
-        results[path][DIFF_B_OUTPUT] = b_outs.get(path)
-        results[path][DIFF_IS_NEW] = path not in a_outs
-        results[path][DIFF_DELETED] = path not in b_outs
-        results[path][DIFF_IS_DIR] = _is_dir(path, a_outs, b_outs)
-    if non_local_cache:
-        logger.warning(
-            "Diff is not supported for non-local outputs. Ignoring: {}".format(
-                non_local_cache
-            )
-        )
-
-    return results
-
-
-def _diff_dir(self, target, diff_dct):
-    result = {DIFF_TARGET: target}
-    result[DIFF_IS_DIR] = True
-    a_entries, b_entries = {}, {}
-    try:
-        a_entries = _extract_dir(
-            self, diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
-        )
-        b_entries = _extract_dir(
-            self, diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
-        )
-        result[DIFF_OLD_FILE], result[DIFF_OLD_CHECKSUM] = _get_dir_info(
-            diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
-        )
-        result[DIFF_NEW_FILE], result[DIFF_NEW_CHECKSUM] = _get_dir_info(
-            diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
-        )
-        result.update(_get_tree_changes(self, a_entries, b_entries))
-    except IOError as e:
-        _file_not_exists(e, result)
-    return result
-
-
-def _diff_file(self, target, diff_dct):
-    result = {DIFF_TARGET: target}
-    size = 0
-    try:
-        if not diff_dct[DIFF_IS_NEW]:
-            result[DIFF_OLD_FILE] = target
-            result[DIFF_OLD_CHECKSUM] = diff_dct[DIFF_A_OUTPUT].checksum
-            size -= os.path.getsize(
-                self.cache.local.get(diff_dct[DIFF_A_OUTPUT].checksum)
-            )
-        if not diff_dct[DIFF_DELETED]:
-            result[DIFF_NEW_FILE] = target
-            result[DIFF_NEW_CHECKSUM] = diff_dct[DIFF_B_OUTPUT].checksum
-            size += os.path.getsize(
-                self.cache.local.get(diff_dct[DIFF_B_OUTPUT].checksum)
-            )
-        result[DIFF_SIZE] = size
-    except IOError as e:
-        _file_not_exists(e, result)
-    return result
-
-
-def _diff_royal(self, target, diff_dct):
-    if diff_dct[DIFF_IS_DIR]:
-        return _diff_dir(self, target, diff_dct)
-    return _diff_file(self, target, diff_dct)
+from dvc.exceptions import DvcException
+from dvc.repo import locked
+from dvc.scm.git import Git
 
 
 @locked
-def diff(self, a_ref, target=None, b_ref=None):
-    """Generates diff message string output
-
-    Args:
-        target(str) - file/directory to check diff of
-        a_ref(str) - first tag
-        (optional) b_ref(str) - second git tag
-
-    Returns:
-        string: string of output message with diff info
+def diff(self, a_rev="HEAD", b_rev=None):
     """
-    result = {}
-    diff_dct = self.scm.get_diff_trees(a_ref, b_ref=b_ref)
-    result[DIFF_A_REF] = diff_dct[DIFF_A_REF]
-    result[DIFF_B_REF] = diff_dct[DIFF_B_REF]
-    if diff_dct[DIFF_EQUAL]:
-        result[DIFF_EQUAL] = True
+    By default, it compares the working tree with the last commit's tree.
+
+    This implementation differs from `git diff` since DVC doesn't have
+    the concept of `index`, but it keeps the same interface, thus,
+    `dvc diff` would be the same as `dvc diff HEAD`.
+    """
+    if type(self.scm) is not Git:
+        raise DvcException("only supported for Git repositories")
+
+    def _paths_checksums():
+        """
+        A dictionary of checksums addressed by relpaths collected from
+        the current tree outputs.
+
+        Unpack directories to include their entries
+
+        To help distinguish between a directory and a file output,
+        the former one will come with a trailing slash in the path:
+
+            directory: "data/"
+            file:      "data"
+        """
+        result = {}
+
+        for stage in self.stages:
+            for output in stage.outs:
+                if not output.is_dir_checksum:
+                    result.update({str(output): output.checksum})
+                    continue
+
+                result.update({os.path.join(str(output), ""): output.checksum})
+
+                for entry in output.dir_cache:
+                    path = str(output.path_info / entry["relpath"])
+                    result.update({path: entry["md5"]})
+
         return result
-    result[DIFF_LIST] = []
-    diff_outs = _get_diff_outs(self, diff_dct)
-    if target is None:
-        result[DIFF_LIST] = [
-            _diff_royal(self, path, diff_outs[path]) for path in diff_outs
-        ]
-    elif target in diff_outs:
-        result[DIFF_LIST] = [_diff_royal(self, target, diff_outs[target])]
-    else:
-        msg = "Have not found file/directory '{}' in the commits"
-        raise FileNotInCommitError(msg.format(target))
-    return result
+
+    working_tree = self.tree
+    a_tree = self.scm.get_tree(a_rev)
+    b_tree = self.scm.get_tree(b_rev) if b_rev else working_tree
+
+    try:
+        self.tree = a_tree
+        old = _paths_checksums()
+
+        self.tree = b_tree
+        new = _paths_checksums()
+    finally:
+        self.tree = working_tree
+
+    # Compare paths between the old and new tree.
+    # set() efficiently converts dict keys to a set
+    added = sorted(set(new) - set(old))
+    deleted = sorted(set(old) - set(new))
+    modified = sorted(set(old) & set(new))
+
+    return {
+        "added": [{"path": path, "checksum": new[path]} for path in added],
+        "deleted": [{"path": path, "checksum": old[path]} for path in deleted],
+        "modified": [
+            {"path": path, "checksum": {"old": old[path], "new": new[path]}}
+            for path in modified
+            if old[path] != new[path]
+        ],
+    }

dvc/repo/get_url.py

@@ -14,6 +14,6 @@ def get_url(url, out=None):
 
     out = os.path.abspath(out)
 
-    dep, = dependency.loads_from(None, [url])
-    out, = output.loads_from(None, [out], use_cache=False)
+    (dep,) = dependency.loads_from(None, [url])
+    (out,) = output.loads_from(None, [out], use_cache=False)
     dep.download(out)

dvc/repo/imp_url.py

@@ -11,7 +11,13 @@ def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
     out = resolve_output(url, out)
 
     stage = Stage.create(
-        self, cmd=None, deps=[url], outs=[out], fname=fname, erepo=erepo
+        self,
+        cmd=None,
+        deps=[url],
+        outs=[out],
+        fname=fname,
+        erepo=erepo,
+        accompany_outs=True,
     )
 
     if stage is None:

dvc/repo/init.py

@@ -89,7 +89,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
     config = Config.init(dvc_dir)
     proj = Repo(root_dir)
 
-    scm.add([config.config_file])
+    scm.add([config.files["repo"]])
 
     if scm.ignore_file:
         scm.add([os.path.join(dvc_dir, scm.ignore_file)])

dvc/repo/ls.py

@@ -0,0 +1,84 @@
+import os
+
+from dvc.exceptions import PathMissingError, OutputNotFoundError
+
+
+@staticmethod
+def ls(url, target=None, rev=None, recursive=None, outs_only=False):
+    from dvc.external_repo import external_repo
+    from dvc.repo import Repo
+    from dvc.utils import relpath
+
+    with external_repo(url, rev) as repo:
+        target_path_info = _get_target_path_info(repo, target)
+        result = []
+        if isinstance(repo, Repo):
+            result.extend(_ls_outs_repo(repo, target_path_info, recursive))
+
+        if not outs_only:
+            result.extend(_ls_files_repo(target_path_info, recursive))
+
+        if target and not result:
+            raise PathMissingError(target, repo, output_only=outs_only)
+
+        def prettify(path_info):
+            if path_info == target_path_info:
+                return path_info.name
+            return relpath(path_info, target_path_info)
+
+        result = list(set(map(prettify, result)))
+        result.sort()
+    return result
+
+
+def _ls_files_repo(target_path_info, recursive=None):
+    from dvc.compat import fspath
+    from dvc.ignore import CleanTree
+    from dvc.path_info import PathInfo
+    from dvc.scm.tree import WorkingTree
+
+    if not os.path.exists(fspath(target_path_info)):
+        return []
+
+    files = []
+    tree = CleanTree(WorkingTree(target_path_info))
+    try:
+        for dirpath, dirnames, filenames in tree.walk(target_path_info):
+            files.extend(map(lambda f: PathInfo(dirpath, f), filenames))
+            if not recursive:
+                files.extend(map(lambda d: PathInfo(dirpath, d), dirnames))
+                break
+    except NotADirectoryError:
+        if os.path.isfile(fspath(target_path_info)):
+            return [target_path_info]
+
+    return files
+
+
+def _ls_outs_repo(repo, target_path_info, recursive=None):
+    from dvc.compat import fspath
+    from dvc.path_info import PathInfo
+
+    try:
+        outs = repo.find_outs_by_path(fspath(target_path_info), recursive=True)
+    except OutputNotFoundError:
+        return []
+
+    if recursive:
+        return [out.path_info for out in outs]
+
+    def get_top_part(path_info):
+        relpath = path_info.relpath(target_path_info)
+        if relpath.parts:
+            return PathInfo(target_path_info, relpath.parts[0])
+        return path_info
+
+    return list({get_top_part(out.path_info) for out in outs})
+
+
+def _get_target_path_info(repo, target=None):
+    from dvc.path_info import PathInfo
+
+    if not target:
+        return PathInfo(repo.root_dir)
+    return PathInfo(repo.root_dir, target)

dvc/repo/metrics/diff.py

@@ -83,9 +83,9 @@ def _get_metrics(repo, *args, rev=None, **kwargs):
         return {}
 
 
-def diff(repo, *args, a_ref=None, b_ref=None, **kwargs):
-    old = _get_metrics(repo, *args, **kwargs, rev=(a_ref or "HEAD"))
-    new = _get_metrics(repo, *args, **kwargs, rev=b_ref)
+def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
+    old = _get_metrics(repo, *args, **kwargs, rev=(a_rev or "HEAD"))
+    new = _get_metrics(repo, *args, **kwargs, rev=b_rev)
 
     paths = set(old.keys())
     paths.update(set(new.keys()))

dvc/repo/reproduce.py

@@ -33,11 +33,18 @@ def _get_active_graph(G):
     for stage in G:
         if not stage.locked:
             continue
-        for st in nx.dfs_postorder_nodes(G, stage):
-            if st == stage:
-                continue
-            if st in active:
-                active.remove_node(st)
+        active.remove_edges_from(G.out_edges(stage))
+        for edge in G.out_edges(stage):
+            _, to_stage = edge
+            for node in nx.dfs_preorder_nodes(G, to_stage):
+                # NOTE: `in_degree` will return InDegreeView({}) if stage
+                # no longer exists in the `active` DAG.
+                if not active.in_degree(node):
+                    # NOTE: if some edge no longer exists `remove_edges_from`
+                    # will ignore it without error.
+                    active.remove_edges_from(G.out_edges(node))
+                    active.remove_node(node)
+
     return active
 
 
@@ -58,11 +65,7 @@ def reproduce(
 
     interactive = kwargs.get("interactive", False)
     if not interactive:
-        config = self.config
-        core = config.config[config.SECTION_CORE]
-        kwargs["interactive"] = core.get(
-            config.SECTION_CORE_INTERACTIVE, False
-        )
+        kwargs["interactive"] = self.config["core"].get("interactive", False)
 
     active_graph = _get_active_graph(self.graph)
     active_pipelines = get_pipelines(active_graph)

dvc/repo/update.py

@@ -2,10 +2,12 @@ from . import locked
 
 
 @locked
-def update(self, target):
+def update(self, target, rev=None):
     from dvc.stage import Stage
 
     stage = Stage.load(self, target)
-    stage.update()
+    stage.update(rev)
 
     stage.dump()
+
+    return stage

dvc/rwlock.py

@@ -4,16 +4,20 @@ import json
 from collections import defaultdict
 from contextlib import contextmanager
 
-from voluptuous import Schema, Invalid
-from funcy.py3 import lmap, lfilter, lmapcat
+from voluptuous import Schema, Invalid, Required, Optional
 
 from .exceptions import DvcException
 from .lock import LockError
 from .utils import relpath
 
-INFO_SCHEMA = {"pid": int, "cmd": str}
+INFO_SCHEMA = {Required("pid"): int, Required("cmd"): str}
 
-SCHEMA = Schema({"write": {str: INFO_SCHEMA}, "read": {str: [INFO_SCHEMA]}})
+SCHEMA = Schema(
+    {
+        Optional("write", default={}): {str: INFO_SCHEMA},
+        Optional("read", default={}): {str: [INFO_SCHEMA]},
+    }
+)
 
 
 class RWLockFileCorruptedError(DvcException):
@@ -36,15 +40,13 @@ def _edit_rwlock(lock_dir):
     path = os.path.join(lock_dir, "rwlock")
     try:
         with open(path, "r") as fobj:
-            lock = json.load(fobj)
-        lock = SCHEMA(lock)
+            lock = SCHEMA(json.load(fobj))
     except FileNotFoundError:
         lock = SCHEMA({})
     except json.JSONDecodeError as exc:
         raise RWLockFileCorruptedError(path) from exc
     except Invalid as exc:
         raise RWLockFileFormatError(path) from exc
-    lock = defaultdict(dict, lock)
     lock["read"] = defaultdict(list, lock["read"])
     lock["write"] = defaultdict(dict, lock["write"])
     yield lock
@@ -61,38 +63,26 @@ def _infos_to_str(infos):
     )
 
 
-def _check_no_writers(lock, info, path_infos):
-    for path_info in path_infos:
-        blocking_urls = lfilter(path_info.overlaps, lock["write"])
-        if not blocking_urls:
-            continue
+def _check_blockers(lock, info, *, mode, waiters):
+    for path_info in waiters:
+        blockers = [
+            blocker
+            for path, infos in lock[mode].items()
+            if path_info.overlaps(path)
+            for blocker in (infos if isinstance(infos, list) else [infos])
+            if blocker != info
+        ]
 
-        writers = lmap(lock["write"].get, blocking_urls)
-        writers = lfilter(lambda i: info != i, writers)
-        if not writers:
+        if not blockers:
             continue
 
         raise LockError(
-            "'{}' is busy, it is being written to by:\n{}".format(
-                str(path_info), _infos_to_str(writers)
-            )
-        )
-
-
-def _check_no_readers(lock, info, path_infos):
-    for path_info in path_infos:
-        blocking_urls = lfilter(path_info.overlaps, lock["read"])
-        if not blocking_urls:
-            continue
-
-        readers = lmapcat(lock["read"].get, blocking_urls)
-        readers = lfilter(lambda i: info != i, readers)
-        if not readers:
-            continue
-
-        raise LockError(
-            "'{}' is busy, it is being read by:\n{}".format(
-                str(path_info), _infos_to_str(readers)
+            "'{path}' is busy, it is being blocked by:\n"
+            "{blockers}\n"
+            "\n"
+            "If there are no processes with such PIDs, you can manually remove"
+            "'.dvc/tmp/rwlock' and try again.".format(
+                path=str(path_info), blockers=_infos_to_str(blockers)
             )
         )
 
@@ -171,8 +161,9 @@ def rwlock(tmp_dir, cmd, read, write):
     info = {"pid": os.getpid(), "cmd": cmd}
 
     with _edit_rwlock(tmp_dir) as lock:
-        _check_no_writers(lock, info, read + write)
-        _check_no_readers(lock, info, write)
+
+        _check_blockers(lock, info, mode="write", waiters=read + write)
+        _check_blockers(lock, info, mode="read", waiters=write)
 
         rchanges = _acquire_read(lock, info, read)
         wchanges = _acquire_write(lock, info, write)

dvc/scm/base.py

@@ -15,22 +15,13 @@ class FileNotInRepoError(SCMError):
     """
 
 
-class FileNotInCommitError(SCMError):
-    """Thrown when trying to find a file/directory that is not
-    in the specified commit in the repository.
-    """
-
-
 class CloneError(SCMError):
     def __init__(self, url, path):
         super().__init__("Failed to clone repo '{}' to '{}'".format(url, path))
 
 
 class RevError(SCMError):
-    def __init__(self, url, rev):
-        super().__init__(
-            "Failed to access revision '{}' for repo '{}'".format(rev, url)
-        )
+    pass
 
 
 class Base(object):

dvc/scm/git/__init__.py

@@ -8,27 +8,14 @@ from pathspec.patterns import GitWildMatchPattern
 
 from dvc.exceptions import GitHookAlreadyExistsError
 from dvc.scm.base import Base
-from dvc.scm.base import CloneError
-from dvc.scm.base import FileNotInRepoError
-from dvc.scm.base import RevError
-from dvc.scm.base import SCMError
+from dvc.scm.base import CloneError, FileNotInRepoError, RevError, SCMError
 from dvc.scm.git.tree import GitTree
-from dvc.utils import fix_env
-from dvc.utils import is_binary
-from dvc.utils import relpath
+from dvc.utils import fix_env, is_binary, relpath
 from dvc.utils.fs import path_isin
 
-
 logger = logging.getLogger(__name__)
 
 
-DIFF_A_TREE = "a_tree"
-DIFF_B_TREE = "b_tree"
-DIFF_A_REF = "a_ref"
-DIFF_B_REF = "b_ref"
-DIFF_EQUAL = "equal"
-
-
 class Git(Base):
     """Class for managing Git."""
 
@@ -94,10 +81,20 @@ class Git(Base):
             try:
                 repo.checkout(rev)
             except git.exc.GitCommandError as exc:
-                raise RevError(url, rev) from exc
+                raise RevError(
+                    "failed to access revision '{}' for repo '{}'".format(
+                        rev, url
+                    )
+                ) from exc
 
         return repo
 
+    @staticmethod
+    def is_sha(rev):
+        import git
+
+        return rev and git.Repo.re_hexsha_shortened.search(rev)
+
     @staticmethod
     def is_repo(root_dir):
         return os.path.isdir(Git._get_git_dir(root_dir))
@@ -136,20 +133,19 @@ class Git(Base):
 
         return entry, gitignore
 
-    @staticmethod
-    def _ignored(entry, gitignore_path):
-        if os.path.exists(gitignore_path):
-            with open(gitignore_path, "r") as fobj:
-                ignore_list = fobj.readlines()
-            return any(
-                filter(lambda x: x.strip() == entry.strip(), ignore_list)
-            )
-        return False
+    def _ignored(self, path):
+        from git.exc import GitCommandError
+
+        try:
+            self.repo.git.check_ignore(path)
+            return True
+        except GitCommandError:
+            return False
 
     def ignore(self, path):
         entry, gitignore = self._get_gitignore(path)
 
-        if self._ignored(entry, gitignore):
+        if self._ignored(path):
             return
 
         msg = "Adding '{}' to '{}'.".format(relpath(path), relpath(gitignore))
@@ -215,6 +211,18 @@ class Git(Base):
         else:
             self.repo.git.checkout(branch)
 
+    def pull(self):
+        infos = self.repo.remote().pull()
+        for info in infos:
+            if info.flags & info.ERROR:
+                raise SCMError("pull failed: {}".format(info.note))
+
+    def push(self):
+        infos = self.repo.remote().push()
+        for info in infos:
+            if info.flags & info.ERROR:
+                raise SCMError("push failed: {}".format(info.summary))
+
     def branch(self, branch):
         self.repo.git.branch(branch)
 
@@ -247,11 +255,12 @@ class Git(Base):
     def list_all_commits(self):
         return [c.hexsha for c in self.repo.iter_commits("--all")]
 
-    def _install_hook(self, name, cmd):
-        command = (
-            '[ "$3" = "0" ]'
-            ' || [ -z "$(git ls-files --full-name .dvc)" ]'
-            " || exec dvc {}".format(cmd)
+    def _install_hook(self, name, preconditions, cmd):
+        # only run in dvc repo
+        in_dvc_repo = '[ -n "$(git ls-files --full-name .dvc)" ]'
+
+        command = "if {}; then exec dvc {}; fi".format(
+            " && ".join([in_dvc_repo] + preconditions), cmd
         )
 
         hook = self._hook_path(name)
@@ -269,9 +278,20 @@ class Git(Base):
     def install(self):
         self._verify_dvc_hooks()
 
-        self._install_hook("post-checkout", "checkout")
-        self._install_hook("pre-commit", "status")
-        self._install_hook("pre-push", "push")
+        self._install_hook(
+            "post-checkout",
+            [
+                # checking out some reference and not specific file.
+                '[ "$3" = "1" ]',
+                # make sure we are not in the middle of a rebase/merge, so we
+                # don't accidentally break it with an unsuccessful checkout.
+                # Note that git hooks are always running in repo root.
+                "[ ! -d .git/rebase-merge ]",
+            ],
+            "checkout",
+        )
+        self._install_hook("pre-commit", [], "status")
+        self._install_hook("pre-push", [], "push")
 
     def cleanup_ignores(self):
         for path in self.ignored_paths:
@@ -304,66 +324,48 @@ class Git(Base):
         return basename == self.ignore_file or Git.GIT_DIR in path_parts
 
     def get_tree(self, rev):
-        return GitTree(self.repo, rev)
-
-    def _get_diff_trees(self, a_ref, b_ref):
-        """Private method for getting the trees and commit hashes of 2 git
-        references. Requires `gitdb` module (from gitpython package).
-
-        Args:
-            a_ref (str): git reference
-            b_ref (str): second git reference. If None, uses HEAD
-
-        Returns:
-            tuple: tuple with elements: (trees, commits)
-        """
-        from gitdb.exc import BadObject, BadName
-
-        trees = {DIFF_A_TREE: None, DIFF_B_TREE: None}
-        commits = []
-        if b_ref is None:
-            b_ref = self.repo.head.commit
-        try:
-            a_commit = self.repo.git.rev_parse(a_ref, short=True)
-            b_commit = self.repo.git.rev_parse(b_ref, short=True)
-            # See https://gitpython.readthedocs.io
-            # /en/2.1.11/reference.html#git.objects.base.Object.__str__
-            commits.append(a_commit)
-            commits.append(b_commit)
-            trees[DIFF_A_TREE] = self.get_tree(commits[0])
-            trees[DIFF_B_TREE] = self.get_tree(commits[1])
-        except (BadName, BadObject) as exc:
-            raise SCMError("git problem") from exc
-        return trees, commits
-
-    def get_diff_trees(self, a_ref, b_ref=None):
-        """Method for getting two repo trees between two git tag commits.
-        Returns the dvc hash names of changed file/directory
-
-        Args:
-            a_ref (str): git reference
-            b_ref (str): optional second git reference, default None
-
-        Returns:
-            dict: dictionary with keys: {a_ref, b_ref, equal}
-                or {a_ref, b_ref, a_tree, b_tree}
-        """
-        diff_dct = {DIFF_EQUAL: False}
-        trees, commits = self._get_diff_trees(a_ref, b_ref)
-        diff_dct[DIFF_A_REF] = commits[0]
-        diff_dct[DIFF_B_REF] = commits[1]
-        if commits[0] == commits[1]:
-            diff_dct[DIFF_EQUAL] = True
-            return diff_dct
-        diff_dct[DIFF_A_TREE] = trees[DIFF_A_TREE]
-        diff_dct[DIFF_B_TREE] = trees[DIFF_B_TREE]
-        return diff_dct
+        return GitTree(self.repo, self.resolve_rev(rev))
 
     def get_rev(self):
-        return self.repo.git.rev_parse("HEAD")
+        return self.repo.rev_parse("HEAD").hexsha
 
     def resolve_rev(self, rev):
-        return self.repo.git.rev_parse(rev)
+        from git.exc import BadName, GitCommandError
+        from contextlib import suppress
+
+        def _resolve_rev(name):
+            with suppress(BadName, GitCommandError):
+                try:
+                    # Try python implementation of rev-parse first, it's faster
+                    return self.repo.rev_parse(name).hexsha
+                except NotImplementedError:
+                    # Fall back to `git rev-parse` for advanced features
+                    return self.repo.git.rev_parse(name)
+
+        # Resolve across local names
+        sha = _resolve_rev(rev)
+        if sha:
+            return sha
+
+        # Try all the remotes and if it resolves unambiguously then take it
+        if not Git.is_sha(rev):
+            shas = {
+                _resolve_rev("{}/{}".format(remote.name, rev))
+                for remote in self.repo.remotes
+            } - {None}
+            if len(shas) > 1:
+                raise RevError("ambiguous Git revision '{}'".format(rev))
+            if len(shas) == 1:
+                return shas.pop()
+
+        raise RevError("unknown Git revision '{}'".format(rev))
+
+    def has_rev(self, rev):
+        try:
+            self.resolve_rev(rev)
+            return True
+        except RevError:
+            return False
 
     def close(self):
         self.repo.close()

dvc/scm/git/tree.py

@@ -102,7 +102,7 @@ class GitTree(BaseTree):
             tree = self.git.tree(self.rev)
         except git.exc.BadName as exc:
             raise DvcException(
-                "revision '{}' not found in git '{}'".format(
+                "revision '{}' not found in Git '{}'".format(
                     self.rev, os.path.relpath(self.git.working_dir)
                 )
             ) from exc

dvc/stage.py

@@ -191,7 +191,7 @@ class Stage(object):
     PARAM_ALWAYS_CHANGED = "always_changed"
 
     SCHEMA = {
-        PARAM_MD5: Any(str, None),
+        PARAM_MD5: output.CHECKSUM_SCHEMA,
         PARAM_CMD: Any(str, None),
         PARAM_WDIR: Any(str, None),
         PARAM_DEPS: Any([dependency.SCHEMA], None),
@@ -405,11 +405,11 @@ class Stage(object):
 
         return self
 
-    def update(self):
+    def update(self, rev=None):
         if not self.is_repo_import and not self.is_import:
             raise StageUpdateError(self.relpath)
 
-        self.deps[0].update()
+        self.deps[0].update(rev=rev)
         locked = self.locked
         self.locked = False
         try:
@@ -502,12 +502,11 @@ class Stage(object):
         return True
 
     @staticmethod
-    def create(repo, **kwargs):
+    def create(repo, accompany_outs=False, **kwargs):
 
         wdir = kwargs.get("wdir", None)
         cwd = kwargs.get("cwd", None)
         fname = kwargs.get("fname", None)
-        add = kwargs.get("add", False)
 
         # Backward compatibility for `cwd` option
         if wdir is None and cwd is not None:
@@ -539,12 +538,12 @@ class Stage(object):
         stage._check_duplicated_arguments()
 
         if not fname:
-            fname = Stage._stage_fname(stage.outs, add)
+            fname = Stage._stage_fname(stage.outs, accompany_outs)
         stage._check_dvc_filename(fname)
 
         # Autodetecting wdir for add, we need to create outs first to do that,
         # so we start with wdir = . and remap out paths later.
-        if add and kwargs.get("wdir") is None and cwd is None:
+        if accompany_outs and kwargs.get("wdir") is None and cwd is None:
             wdir = os.path.dirname(fname)
 
             for out in chain(stage.outs, stage.deps):
@@ -809,7 +808,7 @@ class Stage(object):
             msg += " and " if (changed_deps and changed_outs) else ""
             msg += "outputs {}".format(changed_outs) if changed_outs else ""
             msg += "md5" if not (changed_deps or changed_outs) else ""
-            msg += " of '{}' changed.".format(self.relpath)
+            msg += " of '{}' changed. ".format(self.relpath)
             msg += "Are you sure you want to commit it?"
             if not force and not prompt.confirm(msg):
                 raise StageCommitError(

dvc/state.py

@@ -6,7 +6,6 @@ import re
 import sqlite3
 from urllib.parse import urlunparse, urlencode
 
-from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.utils import current_timestamp
 from dvc.utils import relpath
@@ -23,13 +22,13 @@ logger = logging.getLogger(__name__)
 
 
 class StateVersionTooNewError(DvcException):
-    """Thrown when dvc version is older than the state database version."""
+    """Thrown when DVC version is older than the state database version."""
 
     def __init__(self, dvc_version, expected, actual):
         super().__init__(
-            "you are using an old version '{dvc_version}' of dvc that is "
-            "using state file version '{expected}' which is not compatible "
-            "with the state file version '{actual}' that is used in this "
+            "you are using an old version '{dvc_version}' of DVC that is "
+            "using state file version '{expected}', which is not compatible "
+            "with the state file version '{actual}', that is used in this "
             "repo. Please upgrade right now!".format(
                 dvc_version=dvc_version, expected=expected, actual=actual
             )
@@ -95,18 +94,15 @@ class State(object):  # pylint: disable=too-many-instance-attributes
     MAX_INT = 2 ** 63 - 1
     MAX_UINT = 2 ** 64 - 2
 
-    def __init__(self, repo, config):
+    def __init__(self, repo):
         self.repo = repo
         self.dvc_dir = repo.dvc_dir
         self.root_dir = repo.root_dir
 
-        state_config = config.get(Config.SECTION_STATE, {})
-        self.row_limit = state_config.get(
-            Config.SECTION_STATE_ROW_LIMIT, self.STATE_ROW_LIMIT
-        )
+        state_config = repo.config.get("state", {})
+        self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
         self.row_cleanup_quota = state_config.get(
-            Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
-            self.STATE_ROW_CLEANUP_QUOTA,
+            "row_cleanup_quota", self.STATE_ROW_CLEANUP_QUOTA
         )
 
         if not self.dvc_dir:

dvc/utils/fs.py

@@ -97,8 +97,8 @@ def move(src, dst, mode=None):
     of data is happening.
     """
 
-    src = fspath_py35(src)
-    dst = fspath_py35(dst)
+    src = fspath(src)
+    dst = fspath(dst)
 
     dst = os.path.abspath(dst)
     tmp = "{}.{}".format(dst, uuid())

dvc/version.py

@@ -7,11 +7,11 @@ import os
 import subprocess
 
 
-_BASE_VERSION = "0.82.2"
+_BASE_VERSION = "0.86.2"
 
 
 def _generate_version(base_version):
-    """Generate a version with information about the git repository"""
+    """Generate a version with information about the Git repository."""
     pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
     if not _is_git_repo(pkg_dir) or not _have_git():
@@ -28,7 +28,7 @@ def _generate_version(base_version):
 
 
 def _is_git_repo(dir_path):
-    """Is the given directory version-controlled with git?"""
+    """Is the given directory version-controlled with Git?"""
     return os.path.exists(os.path.join(dir_path, ".git"))
 
 
@@ -57,7 +57,7 @@ def _is_release(dir_path, base_version):
 
 
 def _git_revision(dir_path):
-    """Get the SHA-1 of the HEAD of a git repository."""
+    """Get SHA of the HEAD of a Git repository."""
     return subprocess.check_output(
         ["git", "rev-parse", "HEAD"], cwd=dir_path
     ).strip()

scripts/build_package.sh

@@ -4,26 +4,26 @@ set -e
 set -x
 
 if [ ! -d "dvc" ]; then
-    echo "Please run this script from repository root" >&2
-    exit 1
+  echo "Please run this script from repository root" >&2
+  exit 1
 fi
 
-echo 'PKG = "pip"' > dvc/utils/build.py
+echo 'PKG = "pip"' >dvc/utils/build.py
 
 python setup.py sdist
 python setup.py bdist_wheel --universal
 
 # Make sure we have a correct version
 if [[ -n "$TRAVIS_TAG" ]]; then
-    pip uninstall -y dvc
-    if which dvc; then
-        echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2
-	exit 1
-    fi
-    pip install dist/dvc-*.whl
-    if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then
-        echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2
-        exit 1
-    fi
-    pip uninstall -y dvc
+  pip uninstall -y dvc
+  if which dvc; then
+    echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2
+    exit 1
+  fi
+  pip install dist/dvc-*.whl
+  if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then
+    echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2
+    exit 1
+  fi
+  pip uninstall -y dvc
 fi

scripts/build_posix.sh

@@ -4,13 +4,13 @@ set -e
 set -x
 
 if [[ "$(uname)" == 'Linux' ]]; then
-	INSTALL_DIR=usr
-	FPM_FLAGS=
+  INSTALL_DIR=usr
+  FPM_FLAGS=
 else
-	INSTALL_DIR=usr/local
-	FPM_FLAGS='--osxpkg-identifier-prefix com.iterative'
-	FPM_FLAGS+=' --after-install scripts/fpm/after-install.sh'
-	FPM_FLAGS+=' --after-remove scripts/fpm/after-remove.sh'
+  INSTALL_DIR=usr/local
+  FPM_FLAGS='--osxpkg-identifier-prefix com.iterative'
+  FPM_FLAGS+=' --after-install scripts/fpm/after-install.sh'
+  FPM_FLAGS+=' --after-remove scripts/fpm/after-remove.sh'
 fi
 
 BUILD_DIR=build
@@ -21,129 +21,121 @@ LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib
 FPM_PACKAGE_DIRS="usr"
 ZSH_CMPLT_DIR=usr/share/zsh/site-functions/_dvc
 if [[ "$(uname)" == 'Linux' ]]; then
-	BASH_CMPLT_DIR=etc/bash_completion.d
-	FPM_PACKAGE_DIRS="$FPM_PACKAGE_DIRS etc"
+  BASH_CMPLT_DIR=etc/bash_completion.d
+  FPM_PACKAGE_DIRS="$FPM_PACKAGE_DIRS etc"
 else
-	BASH_CMPLT_DIR=usr/local/etc/bash_completion.d
+  BASH_CMPLT_DIR=usr/local/etc/bash_completion.d
 fi
 
-print_error()
-{
-	echo -e "\e[31m$1\e[0m" >&2
+print_error() {
+  echo -e "\e[31m$1\e[0m" >&2
 }
 
 if [ ! -d "dvc" ]; then
-	print_error "Please run this script from repository root"
-	exit 1
+  print_error "Please run this script from repository root"
+  exit 1
 fi
 
 trap 'print_error "FAIL"; exit 1' ERR
 
-print_info()
-{
-	echo -e "\e[32m$1\e[0m"
+print_info() {
+  echo -e "\e[32m$1\e[0m"
 }
 
-command_exists()
-{
-	command -v $1 > /dev/null 2>&1
+command_exists() {
+  command -v $1 >/dev/null 2>&1
 }
 
-fpm_build()
-{
-	print_info "Building $1..."
-	VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))")
-	fpm -s dir \
-	    -f \
-	    -t $1 \
-            --description "$DESC" \
-	    $FPM_FLAGS \
-	    -n dvc \
-	    -v $VERSION \
-	    -C $BUILD_DIR \
-	    $FPM_PACKAGE_DIRS
+fpm_build() {
+  print_info "Building $1..."
+  VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))")
+  fpm -s dir \
+    -f \
+    -t $1 \
+    --description "$DESC" \
+    $FPM_FLAGS \
+    -n dvc \
+    -v $VERSION \
+    -C $BUILD_DIR \
+    $FPM_PACKAGE_DIRS
 }
 
-cleanup()
-{
-	print_info "Cleaning up..."
-	rm -rf build
+cleanup() {
+  print_info "Cleaning up..."
+  rm -rf build
 }
 
-install_dependencies()
-{
-	print_info "Installing fpm..."
-	if command_exists dnf; then
-		sudo dnf install ruby-devel gcc make rpm-build
-	elif command_exists yum; then
-		sudo yum install ruby-devel gcc make rpm-build
-	elif command_exists apt-get; then
-		sudo apt-get update -y
-		sudo apt-get install ruby-dev build-essential rpm python-pip python-dev
-	elif command_exists brew; then
-		brew install ruby
-	else
-		echo "Unable to install fpm dependencies" && exit 1
-	fi
+install_dependencies() {
+  print_info "Installing fpm..."
+  if command_exists dnf; then
+    sudo dnf install ruby-devel gcc make rpm-build
+  elif command_exists yum; then
+    sudo yum install ruby-devel gcc make rpm-build
+  elif command_exists apt-get; then
+    sudo apt-get update -y
+    sudo apt-get install ruby-dev build-essential rpm python-pip python-dev
+  elif command_exists brew; then
+    brew install ruby
+  else
+    echo "Unable to install fpm dependencies" && exit 1
+  fi
 
-	gem install --no-document fpm
-	
-	# https://github.com/iterative/dvc/issues/2970
-	gem uninstall -i /Users/travis/.rvm/gems/ruby-2.4.3@global rubygems-bundler
+  gem install --no-document fpm
 
-	print_info "Installing requirements..."
-	pip install .[all]
-	pip install -r scripts/build-requirements.txt
+  # https://github.com/iterative/dvc/issues/2970
+  gem uninstall -i /Users/travis/.rvm/gems/ruby-2.4.3@global rubygems-bundler
+
+  print_info "Installing requirements..."
+  pip install .[all]
+  pip install -r scripts/build-requirements.txt
 }
 
-build_dvc()
-{
-	print_info "Building dvc binary..."
-	pyinstaller \
-            --additional-hooks-dir $(pwd)/scripts/hooks dvc/__main__.py \
-            --name dvc \
-            --distpath $LIB_DIR \
-            --specpath $BUILD_DIR
+build_dvc() {
+  print_info "Building dvc binary..."
+  pyinstaller \
+    --additional-hooks-dir $(pwd)/scripts/hooks dvc/__main__.py \
+    --name dvc \
+    --distpath $LIB_DIR \
+    --specpath $BUILD_DIR
 
-	$LIB_DIR/dvc/dvc --help
+  $LIB_DIR/dvc/dvc --help
 
-	# NOTE: in osxpkg fpm replaces symlinks with actual file that it
-	# points to, so we need to use after-install hook. See FPM_FLAGS
-	# above.
-	if [[ "$(uname)" == 'Linux' ]]; then
-		mkdir -p $BIN_DIR
-		pushd $BIN_DIR
-		ln -s ../lib/dvc/dvc dvc
-		popd
-		$BIN_DIR/dvc --help
-	fi
+  # NOTE: in osxpkg fpm replaces symlinks with actual file that it
+  # points to, so we need to use after-install hook. See FPM_FLAGS
+  # above.
+  if [[ "$(uname)" == 'Linux' ]]; then
+    mkdir -p $BIN_DIR
+    pushd $BIN_DIR
+    ln -s ../lib/dvc/dvc dvc
+    popd
+    $BIN_DIR/dvc --help
+  fi
 
-        # NOTE: temporarily not adding scripts to mac package. See [1]
-        # [1] https://github.com/iterative/dvc/issues/2585
-        if [[ "$(uname)" == 'Linux' ]]; then
-            mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
-            cp scripts/completion/dvc.bash $BUILD_DIR/$BASH_CMPLT_DIR/dvc
+  # NOTE: temporarily not adding scripts to mac package. See [1]
+  # [1] https://github.com/iterative/dvc/issues/2585
+  if [[ "$(uname)" == 'Linux' ]]; then
+    mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
+    cp scripts/completion/dvc.bash $BUILD_DIR/$BASH_CMPLT_DIR/dvc
 
-            mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
-            cp scripts/completion/dvc.zsh $BUILD_DIR/$ZSH_CMPLT_DIR
-        fi
+    mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
+    cp scripts/completion/dvc.zsh $BUILD_DIR/$ZSH_CMPLT_DIR
+  fi
 }
 
-build()
-{
-	cleanup
-	echo "PKG = \"$1\"" > dvc/utils/build.py
-	build_dvc
-	fpm_build $1
+build() {
+  cleanup
+  echo "PKG = \"$1\"" >dvc/utils/build.py
+  build_dvc
+  fpm_build $1
 }
 
 install_dependencies
 
 if [[ "$(uname)" == 'Linux' ]]; then
-	build rpm
-	build deb
+  build rpm
+  build deb
 else
-	build osxpkg
+  build osxpkg
 fi
 
 cleanup

scripts/build_snap.sh

@@ -18,8 +18,8 @@ sudo snap install --dangerous --classic dvc_*.snap
 if [[ -n "$TRAVIS_TAG" ]]; then
   # Make sure we have a correct version
   if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then
-      echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2
-      exit 1
+    echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2
+    exit 1
   fi
 fi
 # ensure basic commands can run

scripts/ci/before_install.sh

@@ -6,53 +6,53 @@ set -e
 scriptdir="$(dirname $0)"
 
 if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    ulimit -a
-    sudo sysctl -w kern.maxproc=2048
-    sudo sysctl -w kern.maxprocperuid=2048
-    echo '\nulimit -u 2048' >> ~/.bash_profile
-    ulimit -a
+  ulimit -a
+  sudo sysctl -w kern.maxproc=2048
+  sudo sysctl -w kern.maxprocperuid=2048
+  echo '\nulimit -u 2048' >>~/.bash_profile
+  ulimit -a
 fi
 
-echo > env.sh
+echo >env.sh
 
 if [[ "$TRAVIS_BUILD_STAGE_NAME" == "Test" ]]; then
-    if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then
-        # NOTE: ssh keys for ssh test to be able to ssh to the localhost
-        ssh-keygen -t rsa -N "" -f mykey
-        mkdir -p ~/.ssh
-        cp mykey ~/.ssh/id_rsa
-        cp mykey.pub ~/.ssh/id_rsa.pub
-        cat mykey.pub >> ~/.ssh/authorized_keys
-        ssh-keyscan localhost >> ~/.ssh/known_hosts
-        ssh localhost ls &> /dev/null
-        ssh-keyscan 127.0.0.1 >> ~/.ssh/known_hosts
-        ssh 127.0.0.1 ls &> /dev/null
-        ssh-keyscan 0.0.0.0 >> ~/.ssh/known_hosts
-        ssh 0.0.0.0 ls &> /dev/null
-    fi
+  if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then
+    # NOTE: ssh keys for ssh test to be able to ssh to the localhost
+    ssh-keygen -t rsa -N "" -f mykey
+    mkdir -p ~/.ssh
+    cp mykey ~/.ssh/id_rsa
+    cp mykey.pub ~/.ssh/id_rsa.pub
+    cat mykey.pub >>~/.ssh/authorized_keys
+    ssh-keyscan localhost >>~/.ssh/known_hosts
+    ssh localhost ls &>/dev/null
+    ssh-keyscan 127.0.0.1 >>~/.ssh/known_hosts
+    ssh 127.0.0.1 ls &>/dev/null
+    ssh-keyscan 0.0.0.0 >>~/.ssh/known_hosts
+    ssh 0.0.0.0 ls &>/dev/null
+  fi
 
-    if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-      bash "$scriptdir/install_azurite.sh"
-      bash "$scriptdir/install_oss.sh"
-      bash "$scriptdir/install_hadoop.sh"
-    fi
+  if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+    bash "$scriptdir/install_azurite.sh"
+    bash "$scriptdir/install_oss.sh"
+    bash "$scriptdir/install_hadoop.sh"
+  fi
 
-    if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
-        brew install openssl
-        $scriptdir/retry.sh brew cask install google-cloud-sdk
-    fi
+  if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
+    brew install openssl
+    $scriptdir/retry.sh brew cask install google-cloud-sdk
+  fi
 fi
 
 if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
-    $scriptdir/retry.sh choco install python --version 3.7.5
-    echo 'PATH="/c/Python37:/c/Python37/Scripts:$PATH"' >> env.sh
+  $scriptdir/retry.sh choco install python --version 3.7.5
+  echo 'PATH="/c/Python37:/c/Python37/Scripts:$PATH"' >>env.sh
 elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    ln -s -f /usr/local/bin/python3 /usr/local/bin/python
-    ln -s -f /usr/local/bin/pip3 /usr/local/bin/pip
+  ln -s -f /usr/local/bin/python3 /usr/local/bin/python
+  ln -s -f /usr/local/bin/pip3 /usr/local/bin/pip
 fi
 
 if [[ -n "$TRAVIS_TAG" ]]; then
-    echo "export SNAP_CHANNEL=stable" >> env.sh
+  echo "export SNAP_CHANNEL=stable" >>env.sh
 else
-    echo "export SNAP_CHANNEL=edge" >> env.sh
+  echo "export SNAP_CHANNEL=edge" >>env.sh
 fi

scripts/ci/check_patch.sh

@@ -3,7 +3,7 @@
 set -x
 set -e
 
-pip install Pygments collective.checkdocs pre-commit 
+pip install Pygments collective.checkdocs pre-commit
 
 # stop the build if there are any readme formatting errors
 python setup.py checkdocs

scripts/ci/check_python.sh

@@ -4,11 +4,11 @@ set -e
 set -x
 
 if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
-    where python
-    where pip
+  where python
+  where pip
 else
-    which python
-    which pip
+  which python
+  which pip
 fi
 
 py_ver="$(python -c 'import sys; print(sys.version[0])')"

scripts/ci/deploy_condition.sh

@@ -3,7 +3,7 @@
 set -e
 
 if [[ $TRAVIS_EVENT_TYPE = pull_request || $TRAVIS_EVENT_TPYE = cron ]]; then
-    exit 2
+  exit 2
 fi
 
 # positional args are assumed to be file glob patterns to deploy

scripts/ci/install.sh

@@ -14,9 +14,9 @@ git config --global user.email "[email protected]"
 git config --global user.name "DVC Tester"
 
 if [[ "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
-	aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
-	aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
-	aws configure set region us-east-2
+  aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
+  aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+  aws configure set region us-east-2
 
-	openssl enc -d -aes-256-cbc -md md5 -k $GCP_CREDS -in scripts/ci/gcp-creds.json.enc -out scripts/ci/gcp-creds.json
+  openssl enc -d -aes-256-cbc -md md5 -k $GCP_CREDS -in scripts/ci/gcp-creds.json.enc -out scripts/ci/gcp-creds.json
 fi

scripts/ci/install_azurite.sh

@@ -4,18 +4,18 @@ set -e
 set -x
 
 # install docker
- export DEBIAN_FRONTEND=noninteractive
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
- sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
- sudo apt-get update
- sudo apt-get install -y docker-ce
+export DEBIAN_FRONTEND=noninteractive
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+sudo apt-get update
+sudo apt-get install -y docker-ce
 
 # run azurite
 sudo docker run -d --restart always -e executable=blob -p 10000:10000 \
-	--tmpfs /opt/azurite/folder \
-	mcr.microsoft.com/azure-storage/azurite:3.3.0-preview \
-	azurite -l /data --blobHost 0.0.0.0
+  --tmpfs /opt/azurite/folder \
+  mcr.microsoft.com/azure-storage/azurite:3.3.0-preview \
+  azurite -l /data --blobHost 0.0.0.0
 
 # save secrets
-echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >> env.sh
-echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >> env.sh
+echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >>env.sh
+echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >>env.sh

scripts/ci/install_hadoop.sh

@@ -17,14 +17,14 @@ sudo mkdir hadoop
 sudo mv hadoop-2.6.5/* hadoop/
 popd
 
-echo "export HADOOP_HOME=/usr/local/hadoop" >> env.sh
-echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >> env.sh
-echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >> env.sh
-echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >> env.sh
-echo "export YARN_HOME=/usr/local/hadoop" >> env.sh
-echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >> env.sh
-echo "export JAVA_HOME=/usr/" >> env.sh
-echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> env.sh
+echo "export HADOOP_HOME=/usr/local/hadoop" >>env.sh
+echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >>env.sh
+echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >>env.sh
+echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >>env.sh
+echo "export YARN_HOME=/usr/local/hadoop" >>env.sh
+echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >>env.sh
+echo "export JAVA_HOME=/usr/" >>env.sh
+echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >>env.sh
 
 source env.sh
 

scripts/ci/install_oss.sh

@@ -5,6 +5,6 @@ set -euo pipefail
 git clone https://github.com/iterative/oss-emulator.git
 sudo docker image build -t oss:1.0 oss-emulator
 sudo docker run --detach --restart always -p 8880:8880 --name oss-emulator oss:1.0
-echo "export OSS_ENDPOINT='localhost:8880'" >> env.sh
-echo "export OSS_ACCESS_KEY_ID='AccessKeyID'" >> env.sh
-echo "export OSS_ACCESS_KEY_SECRET='AccessKeySecret'" >> env.sh
+echo "export OSS_ENDPOINT='localhost:8880'" >>env.sh
+echo "export OSS_ACCESS_KEY_ID='AccessKeyID'" >>env.sh
+echo "export OSS_ACCESS_KEY_SECRET='AccessKeySecret'" >>env.sh

scripts/ci/retry.sh

@@ -4,8 +4,8 @@ set -e
 
 N_RETRIES=3
 for i in $(seq $N_RETRIES); do
-    echo "Attempt #$i"
-    $@ && exit $?
+  echo "Attempt #$i"
+  $@ && exit $?
 done
 
 exit $?

scripts/ci/script.sh

@@ -5,8 +5,7 @@ set -e
 
 python -mtests
 
-if [[ "$TRAVIS_PULL_REQUEST" == "false" && \
-      "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
-	pip install codecov
-	codecov
+if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
+  pip install codecov
+  codecov
 fi

scripts/completion/dvc.bash

@@ -9,8 +9,8 @@
 #----------------------------------------------------------
 
 _dvc_commands='add cache checkout commit config destroy diff fetch get-url get gc \
-              import-url import init install lock metrics move pipeline pull push \
-              remote remove repro root run status unlock unprotect update version'
+  import-url import init install lock list metrics move pipeline pull push \
+  remote remove repro root run status unlock unprotect update version'
 
 _dvc_options='-h --help -V --version'
 _dvc_global_options='-h --help -q --quiet -v --verbose'
@@ -22,7 +22,7 @@ _dvc_checkout='-d --with-deps -R --recursive -f --force --relink $(compgen -G *.
 _dvc_commit='-f --force -d --with-deps -R --recursive $(compgen -G *.dvc)'
 _dvc_config='-u --unset --local --system --global'
 _dvc_destroy='-f --force'
-_dvc_diff='-t --target'
+_dvc_diff='-t --show-json --checksums'
 _dvc_fetch='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive $(compgen -G *.dvc)'
 _dvc_get_url=''
 _dvc_get='-o --out --rev --show-url'
@@ -31,12 +31,14 @@ _dvc_import_url='-f --file'
 _dvc_import='-o --out --rev'
 _dvc_init='--no-scm -f --force'
 _dvc_install=''
+_dvc_list='-R --recursive --outs-only --rev $(compgen -G *)'
 _dvc_lock='$(compgen -G *.dvc)'
 _dvc_metrics='add modify rmeove show'
 _dvc_metrics_add='-t --type -x --xpath $(compgen -G *)'
+_dvc_metrics_show='$(-t --type -x --xpath -a --all-branches -T --all-tags -R --recursive $(compgen -G *)'
+_dvc_metrics_diff='--targets -t --type -x --xpath -R --show-json'
 _dvc_metrics_modify='-t --type -x --xpath $(compgen -G *)'
 _dvc_metrics_remove='$(compgen -G *)'
-_dvc_metrics_show='$(-t --type -x --xpath -a --all-branches -T --all-tags -R --recursive $(compgen -G *)'
 _dvc_move='$(compgen -G *)'
 _dvc_pipeline='list show'
 _dvc_pipeline_list=''
@@ -56,9 +58,29 @@ _dvc_run='--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache --
 _dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud $(compgen -G *.dvc)'
 _dvc_unlock='$(compgen -G *.dvc)'
 _dvc_unprotect='$(compgen -G *)'
-_dvc_update='$(compgen -G *.dvc)'
+_dvc_update='--rev $(compgen -G *.dvc)'
 _dvc_version=''
 
+# Params
+# $1 - COMP_WORDS[1]
+comp_command() {
+  local options_list="_dvc_$(replace_hyphen $1)"
+
+  COMPREPLY=( $(compgen -W "$_dvc_global_options ${!options_list}" -- "$word") )
+}
+
+# Params
+# $1 - COMP_WORDS[1]
+# $1 - COMP_WORDS[2]
+comp_subcommand() {
+  local options_list="_dvc_$(replace_hyphen $1)_$(replace_hyphen $2)"
+  if [ -z "${!options_list}" ]; then
+    comp_command $1
+  else
+    COMPREPLY=( $(compgen -W "$_dvc_global_options ${!options_list}" -- "$word") )
+  fi
+}
+
 # Notes:
 #
 # `COMPREPLY` contains what will be rendered after completion is triggered
@@ -71,11 +93,10 @@ _dvc_version=''
 #       x="hello"
 #       ${!x} ->  ${hello} ->  "world"
 #
-_dvc () {
-  replace_hyphen () {
+_dvc() {
+  replace_hyphen() {
     echo $(echo $1 | sed 's/-/_/g')
   }
-
   local word="${COMP_WORDS[COMP_CWORD]}"
 
   COMPREPLY=()
@@ -83,16 +104,12 @@ _dvc () {
   if [ "${COMP_CWORD}" -eq 1 ]; then
     case "$word" in
       -*) COMPREPLY=($(compgen -W "$_dvc_options" -- "$word")) ;;
-      *)  COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
+      *) COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
     esac
   elif [ "${COMP_CWORD}" -eq 2 ]; then
-    local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})"
-
-    COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
+    comp_command ${COMP_WORDS[1]}
   elif [ "${COMP_CWORD}" -eq 3 ]; then
-    local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})_$(replace_hyphen ${COMP_WORDS[2]})"
-
-    COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
+    comp_subcommand ${COMP_WORDS[1]} ${COMP_WORDS[2]}
   fi
 
   return 0

scripts/completion/dvc.zsh

@@ -12,21 +12,22 @@
 
 _dvc_commands() {
   local _commands=(
-    "add:Take data files or directories under DVC control."
+    "add:Track data files or directories with DVC."
     "cache:Manage cache settings."
     "checkout:Checkout data files from cache."
     "commit:Save changed data to cache and update DVC-files."
     "config:Get or set config settings."
     "destroy:Remove DVC-files, local DVC config and data cache."
-    "diff:Show a diff of a DVC controlled data file or a directory."
-    "fetch:Fetch data files from a DVC remote storage."
+    "diff:Show changes between commits in the DVC repository, or between a commit and the workspace."
+    "fetch:Get files or directories tracked by DVC from remote storage into the cache."
     "get-url:Download or copy files from URL."
     "get:Download data from DVC repository."
-    "gc:Collect unused data from DVC cache or a remote storage."
+    "gc:Garbage collect unused objects from cache or remote storage."
     "import-url:Download or copy file from URL and take it under DVC control."
     "import:Download data from DVC repository and take it under DVC control."
     "init:Initialize DVC in the current directory."
     "install:Install DVC git hooks into the repository."
+    "list:List files."
     "lock:Lock DVC-file."
     "metrics:Commands to add, manage, collect and display metrics."
     "move:Rename or move a DVC controlled data file or a directory."
@@ -97,7 +98,10 @@ _dvc_destroy=(
 )
 
 _dvc_diff=(
-  {-t,--target}"[Source path to a data file or directory.]:Target files:"
+  "--show-json[Format the output into a JSON]"
+  "--checksums[Display checksums for each entry]"
+  "1:Old Git commit to compare (defaults to HEAD):"
+  "2:New Git commit to compare (defaults to the current workspace):"
 )
 
 _dvc_fetch=(
@@ -117,7 +121,7 @@ _dvc_geturl=(
 
 _dvc_get=(
   {-o,--out}"[Destination path to put data to.]:OUT:_files -/"
-  "--rev[Git revision (e.g. branch, tag, SHA)]:Revision:"
+  "--rev[Git revision (e.g. SHA, branch, tag)]:Revision:"
   "--show-url[Returns path/url to the location in remote for given path]"
   "1:URL:"
   "2:Path:"
@@ -141,7 +145,7 @@ _dvc_importurl=(
 
 _dvc_import=(
   {-o,--out}"[Destination path to put data to.]:OUT:_files -/"
-  "--rev[DVC repository git revision.]:Commit hash:"
+  "--rev[Git revision (e.g. SHA, branch, tag)]:Commit hash:"
   "1:URL:"
   "2:Path:"
 )
@@ -157,8 +161,16 @@ _dvc_lock=(
   "*:Stages:_files -g '(*.dvc|Dvcfile)'"
 )
 
+_dvc_list=(
+  "--rev[Git revision (e.g. branch, tag, SHA)]:Revision:"
+  {-R,--recursive}"[Recursively add each file under the directory.]"
+  "--outs-only[Only outputs DVC-outs.]"
+  "1:URL:"
+  "2:Target:"
+)
+
 _dvc_metrics=(
-  "1:Sub command:(show add modify remove)"
+  "1:Sub command:(add show diff modify remove)"
 )
 
 _dvc_move=(
@@ -261,6 +273,7 @@ _dvc_unprotect=(
 )
 
 _dvc_update=(
+  "--rev[Git revision (e.g. SHA, branch, tag)]:Revision:"
   "*:Stages:_files -g '(*.dvc|Dvcfile)'"
 )
 
@@ -273,34 +286,35 @@ _arguments \
   '*::args:->args'
 
 case $words[1] in
-  add)        _arguments $_dvc_global_options   $_dvc_add       ;;
-  cache)      _arguments $_dvc_global_options   $_dvc_cache     ;;
-  checkout)   _arguments $_dvc_global_options   $_dvc_checkout  ;;
-  commit)     _arguments $_dvc_global_options   $_dvc_commit    ;;
-  config)     _arguments $_dvc_global_options   $_dvc_config    ;;
-  destroy)    _arguments $_dvc_global_options   $_dvc_destroy   ;;
-  diff)       _arguments $_dvc_global_options   $_dvc_diff      ;;
-  fetch)      _arguments $_dvc_global_options   $_dvc_fetch     ;;
-  get-url)    _arguments $_dvc_global_options   $_dvc_geturl    ;;
-  get)        _arguments $_dvc_global_options   $_dvc_get       ;;
-  gc)         _arguments $_dvc_global_options   $_dvc_gc        ;;
-  import-url) _arguments $_dvc_global_options   $_dvc_importurl ;;
-  import)     _arguments $_dvc_global_options   $_dvc_import    ;;
-  init)       _arguments $_dvc_global_options   $_dvc_init      ;;
-  install)    _arguments $_dvc_global_options   $_dvc_install   ;;
-  lock)       _arguments $_dvc_global_options   $_dvc_lock      ;;
-  metrics)    _arguments $_dvc_global_options   $_dvc_metrics   ;;
-  move)       _arguments $_dvc_global_options   $_dvc_move      ;;
-  pipeline)   _arguments $_dvc_global_options   $_dvc_pipeline  ;;
-  pull)       _arguments $_dvc_global_options   $_dvc_pull      ;;
-  push)       _arguments $_dvc_global_options   $_dvc_push      ;;
-  remote)     _arguments $_dvc_global_options   $_dvc_remote    ;;
-  remove)     _arguments $_dvc_global_options   $_dvc_remove    ;;
-  repro)      _arguments $_dvc_global_options   $_dvc_repro     ;;
-  root)       _arguments $_dvc_global_options   $_dvc_root      ;;
-  run)        _arguments $_dvc_global_options   $_dvc_run       ;;
-  status)     _arguments $_dvc_global_options   $_dvc_status    ;;
-  unlock)     _arguments $_dvc_global_options   $_dvc_unlock    ;;
-  unprotect)  _arguments $_dvc_global_options   $_dvc_unprotect ;;
-  update)     _arguments $_dvc_global_options   $_dvc_update    ;;
+  add) _arguments $_dvc_global_options $_dvc_add ;;
+  cache) _arguments $_dvc_global_options $_dvc_cache ;;
+  checkout) _arguments $_dvc_global_options $_dvc_checkout ;;
+  commit) _arguments $_dvc_global_options $_dvc_commit ;;
+  config) _arguments $_dvc_global_options $_dvc_config ;;
+  destroy) _arguments $_dvc_global_options $_dvc_destroy ;;
+  diff) _arguments $_dvc_global_options $_dvc_diff ;;
+  fetch) _arguments $_dvc_global_options $_dvc_fetch ;;
+  get-url) _arguments $_dvc_global_options $_dvc_geturl ;;
+  get) _arguments $_dvc_global_options $_dvc_get ;;
+  gc) _arguments $_dvc_global_options $_dvc_gc ;;
+  import-url) _arguments $_dvc_global_options $_dvc_importurl ;;
+  import) _arguments $_dvc_global_options $_dvc_import ;;
+  init) _arguments $_dvc_global_options $_dvc_init ;;
+  install) _arguments $_dvc_global_options $_dvc_install ;;
+  lock) _arguments $_dvc_global_options $_dvc_lock ;;
+  list) _arguments $_dvc_global_options $_dvc_list ;;
+  metrics) _arguments $_dvc_global_options $_dvc_metrics ;;
+  move) _arguments $_dvc_global_options $_dvc_move ;;
+  pipeline) _arguments $_dvc_global_options $_dvc_pipeline ;;
+  pull) _arguments $_dvc_global_options $_dvc_pull ;;
+  push) _arguments $_dvc_global_options $_dvc_push ;;
+  remote) _arguments $_dvc_global_options $_dvc_remote ;;
+  remove) _arguments $_dvc_global_options $_dvc_remove ;;
+  repro) _arguments $_dvc_global_options $_dvc_repro ;;
+  root) _arguments $_dvc_global_options $_dvc_root ;;
+  run) _arguments $_dvc_global_options $_dvc_run ;;
+  status) _arguments $_dvc_global_options $_dvc_status ;;
+  unlock) _arguments $_dvc_global_options $_dvc_unlock ;;
+  unprotect) _arguments $_dvc_global_options $_dvc_unprotect ;;
+  update) _arguments $_dvc_global_options $_dvc_update ;;
 esac

scripts/pypi_upload.sh

@@ -3,8 +3,8 @@
 set -e
 
 if [ ! -d "dvc" ]; then
-	echo "Please run this script from repository root"
-	exit 1
+  echo "Please run this script from repository root"
+  exit 1
 fi
 
 rm -rf dist/

setup.py

@@ -48,8 +48,7 @@ install_requires = [
     "ply>=3.9",  # See https://github.com/pyinstaller/pyinstaller/issues/1945
     "colorama>=0.3.9",
     "configobj>=5.0.6",
-    "gitpython>=2.1.8",
-    "gitdb2>=2.0.6",  # See https://github.com/iterative/dvc/issues/1880
+    "gitpython>3",
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
@@ -74,6 +73,7 @@ install_requires = [
     "win-unicode-console>=0.5; sys_platform == 'win32'",
     "pywin32>=225; sys_platform == 'win32'",
     "networkx>=2.1,<2.4",
+    "pydot>=1.2.4",
     "speedcopy>=2.0.1",
     "pyfastcopy>=1.0.3",
     "flatten_json>=0.1.6",
@@ -84,10 +84,7 @@ install_requires = [
 # Extra dependencies for remote integrations
 
 gs = ["google-cloud-storage==1.19.0"]
-# google-api-python-client is internal dependency of pydrive. After merge of
-# https://github.com/gsuitedevs/PyDrive/pull/180 into pydrive's master,
-# usage of google-api-python-client can be removed from DVC.
-gdrive = ["pydrive2>=1.4.0", "google-api-python-client>=1.2"]
+gdrive = ["pydrive2>=1.4.5"]
 s3 = ["boto3>=1.9.201"]
 azure = ["azure-storage-blob==2.1.0"]
 oss = ["oss2==2.6.1"]
@@ -103,7 +100,6 @@ all_remotes = gs + s3 + azure + ssh + oss + gdrive + hdfs
 # Extra dependecies to run tests
 tests_requirements = [
     "wheel>=0.31.1",
-    "pydot>=1.2.4",
     # Test requirements:
     "pytest>=4.6.0",
     "pytest-timeout>=1.3.3",
@@ -128,7 +124,7 @@ tests_requirements = [
 ]
 
 if (sys.version_info) >= (3, 6):
-    tests_requirements.append("black==19.3b0")
+    tests_requirements.append("black==19.10b0")
 
 setup(
     name="dvc",
@@ -136,7 +132,7 @@ setup(
     description="Git for data scientists - manage your code and data together",
     long_description=open("README.rst", "r").read(),
     author="Dmitry Petrov",
-    author_email="dmitry@dataversioncontrol.com",
+    author_email="dmitry@dvc.org",
     download_url="https://github.com/iterative/dvc",
     license="Apache License 2.0",
     install_requires=install_requires,

snap/snapcraft.yaml

@@ -20,6 +20,8 @@ parts:
     plugin: python
     python-packages: [wheel]
     source: .
+    build-snaps:
+    - snapd
     build-packages:
     - git
     - libc-dev

tests/dir_helpers.py

@@ -45,6 +45,7 @@ from global repo template to creating everything inplace, which:
 
 import os
 import pathlib
+import logging
 from contextlib import contextmanager
 
 import pytest
@@ -65,6 +66,11 @@ __all__ = [
 ]
 
 
+# see https://github.com/iterative/dvc/issues/3167
+git_logger = logging.getLogger("git")
+git_logger.setLevel(logging.CRITICAL)
+
+
 class TmpDir(pathlib.Path):
     def __new__(cls, *args, **kwargs):
         if cls is TmpDir:
@@ -258,15 +264,15 @@ def run_copy(tmp_dir, dvc):
 
 @pytest.fixture
 def erepo_dir(make_tmp_dir):
-    from dvc.remote.config import RemoteConfig
-
     path = make_tmp_dir("erepo", scm=True, dvc=True)
 
     # Chdir for git and dvc to work locally
     with path.chdir():
-        rconfig = RemoteConfig(path.dvc.config)
-        rconfig.add("upstream", path.dvc.cache.local.cache_dir, default=True)
-        path.scm_add([path.dvc.config.config_file], commit="add remote")
+        with path.dvc.config.edit() as conf:
+            cache_dir = path.dvc.cache.local.cache_dir
+            conf["remote"]["upstream"] = {"url": cache_dir}
+            conf["core"]["remote"] = "upstream"
+        path.scm_add([path.dvc.config.files["repo"]], commit="add remote")
 
     return path
 

tests/func/remote/test_local.py

@@ -8,7 +8,7 @@ from tests.utils import trees_equal
 
 
 def test_dont_fail_on_unpacked_create_fail(tmp_dir, dvc):
-    stage, = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
+    (stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
 
     with mock.patch.object(
         RemoteLOCAL, "_create_unpacked_dir", side_effect=DvcException("msg")
@@ -18,7 +18,7 @@ def test_dont_fail_on_unpacked_create_fail(tmp_dir, dvc):
 
 
 def test_remove_unpacked_on_create_fail(tmp_dir, dvc):
-    stage, = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
+    (stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
     unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX
 
     # artificial unpacked dir for test purpose
@@ -34,7 +34,7 @@ def test_remove_unpacked_on_create_fail(tmp_dir, dvc):
 
 
 def test_create_unpacked_on_status(tmp_dir, dvc):
-    stage, = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
+    (stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
     unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX
     assert not os.path.exists(unpacked_dir)
 
@@ -45,7 +45,7 @@ def test_create_unpacked_on_status(tmp_dir, dvc):
 
 
 def test_dir_cache_changed_on_single_cache_file_modification(tmp_dir, dvc):
-    stage, = tmp_dir.dvc_gen(
+    (stage,) = tmp_dir.dvc_gen(
         {"dir": {"file1": "file1 content", "file2": "file2 content"}}
     )
     unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX

tests/func/test_add.py

@@ -11,7 +11,7 @@ from mock import patch
 
 import dvc as dvc_module
 from dvc.cache import Cache
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, OverlappingOutputPathsError
 from dvc.exceptions import RecursiveAddingWhileUsingFilename
 from dvc.exceptions import StageFileCorruptedError
 from dvc.main import main
@@ -31,7 +31,7 @@ from tests.utils import get_gitignore_content
 
 
 def test_add(tmp_dir, dvc):
-    stage, = tmp_dir.dvc_gen({"foo": "foo"})
+    (stage,) = tmp_dir.dvc_gen({"foo": "foo"})
     md5, _ = file_md5("foo")
 
     assert stage is not None
@@ -49,7 +49,7 @@ def test_add_unicode(tmp_dir, dvc):
     with open("\xe1", "wb") as fd:
         fd.write("something".encode("utf-8"))
 
-    stage, = dvc.add("\xe1")
+    (stage,) = dvc.add("\xe1")
 
     assert os.path.isfile(stage.path)
 
@@ -60,7 +60,7 @@ def test_add_unsupported_file(dvc):
 
 
 def test_add_directory(tmp_dir, dvc):
-    stage, = tmp_dir.dvc_gen({"dir": {"file": "file"}})
+    (stage,) = tmp_dir.dvc_gen({"dir": {"file": "file"}})
 
     assert stage is not None
     assert len(stage.deps) == 0
@@ -161,7 +161,7 @@ def test_add_file_in_dir(tmp_dir, dvc):
     tmp_dir.gen({"dir": {"subdir": {"subdata": "subdata content"}}})
     subdir_path = os.path.join("dir", "subdir", "subdata")
 
-    stage, = dvc.add(subdir_path)
+    (stage,) = dvc.add(subdir_path)
 
     assert stage is not None
     assert len(stage.deps) == 0
@@ -216,7 +216,7 @@ class TestAddLocalRemoteFile(TestDvc):
         self.assertEqual(ret, 0)
 
         d = load_stage_file("bar.dvc")
-        self.assertEqual(d["outs"][0]["path"], bar)
+        self.assertEqual(d["outs"][0]["path"], self.BAR)
 
 
 class TestCmdAdd(TestDvc):
@@ -546,10 +546,10 @@ def temporary_windows_drive(tmp_path_factory):
 def test_windows_should_add_when_cache_on_different_drive(
     tmp_dir, dvc, temporary_windows_drive
 ):
-    dvc.config.set("cache", "dir", temporary_windows_drive)
+    dvc.config["cache"]["dir"] = temporary_windows_drive
     dvc.cache = Cache(dvc)
 
-    stage, = tmp_dir.dvc_gen({"file": "file"})
+    (stage,) = tmp_dir.dvc_gen({"file": "file"})
     cache_path = stage.outs[0].cache_path
 
     assert path_isin(cache_path, temporary_windows_drive)
@@ -601,7 +601,7 @@ def test_should_relink_on_repeated_add(
 ):
     from dvc.path_info import PathInfo
 
-    dvc.config.set("cache", "type", link)
+    dvc.config["cache"]["type"] = link
 
     tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
 
@@ -641,3 +641,17 @@ def test_escape_gitignore_entries(tmp_dir, scm, dvc):
 
     tmp_dir.dvc_gen(fname, "...")
     assert ignored_fname in get_gitignore_content()
+
+
+def test_add_from_data_dir(tmp_dir, scm, dvc):
+    tmp_dir.dvc_gen({"dir": {"file1": "file1 content"}})
+
+    tmp_dir.gen({"dir": {"file2": "file2 content"}})
+
+    with pytest.raises(OverlappingOutputPathsError) as e:
+        dvc.add(os.path.join("dir", "file2"))
+    assert str(e.value) == (
+        "Cannot add '{out}', because it is overlapping with other DVC "
+        "tracked output: 'dir'.\n"
+        "To include '{out}' in 'dir', run 'dvc commit dir.dvc'"
+    ).format(out=os.path.join("dir", "file2"))

tests/func/test_api.py

@@ -1,17 +1,14 @@
 import os
 import shutil
-import copy
 
-import ruamel.yaml
 import pytest
 
 from dvc import api
-from dvc.api import SummonError, UrlNotDvcRepoError
+from dvc.api import UrlNotDvcRepoError
 from dvc.compat import fspath
 from dvc.exceptions import FileMissingError
 from dvc.main import main
 from dvc.path_info import URLInfo
-from dvc.remote.config import RemoteConfig
 from tests.remotes import Azure, GCP, HDFS, Local, OSS, S3, SSH
 
 
@@ -111,9 +108,9 @@ def test_missing(remote_url, tmp_dir, dvc):
 
 
 def _set_remote_url_and_commit(repo, remote_url):
-    rconfig = RemoteConfig(repo.config)
-    rconfig.modify("upstream", "url", remote_url)
-    repo.scm.add([repo.config.config_file])
+    with repo.config.edit() as conf:
+        conf["remote"]["upstream"]["url"] = remote_url
+    repo.scm.add([repo.config.files["repo"]])
     repo.scm.commit("modify remote")
 
 
@@ -141,68 +138,3 @@ def test_open_not_cached(dvc):
     os.remove(metric_file)
     with pytest.raises(FileMissingError):
         api.read(metric_file)
-
-
-def test_summon(tmp_dir, dvc, erepo_dir):
-    objects = {
-        "objects": [
-            {
-                "name": "sum",
-                "meta": {"description": "Add <x> to <number>"},
-                "summon": {
-                    "type": "python",
-                    "call": "calculator.add_to_num",
-                    "args": {"x": 1},
-                    "deps": ["number"],
-                },
-            }
-        ]
-    }
-
-    other_objects = copy.deepcopy(objects)
-    other_objects["objects"][0]["summon"]["args"]["x"] = 100
-
-    dup_objects = copy.deepcopy(objects)
-    dup_objects["objects"] *= 2
-
-    with erepo_dir.chdir():
-        erepo_dir.dvc_gen("number", "100", commit="Add number.dvc")
-        erepo_dir.scm_gen("dvcsummon.yaml", ruamel.yaml.dump(objects))
-        erepo_dir.scm_gen("other.yaml", ruamel.yaml.dump(other_objects))
-        erepo_dir.scm_gen("dup.yaml", ruamel.yaml.dump(dup_objects))
-        erepo_dir.scm_gen("invalid.yaml", ruamel.yaml.dump({"name": "sum"}))
-        erepo_dir.scm_gen("not_yaml.yaml", "a: - this is not a YAML file")
-        erepo_dir.scm_gen(
-            "calculator.py",
-            "def add_to_num(x): return x + int(open('number').read())",
-        )
-        erepo_dir.scm.commit("Add files")
-
-    repo_url = "file://{}".format(erepo_dir)
-
-    assert api.summon("sum", repo=repo_url) == 101
-    assert api.summon("sum", repo=repo_url, args={"x": 2}) == 102
-    assert api.summon("sum", repo=repo_url, summon_file="other.yaml") == 200
-
-    try:
-        api.summon("sum", repo=repo_url, summon_file="missing.yaml")
-    except SummonError as exc:
-        assert "Summon file not found" in str(exc)
-        assert "missing.yaml" in str(exc)
-        assert repo_url in str(exc)
-    else:
-        pytest.fail("Did not raise on missing summon file")
-
-    with pytest.raises(SummonError, match=r"No object with name 'missing'"):
-        api.summon("missing", repo=repo_url)
-
-    with pytest.raises(
-        SummonError, match=r"More than one object with name 'sum'"
-    ):
-        api.summon("sum", repo=repo_url, summon_file="dup.yaml")
-
-    with pytest.raises(SummonError, match=r"extra keys not allowed"):
-        api.summon("sum", repo=repo_url, summon_file="invalid.yaml")
-
-    with pytest.raises(SummonError, match=r"Failed to parse summon file"):
-        api.summon("sum", repo=repo_url, summon_file="not_yaml.yaml")

tests/func/test_cache.py

@@ -154,7 +154,7 @@ class TestCmdCacheDir(TestDvc):
         ret = main(["cache", "dir", dname])
         self.assertEqual(ret, 0)
 
-        config = configobj.ConfigObj(self.dvc.config.config_file)
+        config = configobj.ConfigObj(self.dvc.config.files["repo"])
         self.assertEqual(config["cache"]["dir"], dname)
 
     def test_relative_path(self):
@@ -166,7 +166,7 @@ class TestCmdCacheDir(TestDvc):
         # NOTE: we are in the repo's root and config is in .dvc/, so
         # dir path written to config should be just one level above.
         rel = os.path.join("..", dname)
-        config = configobj.ConfigObj(self.dvc.config.config_file)
+        config = configobj.ConfigObj(self.dvc.config.files["repo"])
         self.assertEqual(config["cache"]["dir"], rel)
 
         ret = main(["add", self.FOO])
@@ -178,9 +178,8 @@ class TestCmdCacheDir(TestDvc):
         self.assertEqual(len(files), 1)
 
 
-class TestShouldCacheBeReflinkOrCopyByDefault(TestDvc):
-    def test(self):
-        self.assertEqual(self.dvc.cache.local.cache_types, ["reflink", "copy"])
+def test_default_cache_type(dvc):
+    assert dvc.cache.local.cache_types == ["reflink", "copy"]
 
 
 @pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
@@ -189,8 +188,8 @@ class TestShouldCacheBeReflinkOrCopyByDefault(TestDvc):
     [(False, 0o775, 0o664), (True, 0o775, 0o444)],
 )
 def test_shared_cache(tmp_dir, dvc, protected, dir_mode, file_mode):
-    dvc.config.set("cache", "shared", "group")
-    dvc.config.set("cache", "protected", str(protected))
+    with dvc.config.edit() as conf:
+        conf["cache"].update({"shared": "group", "protected": str(protected)})
     dvc.cache = Cache(dvc)
 
     tmp_dir.dvc_gen(

tests/func/test_commit.py

@@ -18,7 +18,7 @@ def test_commit_recursive(tmp_dir, dvc):
 
 def test_commit_force(tmp_dir, dvc):
     tmp_dir.gen({"dir": {"file": "text1", "file2": "text2"}})
-    stage, = dvc.add("dir", no_commit=True)
+    (stage,) = dvc.add("dir", no_commit=True)
 
     with dvc.state:
         assert stage.outs[0].changed_cache()
@@ -40,7 +40,7 @@ def test_commit_force(tmp_dir, dvc):
 
 def test_commit_with_deps(tmp_dir, dvc, run_copy):
     tmp_dir.gen("foo", "foo")
-    foo_stage, = dvc.add("foo", no_commit=True)
+    (foo_stage,) = dvc.add("foo", no_commit=True)
     assert foo_stage is not None
     assert len(foo_stage.outs) == 1
 
@@ -60,7 +60,7 @@ def test_commit_with_deps(tmp_dir, dvc, run_copy):
 
 def test_commit_changed_md5(tmp_dir, dvc):
     tmp_dir.gen({"file": "file content"})
-    stage, = dvc.add("file", no_commit=True)
+    (stage,) = dvc.add("file", no_commit=True)
 
     stage_file_content = load_stage_file(stage.path)
     stage_file_content["md5"] = "1111111111"

tests/func/test_config.py

@@ -1,15 +1,14 @@
+import pytest
 import configobj
 
 from dvc.main import main
+from dvc.config import Config, ConfigError
 from tests.basic_env import TestDvc
 
 
 class TestConfigCLI(TestDvc):
     def _contains(self, section, field, value, local=False):
-        if local:
-            fname = self.dvc.config.config_local_file
-        else:
-            fname = self.dvc.config.config_file
+        fname = self.dvc.config.files["local" if local else "repo"]
 
         config = configobj.ConfigObj(fname)
         if section not in config.keys():
@@ -33,11 +32,11 @@ class TestConfigCLI(TestDvc):
         self.assertEqual(ret, 0)
 
     def _do_test(self, local=False):
-        section = "setsection"
-        field = "setfield"
+        section = "core"
+        field = "analytics"
         section_field = "{}.{}".format(section, field)
-        value = "setvalue"
-        newvalue = "setnewvalue"
+        value = "True"
+        newvalue = "False"
 
         base = ["config"]
         if local:
@@ -83,3 +82,27 @@ class TestConfigCLI(TestDvc):
 
         ret = main(["config", "core.non_existing_field", "-u"])
         self.assertEqual(ret, 251)
+
+
+def test_set_invalid_key(dvc):
+    with pytest.raises(ConfigError, match=r"extra keys not allowed"):
+        with dvc.config.edit() as conf:
+            conf["core"]["invalid_key"] = "value"
+
+
+def test_merging_two_levels(dvc):
+    with dvc.config.edit() as conf:
+        conf["remote"]["test"] = {"url": "ssh://example.com"}
+
+    with dvc.config.edit("local") as conf:
+        conf["remote"]["test"] = {"password": "1"}
+
+    assert dvc.config["remote"]["test"] == {
+        "url": "ssh://example.com",
+        "password": "1",
+    }
+
+
+def test_config_loads_without_error_for_non_dvc_repo(tmp_dir):
+    # regression testing for https://github.com/iterative/dvc/issues/3328
+    Config(validate=True)

tests/func/test_data_cloud.py

@@ -7,12 +7,11 @@ from unittest import SkipTest
 
 import pytest
 
-from dvc.cache import NamedCache
 from dvc.compat import fspath
-from dvc.config import Config
+from dvc.cache import NamedCache
 from dvc.data_cloud import DataCloud
 from dvc.main import main
-from dvc.remote import RemoteAZURE, RemoteConfig
+from dvc.remote import RemoteAZURE
 from dvc.remote import RemoteGDrive
 from dvc.remote import RemoteGS
 from dvc.remote import RemoteHDFS
@@ -21,12 +20,9 @@ from dvc.remote import RemoteLOCAL
 from dvc.remote import RemoteOSS
 from dvc.remote import RemoteS3
 from dvc.remote import RemoteSSH
-from dvc.remote.base import STATUS_DELETED
-from dvc.remote.base import STATUS_NEW
-from dvc.remote.base import STATUS_OK
+from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
 from dvc.utils import file_md5
-from dvc.utils.stage import dump_stage_file
-from dvc.utils.stage import load_stage_file
+from dvc.utils.stage import dump_stage_file, load_stage_file
 from tests.basic_env import TestDvc
 
 from tests.remotes import (
@@ -39,7 +35,6 @@ from tests.remotes import (
     SSHMocked,
     OSS,
     TEST_CONFIG,
-    TEST_SECTION,
     TEST_GCP_CREDS_FILE,
     TEST_GDRIVE_CLIENT_ID,
     TEST_GDRIVE_CLIENT_SECRET,
@@ -49,7 +44,7 @@ from tests.remotes import (
 
 class TestDataCloud(TestDvc):
     def _test_cloud(self, config, cl):
-        self.dvc.config.config = config
+        self.dvc.config = config
         cloud = DataCloud(self.dvc)
         self.assertIsInstance(cloud.get_remote(), cl)
 
@@ -68,7 +63,7 @@ class TestDataCloud(TestDvc):
 
         for scheme, cl in clist:
             remote_url = scheme + str(uuid.uuid4())
-            config[TEST_SECTION][Config.SECTION_REMOTE_URL] = remote_url
+            config["remote"][TEST_REMOTE]["url"] = remote_url
             self._test_cloud(config, cl)
 
 
@@ -100,9 +95,8 @@ class TestDataCloudBase(TestDvc):
         keyfile = self._get_keyfile()
 
         config = copy.deepcopy(TEST_CONFIG)
-        config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
-        config[TEST_SECTION][Config.SECTION_REMOTE_KEY_FILE] = keyfile
-        self.dvc.config.config = config
+        config["remote"][TEST_REMOTE] = {"url": repo, "keyfile": keyfile}
+        self.dvc.config = config
         self.cloud = DataCloud(self.dvc)
 
         self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
@@ -194,24 +188,28 @@ class TestRemoteS3(S3, TestDataCloudBase):
         return RemoteS3
 
 
+def setup_gdrive_cloud(remote_url, dvc):
+    config = copy.deepcopy(TEST_CONFIG)
+    config["remote"][TEST_REMOTE] = {
+        "url": remote_url,
+        "gdrive_client_id": TEST_GDRIVE_CLIENT_ID,
+        "gdrive_client_secret": TEST_GDRIVE_CLIENT_SECRET,
+    }
+
+    dvc.config = config
+    remote = DataCloud(dvc).get_remote()
+    remote._create_remote_dir("root", remote.path_info.path)
+
+
 class TestRemoteGDrive(GDrive, TestDataCloudBase):
     def _setup_cloud(self):
         self._ensure_should_run()
 
-        repo = self.get_url()
+        setup_gdrive_cloud(self.get_url(), self.dvc)
 
-        config = copy.deepcopy(TEST_CONFIG)
-        config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
-        config[TEST_SECTION][
-            Config.SECTION_GDRIVE_CLIENT_ID
-        ] = TEST_GDRIVE_CLIENT_ID
-        config[TEST_SECTION][
-            Config.SECTION_GDRIVE_CLIENT_SECRET
-        ] = TEST_GDRIVE_CLIENT_SECRET
-        self.dvc.config.config = config
         self.cloud = DataCloud(self.dvc)
-
-        self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
+        remote = self.cloud.get_remote()
+        self.assertIsInstance(remote, self._get_cloud_class())
 
     def _get_cloud_class(self):
         return RemoteGDrive
@@ -224,11 +222,11 @@ class TestRemoteGS(GCP, TestDataCloudBase):
         repo = self.get_url()
 
         config = copy.deepcopy(TEST_CONFIG)
-        config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
-        config[TEST_SECTION][
-            Config.SECTION_GCP_CREDENTIALPATH
-        ] = TEST_GCP_CREDS_FILE
-        self.dvc.config.config = config
+        config["remote"][TEST_REMOTE] = {
+            "url": repo,
+            "credentialpath": TEST_GCP_CREDS_FILE,
+        }
+        self.dvc.config = config
         self.cloud = DataCloud(self.dvc)
 
         self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
@@ -266,10 +264,12 @@ class TestRemoteSSHMocked(SSHMocked, TestDataCloudBase):
         keyfile = self._get_keyfile()
 
         config = copy.deepcopy(TEST_CONFIG)
-        config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
-        config[TEST_SECTION][Config.SECTION_REMOTE_KEY_FILE] = keyfile
-        config[TEST_SECTION][Config.SECTION_REMOTE_NO_TRAVERSE] = False
-        self.dvc.config.config = config
+        config["remote"][TEST_REMOTE] = {
+            "url": repo,
+            "keyfile": keyfile,
+            "no_traverse": False,
+        }
+        self.dvc.config = config
         self.cloud = DataCloud(self.dvc)
 
         self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
@@ -303,7 +303,12 @@ class TestDataCloudCLIBase(TestDvc):
     def get_url():
         raise NotImplementedError
 
+    def _setup_cloud(self):
+        pass
+
     def _test_cloud(self, remote=None):
+        self._setup_cloud()
+
         args = ["-v", "-j", "2"]
         if remote:
             args += ["-r", remote]
@@ -402,6 +407,9 @@ class TestRemoteS3CLI(S3, TestDataCloudCLIBase):
 
 
 class TestRemoteGDriveCLI(GDrive, TestDataCloudCLIBase):
+    def _setup_cloud(self):
+        setup_gdrive_cloud(self.get_url(), self.dvc)
+
     def _test(self):
         url = self.get_url()
 
@@ -411,7 +419,7 @@ class TestRemoteGDriveCLI(GDrive, TestDataCloudCLIBase):
                 "remote",
                 "modify",
                 TEST_REMOTE,
-                Config.SECTION_GDRIVE_CLIENT_ID,
+                "gdrive_client_id",
                 TEST_GDRIVE_CLIENT_ID,
             ]
         )
@@ -420,7 +428,7 @@ class TestRemoteGDriveCLI(GDrive, TestDataCloudCLIBase):
                 "remote",
                 "modify",
                 TEST_REMOTE,
-                Config.SECTION_GDRIVE_CLIENT_SECRET,
+                "gdrive_client_secret",
                 TEST_GDRIVE_CLIENT_SECRET,
             ]
         )
@@ -498,9 +506,9 @@ class TestWarnOnOutdatedStage(TestDvc):
             self._caplog.clear()
             self.main(["status", "-c"])
             expected_warning = (
-                "Output 'bar'(Stage: 'bar.dvc') is missing version info."
-                " Cache for it will not be collected."
-                " Use dvc repro to get your pipeline up to date."
+                "Output 'bar'(Stage: 'bar.dvc') is missing version info. "
+                "Cache for it will not be collected. "
+                "Use `dvc repro` to get your pipeline up to date."
             )
 
             assert expected_warning in self._caplog.text
@@ -655,11 +663,10 @@ def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory):
     tmp_dir.dvc_gen({"file": "file1 content"}, commit="add file")
     tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir")
 
-    RemoteConfig(dvc.config).add(
-        "local_remote",
-        fspath(tmp_path_factory.mktemp("local_remote")),
-        default=True,
-    )
+    dvc.config["remote"]["local_remote"] = {
+        "url": fspath(tmp_path_factory.mktemp("local_remote"))
+    }
+    dvc.config["core"]["remote"] = "local_remote"
     dvc.push()
 
     # remove artifacts and cache to trigger fetching
@@ -675,11 +682,7 @@ def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory):
     # Removing cache will invalidate existing state entries
     shutil.rmtree(dvc.cache.local.cache_dir)
 
-    dvc.config.set(
-        Config.SECTION_REMOTE_FMT.format("local_remote"),
-        Config.SECTION_REMOTE_VERIFY,
-        "True",
-    )
+    dvc.config["remote"]["local_remote"]["verify"] = True
 
     dvc.pull()
     assert checksum_spy.call_count == 3

tests/func/test_diff.py

@@ -1,267 +1,172 @@
+import hashlib
 import os
+import pytest
+import shutil
 
-import dvc.repo.diff as diff
-from dvc.main import main
-from dvc.scm.base import FileNotInCommitError
-from tests.basic_env import TestDvcGit
+from dvc.compat import fspath
+from dvc.exceptions import DvcException
 
 
-class TestDiff(TestDvcGit):
-    def setUp(self):
-        super().setUp()
-
-        self.new_file = "new_test_file"
-        self.create(self.new_file, self.new_file)
-        stage = self.dvc.add(self.new_file)[0]
-        self.a_ref = self.git.git.rev_parse(self.git.head.commit, short=True)
-        self.new_checksum = stage.outs[0].checksum
-        self.git.index.add([self.new_file + ".dvc"])
-        self.git.index.commit("adds new_file")
-        self.test_dct = {
-            diff.DIFF_A_REF: self.a_ref,
-            diff.DIFF_B_REF: self.git.git.rev_parse(
-                self.git.head.commit, short=True
-            ),
-            diff.DIFF_LIST: [
-                {
-                    diff.DIFF_TARGET: self.new_file,
-                    diff.DIFF_NEW_FILE: self.new_file,
-                    diff.DIFF_NEW_CHECKSUM: self.new_checksum,
-                    diff.DIFF_SIZE: 13,
-                }
-            ],
-        }
-
-    def test(self):
-        out = self.dvc.scm.get_diff_trees(self.a_ref)
-        self.assertFalse(out[diff.DIFF_EQUAL])
-        self.assertEqual(self.a_ref, out[diff.DIFF_A_REF])
-        self.assertEqual(
-            self.git.git.rev_parse(self.git.head.commit, short=True),
-            out[diff.DIFF_B_REF],
-        )
+def digest(text):
+    return hashlib.md5(bytes(text, "utf-8")).hexdigest()
 
 
-class TestDiffRepo(TestDiff):
-    def test(self):
-        result = self.dvc.diff(self.a_ref, target=self.new_file)
-        self.assertEqual(self.test_dct, result)
+def test_no_scm(tmp_dir, dvc):
+    tmp_dir.dvc_gen("file", "text")
+
+    with pytest.raises(DvcException, match=r"only supported for Git repos"):
+        dvc.diff()
 
 
-class TestDiffCmdLine(TestDiff):
-    def test(self):
-        ret = main(["diff", "-t", self.new_file, self.a_ref])
-        self.assertEqual(ret, 0)
+def test_added(tmp_dir, scm, dvc):
+    tmp_dir.dvc_gen("file", "text")
+
+    assert dvc.diff() == {
+        "added": [{"path": "file", "checksum": digest("text")}],
+        "deleted": [],
+        "modified": [],
+    }
 
 
-class TestDiffDir(TestDvcGit):
-    def setUp(self):
-        super().setUp()
+def test_no_cache_entry(tmp_dir, scm, dvc):
+    tmp_dir.dvc_gen("file", "first", commit="add a file")
 
-        stage = self.dvc.add(self.DATA_DIR)[0]
-        self.git.index.add([self.DATA_DIR + ".dvc"])
-        self.git.index.commit("adds data_dir")
-        self.a_ref = self.git.git.rev_parse(
-            self.dvc.scm.repo.head.commit, short=True
-        )
-        self.old_checksum = stage.outs[0].checksum
-        self.new_file = os.path.join(self.DATA_SUB_DIR, diff.DIFF_NEW_FILE)
-        self.create(self.new_file, self.new_file)
-        stage = self.dvc.add(self.DATA_DIR)[0]
-        self.git.index.add([self.DATA_DIR + ".dvc"])
-        self.git.index.commit(message="adds data_dir with new_file")
-        self.new_checksum = stage.outs[0].checksum
+    tmp_dir.dvc_gen({"dir": {"1": "1", "2": "2"}})
+    tmp_dir.dvc_gen("file", "second")
 
-    def test(self):
-        out = self.dvc.scm.get_diff_trees(self.a_ref)
-        self.assertFalse(out[diff.DIFF_EQUAL])
-        self.assertEqual(self.a_ref, out[diff.DIFF_A_REF])
-        self.assertEqual(
-            self.git.git.rev_parse(self.git.head.commit, short=True),
-            out[diff.DIFF_B_REF],
-        )
+    shutil.rmtree(fspath(tmp_dir / ".dvc" / "cache"))
+    (tmp_dir / ".dvc" / "state").unlink()
+
+    dir_checksum = "5fb6b29836c388e093ca0715c872fe2a.dir"
+
+    assert dvc.diff() == {
+        "added": [
+            {"path": os.path.join("dir", ""), "checksum": dir_checksum},
+            {"path": os.path.join("dir", "1"), "checksum": digest("1")},
+            {"path": os.path.join("dir", "2"), "checksum": digest("2")},
+        ],
+        "deleted": [],
+        "modified": [
+            {
+                "path": "file",
+                "checksum": {"old": digest("first"), "new": digest("second")},
+            }
+        ],
+    }
 
 
-class TestDiffDirRepo(TestDiffDir):
-    maxDiff = None
+def test_deleted(tmp_dir, scm, dvc):
+    tmp_dir.dvc_gen("file", "text", commit="add file")
+    (tmp_dir / "file.dvc").unlink()
 
-    def test(self):
-        result = self.dvc.diff(self.a_ref, target=self.DATA_DIR)
-        test_dct = {
-            diff.DIFF_A_REF: self.git.git.rev_parse(self.a_ref, short=True),
-            diff.DIFF_B_REF: self.git.git.rev_parse(
-                self.git.head.commit, short=True
-            ),
-            diff.DIFF_LIST: [
-                {
-                    diff.DIFF_CHANGE: 0,
-                    diff.DIFF_DEL: 0,
-                    diff.DIFF_IDENT: 2,
-                    diff.DIFF_MOVE: 0,
-                    diff.DIFF_NEW: 1,
-                    diff.DIFF_IS_DIR: True,
-                    diff.DIFF_TARGET: self.DATA_DIR,
-                    diff.DIFF_NEW_FILE: self.DATA_DIR,
-                    diff.DIFF_NEW_CHECKSUM: self.new_checksum,
-                    diff.DIFF_OLD_FILE: self.DATA_DIR,
-                    diff.DIFF_OLD_CHECKSUM: self.old_checksum,
-                    diff.DIFF_SIZE: 30,
-                }
-            ],
-        }
-        self.assertEqual(test_dct, result)
+    assert dvc.diff() == {
+        "added": [],
+        "deleted": [{"path": "file", "checksum": digest("text")}],
+        "modified": [],
+    }
 
 
-class TestDiffDirRepoDeletedFile(TestDiffDir):
-    maxDiff = None
+def test_modified(tmp_dir, scm, dvc):
+    tmp_dir.dvc_gen("file", "first", commit="first version")
+    tmp_dir.dvc_gen("file", "second")
 
-    def setUp(self):
-        super().setUp()
-
-        self.b_ref = self.a_ref
-        tmp = self.new_checksum
-        self.new_checksum = self.old_checksum
-        self.a_ref = str(self.dvc.scm.repo.head.commit)
-        self.old_checksum = tmp
-
-    def test(self):
-        result = self.dvc.diff(
-            self.a_ref, b_ref=self.b_ref, target=self.DATA_DIR
-        )
-        test_dct = {
-            diff.DIFF_A_REF: self.git.git.rev_parse(self.a_ref, short=True),
-            diff.DIFF_B_REF: self.git.git.rev_parse(self.b_ref, short=True),
-            diff.DIFF_LIST: [
-                {
-                    diff.DIFF_CHANGE: 0,
-                    diff.DIFF_DEL: 1,
-                    diff.DIFF_IDENT: 2,
-                    diff.DIFF_MOVE: 0,
-                    diff.DIFF_NEW: 0,
-                    diff.DIFF_IS_DIR: True,
-                    diff.DIFF_TARGET: self.DATA_DIR,
-                    diff.DIFF_NEW_FILE: self.DATA_DIR,
-                    diff.DIFF_NEW_CHECKSUM: self.new_checksum,
-                    diff.DIFF_OLD_FILE: self.DATA_DIR,
-                    diff.DIFF_OLD_CHECKSUM: self.old_checksum,
-                    diff.DIFF_SIZE: -30,
-                }
-            ],
-        }
-        self.assertEqual(test_dct, result)
+    assert dvc.diff() == {
+        "added": [],
+        "deleted": [],
+        "modified": [
+            {
+                "path": "file",
+                "checksum": {"old": digest("first"), "new": digest("second")},
+            }
+        ],
+    }
 
 
-class TestDiffFileNotFound(TestDiffDir):
-    def setUp(self):
-        super().setUp()
-        self.unknown_file = "unknown_file_" + str(id(self))
+def test_refs(tmp_dir, scm, dvc):
+    tmp_dir.dvc_gen("file", "first", commit="first version")
+    tmp_dir.dvc_gen("file", "second", commit="second version")
+    tmp_dir.dvc_gen("file", "third", commit="third version")
 
-    def test(self):
-        with self.assertRaises(FileNotInCommitError):
-            self.dvc.diff(self.a_ref, target=self.unknown_file)
+    HEAD_2 = digest("first")
+    HEAD_1 = digest("second")
+    HEAD = digest("third")
+
+    assert dvc.diff("HEAD~1") == {
+        "added": [],
+        "deleted": [],
+        "modified": [
+            {"path": "file", "checksum": {"old": HEAD_1, "new": HEAD}}
+        ],
+    }
+
+    assert dvc.diff("HEAD~2", "HEAD~1") == {
+        "added": [],
+        "deleted": [],
+        "modified": [
+            {"path": "file", "checksum": {"old": HEAD_2, "new": HEAD_1}}
+        ],
+    }
+
+    with pytest.raises(DvcException, match=r"unknown Git revision 'missing'"):
+        dvc.diff("missing")
 
 
-class TestDiffModifiedFile(TestDiff):
-    maxDiff = None
+def test_directories(tmp_dir, scm, dvc):
+    tmp_dir.dvc_gen({"dir": {"1": "1", "2": "2"}}, commit="add a directory")
+    tmp_dir.dvc_gen({"dir": {"3": "3"}}, commit="add a file")
+    tmp_dir.dvc_gen({"dir": {"2": "two"}}, commit="modify a file")
 
-    def setUp(self):
-        super().setUp()
+    (tmp_dir / "dir" / "2").unlink()
+    dvc.add("dir")
+    scm.add("dir.dvc")
+    scm.commit("delete a file")
 
-        self.old_checksum = self.new_checksum
-        self.new_file_content = "new_test_file_bigger_content_123456789"
-        self.diff_len = len(self.new_file) + len(self.new_file_content)
-        self.create(self.new_file, self.new_file_content)
-        stage = self.dvc.add(self.new_file)[0]
-        self.git.index.add([self.new_file + ".dvc"])
-        self.git.index.commit("change new_file content to be bigger")
-        self.new_checksum = stage.outs[0].checksum
-        self.b_ref = self.git.git.rev_parse(self.git.head.commit, short=True)
+    # The ":/<text>" format is a way to specify revisions by commit message:
+    #       https://git-scm.com/docs/revisions
+    #
+    assert dvc.diff(":/init", ":/directory") == {
+        "added": [
+            {
+                "path": os.path.join("dir", ""),
+                "checksum": "5fb6b29836c388e093ca0715c872fe2a.dir",
+            },
+            {"path": os.path.join("dir", "1"), "checksum": digest("1")},
+            {"path": os.path.join("dir", "2"), "checksum": digest("2")},
+        ],
+        "deleted": [],
+        "modified": [],
+    }
 
-    def test(self):
-        result = self.dvc.diff(
-            self.a_ref, b_ref=self.b_ref, target=self.new_file
-        )
-        test_dct = {
-            diff.DIFF_A_REF: self.git.git.rev_parse(self.a_ref, short=True),
-            diff.DIFF_B_REF: self.git.git.rev_parse(self.b_ref, short=True),
-            diff.DIFF_LIST: [
-                {
-                    diff.DIFF_NEW_CHECKSUM: self.new_checksum,
-                    diff.DIFF_NEW_FILE: self.new_file,
-                    diff.DIFF_TARGET: self.new_file,
-                    diff.DIFF_SIZE: self.diff_len,
-                }
-            ],
-        }
-        self.assertEqual(test_dct, result)
+    assert dvc.diff(":/directory", ":/modify") == {
+        "added": [{"path": os.path.join("dir", "3"), "checksum": digest("3")}],
+        "deleted": [],
+        "modified": [
+            {
+                "path": os.path.join("dir", ""),
+                "checksum": {
+                    "old": "5fb6b29836c388e093ca0715c872fe2a.dir",
+                    "new": "9b5faf37366b3370fd98e3e60ca439c1.dir",
+                },
+            },
+            {
+                "path": os.path.join("dir", "2"),
+                "checksum": {"old": digest("2"), "new": digest("two")},
+            },
+        ],
+    }
 
-
-class TestDiffDirWithFile(TestDiffDir):
-    maxDiff = None
-
-    def setUp(self):
-        super().setUp()
-
-        self.a_ref = self.git.git.rev_parse(self.git.head.commit, short=True)
-        self.old_checksum = self.new_checksum
-        self.new_file_content = "new_test_file_bigger_content_123456789"
-        self.diff_len = len(self.new_file_content)
-        self.create(self.new_file, self.new_file_content)
-        stage = self.dvc.add(self.DATA_DIR)[0]
-        self.git.index.add([self.DATA_DIR + ".dvc"])
-        self.git.index.commit(message="modify file in the data dir")
-        self.new_checksum = stage.outs[0].checksum
-        self.b_ref = self.git.git.rev_parse(self.git.head.commit, short=True)
-
-    def test(self):
-        result = self.dvc.diff(self.a_ref, target=self.DATA_DIR)
-        test_dct = {
-            diff.DIFF_A_REF: self.git.git.rev_parse(self.a_ref, short=True),
-            diff.DIFF_B_REF: self.git.git.rev_parse(self.b_ref, short=True),
-            diff.DIFF_LIST: [
-                {
-                    diff.DIFF_IDENT: 2,
-                    diff.DIFF_CHANGE: 1,
-                    diff.DIFF_DEL: 0,
-                    diff.DIFF_MOVE: 0,
-                    diff.DIFF_NEW: 0,
-                    diff.DIFF_IS_DIR: True,
-                    diff.DIFF_TARGET: self.DATA_DIR,
-                    diff.DIFF_NEW_FILE: self.DATA_DIR,
-                    diff.DIFF_NEW_CHECKSUM: self.new_checksum,
-                    diff.DIFF_OLD_FILE: self.DATA_DIR,
-                    diff.DIFF_OLD_CHECKSUM: self.old_checksum,
-                    diff.DIFF_SIZE: self.diff_len,
-                }
-            ],
-        }
-        self.assertEqual(test_dct, result)
-
-
-class TestDiffCmdMessage(TestDiff):
-    maxDiff = None
-
-    def test(self):
-        ret = main(
-            [
-                "diff",
-                self.test_dct[diff.DIFF_A_REF],
-                self.test_dct[diff.DIFF_B_REF],
-            ]
-        )
-        self.assertEqual(ret, 0)
-
-        msg1 = "dvc diff from {0} to {1}".format(
-            self.git.git.rev_parse(self.test_dct[diff.DIFF_A_REF], short=True),
-            self.git.git.rev_parse(self.test_dct[diff.DIFF_B_REF], short=True),
-        )
-        msg2 = "diff for '{0}'".format(
-            self.test_dct[diff.DIFF_LIST][0][diff.DIFF_TARGET]
-        )
-        msg3 = "+{0} with md5 {1}".format(
-            self.test_dct[diff.DIFF_LIST][0][diff.DIFF_TARGET],
-            self.test_dct[diff.DIFF_LIST][0][diff.DIFF_NEW_CHECKSUM],
-        )
-        msg4 = "added file with size 13 Bytes"
-        for m in [msg1, msg2, msg3, msg4]:
-            assert m in self._caplog.text
+    assert dvc.diff(":/modify", ":/delete") == {
+        "added": [],
+        "deleted": [
+            {"path": os.path.join("dir", "2"), "checksum": digest("two")}
+        ],
+        "modified": [
+            {
+                "path": os.path.join("dir", ""),
+                "checksum": {
+                    "old": "9b5faf37366b3370fd98e3e60ca439c1.dir",
+                    "new": "83ae82fb367ac9926455870773ff09e6.dir",
+                },
+            }
+        ],
+    }

tests/func/test_external_repo.py

@@ -1,8 +1,11 @@
+import os
+
 from mock import patch
 from dvc.compat import fspath
 
 from dvc.external_repo import external_repo
 from dvc.scm.git import Git
+from dvc.remote import RemoteLOCAL
 
 
 def test_external_repo(erepo_dir):
@@ -23,3 +26,65 @@ def test_external_repo(erepo_dir):
                 assert fd.read() == "branch"
 
         assert mock.call_count == 1
+
+
+def test_source_change(erepo_dir):
+    url = fspath(erepo_dir)
+    with external_repo(url) as repo:
+        old_rev = repo.scm.get_rev()
+
+    erepo_dir.scm_gen("file", "text", commit="a change")
+
+    with external_repo(url) as repo:
+        new_rev = repo.scm.get_rev()
+
+    assert old_rev != new_rev
+
+
+def test_cache_reused(erepo_dir, mocker):
+    with erepo_dir.chdir():
+        erepo_dir.dvc_gen("file", "text", commit="add file")
+
+    download_spy = mocker.spy(RemoteLOCAL, "download")
+
+    # Use URL to prevent any fishy optimizations
+    url = "file://{}".format(erepo_dir)
+    with external_repo(url) as repo:
+        repo.fetch()
+        assert download_spy.mock.call_count == 1
+
+    # Should not download second time
+    erepo_dir.scm.branch("branch")
+    with external_repo(url, "branch") as repo:
+        repo.fetch()
+        assert download_spy.mock.call_count == 1
+
+
+def test_known_sha(erepo_dir):
+    url = "file://{}".format(erepo_dir)
+    with external_repo(url) as repo:
+        rev = repo.scm.get_rev()
+        prev_rev = repo.scm.resolve_rev("HEAD^")
+
+    # Hits cache
+    with external_repo(url, rev) as repo:
+        pass
+
+    # No clone, no pull, copies a repo, checks out the known sha
+    with external_repo(url, prev_rev) as repo:
+        pass
+
+
+def test_pull_subdir_file(tmp_dir, erepo_dir):
+    with erepo_dir.chdir():
+        subdir = erepo_dir / "subdir"
+        subdir.mkdir()
+        (subdir / "file").write_text("contents")
+        erepo_dir.dvc_add(subdir / "file", commit="create file")
+
+    dest = tmp_dir / "file"
+    with external_repo(fspath(erepo_dir)) as repo:
+        repo.pull_to(os.path.join("subdir", "file"), dest)
+
+    assert dest.is_file()
+    assert dest.read_text() == "contents"