dvc: bump to 0.86.2

* get/import: update cmd and url arg descs with correct behavior

* get/import: update path arg desc as well (See previous commit)

* get/imp: udpate help output with simpler terms
per https://github.com/iterative/dvc/pull/3277#pullrequestreview-352650016

* api: url->URL in a docstring

* metrics show: update help output per
per https://github.com/iterative/dvc.org/pull/933#pullrequestreview-351862579

* diff: update `diff` and `metrics diff` help desc to match docs
per https://github.com/iterative/dvc.org/pull/933#pullrequestreview-356890960

* diff: update autocomplete scripts
per https://github.com/iterative/dvc/pull/3308#issuecomment-585220300

* diff: update help output again
per https://github.com/iterative/dvc.org/pull/933#pullrequestreview-356890960

* diff/metrics diff: update help output yet again
per https://github.com/iterative/dvc.org/pull/933#discussion_r378654316

Co-authored-by: Ruslan Kuprieiev <[email protected]>

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,4 +5,3 @@
 * [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect everything to be addressed. Please review them carefully and fix those that actually improve code or fix bugs.
 
 Thank you for the contribution - we'll try to review it as soon as possible. 🙏
-

.gitignore

@@ -26,7 +26,7 @@ innosetup/config.ini
 .coverage
 .coverage.*
 
-*.swp
+*.sw?
 
 pip-wheel-metadata/
 .vscode/

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: 19.3b0
+    rev: 19.10b0
     hooks:
     - id: black
       language_version: python3
@@ -9,3 +9,9 @@ repos:
     hooks:
     - id: flake8
       language_version: python3
+-   repo: https://github.com/lovesegfault/beautysh
+    rev: master
+    hooks:
+    - id: beautysh
+      language_version: python3
+      args: [-i, '2']  # 2-space indentaion

.restyled.yaml

@@ -1,7 +1,7 @@
 ---
 restylers:
   - name: black
-    image: restyled/restyler-black:v19.3b0
+    image: restyled/restyler-black:v19.10b0
     command:
     - black
     arguments: []

.travis.yml

@@ -74,22 +74,25 @@ jobs:
     python: 3.7
     script: ./scripts/build_package.sh
   - name: Snapcraft snap
+    language: python
+    python: 3.7
     addons:
      snaps:
      - name: snapcraft
        channel: stable
        confinement: classic
+     - name: review-tools
+       channel: stable
      - name: lxd
        channel: stable
     env:
     - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}'
-    before_install:
+    install:
     - sudo /snap/bin/lxd.migrate -yes
     - sudo /snap/bin/lxd waitready
     - sudo /snap/bin/lxd init --auto
-    install:
     script:
-    - sudo snapcraft --use-lxd
+    - ./scripts/build_snap.sh
     after_failure:
     - sudo journalctl -u snapd
 
@@ -135,7 +138,7 @@ deploy:
       stage: build
   - provider: snap
     snap: dvc_*.snap
-    channel: ${TRAVIS_TAG:+stable}${TRAVIS_TAG:-edge}
+    channel: $SNAP_CHANNEL
     skip_cleanup: true
     on:
       all_branches: true

README.rst

@@ -10,9 +10,13 @@
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
-.. image:: https://travis-ci.com/iterative/dvc.svg?branch=master
+.. image:: https://img.shields.io/badge/release-ok-brightgreen
   :target: https://travis-ci.com/iterative/dvc
-  :alt: Travis
+  :alt: Release
+
+.. image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev
+  :target: https://travis-ci.com/iterative/dvc
+  :alt: Travis dev branch
 
 .. image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
   :target: https://codeclimate.com/github/iterative/dvc

debian/changelog

@@ -2,4 +2,4 @@ dvc (0.75.b1) experimental; urgency=low
 
   * First native Debian package version.
 
- -- Dmitry Petrov <dmitry@dataversioncontrol.com>  Sat, 07 Dec 2019 09:49:28 +0300
+ -- Dmitry Petrov <dmitry@dvc.org>  Sat, 07 Dec 2019 09:49:28 +0300

debian/control

@@ -1,5 +1,5 @@
 Source: dvc
-Maintainer: Dmitry Petrov <dmitry@dataversioncontrol.com>
+Maintainer: Dmitry Petrov <dmitry@dvc.org>
 Section: python
 Priority: optional
 Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)

debian/copyright

@@ -1,10 +1,10 @@
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: DVC
-Upstream-Contact: Dmitry Petrov <dmitry@dataversioncontrol.com>
+Upstream-Contact: Dmitry Petrov <dmitry@dvc.org>
 Source: https://github.com/iterative/dvc
 
 Files: *
-Copyright: Dmitry Petrov <dmitry@dataversioncontrol.com>
+Copyright: Dmitry Petrov <dmitry@dvc.org>
 License: Apache 2.0
  On Debian systems, the full text of the Apache 2.0
  License can be found in the file

dvc/__main__.py

@@ -1,4 +1,4 @@
-"""Main entry point for dvc command line tool."""
+"""Main entry point for DVC command line tool."""
 import sys
 
 from dvc.main import main

dvc/analytics.py

@@ -54,11 +54,8 @@ def is_enabled():
         return False
 
     enabled = to_bool(
-        Config(validate=False)
-        .config.get(Config.SECTION_CORE, {})
-        .get(Config.SECTION_CORE_ANALYTICS, "true")
+        Config(validate=False).get("core", {}).get("analytics", "true")
     )
-
     logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
 
     return enabled
@@ -76,7 +73,10 @@ def send(report):
     headers = {"content-type": "application/json"}
 
     with open(report, "rb") as fobj:
-        requests.post(url, data=fobj, headers=headers, timeout=5)
+        try:
+            requests.post(url, data=fobj, headers=headers, timeout=5)
+        except requests.exceptions.RequestException:
+            logger.debug("failed to send analytics report", exc_info=True)
 
     os.remove(report)
 
@@ -141,7 +141,7 @@ def _find_or_create_user_id():
 
     IDs are generated randomly with UUID.
     """
-    config_dir = Config.get_global_config_dir()
+    config_dir = Config.get_dir("global")
     fname = os.path.join(config_dir, "user_id")
     lockfile = os.path.join(config_dir, "user_id.lock")
 

dvc/api.py

@@ -1,57 +1,20 @@
-from builtins import open as builtin_open
-import importlib
 import os
-import sys
 from contextlib import contextmanager, _GeneratorContextManager as GCM
-import threading
-
-from funcy import wrap_with
-import ruamel.yaml
-from voluptuous import Schema, Required, Invalid
 
 from dvc.repo import Repo
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.external_repo import external_repo
 
 
-SUMMON_FILE_SCHEMA = Schema(
-    {
-        Required("objects"): [
-            {
-                Required("name"): str,
-                "meta": dict,
-                Required("summon"): {
-                    Required("type"): str,
-                    "deps": [str],
-                    str: object,
-                },
-            }
-        ]
-    }
-)
-SUMMON_PYTHON_SCHEMA = Schema(
-    {
-        Required("type"): "python",
-        Required("call"): str,
-        "args": dict,
-        "deps": [str],
-    }
-)
-
-
-class SummonError(DvcException):
-    pass
-
-
 class UrlNotDvcRepoError(DvcException):
-    """Thrown if given url is not a DVC repository.
+    """Thrown if given URL is not a DVC repository.
 
     Args:
-        url (str): url to the repository.
+        url (str): URL to the repository
     """
 
     def __init__(self, url):
-        super().__init__("URL '{}' is not a dvc repository.".format(url))
+        super().__init__("'{}' is not a DVC repository.".format(url))
 
 
 def get_url(path, repo=None, rev=None, remote=None):
@@ -60,14 +23,11 @@ def get_url(path, repo=None, rev=None, remote=None):
     `repo`.
     NOTE: There is no guarantee that the file actually exists in that location.
     """
-    try:
-        with _make_repo(repo, rev=rev) as _repo:
-            abspath = os.path.join(_repo.root_dir, path)
-            out, = _repo.find_outs_by_path(abspath)
-            remote_obj = _repo.cloud.get_remote(remote)
-            return str(remote_obj.checksum_to_path_info(out.checksum))
-    except NotDvcRepoError:
-        raise UrlNotDvcRepoError(repo)
+    with _make_repo(repo, rev=rev) as _repo:
+        _require_dvc(_repo)
+        out = _repo.find_out_by_relpath(path)
+        remote_obj = _repo.cloud.get_remote(remote)
+        return str(remote_obj.checksum_to_path_info(out.checksum))
 
 
 def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
@@ -96,9 +56,8 @@ class _OpenContextManager(GCM):
 
 def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
     with _make_repo(repo, rev=rev) as _repo:
-        abspath = os.path.join(_repo.root_dir, path)
-        with _repo.open(
-            abspath, remote=remote, mode=mode, encoding=encoding
+        with _repo.open_by_relpath(
+            path, remote=remote, mode=mode, encoding=encoding
         ) as fd:
             yield fd
 
@@ -112,133 +71,18 @@ def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
 
 
 @contextmanager
-def _make_repo(repo_url, rev=None):
-    if not repo_url or os.path.exists(repo_url):
-        assert (
-            rev is None
-        ), "Git revisions are not supported for local DVC projects."
-        yield Repo(repo_url)
-    else:
-        with external_repo(url=repo_url, rev=rev) as repo:
-            yield repo
-
-
-def summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml", args=None):
-    """Instantiate an object described in the `summon_file`."""
-    with prepare_summon(
-        name, repo=repo, rev=rev, summon_file=summon_file
-    ) as desc:
+def _make_repo(repo_url=None, rev=None):
+    repo_url = repo_url or os.getcwd()
+    if rev is None and os.path.exists(repo_url):
         try:
-            summon_dict = SUMMON_PYTHON_SCHEMA(desc.obj["summon"])
-        except Invalid as exc:
-            raise SummonError(str(exc)) from exc
-
-        _args = {**summon_dict.get("args", {}), **(args or {})}
-        return _invoke_method(summon_dict["call"], _args, desc.repo.root_dir)
-
-
-@contextmanager
-def prepare_summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml"):
-    """Does a couple of things every summon needs as a prerequisite:
-    clones the repo, parses the summon file and pulls the deps.
-
-    Calling code is expected to complete the summon logic following
-    instructions stated in "summon" dict of the object spec.
-
-    Returns a SummonDesc instance, which contains references to a Repo object,
-    named object specification and resolved paths to deps.
-    """
-    with _make_repo(repo, rev=rev) as _repo:
-        try:
-            path = os.path.join(_repo.root_dir, summon_file)
-            obj = _get_object_spec(name, path)
-            yield SummonDesc(_repo, obj)
-        except SummonError as exc:
-            raise SummonError(
-                str(exc) + " at '{}' in '{}'".format(summon_file, repo)
-            ) from exc.__cause__
-
-
-class SummonDesc:
-    def __init__(self, repo, obj):
-        self.repo = repo
-        self.obj = obj
-        self._pull_deps()
-
-    @property
-    def deps(self):
-        return [os.path.join(self.repo.root_dir, d) for d in self._deps]
-
-    @property
-    def _deps(self):
-        return self.obj["summon"].get("deps", [])
-
-    def _pull_deps(self):
-        if not self._deps:
+            yield Repo(repo_url)
             return
-
-        outs = [self.repo.find_out_by_relpath(d) for d in self._deps]
-
-        with self.repo.state:
-            for out in outs:
-                self.repo.cloud.pull(out.get_used_cache())
-                out.checkout()
+        except NotDvcRepoError:
+            pass  # fallthrough to external_repo
+    with external_repo(url=repo_url, rev=rev) as repo:
+        yield repo
 
 
-def _get_object_spec(name, path):
-    """
-    Given a summonable object's name, search for it on the given file
-    and return its description.
-    """
-    try:
-        with builtin_open(path, "r") as fobj:
-            content = SUMMON_FILE_SCHEMA(ruamel.yaml.safe_load(fobj.read()))
-            objects = [x for x in content["objects"] if x["name"] == name]
-
-        if not objects:
-            raise SummonError("No object with name '{}'".format(name))
-        elif len(objects) >= 2:
-            raise SummonError(
-                "More than one object with name '{}'".format(name)
-            )
-
-        return objects[0]
-
-    except FileNotFoundError as exc:
-        raise SummonError("Summon file not found") from exc
-    except ruamel.yaml.YAMLError as exc:
-        raise SummonError("Failed to parse summon file") from exc
-    except Invalid as exc:
-        raise SummonError(str(exc)) from exc
-
-
-@wrap_with(threading.Lock())
-def _invoke_method(call, args, path):
-    # XXX: Some issues with this approach:
-    #   * Import will pollute sys.modules
-    #   * sys.path manipulation is "theoretically" not needed,
-    #     but tests are failing for an unknown reason.
-    cwd = os.getcwd()
-
-    try:
-        os.chdir(path)
-        sys.path.insert(0, path)
-        method = _import_string(call)
-        return method(**args)
-    finally:
-        os.chdir(cwd)
-        sys.path.pop(0)
-
-
-def _import_string(import_name):
-    """Imports an object based on a string.
-    Useful to delay import to not load everything on startup.
-    Use dotted notaion in `import_name`, e.g. 'dvc.remote.gs.RemoteGS'.
-
-    :return: imported object
-    """
-    if "." in import_name:
-        module, obj = import_name.rsplit(".", 1)
-    else:
-        return importlib.import_module(import_name)
-    return getattr(importlib.import_module(module), obj)
+def _require_dvc(repo):
+    if not isinstance(repo, Repo):
+        raise UrlNotDvcRepoError(repo.url)

dvc/cache.py

@@ -1,25 +1,8 @@
-"""Manages cache of a dvc repo."""
-import os
+"""Manages cache of a DVC repo."""
 from collections import defaultdict
 
 from funcy import cached_property
 
-from dvc.config import Config
-
-
-class CacheConfig(object):
-    def __init__(self, config):
-        self.config = config
-
-    def set_dir(self, dname, level=None):
-        from dvc.remote.config import RemoteConfig
-
-        configobj = self.config.get_configobj(level)
-        path = RemoteConfig.resolve_path(dname, configobj.filename)
-        self.config.set(
-            Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level
-        )
-
 
 def _make_remote_property(name):
     """
@@ -58,7 +41,7 @@ def _make_remote_property(name):
 
 
 class Cache(object):
-    """Class that manages cache locations of a dvc repo.
+    """Class that manages cache locations of a DVC repo.
 
     Args:
         repo (dvc.repo.Repo): repo instance that this cache belongs to.
@@ -70,37 +53,22 @@ class Cache(object):
         from dvc.remote import Remote
 
         self.repo = repo
+        self.config = config = repo.config["cache"]
 
-        self.config = config = repo.config.config[Config.SECTION_CACHE]
-        local = config.get(Config.SECTION_CACHE_LOCAL)
+        local = config.get("local")
 
         if local:
-            name = Config.SECTION_REMOTE_FMT.format(local)
-            settings = repo.config.config[name]
+            settings = {"name": local}
         else:
-            default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
-            cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
-            cache_type = config.get(Config.SECTION_CACHE_TYPE)
-            protected = config.get(Config.SECTION_CACHE_PROTECTED)
-            shared = config.get(Config.SECTION_CACHE_SHARED)
-
-            settings = {
-                Config.PRIVATE_CWD: config.get(
-                    Config.PRIVATE_CWD, repo.dvc_dir
-                ),
-                Config.SECTION_REMOTE_URL: cache_dir,
-                Config.SECTION_CACHE_TYPE: cache_type,
-                Config.SECTION_CACHE_PROTECTED: protected,
-                Config.SECTION_CACHE_SHARED: shared,
-            }
+            settings = {**config, "url": config["dir"]}
 
         self.local = Remote(repo, **settings)
 
-    s3 = _make_remote_property(Config.SECTION_CACHE_S3)
-    gs = _make_remote_property(Config.SECTION_CACHE_GS)
-    ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
-    hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
-    azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
+    s3 = _make_remote_property("s3")
+    gs = _make_remote_property("gs")
+    ssh = _make_remote_property("ssh")
+    hdfs = _make_remote_property("hdfs")
+    azure = _make_remote_property("azure")
 
 
 class NamedCache(object):

dvc/cli.py

@@ -1,39 +1,42 @@
-"""Command line interface for dvc."""
+"""DVC command line interface"""
 import argparse
 import logging
 import sys
 
-import dvc.command.add as add
-import dvc.command.cache as cache
-import dvc.command.checkout as checkout
-import dvc.command.commit as commit
-import dvc.command.config as config
-import dvc.command.daemon as daemon
-import dvc.command.data_sync as data_sync
-import dvc.command.destroy as destroy
-import dvc.command.diff as diff
-import dvc.command.gc as gc
-import dvc.command.get as get
-import dvc.command.get_url as get_url
-import dvc.command.imp as imp
-import dvc.command.imp_url as imp_url
-import dvc.command.init as init
-import dvc.command.install as install
-import dvc.command.lock as lock
-import dvc.command.metrics as metrics
-import dvc.command.move as move
-import dvc.command.pipeline as pipeline
-import dvc.command.remote as remote
-import dvc.command.remove as remove
-import dvc.command.repro as repro
-import dvc.command.root as root
-import dvc.command.run as run
-import dvc.command.tag as tag
-import dvc.command.unprotect as unprotect
-import dvc.command.update as update
-import dvc.command.version as version
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import DvcParserError
+from .command import (
+    add,
+    cache,
+    checkout,
+    commit,
+    config,
+    daemon,
+    data_sync,
+    destroy,
+    diff,
+    gc,
+    get,
+    get_url,
+    imp,
+    imp_url,
+    init,
+    install,
+    ls,
+    lock,
+    metrics,
+    move,
+    pipeline,
+    remote,
+    remove,
+    repro,
+    root,
+    run,
+    tag,
+    unprotect,
+    update,
+    version,
+)
+from .command.base import fix_subparsers
+from .exceptions import DvcParserError
 
 
 logger = logging.getLogger(__name__)
@@ -60,6 +63,7 @@ COMMANDS = [
     metrics,
     install,
     root,
+    ls,
     lock,
     pipeline,
     daemon,
@@ -89,7 +93,7 @@ class DvcParser(argparse.ArgumentParser):
 
 class VersionAction(argparse.Action):  # pragma: no cover
     # pylint: disable=too-few-public-methods
-    """Shows dvc version and exits."""
+    """Shows DVC version and exits."""
 
     def __call__(self, parser, namespace, values, option_string=None):
         from dvc import __version__
@@ -170,7 +174,7 @@ def parse_args(argv=None):
         title="Available Commands",
         metavar="COMMAND",
         dest="cmd",
-        help="Use dvc COMMAND --help for command-specific help.",
+        help="Use `dvc COMMAND --help` for command-specific help.",
     )
 
     fix_subparsers(subparsers)

dvc/command/add.py

@@ -1,11 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.exceptions import DvcException
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
-
+from dvc.command.base import append_doc_link, CmdBase
+from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
 logger = logging.getLogger(__name__)
 

dvc/command/base.py

@@ -35,7 +35,7 @@ class CmdBase(object):
         self.repo = Repo()
         self.config = self.repo.config
         self.args = args
-        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        hardlink_lock = self.config["core"].get("hardlink_lock", False)
         updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
         updater.check()
 

dvc/command/cache.py

@@ -1,18 +1,13 @@
 import argparse
 
-from dvc.cache import CacheConfig
-from dvc.command.base import append_doc_link
-from dvc.command.base import fix_subparsers
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
 
 
 class CmdCacheDir(CmdConfig):
-    def __init__(self, args):
-        super().__init__(args)
-        self.cache_config = CacheConfig(self.config)
-
     def run(self):
-        self.cache_config.set_dir(self.args.value, level=self.args.level)
+        with self.config.edit(level=self.args.level) as edit:
+            edit["cache"]["dir"] = self.args.value
         return 0
 
 
@@ -30,7 +25,8 @@ def add_parser(subparsers, parent_parser):
     )
 
     cache_subparsers = cache_parser.add_subparsers(
-        dest="cmd", help="Use dvc cache CMD --help for command-specific help."
+        dest="cmd",
+        help="Use `dvc cache CMD --help` for command-specific " "help.",
     )
 
     fix_subparsers(cache_subparsers)

dvc/command/commit.py

@@ -47,7 +47,7 @@ def add_parser(subparsers, parent_parser):
         "--force",
         action="store_true",
         default=False,
-        help="Commit even if checksums for dependencies/outputs changed.",
+        help="Commit even if hash value for dependencies/outputs changed.",
     )
     commit_parser.add_argument(
         "-d",

dvc/command/config.py

@@ -1,10 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
-from dvc.config import Config
-
+from dvc.command.base import append_doc_link, CmdBaseNoRepo
+from dvc.config import Config, ConfigError
 
 logger = logging.getLogger(__name__)
 
@@ -18,17 +16,21 @@ class CmdConfig(CmdBaseNoRepo):
     def run(self):
         section, opt = self.args.name.lower().strip().split(".", 1)
 
-        if self.args.unset:
-            self.config.unset(section, opt, level=self.args.level)
-        elif self.args.value is None:
-            logger.info(self.config.get(section, opt, level=self.args.level))
-        else:
-            self.config.set(
-                section, opt, self.args.value, level=self.args.level
-            )
+        if self.args.value is None and not self.args.unset:
+            conf = self.config.load_one(self.args.level)
+            self._check(conf, section, opt)
+            logger.info(conf[section][opt])
+            return 0
 
-        is_write = self.args.unset or self.args.value is not None
-        if is_write and self.args.name == "cache.type":
+        with self.config.edit(self.args.level) as conf:
+            if self.args.unset:
+                self._check(conf, section, opt)
+                del conf[section][opt]
+            else:
+                self._check(conf, section)
+                conf[section][opt] = self.args.value
+
+        if self.args.name == "cache.type":
             logger.warning(
                 "You have changed the 'cache.type' option. This doesn't update"
                 " any existing workspace file links, but it can be done with:"
@@ -37,30 +39,39 @@ class CmdConfig(CmdBaseNoRepo):
 
         return 0
 
+    def _check(self, conf, section, opt=None):
+        if section not in conf:
+            msg = "section {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+
+        if opt and opt not in conf[section]:
+            msg = "option {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+
 
 parent_config_parser = argparse.ArgumentParser(add_help=False)
 parent_config_parser.add_argument(
     "--global",
     dest="level",
     action="store_const",
-    const=Config.LEVEL_GLOBAL,
+    const="global",
     help="Use global config.",
 )
 parent_config_parser.add_argument(
     "--system",
     dest="level",
     action="store_const",
-    const=Config.LEVEL_SYSTEM,
+    const="system",
     help="Use system config.",
 )
 parent_config_parser.add_argument(
     "--local",
     dest="level",
     action="store_const",
-    const=Config.LEVEL_LOCAL,
+    const="local",
     help="Use local config.",
 )
-parent_config_parser.set_defaults(level=Config.LEVEL_REPO)
+parent_config_parser.set_defaults(level="repo")
 
 
 def add_parser(subparsers, parent_parser):

dvc/command/daemon.py

@@ -16,9 +16,7 @@ class CmdDaemonUpdater(CmdDaemonBase):
         root_dir = Repo.find_root()
         dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
         config = Config(dvc_dir, validate=False)
-        hardlink_lock = config.config.get("core", {}).get(
-            "hardlink_lock", False
-        )
+        hardlink_lock = config.get("core", {}).get("hardlink_lock", False)
         updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
         updater.fetch(detach=False)
 
@@ -44,7 +42,8 @@ def add_parser(subparsers, parent_parser):
     )
 
     daemon_subparsers = daemon_parser.add_subparsers(
-        dest="cmd", help="Use dvc daemon CMD --help for command-specific help."
+        dest="cmd",
+        help="Use `dvc daemon CMD --help` for command-specific " "help.",
     )
 
     fix_subparsers(daemon_subparsers)

dvc/command/data_sync.py

@@ -148,7 +148,8 @@ def add_parser(subparsers, _parent_parser):
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
-    PUSH_HELP = "Push data files to a DVC remote storage."
+    PUSH_HELP = "Uploads files or directories tracked by DVC"
+    " to remote storage."
 
     push_parser = subparsers.add_parser(
         "push",
@@ -191,7 +192,8 @@ def add_parser(subparsers, _parent_parser):
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
-    FETCH_HELP = "Fetch data files from a DVC remote storage."
+    FETCH_HELP = "Get files or directories tracked by DVC from remote storage"
+    " into the cache."
 
     fetch_parser = subparsers.add_parser(
         "fetch",

dvc/command/diff.py

@@ -1,12 +1,11 @@
 import argparse
+import json
 import logging
+import os
 
-import humanize
-import inflect
-from funcy import compact
+import colorama
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 
@@ -15,156 +14,144 @@ logger = logging.getLogger(__name__)
 
 class CmdDiff(CmdBase):
     @staticmethod
-    def _print_size(size):
-        if size < 0:
-            change = "decreased by {}"
-        elif size > 0:
-            change = "increased by {}"
-        else:
-            change = "not changed"
-        natur_size = humanize.naturalsize(abs(size))
-        return change.format(natur_size)
+    def _format(diff):
+        """
+        Given a diff structure, generate a string of paths separated
+        by new lines and grouped together by their state.
 
-    @staticmethod
-    def _get_md5_string(sign, file_name, checksum):
-        sample_msg = ""
-        if file_name:
-            sample_msg = "{}{} with md5 {}\n"
-            sample_msg = sample_msg.format(sign, file_name, checksum)
-        return sample_msg
+        A group's header is colored and its entries are sorted to enhance
+        readability, for example:
 
-    @classmethod
-    def _get_dir_changes(cls, dct):
-        import dvc.repo.diff as diff
+            Added:
+                another_file.txt
+                backup.tar
+                dir/
+                dir/1
 
-        engine = inflect.engine()
-        changes_msg = (
-            "{} {} untouched, {} {} modified, {} {} added, "
-            "{} {} deleted, size was {}"
+        An example of a diff formatted when entries contain checksums:
+
+            Added:
+                d3b07384 foo
+
+            Modified:
+                c157a790..f98bf6f1 bar
+
+        If a group has no entries, it won't be included in the result.
+
+        At the bottom, include a summary with the number of files per state.
+        """
+
+        def _digest(checksum):
+            if type(checksum) is str:
+                return checksum[0:8]
+            return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
+
+        colors = {
+            "added": colorama.Fore.GREEN,
+            "modified": colorama.Fore.YELLOW,
+            "deleted": colorama.Fore.RED,
+        }
+
+        summary = {}
+        groups = []
+
+        for state in ["added", "deleted", "modified"]:
+            summary[state] = 0
+            entries = diff[state]
+
+            if not entries:
+                continue
+
+            content = []
+
+            for entry in entries:
+                path = entry["path"]
+                checksum = entry.get("checksum")
+                summary[state] += 1 if not path.endswith(os.sep) else 0
+                content.append(
+                    "{space}{checksum}{separator}{path}".format(
+                        space="    ",
+                        checksum=_digest(checksum) if checksum else "",
+                        separator="  " if checksum else "",
+                        path=entry["path"],
+                    )
+                )
+
+            groups.append(
+                "{color}{header}{nc}:\n{content}".format(
+                    color=colors[state],
+                    header=state.capitalize(),
+                    nc=colorama.Fore.RESET,
+                    content="\n".join(content),
+                )
+            )
+
+        groups.append(
+            "files summary: {added} added, {deleted} deleted,"
+            " {modified} modified".format_map(summary)
         )
-        changes_msg = changes_msg.format(
-            dct[diff.DIFF_IDENT],
-            engine.plural("file", dct[diff.DIFF_IDENT]),
-            dct[diff.DIFF_CHANGE],
-            engine.plural("file", dct[diff.DIFF_CHANGE]),
-            dct[diff.DIFF_NEW],
-            engine.plural("file", dct[diff.DIFF_NEW]),
-            dct[diff.DIFF_DEL],
-            engine.plural("file", dct[diff.DIFF_DEL]),
-            cls._print_size(dct[diff.DIFF_SIZE]),
-        )
-        return changes_msg
 
-    @classmethod
-    def _get_file_changes(cls, dct):
-        import dvc.repo.diff as diff
-
-        if (
-            dct.get(diff.DIFF_OLD_FILE)
-            and dct.get(diff.DIFF_NEW_FILE)
-            and dct[diff.DIFF_SIZE] == 0
-        ):
-            msg = "file size was not changed"
-        elif dct.get(diff.DIFF_NEW_FILE):
-            msg = "added file with size {}".format(
-                humanize.naturalsize(dct[diff.DIFF_SIZE])
-            )
-        elif dct.get(diff.DIFF_OLD_FILE):
-            msg = "deleted file with size {}".format(
-                humanize.naturalsize(abs(dct[diff.DIFF_SIZE]))
-            )
-        else:
-            msg = "file was modified, file size {}".format(
-                cls._print_size(dct[diff.DIFF_SIZE])
-            )
-        return msg
-
-    @classmethod
-    def _get_royal_changes(cls, dct):
-        import dvc.repo.diff as diff
-
-        if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
-            if dct.get("is_dir"):
-                return cls._get_dir_changes(dct)
-            else:
-                return cls._get_file_changes(dct)
-        return "size is ?"
-
-    @classmethod
-    def _show(cls, diff_dct):
-        import dvc.repo.diff as diff
-
-        msg = "dvc diff from {} to {}".format(
-            diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
-        )
-        if diff_dct.get(diff.DIFF_EQUAL):
-            logger.info(msg)
-            return
-        for dct in diff_dct[diff.DIFF_LIST]:
-            msg += "\n\ndiff for '{}'\n".format(dct[diff.DIFF_TARGET])
-            msg += cls._get_md5_string(
-                "-",
-                dct.get(diff.DIFF_OLD_FILE),
-                dct.get(diff.DIFF_OLD_CHECKSUM),
-            )
-            msg += cls._get_md5_string(
-                "+",
-                dct.get(diff.DIFF_NEW_FILE),
-                dct.get(diff.DIFF_NEW_CHECKSUM),
-            )
-            msg += "\n"
-            msg += cls._get_royal_changes(dct)
-        logger.info(msg)
-        return msg
+        return "\n\n".join(groups)
 
     def run(self):
         try:
-            msg = self.repo.diff(
-                self.args.a_ref, target=self.args.target, b_ref=self.args.b_ref
-            )
-            self._show(msg)
+            diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
+
+            if not any(diff.values()):
+                return 0
+
+            if not self.args.checksums:
+                for _, entries in diff.items():
+                    for entry in entries:
+                        del entry["checksum"]
+
+            if self.args.show_json:
+                res = json.dumps(diff)
+            else:
+                res = self._format(diff)
+
+            logger.info(res)
+
         except DvcException:
-            msg = "failed to get 'diff {}'"
-            args = " ".join(
-                compact([self.args.target, self.args.a_ref, self.args.b_ref])
-            )
-            msg = msg.format(args)
-            logger.exception(msg)
+            logger.exception("failed to get diff")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     DIFF_DESCRIPTION = (
-        "Show diff of a data file or a directory that is under DVC control.\n"
-        "Some basic statistics summary, how many files were deleted/changed."
+        "Show changes between commits in the DVC repository,"
+        " or between a commit and the workspace."
+        "Prints the list of paths added, modified, or deleted."
     )
-    DIFF_HELP = "Show a diff of a DVC controlled data file or a directory."
     diff_parser = subparsers.add_parser(
         "diff",
         parents=[parent_parser],
         description=append_doc_link(DIFF_DESCRIPTION, "diff"),
-        help=DIFF_HELP,
+        help=DIFF_DESCRIPTION,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     diff_parser.add_argument(
-        "-t",
-        "--target",
-        help=(
-            "Source path to a data file or directory. Default None. "
-            "If not specified, compares all files and directories "
-            "that are under DVC control in the current working space."
-        ),
+        "a_rev",
+        help="Old Git commit to compare (defaults to HEAD)",
+        nargs="?",
+        default="HEAD",
     )
     diff_parser.add_argument(
-        "a_ref", help="Git reference from which diff calculates"
-    )
-    diff_parser.add_argument(
-        "b_ref",
-        help=(
-            "Git reference until which diff calculates, if omitted "
-            "diff shows the difference between current HEAD and a_ref"
-        ),
+        "b_rev",
+        help=("New Git commit to compare (defaults to the current workspace)"),
         nargs="?",
     )
+    diff_parser.add_argument(
+        "--show-json",
+        help="Format the output into a JSON",
+        action="store_true",
+        default=False,
+    )
+    diff_parser.add_argument(
+        "--checksums",
+        help="Display hash value for each entry",
+        action="store_true",
+        default=False,
+    )
     diff_parser.set_defaults(func=CmdDiff)

dvc/command/gc.py

@@ -52,10 +52,10 @@ class CmdGC(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    GC_HELP = "Collect unused data from DVC cache or a remote storage."
+    GC_HELP = "Garbage collect unused objects from cache or remote storage."
     GC_DESCRIPTION = (
-        "Deletes all files in the cache or a remote which are not in\n"
-        "use by the specified git references (defaults to just HEAD)."
+        "Removes all files in the cache or a remote which are not in\n"
+        "use by the specified Git revisions (defaults to just HEAD)."
     )
     gc_parser = subparsers.add_parser(
         "gc",

dvc/command/get.py

@@ -19,7 +19,7 @@ class CmdGet(CmdBaseNoRepo):
             )
             logger.info(url)
         except DvcException:
-            logger.exception("failed to show url")
+            logger.exception("failed to show URL")
             return 1
 
         return 0
@@ -51,9 +51,7 @@ class CmdGet(CmdBaseNoRepo):
 
 
 def add_parser(subparsers, parent_parser):
-    GET_HELP = (
-        "Download a file or directory from any DVC project or Git repository."
-    )
+    GET_HELP = "Download file or directory tracked by DVC or by Git."
     get_parser = subparsers.add_parser(
         "get",
         parents=[parent_parser],
@@ -62,22 +60,21 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     get_parser.add_argument(
-        "url",
-        help="Location of DVC project or Git repository to download from",
+        "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the project or repository",
+        "path", help="Path to a file or directory within the repository"
     )
     get_parser.add_argument(
         "-o", "--out", nargs="?", help="Destination path to download files to"
     )
     get_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
+        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
     )
     get_parser.add_argument(
         "--show-url",
         action="store_true",
-        help="Returns path/url to the location in remote for specified path",
+        help="Print the storage location (URL) the target data would be "
+        "downloaded from, and exit.",
     )
     get_parser.set_defaults(func=CmdGet)

dvc/command/imp.py

@@ -30,8 +30,8 @@ class CmdImport(CmdBase):
 
 def add_parser(subparsers, parent_parser):
     IMPORT_HELP = (
-        "Download a file or directory from any DVC project or Git repository "
-        "and take it under DVC control."
+        "Download file or directory tracked by DVC or by Git "
+        "into the workspace, and track it."
     )
 
     import_parser = subparsers.add_parser(
@@ -42,17 +42,15 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawTextHelpFormatter,
     )
     import_parser.add_argument(
-        "url",
-        help="Location of DVC project or Git repository to download from",
+        "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the project or repository",
+        "path", help="Path to a file or directory within the repository"
     )
     import_parser.add_argument(
         "-o", "--out", nargs="?", help="Destination path to download files to"
     )
     import_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
+        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
     )
     import_parser.set_defaults(func=CmdImport)

dvc/command/init.py

@@ -19,7 +19,7 @@ class CmdInit(CmdBaseNoRepo):
             )
             self.config = self.repo.config
         except InitError:
-            logger.exception("failed to initiate dvc")
+            logger.exception("failed to initiate DVC")
             return 1
         return 0
 
@@ -43,7 +43,7 @@ def add_parser(subparsers, parent_parser):
         "--no-scm",
         action="store_true",
         default=False,
-        help="Initiate dvc in directory that is "
+        help="Initiate DVC in directory that is "
         "not tracked by any scm tool (e.g. git).",
     )
     init_parser.add_argument(

dvc/command/install.py

@@ -13,7 +13,7 @@ class CmdInstall(CmdBase):
         try:
             self.repo.install()
         except Exception:
-            logger.exception("failed to install dvc hooks")
+            logger.exception("failed to install DVC Git hooks")
             return 1
         return 0
 

dvc/command/ls.py

@@ -0,0 +1,68 @@
+import argparse
+import logging
+
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
+from dvc.exceptions import DvcException
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdList(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            nodes = Repo.ls(
+                self.args.url,
+                self.args.target,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+                outs_only=self.args.outs_only,
+            )
+            if nodes:
+                logger.info("\n".join(nodes))
+            return 0
+        except DvcException:
+            logger.exception("failed to list '{}'".format(self.args.url))
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    LIST_HELP = "List files and DVC outputs in the repo."
+    list_parser = subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(LIST_HELP, "list"),
+        help=LIST_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    list_parser.add_argument(
+        "url",
+        help="Supported urls:\n"
+        "/path/to/file\n"
+        "/path/to/directory\n"
+        "C:\\\\path\\to\\file\n"
+        "C:\\\\path\\to\\directory\n"
+        "https://github.com/path/to/repo\n"
+        "[email protected]:path/to/repo.git\n",
+    )
+    list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        help="Recursively list files.",
+    )
+    list_parser.add_argument(
+        "--outs-only", action="store_true", help="Show only DVC outputs."
+    )
+    list_parser.add_argument(
+        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
+    )
+    list_parser.add_argument(
+        "target",
+        nargs="?",
+        help="Path to directory within the repository to list outputs for",
+    )
+    list_parser.set_defaults(func=CmdList)

dvc/command/metrics.py

@@ -136,8 +136,8 @@ class CmdMetricsDiff(CmdBase):
     def run(self):
         try:
             diff = self.repo.metrics.diff(
-                a_ref=self.args.a_ref,
-                b_ref=self.args.b_ref,
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
                 targets=self.args.targets,
                 typ=self.args.type,
                 xpath=self.args.xpath,
@@ -171,7 +171,7 @@ def add_parser(subparsers, parent_parser):
 
     metrics_subparsers = metrics_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc metrics CMD --help to display command-specific help.",
+        help="Use `dvc metrics CMD --help` to display command-specific help.",
     )
 
     fix_subparsers(metrics_subparsers)
@@ -273,7 +273,8 @@ def add_parser(subparsers, parent_parser):
     metrics_remove_parser.add_argument("path", help="Path to a metric file.")
     metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
 
-    METRICS_DIFF_HELP = "Output metric values."
+    METRICS_DIFF_HELP = "Show changes in metrics between commits"
+    " in the DVC repository, or between a commit and the workspace."
     metrics_diff_parser = metrics_subparsers.add_parser(
         "diff",
         parents=[parent_parser],
@@ -282,20 +283,12 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_diff_parser.add_argument(
-        "a_ref",
-        nargs="?",
-        help=(
-            "Git reference from which diff is calculated. "
-            "If omitted `HEAD`(latest commit) is used."
-        ),
+        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
     )
     metrics_diff_parser.add_argument(
-        "b_ref",
+        "b_rev",
         nargs="?",
-        help=(
-            "Git reference to which diff is calculated. "
-            "If omitted current working tree is used."
-        ),
+        help=("New Git commit to compare (defaults to the current workspace)"),
     )
     metrics_diff_parser.add_argument(
         "--targets",

dvc/command/pipeline.py

@@ -31,45 +31,50 @@ class CmdPipelineShow(CmdBase):
             else:
                 logger.info(stage.path_in_repo)
 
-    def __build_graph(self, target, commands, outs):
+    def _build_graph(self, target, commands, outs):
         import networkx
         from dvc.stage import Stage
         from dvc.repo.graph import get_pipeline
 
-        stage = Stage.load(self.repo, target)
-        G = get_pipeline(self.repo.pipelines, stage)
+        target_stage = Stage.load(self.repo, target)
+        G = get_pipeline(self.repo.pipelines, target_stage)
 
-        nodes = []
-        for stage in G:
+        nodes = set()
+        for stage in networkx.dfs_preorder_nodes(G, target_stage):
             if commands:
                 if stage.cmd is None:
                     continue
-                nodes.append(stage.cmd)
+                nodes.add(stage.cmd)
             elif outs:
                 for out in stage.outs:
-                    nodes.append(str(out))
+                    nodes.add(str(out))
+                for dep in stage.deps:
+                    nodes.add(str(dep))
             else:
-                nodes.append(stage.relpath)
+                nodes.add(stage.relpath)
 
         edges = []
-        for from_stage, to_stage in G.edges():
-            if commands:
-                if to_stage.cmd is None:
-                    continue
-                edges.append((from_stage.cmd, to_stage.cmd))
-            elif outs:
-                for from_out in from_stage.outs:
-                    for to_out in to_stage.outs:
-                        edges.append((str(from_out), str(to_out)))
-            else:
-                edges.append((from_stage.relpath, to_stage.relpath))
 
-        return nodes, edges, networkx.is_tree(G)
+        if outs:
+            for stage in networkx.dfs_preorder_nodes(G, target_stage):
+                for dep in stage.deps:
+                    for out in stage.outs:
+                        edges.append((str(out), str(dep)))
+        else:
+            for from_stage, to_stage in networkx.dfs_edges(G, target_stage):
+                if commands:
+                    if to_stage.cmd is None:
+                        continue
+                    edges.append((from_stage.cmd, to_stage.cmd))
+                else:
+                    edges.append((from_stage.relpath, to_stage.relpath))
+
+        return list(nodes), edges, networkx.is_tree(G)
 
     def _show_ascii(self, target, commands, outs):
         from dvc.dagascii import draw
 
-        nodes, edges, _ = self.__build_graph(target, commands, outs)
+        nodes, edges, _ = self._build_graph(target, commands, outs)
 
         if not nodes:
             return
@@ -79,7 +84,7 @@ class CmdPipelineShow(CmdBase):
     def _show_dependencies_tree(self, target, commands, outs):
         from treelib import Tree
 
-        nodes, edges, is_tree = self.__build_graph(target, commands, outs)
+        nodes, edges, is_tree = self._build_graph(target, commands, outs)
         if not nodes:
             return
         if not is_tree:
@@ -100,12 +105,12 @@ class CmdPipelineShow(CmdBase):
             observe_list.pop(0)
         tree.show()
 
-    def __write_dot(self, target, commands, outs):
+    def _write_dot(self, target, commands, outs):
         import io
         import networkx
         from networkx.drawing.nx_pydot import write_dot
 
-        _, edges, _ = self.__build_graph(target, commands, outs)
+        _, edges, _ = self._build_graph(target, commands, outs)
         edges = [edge[::-1] for edge in edges]
 
         simple_g = networkx.DiGraph()
@@ -126,9 +131,7 @@ class CmdPipelineShow(CmdBase):
                         target, self.args.commands, self.args.outs
                     )
                 elif self.args.dot:
-                    self.__write_dot(
-                        target, self.args.commands, self.args.outs
-                    )
+                    self._write_dot(target, self.args.commands, self.args.outs)
                 elif self.args.tree:
                     self._show_dependencies_tree(
                         target, self.args.commands, self.args.outs
@@ -172,7 +175,7 @@ def add_parser(subparsers, parent_parser):
 
     pipeline_subparsers = pipeline_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc pipeline CMD --help for command-specific help.",
+        help="Use `dvc pipeline CMD --help` for command-specific help.",
     )
 
     fix_subparsers(pipeline_subparsers)

dvc/command/remote.py

@@ -1,71 +1,99 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import fix_subparsers
+from dvc.config import ConfigError
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.remote.config import RemoteConfig
+from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
 
-class CmdRemoteConfig(CmdConfig):
+class CmdRemote(CmdConfig):
     def __init__(self, args):
         super().__init__(args)
-        self.remote_config = RemoteConfig(self.config)
+
+        if getattr(self.args, "name", None):
+            self.args.name = self.args.name.lower()
+
+    def _check_exists(self, conf):
+        if self.args.name not in conf["remote"]:
+            raise ConfigError(
+                "remote '{}' doesn't exists.".format(self.args.name)
+            )
 
 
-class CmdRemoteAdd(CmdRemoteConfig):
+class CmdRemoteAdd(CmdRemote):
     def run(self):
         if self.args.default:
             logger.info(
                 "Setting '{}' as a default remote.".format(self.args.name)
             )
-        self.remote_config.add(
-            self.args.name,
-            self.args.url,
-            force=self.args.force,
-            default=self.args.default,
-            level=self.args.level,
-        )
+
+        with self.config.edit(self.args.level) as conf:
+            if self.args.name in conf["remote"] and not self.args.force:
+                raise ConfigError(
+                    "remote '{}' already exists. Use `-f|--force` to "
+                    "overwrite it.".format(self.args.name)
+                )
+
+            conf["remote"][self.args.name] = {"url": self.args.url}
+            if self.args.default:
+                conf["core"]["remote"] = self.args.name
+
         return 0
 
 
-class CmdRemoteRemove(CmdRemoteConfig):
+class CmdRemoteRemove(CmdRemote):
     def run(self):
-        self.remote_config.remove(self.args.name, level=self.args.level)
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            del conf["remote"][self.args.name]
+
+        # Remove core.remote refs to this remote in any shadowing configs
+        for level in reversed(self.config.LEVELS):
+            with self.config.edit(level) as conf:
+                if conf["core"].get("remote") == self.args.name:
+                    del conf["core"]["remote"]
+
+            if level == self.args.level:
+                break
+
         return 0
 
 
-class CmdRemoteModify(CmdRemoteConfig):
+class CmdRemoteModify(CmdRemote):
     def run(self):
-        self.remote_config.modify(
-            self.args.name,
-            self.args.option,
-            self.args.value,
-            level=self.args.level,
-        )
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            conf["remote"][self.args.name][self.args.option] = self.args.value
         return 0
 
 
-class CmdRemoteDefault(CmdRemoteConfig):
+class CmdRemoteDefault(CmdRemote):
     def run(self):
+
         if self.args.name is None and not self.args.unset:
-            name = self.remote_config.get_default(level=self.args.level)
-            print(name)
+            conf = self.config.load_one(self.args.level)
+            try:
+                print(conf["core"]["remote"])
+            except KeyError:
+                logger.info("No default remote set")
+                return 1
         else:
-            self.remote_config.set_default(
-                self.args.name, unset=self.args.unset, level=self.args.level
-            )
+            with self.config.edit(self.args.level) as conf:
+                if self.args.unset:
+                    conf["core"].pop("remote", None)
+                else:
+                    conf["core"]["remote"] = self.args.name
         return 0
 
 
-class CmdRemoteList(CmdRemoteConfig):
+class CmdRemoteList(CmdRemote):
     def run(self):
-        for name, url in self.remote_config.list(
-            level=self.args.level
-        ).items():
-            logger.info("{}\t{}".format(name, url))
+        conf = self.config.load_one(self.args.level)
+        for name, conf in conf["remote"].items():
+            logger.info("{}\t{}".format(name, conf["url"]))
         return 0
 
 
@@ -83,7 +111,7 @@ def add_parser(subparsers, parent_parser):
 
     remote_subparsers = remote_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc remote CMD --help for " "command-specific help.",
+        help="Use `dvc remote CMD --help` for " "command-specific help.",
     )
 
     fix_subparsers(remote_subparsers)
@@ -99,7 +127,9 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser.add_argument("name", help="Name of the remote")
     remote_add_parser.add_argument(
         "url",
-        help="URL. See full list of supported urls at " "man.dvc.org/remote",
+        help="URL. See full list of supported urls at {}".format(
+            format_link("https://man.dvc.org/remote")
+        ),
     )
     remote_add_parser.add_argument(
         "-d",

dvc/command/update.py

@@ -14,7 +14,7 @@ class CmdUpdate(CmdBase):
         ret = 0
         for target in self.args.targets:
             try:
-                self.repo.update(target)
+                self.repo.update(target, self.args.rev)
             except DvcException:
                 logger.exception("failed to update '{}'.".format(target))
                 ret = 1
@@ -33,4 +33,7 @@ def add_parser(subparsers, parent_parser):
     update_parser.add_argument(
         "targets", nargs="+", help="DVC-files to update."
     )
+    update_parser.add_argument(
+        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+    )
     update_parser.set_defaults(func=CmdUpdate)

dvc/command/version.py

@@ -86,9 +86,9 @@ class CmdVersion(CmdBaseNoRepo):
     @staticmethod
     def get_linktype_support_info(repo):
         links = {
-            "reflink": System.reflink,
-            "hardlink": System.hardlink,
-            "symlink": System.symlink,
+            "reflink": (System.reflink, None),
+            "hardlink": (System.hardlink, System.is_hardlink),
+            "symlink": (System.symlink, System.is_symlink),
         }
 
         fname = "." + str(uuid.uuid4())
@@ -98,18 +98,16 @@ class CmdVersion(CmdBaseNoRepo):
 
         cache = []
 
-        for name, link in links.items():
+        for name, (link, is_link) in links.items():
             try:
                 link(src, dst)
+                status = "supported"
+                if is_link and not is_link(dst):
+                    status = "broken"
                 os.unlink(dst)
-                supported = True
             except DvcException:
-                supported = False
-            cache.append(
-                "{name} - {supported}".format(
-                    name=name, supported=True if supported else False
-                )
-            )
+                status = "not supported"
+            cache.append("{name} - {status}".format(name=name, status=status))
         os.remove(src)
 
         return ", ".join(cache)

dvc/config.py

@@ -1,17 +1,17 @@
 """DVC config objects."""
-
-import copy
-import errno
+from contextlib import contextmanager
 import logging
 import os
 import re
+from urllib.parse import urlparse
 
+from funcy import cached_property, re_find, walk_values, compact
 import configobj
-from voluptuous import Schema, Required, Optional, Invalid
-from voluptuous import All, Any, Lower, Range, Coerce, Match
+from voluptuous import Schema, Optional, Invalid, ALLOW_EXTRA
+from voluptuous import All, Any, Lower, Range, Coerce
 
-from dvc.exceptions import DvcException
-from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -24,15 +24,7 @@ class ConfigError(DvcException):
 
 
 class NoRemoteError(ConfigError):
-    def __init__(self, command):
-        msg = (
-            "no remote specified. Setup default remote with\n"
-            "    dvc remote default <name>\n"
-            "or use:\n"
-            "    dvc {} -r <name>\n".format(command)
-        )
-
-        super().__init__(msg)
+    pass
 
 
 def supported_cache_type(types):
@@ -56,13 +48,12 @@ def supported_cache_type(types):
 
 
 # Checks that value is either true or false and converts it to bool
-Bool = All(
+to_bool = Bool = All(
     Lower,
     Any("true", "false"),
     lambda v: v == "true",
     msg="expected true or false",
 )
-to_bool = Schema(Bool)
 
 
 def Choices(*choices):
@@ -75,8 +66,125 @@ def Choices(*choices):
     return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
 
 
-class Config(object):  # pylint: disable=too-many-instance-attributes
-    """Class that manages configuration files for a dvc repo.
+def ByUrl(mapping):
+    schemas = walk_values(Schema, mapping)
+
+    def validate(data):
+        if "url" not in data:
+            raise Invalid("expected 'url'")
+
+        parsed = urlparse(data["url"])
+        # Windows absolute paths should really have scheme == "" (local)
+        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+            return schemas[""](data)
+        if parsed.scheme not in schemas:
+            raise Invalid("Unsupported URL type {}://".format(parsed.scheme))
+
+        return schemas[parsed.scheme](data)
+
+    return validate
+
+
+class RelPath(str):
+    pass
+
+
+REMOTE_COMMON = {
+    "url": str,
+    "checksum_jobs": All(Coerce(int), Range(1)),
+    "no_traverse": Bool,
+    "verify": Bool,
+}
+LOCAL_COMMON = {
+    "type": supported_cache_type,
+    Optional("protected", default=False): Bool,
+    "shared": All(Lower, Choices("group")),
+    Optional("slow_link_warning", default=True): Bool,
+}
+SCHEMA = {
+    "core": {
+        "remote": Lower,
+        "checksum_jobs": All(Coerce(int), Range(1)),
+        Optional("interactive", default=False): Bool,
+        Optional("analytics", default=True): Bool,
+        Optional("hardlink_lock", default=False): Bool,
+    },
+    "cache": {
+        "local": str,
+        "s3": str,
+        "gs": str,
+        "hdfs": str,
+        "ssh": str,
+        "azure": str,
+        # This is for default local cache
+        "dir": str,
+        **LOCAL_COMMON,
+    },
+    "remote": {
+        str: ByUrl(
+            {
+                "": {**LOCAL_COMMON, **REMOTE_COMMON},
+                "s3": {
+                    "region": str,
+                    "profile": str,
+                    "credentialpath": str,
+                    "endpointurl": str,
+                    Optional("listobjects", default=False): Bool,
+                    Optional("use_ssl", default=True): Bool,
+                    "sse": str,
+                    "acl": str,
+                    "grant_read": str,
+                    "grant_read_acp": str,
+                    "grant_write_acp": str,
+                    "grant_full_control": str,
+                    **REMOTE_COMMON,
+                },
+                "gs": {
+                    "projectname": str,
+                    "credentialpath": str,
+                    **REMOTE_COMMON,
+                },
+                "ssh": {
+                    "type": supported_cache_type,
+                    "port": Coerce(int),
+                    "user": str,
+                    "password": str,
+                    "ask_password": Bool,
+                    "keyfile": str,
+                    "timeout": Coerce(int),
+                    "gss_auth": Bool,
+                    **REMOTE_COMMON,
+                },
+                "hdfs": {"user": str, **REMOTE_COMMON},
+                "azure": {"connection_string": str, **REMOTE_COMMON},
+                "oss": {
+                    "oss_key_id": str,
+                    "oss_key_secret": str,
+                    "oss_endpoint": str,
+                    **REMOTE_COMMON,
+                },
+                "gdrive": {
+                    "gdrive_client_id": str,
+                    "gdrive_client_secret": str,
+                    "gdrive_user_credentials_file": str,
+                    **REMOTE_COMMON,
+                },
+                "http": REMOTE_COMMON,
+                "https": REMOTE_COMMON,
+                "remote": {str: object},  # Any of the above options are valid
+            }
+        )
+    },
+    "state": {
+        "row_limit": All(Coerce(int), Range(1)),
+        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
+    },
+}
+COMPILED_SCHEMA = Schema(SCHEMA)
+
+
+class Config(dict):
+    """Class that manages configuration files for a DVC repo.
 
     Args:
         dvc_dir (str): optional path to `.dvc` directory, that is used to
@@ -93,166 +201,14 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     APPNAME = "dvc"
     APPAUTHOR = "iterative"
 
-    # NOTE: used internally in RemoteLOCAL to know config
-    # location, that url should resolved relative to.
-    PRIVATE_CWD = "_cwd"
+    # In the order they shadow each other
+    LEVELS = ("system", "global", "repo", "local")
 
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
-    CREDENTIALPATH = "credentialpath"
-
-    LEVEL_LOCAL = 0
-    LEVEL_REPO = 1
-    LEVEL_GLOBAL = 2
-    LEVEL_SYSTEM = 3
-
-    SECTION_CORE = "core"
-    SECTION_CORE_LOGLEVEL = "loglevel"
-    SECTION_CORE_LOGLEVEL_SCHEMA = All(
-        Lower, Choices("info", "debug", "warning", "error")
-    )
-    SECTION_CORE_REMOTE = "remote"
-    SECTION_CORE_INTERACTIVE = "interactive"
-    SECTION_CORE_ANALYTICS = "analytics"
-    SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_CORE_HARDLINK_LOCK = "hardlink_lock"
-
-    SECTION_CACHE = "cache"
-    SECTION_CACHE_DIR = "dir"
-    SECTION_CACHE_TYPE = "type"
-    SECTION_CACHE_PROTECTED = "protected"
-    SECTION_CACHE_SHARED = "shared"
-    SECTION_CACHE_SHARED_SCHEMA = All(Lower, Choices("group"))
-    SECTION_CACHE_LOCAL = "local"
-    SECTION_CACHE_S3 = "s3"
-    SECTION_CACHE_GS = "gs"
-    SECTION_CACHE_SSH = "ssh"
-    SECTION_CACHE_HDFS = "hdfs"
-    SECTION_CACHE_AZURE = "azure"
-    SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
-    SECTION_CACHE_SCHEMA = {
-        SECTION_CACHE_LOCAL: str,
-        SECTION_CACHE_S3: str,
-        SECTION_CACHE_GS: str,
-        SECTION_CACHE_HDFS: str,
-        SECTION_CACHE_SSH: str,
-        SECTION_CACHE_AZURE: str,
-        SECTION_CACHE_DIR: str,
-        SECTION_CACHE_TYPE: supported_cache_type,
-        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
-        SECTION_CACHE_SHARED: SECTION_CACHE_SHARED_SCHEMA,
-        PRIVATE_CWD: str,
-        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): Bool,
-    }
-
-    SECTION_CORE_SCHEMA = {
-        SECTION_CORE_LOGLEVEL: SECTION_CORE_LOGLEVEL_SCHEMA,
-        SECTION_CORE_REMOTE: Lower,
-        Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
-        Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
-        SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
-        Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool,
-    }
-
-    # aws specific options
-    SECTION_AWS_CREDENTIALPATH = CREDENTIALPATH
-    SECTION_AWS_ENDPOINT_URL = "endpointurl"
-    SECTION_AWS_LIST_OBJECTS = "listobjects"
-    SECTION_AWS_REGION = "region"
-    SECTION_AWS_PROFILE = "profile"
-    SECTION_AWS_USE_SSL = "use_ssl"
-    SECTION_AWS_SSE = "sse"
-    SECTION_AWS_ACL = "acl"
-    SECTION_AWS_GRANT_READ = "grant_read"
-    SECTION_AWS_GRANT_READ_ACP = "grant_read_acp"
-    SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp"
-    SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control"
-
-    # gcp specific options
-    SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH
-    SECTION_GCP_PROJECTNAME = "projectname"
-
-    # azure specific option
-    SECTION_AZURE_CONNECTION_STRING = "connection_string"
-
-    # Alibabacloud oss options
-    SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
-    SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
-    SECTION_OSS_ENDPOINT = "oss_endpoint"
-
-    # GDrive options
-    SECTION_GDRIVE_CLIENT_ID = "gdrive_client_id"
-    SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret"
-    SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file"
-
-    SECTION_REMOTE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
-    SECTION_REMOTE_FMT = 'remote "{}"'
-    SECTION_REMOTE_URL = "url"
-    SECTION_REMOTE_USER = "user"
-    SECTION_REMOTE_PORT = "port"
-    SECTION_REMOTE_KEY_FILE = "keyfile"
-    SECTION_REMOTE_TIMEOUT = "timeout"
-    SECTION_REMOTE_PASSWORD = "password"
-    SECTION_REMOTE_ASK_PASSWORD = "ask_password"
-    SECTION_REMOTE_GSS_AUTH = "gss_auth"
-    SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
-    SECTION_REMOTE_SCHEMA = {
-        Required(SECTION_REMOTE_URL): str,
-        SECTION_AWS_REGION: str,
-        SECTION_AWS_PROFILE: str,
-        SECTION_AWS_CREDENTIALPATH: str,
-        SECTION_AWS_ENDPOINT_URL: str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): Bool,
-        Optional(SECTION_AWS_USE_SSL, default=True): Bool,
-        SECTION_AWS_SSE: str,
-        SECTION_AWS_ACL: str,
-        SECTION_AWS_GRANT_READ: str,
-        SECTION_AWS_GRANT_READ_ACP: str,
-        SECTION_AWS_GRANT_WRITE_ACP: str,
-        SECTION_AWS_GRANT_FULL_CONTROL: str,
-        SECTION_GCP_PROJECTNAME: str,
-        SECTION_CACHE_TYPE: supported_cache_type,
-        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
-        SECTION_REMOTE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
-        SECTION_REMOTE_USER: str,
-        SECTION_REMOTE_PORT: Coerce(int),
-        SECTION_REMOTE_KEY_FILE: str,
-        SECTION_REMOTE_TIMEOUT: Coerce(int),
-        SECTION_REMOTE_PASSWORD: str,
-        SECTION_REMOTE_ASK_PASSWORD: Bool,
-        SECTION_REMOTE_GSS_AUTH: Bool,
-        SECTION_AZURE_CONNECTION_STRING: str,
-        SECTION_OSS_ACCESS_KEY_ID: str,
-        SECTION_OSS_ACCESS_KEY_SECRET: str,
-        SECTION_OSS_ENDPOINT: str,
-        SECTION_GDRIVE_CLIENT_ID: str,
-        SECTION_GDRIVE_CLIENT_SECRET: str,
-        SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
-        PRIVATE_CWD: str,
-        Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): Bool,
-    }
-
-    SECTION_STATE = "state"
-    SECTION_STATE_ROW_LIMIT = "row_limit"
-    SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
-    SECTION_STATE_SCHEMA = {
-        SECTION_STATE_ROW_LIMIT: All(Coerce(int), Range(1)),
-        SECTION_STATE_ROW_CLEANUP_QUOTA: All(Coerce(int), Range(0, 100)),
-    }
-
-    SCHEMA = {
-        Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
-        Match(SECTION_REMOTE_REGEX): SECTION_REMOTE_SCHEMA,
-        Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
-        Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
-    }
-    COMPILED_SCHEMA = Schema(SCHEMA)
-
     def __init__(self, dvc_dir=None, validate=True):
         self.dvc_dir = dvc_dir
-        self.validate = validate
 
         if not dvc_dir:
             try:
@@ -264,33 +220,31 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
-        self.load()
+        self.load(validate=validate)
 
-    @staticmethod
-    def get_global_config_dir():
-        """Returns global config location. E.g. ~/.config/dvc/config.
+    @classmethod
+    def get_dir(cls, level):
+        from appdirs import user_config_dir, site_config_dir
 
-        Returns:
-            str: path to the global config directory.
-        """
-        from appdirs import user_config_dir
+        assert level in ("global", "system")
 
-        return user_config_dir(
-            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
-        )
+        if level == "global":
+            return user_config_dir(cls.APPNAME, cls.APPAUTHOR)
+        if level == "system":
+            return site_config_dir(cls.APPNAME, cls.APPAUTHOR)
 
-    @staticmethod
-    def get_system_config_dir():
-        """Returns system config location. E.g. /etc/dvc.conf.
+    @cached_property
+    def files(self):
+        files = {
+            level: os.path.join(self.get_dir(level), self.CONFIG)
+            for level in ("system", "global")
+        }
 
-        Returns:
-            str: path to the system config directory.
-        """
-        from appdirs import site_config_dir
+        if self.dvc_dir is not None:
+            files["repo"] = os.path.join(self.dvc_dir, self.CONFIG)
+            files["local"] = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
 
-        return site_config_dir(
-            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
-        )
+        return files
 
     @staticmethod
     def init(dvc_dir):
@@ -306,243 +260,137 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         open(config_file, "w+").close()
         return Config(dvc_dir)
 
-    def _resolve_cache_path(self, config):
-        cache = config.get(self.SECTION_CACHE)
-        if cache is None:
-            return
-
-        cache_dir = cache.get(self.SECTION_CACHE_DIR)
-        if cache_dir is None:
-            return
-
-        cache[self.PRIVATE_CWD] = os.path.dirname(config.filename)
-
-    def _resolve_paths(self, config):
-        if config.filename is None:
-            return config
-
-        ret = copy.deepcopy(config)
-        self._resolve_cache_path(ret)
-
-        for section in ret.values():
-            if self.SECTION_REMOTE_URL not in section.keys():
-                continue
-
-            section[self.PRIVATE_CWD] = os.path.dirname(ret.filename)
-
-        return ret
-
-    def _load_configs(self):
-        system_config_file = os.path.join(
-            self.get_system_config_dir(), self.CONFIG
-        )
-
-        global_config_file = os.path.join(
-            self.get_global_config_dir(), self.CONFIG
-        )
-
-        self._system_config = configobj.ConfigObj(system_config_file)
-        self._global_config = configobj.ConfigObj(global_config_file)
-        self._repo_config = configobj.ConfigObj()
-        self._local_config = configobj.ConfigObj()
-
-        if not self.dvc_dir:
-            return
-
-        config_file = os.path.join(self.dvc_dir, self.CONFIG)
-        config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
-
-        self._repo_config = configobj.ConfigObj(config_file)
-        self._local_config = configobj.ConfigObj(config_local_file)
-
-    @property
-    def config_local_file(self):
-        return self._local_config.filename
-
-    @property
-    def config_file(self):
-        return self._repo_config.filename
-
-    def load(self):
+    def load(self, validate=True):
         """Loads config from all the config files.
 
         Raises:
             dvc.config.ConfigError: thrown if config has invalid format.
         """
-        self._load_configs()
+        conf = {}
+        for level in self.LEVELS:
+            if level in self.files:
+                _merge(conf, self.load_one(level))
 
-        self.config = configobj.ConfigObj()
-        for c in [
-            self._system_config,
-            self._global_config,
-            self._repo_config,
-            self._local_config,
-        ]:
-            c = self._resolve_paths(c)
-            c = self._lower(c)
-            self.config.merge(c)
+        if validate:
+            conf = self.validate(conf)
 
-        if not self.validate:
-            return
+        self.clear()
+        self.update(conf)
 
-        d = self.config.dict()
-        try:
-            d = self.COMPILED_SCHEMA(d)
-        except Invalid as exc:
-            raise ConfigError(str(exc)) from exc
-        self.config = configobj.ConfigObj(d, write_empty_values=True)
+        # Add resolved default cache.dir
+        if not self["cache"].get("dir") and self.dvc_dir:
+            self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
-    def save(self, config=None):
-        """Saves config to config files.
+    def load_one(self, level):
+        conf = _load_config(self.files[level])
+        conf = self._load_paths(conf, self.files[level])
 
-        Raises:
-            dvc.config.ConfigError: thrown if failed to write config file.
-        """
-        if config is not None:
-            clist = [config]
-        else:
-            clist = [
-                self._system_config,
-                self._global_config,
-                self._repo_config,
-                self._local_config,
-            ]
+        # Autovivify sections
+        for key in COMPILED_SCHEMA.schema:
+            conf.setdefault(key, {})
 
-        for conf in clist:
-            self._save(conf)
+        return conf
 
+    @staticmethod
+    def _load_paths(conf, filename):
+        abs_conf_dir = os.path.abspath(os.path.dirname(filename))
+
+        def resolve(path):
+            if os.path.isabs(path) or re.match(r"\w+://", path):
+                return path
+            return RelPath(os.path.join(abs_conf_dir, path))
+
+        return Config._map_dirs(conf, resolve)
+
+    @staticmethod
+    def _save_paths(conf, filename):
+        conf_dir = os.path.dirname(filename)
+
+        def rel(path):
+            if re.match(r"\w+://", path):
+                return path
+
+            if isinstance(path, RelPath) or not os.path.isabs(path):
+                return relpath(path, conf_dir)
+            return path
+
+        return Config._map_dirs(conf, rel)
+
+    @staticmethod
+    def _map_dirs(conf, func):
+        dirs_schema = {"cache": {"dir": func}, "remote": {str: {"url": func}}}
+        return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
+
+    @contextmanager
+    def edit(self, level="repo"):
+        if level in {"repo", "local"} and self.dvc_dir is None:
+            raise ConfigError("Not inside a dvc repo")
+
+        conf = self.load_one(level)
+        yield conf
+
+        conf = self._save_paths(conf, self.files[level])
+        _save_config(self.files[level], conf)
         self.load()
 
     @staticmethod
-    def _save(config):
-        if config.filename is None:
-            return
-
-        logger.debug("Writing '{}'.".format(config.filename))
-        dname = os.path.dirname(os.path.abspath(config.filename))
+    def validate(data):
         try:
-            os.makedirs(dname)
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                raise
-        config.write()
+            return COMPILED_SCHEMA(data)
+        except Invalid as exc:
+            raise ConfigError(str(exc)) from None
 
-    def unset(self, section, opt=None, level=None, force=False):
-        """Unsets specified option and/or section in the config.
 
-        Args:
-            section (str): section name.
-            opt (str): optional option name.
-            level (int): config level to use.
-            force (bool): don't error-out even if section doesn't exist. False
-                by default.
+def _load_config(filename):
+    conf_obj = configobj.ConfigObj(filename)
+    return _parse_remotes(_lower_keys(conf_obj.dict()))
 
-        Raises:
-            dvc.config.ConfigError: thrown if section doesn't exist and
-                `force != True`.
-        """
-        config = self.get_configobj(level)
 
-        if section not in config.keys():
-            if force:
-                return
-            raise ConfigError("section '{}' doesn't exist".format(section))
+def _save_config(filename, conf_dict):
+    logger.debug("Writing '{}'.".format(filename))
+    os.makedirs(os.path.dirname(filename), exist_ok=True)
 
-        if opt:
-            if opt not in config[section].keys():
-                if force:
-                    return
-                raise ConfigError(
-                    "option '{}.{}' doesn't exist".format(section, opt)
-                )
-            del config[section][opt]
+    config = configobj.ConfigObj(_pack_remotes(conf_dict))
+    config.filename = filename
+    config.write()
 
-            if not config[section]:
-                del config[section]
+
+def _parse_remotes(conf):
+    result = {"remote": {}}
+
+    for section, val in conf.items():
+        name = re_find(r'^\s*remote\s*"(.*)"\s*$', section)
+        if name:
+            result["remote"][name] = val
         else:
-            del config[section]
+            result[section] = val
 
-        self.save(config)
+    return result
 
-    def set(self, section, opt, value, level=None, force=True):
-        """Sets specified option in the config.
 
-        Args:
-            section (str): section name.
-            opt (str): option name.
-            value: value to set option to.
-            level (int): config level to use.
-            force (bool): set option even if section already exists. True by
-                default.
+def _pack_remotes(conf):
+    # Drop empty sections
+    result = compact(conf)
 
-        Raises:
-            dvc.config.ConfigError: thrown if section already exists and
-                `force != True`.
+    # Transform remote.name -> 'remote "name"'
+    for name, val in conf["remote"].items():
+        result['remote "{}"'.format(name)] = val
+    result.pop("remote", None)
 
-        """
-        config = self.get_configobj(level)
+    return result
 
-        if section not in config.keys():
-            config[section] = {}
-        elif not force:
-            raise ConfigError(
-                "Section '{}' already exists. Use `-f|--force` to overwrite "
-                "section with new value.".format(section)
-            )
 
-        config[section][opt] = value
-        self.save(config)
+def _merge(into, update):
+    """Merges second dict into first recursively"""
+    for key, val in update.items():
+        if isinstance(into.get(key), dict) and isinstance(val, dict):
+            _merge(into[key], val)
+        else:
+            into[key] = val
 
-    def get(self, section, opt=None, level=None):
-        """Return option value from the config.
 
-        Args:
-            section (str): section name.
-            opt (str): option name.
-            level (int): config level to use.
-
-        Returns:
-            value (str, int): option value.
-        """
-        config = self.get_configobj(level)
-
-        if section not in config.keys():
-            raise ConfigError("section '{}' doesn't exist".format(section))
-
-        if opt not in config[section].keys():
-            raise ConfigError(
-                "option '{}.{}' doesn't exist".format(section, opt)
-            )
-
-        return config[section][opt]
-
-    @staticmethod
-    def _lower(config):
-        new_config = configobj.ConfigObj()
-        for s_key, s_value in config.items():
-            new_s = {}
-            for key, value in s_value.items():
-                new_s[key.lower()] = str(value)
-            new_config[s_key.lower()] = new_s
-        return new_config
-
-    def get_configobj(self, level):
-        configs = {
-            self.LEVEL_LOCAL: self._local_config,
-            self.LEVEL_REPO: self._repo_config,
-            self.LEVEL_GLOBAL: self._global_config,
-            self.LEVEL_SYSTEM: self._system_config,
-        }
-
-        return configs.get(level, self._repo_config)
-
-    def list_options(self, section_regex, option, level=None):
-        ret = {}
-        config = self.get_configobj(level)
-        for section in config.keys():
-            r = re.match(section_regex, section)
-            if r:
-                name = r.group("name")
-                value = config[section].get(option, "")
-                ret[name] = value
-        return ret
+def _lower_keys(data):
+    return {
+        k.lower(): _lower_keys(v) if isinstance(v, dict) else v
+        for k, v in data.items()
+    }

dvc/dagascii.py

@@ -14,7 +14,7 @@ from grandalf.routing import EdgeViewer
 from grandalf.routing import route_with_lines
 
 from dvc.env import DVC_PAGER
-
+from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
@@ -45,7 +45,9 @@ def find_pager():
 
     logger.warning(
         "Unable to find `less` in the PATH. Check out "
-        "man.dvc.org/doc/command-reference/pipeline/show for more info."
+        "{} for more info.".format(
+            format_link("https://man.dvc.org/pipeline/show")
+        )
     )
     return pydoc.plainpager
 

dvc/data_cloud.py

@@ -2,7 +2,6 @@
 
 import logging
 
-from dvc.config import Config
 from dvc.config import NoRemoteError
 from dvc.remote import Remote
 
@@ -24,22 +23,27 @@ class DataCloud(object):
     def __init__(self, repo):
         self.repo = repo
 
-    @property
-    def _config(self):
-        return self.repo.config.config
-
-    @property
-    def _core(self):
-        return self._config.get(Config.SECTION_CORE, {})
-
     def get_remote(self, remote=None, command="<command>"):
         if not remote:
-            remote = self._core.get(Config.SECTION_CORE_REMOTE)
+            remote = self.repo.config["core"].get("remote")
 
         if remote:
             return self._init_remote(remote)
 
-        raise NoRemoteError(command)
+        if bool(self.repo.config["remote"]):
+            error_msg = (
+                "no remote specified. Setup default remote with\n"
+                "    dvc remote default <remote name>\n"
+                "or use:\n"
+                "    dvc {} -r <remote name>".format(command)
+            )
+        else:
+            error_msg = (
+                "no remote specified. Create a default remote with\n"
+                "    dvc remote add -d <remote name> <remote url>"
+            )
+
+        raise NoRemoteError(error_msg)
 
     def _init_remote(self, remote):
         return Remote(self.repo, name=remote)
@@ -73,13 +77,25 @@ class DataCloud(object):
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
-        return self.repo.cache.local.pull(
-            cache,
-            jobs=jobs,
-            remote=self.get_remote(remote, "pull"),
-            show_checksums=show_checksums,
+        remote = self.get_remote(remote, "pull")
+        downloaded_items_num = self.repo.cache.local.pull(
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )
 
+        if not remote.verify:
+            self._save_pulled_checksums(cache)
+
+        return downloaded_items_num
+
+    def _save_pulled_checksums(self, cache):
+        for checksum in cache["local"].keys():
+            cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
+            if self.repo.cache.local.exists(cache_file):
+                # We can safely save here, as existing corrupted files will be
+                # removed upon status, while files corrupted during download
+                # will not be moved from tmp_file (see `RemoteBASE.download()`)
+                self.repo.state.save(cache_file, checksum)
+
     def status(self, cache, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
 

dvc/dependency/base.py

@@ -27,5 +27,5 @@ class DependencyBase(object):
     IsNotFileOrDirError = DependencyIsNotFileOrDirError
     IsStageFileError = DependencyIsStageFileError
 
-    def update(self):
+    def update(self, rev=None):
         pass

dvc/dependency/repo.py

@@ -1,20 +1,10 @@
-import copy
 import os
-from contextlib import contextmanager
 
-from funcy import merge
+from voluptuous import Required
 
 from .local import DependencyLOCAL
-from dvc.external_repo import cached_clone
-from dvc.external_repo import external_repo
-from dvc.exceptions import NotDvcRepoError
 from dvc.exceptions import OutputNotFoundError
-from dvc.exceptions import NoOutputInExternalRepoError
-from dvc.exceptions import PathMissingError
-from dvc.config import NoRemoteError
-from dvc.utils.fs import fs_copy
 from dvc.path_info import PathInfo
-from dvc.scm import SCM
 
 
 class DependencyREPO(DependencyLOCAL):
@@ -23,7 +13,11 @@ class DependencyREPO(DependencyLOCAL):
     PARAM_REV = "rev"
     PARAM_REV_LOCK = "rev_lock"
 
-    REPO_SCHEMA = {PARAM_URL: str, PARAM_REV: str, PARAM_REV_LOCK: str}
+    REPO_SCHEMA = {
+        Required(PARAM_URL): str,
+        PARAM_REV: str,
+        PARAM_REV_LOCK: str,
+    }
 
     def __init__(self, def_repo, stage, *args, **kwargs):
         self.def_repo = def_repo
@@ -39,39 +33,31 @@ class DependencyREPO(DependencyLOCAL):
     @property
     def repo_pair(self):
         d = self.def_repo
-        return d[self.PARAM_URL], d[self.PARAM_REV_LOCK] or d[self.PARAM_REV]
+        rev = d.get(self.PARAM_REV_LOCK) or d.get(self.PARAM_REV)
+        return d[self.PARAM_URL], rev
 
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 
-    @contextmanager
-    def _make_repo(self, **overrides):
-        with external_repo(**merge(self.def_repo, overrides)) as repo:
-            yield repo
+    def _make_repo(self, *, locked=True):
+        from dvc.external_repo import external_repo
 
-    def _get_checksum(self, updated=False):
-        rev_lock = None
-        if not updated:
-            rev_lock = self.def_repo.get(self.PARAM_REV_LOCK)
+        d = self.def_repo
+        rev = (d.get("rev_lock") if locked else None) or d.get("rev")
+        return external_repo(d["url"], rev=rev)
 
-        try:
-            with self._make_repo(rev_lock=rev_lock) as repo:
+    def _get_checksum(self, locked=True):
+        with self._make_repo(locked=locked) as repo:
+            try:
                 return repo.find_out_by_relpath(self.def_path).info["md5"]
-        except (NotDvcRepoError, NoOutputInExternalRepoError):
-            # Fall through and clone
-            pass
-
-        repo_path = cached_clone(
-            self.def_repo[self.PARAM_URL],
-            rev=rev_lock or self.def_repo.get(self.PARAM_REV),
-        )
-        path = PathInfo(os.path.join(repo_path, self.def_path))
-
-        return self.repo.cache.local.get_checksum(path)
+            except OutputNotFoundError:
+                path = PathInfo(os.path.join(repo.root_dir, self.def_path))
+                # We are polluting our repo cache with some dir listing here
+                return self.repo.cache.local.get_checksum(path)
 
     def status(self):
-        current_checksum = self._get_checksum(updated=False)
-        updated_checksum = self._get_checksum(updated=True)
+        current_checksum = self._get_checksum(locked=True)
+        updated_checksum = self._get_checksum(locked=False)
 
         if current_checksum != updated_checksum:
             return {str(self): "update available"}
@@ -84,71 +70,19 @@ class DependencyREPO(DependencyLOCAL):
     def dumpd(self):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
-    def fetch(self):
-        with self._make_repo(
-            cache_dir=self.repo.cache.local.cache_dir
-        ) as repo:
-            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
-
-            out = repo.find_out_by_relpath(self.def_path)
-            with repo.state:
-                try:
-                    repo.cloud.pull(out.get_used_cache())
-                except NoRemoteError:
-                    # It would not be good idea to raise exception if the
-                    # file is already present in the cache
-                    if not self.repo.cache.local.changed_cache(out.checksum):
-                        return out
-                    raise
-
-        return out
-
-    @staticmethod
-    def _is_git_file(repo_dir, path):
-        from dvc.repo import Repo
-
-        if os.path.isabs(path):
-            return False
-
-        try:
-            repo = Repo(repo_dir)
-        except NotDvcRepoError:
-            return True
-
-        try:
-            output = repo.find_out_by_relpath(path)
-            return not output.use_cache
-        except OutputNotFoundError:
-            return True
-        finally:
-            repo.close()
-
-    def _copy_if_git_file(self, to_path):
-        src_path = self.def_path
-        repo_dir = cached_clone(**self.def_repo)
-
-        if not self._is_git_file(repo_dir, src_path):
-            return False
-
-        src_full_path = os.path.join(repo_dir, src_path)
-        dst_full_path = os.path.abspath(to_path)
-        fs_copy(src_full_path, dst_full_path)
-        self.def_repo[self.PARAM_REV_LOCK] = SCM(repo_dir).get_rev()
-        return True
-
     def download(self, to):
-        try:
-            if self._copy_if_git_file(to.fspath):
-                return
+        with self._make_repo() as repo:
+            if self.def_repo.get(self.PARAM_REV_LOCK) is None:
+                self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
 
-            out = self.fetch()
-            to.info = copy.copy(out.info)
-            to.checkout()
-        except (FileNotFoundError):
-            raise PathMissingError(
-                self.def_path, self.def_repo[self.PARAM_URL]
-            )
+            if hasattr(repo, "cache"):
+                repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
 
-    def update(self):
-        with self._make_repo(rev_lock=None) as repo:
+            repo.pull_to(self.def_path, to.path_info)
+
+    def update(self, rev=None):
+        if rev:
+            self.def_repo[self.PARAM_REV] = rev
+
+        with self._make_repo(locked=False) as repo:
             self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()

dvc/exceptions.py

@@ -1,6 +1,6 @@
 """Exceptions raised by the dvc."""
 
-from dvc.utils import relpath
+from dvc.utils import relpath, format_link
 
 
 class DvcException(Exception):
@@ -38,7 +38,7 @@ class OutputNotFoundError(DvcException):
         self.output = output
         self.repo = repo
         super().__init__(
-            "unable to find DVC-file with output '{path}'".format(
+            "Unable to find DVC-file with output '{path}'".format(
                 path=relpath(self.output)
             )
         )
@@ -112,18 +112,7 @@ class MoveNotDataSourceError(DvcException):
 
 
 class NotDvcRepoError(DvcException):
-    """Thrown if a directory is not a dvc repo.
-
-    Args:
-        root (str): path to the directory.
-    """
-
-    def __init__(self, root):
-        msg = (
-            "you are not inside of a dvc repository "
-            "(checked up to mount point '{}')"
-        )
-        super().__init__(msg.format(root))
+    """Thrown if a directory is not a DVC repo"""
 
 
 class DvcParserError(DvcException):
@@ -199,17 +188,10 @@ class RecursiveAddingWhileUsingFilename(DvcException):
 
 
 class OverlappingOutputPathsError(DvcException):
-    def __init__(self, out_1, out_2):
-        super().__init__(
-            "Paths for outs:\n'{}'('{}')\n'{}'('{}')\noverlap. To avoid "
-            "unpredictable behaviour, rerun command with non overlapping outs "
-            "paths.".format(
-                str(out_1),
-                out_1.stage.relpath,
-                str(out_2),
-                out_2.stage.relpath,
-            )
-        )
+    def __init__(self, parent, overlapping_out, message):
+        self.parent = parent
+        self.overlapping_out = overlapping_out
+        super().__init__(message)
 
 
 class CheckoutErrorSuggestGit(DvcException):
@@ -227,6 +209,7 @@ class ETagMismatchError(DvcException):
 
 class FileMissingError(DvcException):
     def __init__(self, path):
+        self.path = path
         super().__init__(
             "Can't find '{}' neither locally nor on remote".format(path)
         )
@@ -243,9 +226,10 @@ class DvcIgnoreInCollectedDirError(DvcException):
 class GitHookAlreadyExistsError(DvcException):
     def __init__(self, hook_name):
         super().__init__(
-            "Hook '{}' already exists. Please refer to "
-            "https://man.dvc.org/install "
-            "for more info.".format(hook_name)
+            "Hook '{}' already exists. Please refer to {} for more "
+            "info.".format(
+                hook_name, format_link("https://man.dvc.org/install")
+            )
         )
 
 
@@ -303,9 +287,35 @@ class HTTPError(DvcException):
 
 
 class PathMissingError(DvcException):
-    def __init__(self, path, repo):
+    default_msg = (
+        "The path '{}' does not exist in the target repository '{}'"
+        " neither as an output nor a git-handled file."
+    )
+    default_msg_output_only = (
+        "The path '{}' does not exist in the target repository '{}'"
+        " as an output."
+    )
+
+    def __init__(self, path, repo, output_only=False):
         msg = (
-            "The path '{}' does not exist in the target repository '{}'"
-            " neighther as an output nor a git-handled file."
+            self.default_msg
+            if not output_only
+            else self.default_msg_output_only
         )
         super().__init__(msg.format(path, repo))
+        self.output_only = output_only
+
+
+class RemoteCacheRequiredError(DvcException):
+    def __init__(self, path_info):
+        super().__init__(
+            (
+                "Current operation was unsuccessful because '{}' requires "
+                "existing cache on '{}' remote. See {} for information on how "
+                "to set up remote cache."
+            ).format(
+                path_info,
+                path_info.scheme,
+                format_link("https://man.dvc.org/config#cache"),
+            )
+        )

dvc/external_repo.py

@@ -1,27 +1,36 @@
+import logging
 import os
 import tempfile
 from contextlib import contextmanager
 from distutils.dir_util import copy_tree
+import threading
 
-from funcy import retry
+from funcy import retry, suppress, wrap_with, cached_property
 
-from dvc.config import NoRemoteError, ConfigError
+from dvc.path_info import PathInfo
+from dvc.compat import fspath
+from dvc.repo import Repo
+from dvc.config import NoRemoteError, NotDvcRepoError
 from dvc.exceptions import NoRemoteInExternalRepoError
-from dvc.remote import RemoteConfig
-from dvc.exceptions import NoOutputInExternalRepoError
-from dvc.exceptions import OutputNotFoundError
-from dvc.utils.fs import remove
+from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
+from dvc.exceptions import FileMissingError, PathMissingError
+from dvc.utils.fs import remove, fs_copy, move
+from dvc.utils import tmp_fname
+from dvc.scm.git import Git
 
 
-REPO_CACHE = {}
+logger = logging.getLogger(__name__)
 
 
 @contextmanager
-def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
-    from dvc.repo import Repo
+def external_repo(url, rev=None, for_write=False):
+    logger.debug("Creating external repo {}@{}", url, rev)
+    path = _cached_clone(url, rev, for_write=for_write)
+    try:
+        repo = ExternalRepo(path, url)
+    except NotDvcRepoError:
+        repo = ExternalGitRepo(path, url)
 
-    path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
-    repo = Repo(path)
     try:
         yield repo
     except NoRemoteError:
@@ -30,103 +39,206 @@ def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
         if exc.repo is repo:
             raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url)
         raise
-    repo.close()
-
-
-def cached_clone(url, rev=None, **_ignored_kwargs):
-    """Clone an external git repo to a temporary directory.
-
-    Returns the path to a local temporary directory with the specified
-    revision checked out.
-
-    Uses the REPO_CACHE to avoid accessing the remote server again if
-    cloning from the same URL twice in the same session.
-
-    """
-
-    new_path = tempfile.mkdtemp("dvc-erepo")
-
-    # Copy and adjust existing clean clone
-    if (url, None, None) in REPO_CACHE:
-        old_path = REPO_CACHE[url, None, None]
-
-        # This one unlike shutil.copytree() works with an existing dir
-        copy_tree(old_path, new_path)
-    else:
-        # Create a new clone
-        _clone_repo(url, new_path)
-
-        # Save clean clone dir so that we will have access to a default branch
-        clean_clone_path = tempfile.mkdtemp("dvc-erepo")
-        copy_tree(new_path, clean_clone_path)
-        REPO_CACHE[url, None, None] = clean_clone_path
-
-    # Check out the specified revision
-    if rev is not None:
-        _git_checkout(new_path, rev)
-
-    return new_path
-
-
-def _external_repo(url=None, rev=None, cache_dir=None):
-    from dvc.config import Config
-    from dvc.cache import CacheConfig
-    from dvc.repo import Repo
-
-    key = (url, rev, cache_dir)
-    if key in REPO_CACHE:
-        return REPO_CACHE[key]
-
-    new_path = cached_clone(url, rev=rev)
-
-    repo = Repo(new_path)
-    try:
-        # check if the URL is local and no default remote is present
-        # add default remote pointing to the original repo's cache location
-        if os.path.isdir(url):
-            rconfig = RemoteConfig(repo.config)
-            if not _default_remote_set(rconfig):
-                original_repo = Repo(url)
-                try:
-                    rconfig.add(
-                        "auto-generated-upstream",
-                        original_repo.cache.local.cache_dir,
-                        default=True,
-                        level=Config.LEVEL_LOCAL,
-                    )
-                finally:
-                    original_repo.close()
-
-        if cache_dir is not None:
-            cache_config = CacheConfig(repo.config)
-            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
+    except FileMissingError as exc:
+        raise PathMissingError(exc.path, url)
     finally:
-        # Need to close/reopen repo to force config reread
         repo.close()
-
-    REPO_CACHE[key] = new_path
-    return new_path
+        if for_write:
+            _remove(path)
 
 
-def _git_checkout(repo_path, revision):
-    from dvc.scm import Git
-
-    git = Git(repo_path)
-    try:
-        git.checkout(revision)
-    finally:
-        git.close()
+CLONES = {}
+CACHE_DIRS = {}
 
 
 def clean_repos():
     # Outside code should not see cache while we are removing
-    repo_paths = list(REPO_CACHE.values())
-    REPO_CACHE.clear()
+    paths = list(CLONES.values()) + list(CACHE_DIRS.values())
+    CLONES.clear()
+    CACHE_DIRS.clear()
 
-    for path in repo_paths:
+    for path in paths:
         _remove(path)
 
 
+class ExternalRepo(Repo):
+    def __init__(self, root_dir, url):
+        super().__init__(root_dir)
+        self.url = url
+        self._set_cache_dir()
+        self._set_upstream()
+
+    def pull_to(self, path, to_info):
+        """
+        Pull the corresponding file or directory specified by `path` and
+        checkout it into `to_info`.
+
+        It works with files tracked by Git and DVC, and also local files
+        outside the repository.
+        """
+        out = None
+        path_info = PathInfo(self.root_dir) / path
+
+        with suppress(OutputNotFoundError):
+            (out,) = self.find_outs_by_path(fspath(path_info), strict=False)
+
+        try:
+            if out and out.use_cache:
+                self._pull_cached(out, path_info, to_info)
+                return
+
+            # Check if it is handled by Git (it can't have an absolute path)
+            if os.path.isabs(path):
+                raise FileNotFoundError
+
+            fs_copy(fspath(path_info), fspath(to_info))
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+    def _pull_cached(self, out, path_info, dest):
+        with self.state:
+            tmp = PathInfo(tmp_fname(dest))
+            src = tmp / path_info.relative_to(out.path_info)
+
+            out.path_info = tmp
+
+            # Only pull unless all needed cache is present
+            if out.changed_cache(filter_info=src):
+                self.cloud.pull(out.get_used_cache(filter_info=src))
+
+            failed = out.checkout(filter_info=src)
+
+            move(src, dest)
+            remove(tmp)
+
+            if failed:
+                raise FileNotFoundError
+
+    @wrap_with(threading.Lock())
+    def _set_cache_dir(self):
+        try:
+            cache_dir = CACHE_DIRS[self.url]
+        except KeyError:
+            cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
+
+        self.cache.local.cache_dir = cache_dir
+
+    def _set_upstream(self):
+        # check if the URL is local and no default remote is present
+        # add default remote pointing to the original repo's cache location
+        if os.path.isdir(self.url):
+            if not self.config["core"].get("remote"):
+                src_repo = Repo(self.url)
+                try:
+                    cache_dir = src_repo.cache.local.cache_dir
+                finally:
+                    src_repo.close()
+
+                self.config["remote"]["auto-generated-upstream"] = {
+                    "url": cache_dir
+                }
+                self.config["core"]["remote"] = "auto-generated-upstream"
+
+
+class ExternalGitRepo:
+    def __init__(self, root_dir, url):
+        self.root_dir = root_dir
+        self.url = url
+
+    @cached_property
+    def scm(self):
+        return Git(self.root_dir)
+
+    def close(self):
+        if "scm" in self.__dict__:
+            self.scm.close()
+
+    def find_out_by_relpath(self, path):
+        raise OutputNotFoundError(path, self)
+
+    def pull_to(self, path, to_info):
+        try:
+            # Git handled files can't have absolute path
+            if os.path.isabs(path):
+                raise FileNotFoundError
+
+            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+    @contextmanager
+    def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
+        try:
+            abs_path = os.path.join(self.root_dir, path)
+            with open(abs_path, mode, encoding=encoding) as fd:
+                yield fd
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+
+def _cached_clone(url, rev, for_write=False):
+    """Clone an external git repo to a temporary directory.
+
+    Returns the path to a local temporary directory with the specified
+    revision checked out. If for_write is set prevents reusing this dir via
+    cache.
+    """
+    if not for_write and Git.is_sha(rev) and (url, rev) in CLONES:
+        return CLONES[url, rev]
+
+    clone_path = _clone_default_branch(url, rev)
+    rev_sha = Git(clone_path).resolve_rev(rev or "HEAD")
+
+    if not for_write and (url, rev_sha) in CLONES:
+        return CLONES[url, rev_sha]
+
+    # Copy to a new dir to keep the clone clean
+    repo_path = tempfile.mkdtemp("dvc-erepo")
+    copy_tree(clone_path, repo_path)
+
+    # Check out the specified revision
+    if rev is not None:
+        _git_checkout(repo_path, rev)
+
+    if not for_write:
+        CLONES[url, rev_sha] = repo_path
+    return repo_path
+
+
+@wrap_with(threading.Lock())
+def _clone_default_branch(url, rev):
+    """Get or create a clean clone of the url.
+
+    The cloned is reactualized with git pull unless rev is a known sha.
+    """
+    clone_path = CLONES.get(url)
+
+    git = None
+    try:
+        if clone_path:
+            git = Git(clone_path)
+            # Do not pull for known shas, branches and tags might move
+            if not Git.is_sha(rev) or not git.has_rev(rev):
+                git.pull()
+        else:
+            clone_path = tempfile.mkdtemp("dvc-clone")
+            git = Git.clone(url, clone_path)
+            CLONES[url] = clone_path
+    finally:
+        if git:
+            git.close()
+
+    return clone_path
+
+
+def _git_checkout(repo_path, rev):
+    git = Git(repo_path)
+    try:
+        git.checkout(rev)
+    finally:
+        git.close()
+
+
 def _remove(path):
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
@@ -134,26 +246,3 @@ def _remove(path):
         os_retry(remove)(path)
     else:
         remove(path)
-
-
-def _clone_repo(url, path):
-    from dvc.scm.git import Git
-
-    git = Git.clone(url, path)
-    git.close()
-
-
-def _default_remote_set(rconfig):
-    """
-    Checks if default remote config is present.
-    Args:
-        rconfig: a remote config
-
-    Returns:
-        True if the default remote config is set, else False
-    """
-    try:
-        rconfig.get_default()
-        return True
-    except ConfigError:
-        return False

dvc/lock.py

@@ -22,11 +22,11 @@ FAILED_TO_LOCK_MESSAGE = (
 
 
 class LockError(DvcException):
-    """Thrown when unable to acquire the lock for dvc repo."""
+    """Thrown when unable to acquire the lock for DVC repo."""
 
 
 class Lock(object):
-    """Class for dvc repo lock.
+    """Class for DVC repo lock.
 
     Uses zc.lockfile as backend.
     """
@@ -80,7 +80,7 @@ class Lock(object):
 
 
 class HardlinkLock(flufl.lock.Lock):
-    """Class for dvc repo lock.
+    """Class for DVC repo lock.
 
     Args:
         lockfile (str): the lock filename

dvc/logger.py

@@ -1,6 +1,5 @@
-"""Manages logging configuration for dvc repo."""
+"""Manages logging configuration for DVC repo."""
 
-import traceback
 import logging.config
 import logging.handlers
 
@@ -37,21 +36,19 @@ class ExcludeInfoFilter(logging.Filter):
 
 
 class ColorFormatter(logging.Formatter):
-    """Enable color support when logging to a terminal that supports it.
+    """Spit out colored text in supported terminals.
 
-    Color support on Windows versions that do not support ANSI color codes is
-    enabled by use of the colorama__ library.
+    colorama__ makes ANSI escape character sequences work under Windows.
     See the colorama documentation for details.
 
     __ https://pypi.python.org/pypi/colorama
 
-    For records containing `exc_info`, it will use a custom `_walk_exc` to
-    retrieve the whole traceback.
+    If record has an extra `tb_only` attribute, it will not show the
+    exception cause, just the message and the traceback.
     """
 
     color_code = {
         "DEBUG": colorama.Fore.BLUE,
-        "INFO": "",
         "WARNING": colorama.Fore.YELLOW,
         "ERROR": colorama.Fore.RED,
         "CRITICAL": colorama.Fore.RED,
@@ -63,80 +60,40 @@ class ColorFormatter(logging.Formatter):
         if record.levelname == "INFO":
             return msg
 
-        if record.levelname == "ERROR" or record.levelname == "CRITICAL":
-            exception, stack_trace = self._parse_exc(record)
+        if record.exc_info:
+            if getattr(record, "tb_only", False):
+                cause = ""
+            else:
+                cause = ": ".join(_iter_causes(record.exc_info[1]))
 
-            return (
-                "{color}{levelname}{nc}: {description}" "{stack_trace}\n"
-            ).format(
-                color=self.color_code.get(record.levelname, ""),
-                nc=colorama.Fore.RESET,
-                levelname=record.levelname,
-                description=self._description(msg, exception),
-                msg=msg,
-                stack_trace=stack_trace,
+            msg = "{message}{separator}{cause}".format(
+                message=msg or "",
+                separator=" - " if msg and cause else "",
+                cause=cause,
             )
 
-        return "{color}{levelname}{nc}: {msg}".format(
-            color=self.color_code.get(record.levelname, ""),
+            if _is_verbose():
+                msg += _stack_trace(record.exc_info)
+
+        return "{asctime}{color}{levelname}{nc}: {msg}".format(
+            asctime=self.formatTime(record, self.datefmt),
+            color=self.color_code[record.levelname],
             nc=colorama.Fore.RESET,
             levelname=record.levelname,
             msg=msg,
         )
 
-    def _current_level(self):
-        return logging.getLogger("dvc").getEffectiveLevel()
+    def formatTime(self, record, datefmt=None):
+        # only show if current level is set to DEBUG
+        # also, skip INFO as it is used for UI
+        if not _is_verbose() or record.levelno == logging.INFO:
+            return ""
 
-    def _is_visible(self, record):
-        return record.levelno >= self._current_level()
-
-    def _description(self, message, exception):
-        description = ""
-
-        if exception and message:
-            description = "{message} - {exception}"
-        elif exception:
-            description = "{exception}"
-        elif message:
-            description = "{message}"
-
-        return description.format(message=message, exception=exception)
-
-    def _walk_exc(self, exc_info):
-        exc = exc_info[1]
-
-        exc_list = [str(exc)]
-
-        while hasattr(exc, "__cause__") and exc.__cause__:
-            exc_list.append(str(exc.__cause__))
-            exc = exc.__cause__
-
-        return exc_list
-
-    def _parse_exc(self, record):
-        tb_only = getattr(record, "tb_only", False)
-
-        if not record.exc_info:
-            return (None, "")
-
-        exc_list = self._walk_exc(record.exc_info)
-        tb = traceback.format_exception(*record.exc_info)
-
-        exception = None if tb_only else ": ".join(exc_list)
-
-        if self._current_level() == logging.DEBUG:
-            stack_trace = (
-                "\n" "{red}{line}{nc}\n" "{stack_trace}" "{red}{line}{nc}"
-            ).format(
-                red=colorama.Fore.RED,
-                nc=colorama.Fore.RESET,
-                line="-" * 60,
-                stack_trace="".join(tb),
-            )
-        else:
-            stack_trace = ""
-
-        return (exception, stack_trace)
+        return "{green}{date}{nc} ".format(
+            green=colorama.Fore.GREEN,
+            date=super().formatTime(record, datefmt),
+            nc=colorama.Fore.RESET,
+        )
 
 
 class LoggerHandler(logging.StreamHandler):
@@ -158,6 +115,32 @@ class LoggerHandler(logging.StreamHandler):
             self.handleError(record)
 
 
+def _is_verbose():
+    return logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG
+
+
+def _iter_causes(exc):
+    while exc:
+        yield str(exc)
+        exc = exc.__cause__
+
+
+def _stack_trace(exc_info):
+    import traceback
+
+    return (
+        "\n"
+        "{red}{line}{nc}\n"
+        "{trace}"
+        "{red}{line}{nc}".format(
+            red=colorama.Fore.RED,
+            line="-" * 60,
+            trace="".join(traceback.format_exception(*exc_info)),
+            nc=colorama.Fore.RESET,
+        )
+    )
+
+
 def setup(level=logging.INFO):
     colorama.init()
 

dvc/main.py

@@ -11,6 +11,7 @@ from dvc.exceptions import NotDvcRepoError
 from dvc.external_repo import clean_repos
 from dvc.logger import FOOTER
 from dvc.utils import format_link
+from dvc.remote.pool import close_pools
 
 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
@@ -73,6 +74,11 @@ def main(argv=None):
     finally:
         logger.setLevel(outerLogLevel)
 
+        # Closing pools by-hand to prevent wierd messages when closing SSH
+        # connections. See https://github.com/iterative/dvc/issues/3248 for
+        # more info.
+        close_pools()
+
         # Remove cached repos in the end of the call, these are anonymous
         # so won't be reused by any other subsequent run anyway.
         clean_repos()

dvc/output/__init__.py

@@ -1,5 +1,5 @@
 from urllib.parse import urlparse
-from voluptuous import Any, Required
+from voluptuous import Any, Required, Lower, Length, Coerce, And, SetTo
 
 from dvc.output.base import OutputBase
 from dvc.output.gs import OutputGS
@@ -29,6 +29,12 @@ OUTS_MAP = {
     Schemes.LOCAL: OutputLOCAL,
 }
 
+CHECKSUM_SCHEMA = Any(
+    None,
+    And(str, Length(max=0), SetTo(None)),
+    And(Any(str, And(int, Coerce(str))), Length(min=3), Lower),
+)
+
 # NOTE: currently there are only 3 possible checksum names:
 #
 #    1) md5 (LOCAL, SSH, GS);
@@ -37,15 +43,15 @@ OUTS_MAP = {
 #
 # so when a few types of outputs share the same name, we only need
 # specify it once.
-CHECKSUM_SCHEMA = {
-    RemoteLOCAL.PARAM_CHECKSUM: Any(str, None),
-    RemoteS3.PARAM_CHECKSUM: Any(str, None),
-    RemoteHDFS.PARAM_CHECKSUM: Any(str, None),
+CHECKSUMS_SCHEMA = {
+    RemoteLOCAL.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    RemoteS3.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    RemoteHDFS.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
 }
 
-TAGS_SCHEMA = {str: CHECKSUM_SCHEMA}
+TAGS_SCHEMA = {str: CHECKSUMS_SCHEMA}
 
-SCHEMA = CHECKSUM_SCHEMA.copy()
+SCHEMA = CHECKSUMS_SCHEMA.copy()
 SCHEMA[Required(OutputBase.PARAM_PATH)] = str
 SCHEMA[OutputBase.PARAM_CACHE] = bool
 SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA

dvc/output/base.py

@@ -6,7 +6,7 @@ from voluptuous import Any
 
 import dvc.prompt as prompt
 from dvc.cache import NamedCache
-from dvc.exceptions import CollectCacheError
+from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError
 from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteBASE
 
@@ -98,14 +98,9 @@ class OutputBase(object):
         self.persist = persist
         self.tags = None if self.IS_DEPENDENCY else (tags or {})
 
-        if self.use_cache and self.cache is None:
-            raise DvcException(
-                "no cache location setup for '{}' outputs.".format(
-                    self.REMOTE.scheme
-                )
-            )
-
         self.path_info = self._parse_path(remote, path)
+        if self.use_cache and self.cache is None:
+            raise RemoteCacheRequiredError(self.path_info)
 
     def _parse_path(self, remote, path):
         if remote:
@@ -167,11 +162,11 @@ class OutputBase(object):
     def changed_checksum(self):
         return self.checksum != self.remote.get_checksum(self.path_info)
 
-    def changed_cache(self):
+    def changed_cache(self, filter_info=None):
         if not self.use_cache or not self.checksum:
             return True
 
-        return self.cache.changed_cache(self.checksum)
+        return self.cache.changed_cache(self.checksum, filter_info=filter_info)
 
     def status(self):
         if self.checksum and self.use_cache and self.changed_cache():
@@ -410,14 +405,14 @@ class OutputBase(object):
 
         if self.stage.is_repo_import:
             cache = NamedCache()
-            dep, = self.stage.deps
+            (dep,) = self.stage.deps
             cache.external[dep.repo_pair].add(dep.def_path)
             return cache
 
         if not self.info:
             logger.warning(
                 "Output '{}'({}) is missing version info. Cache for it will "
-                "not be collected. Use dvc repro to get your pipeline up to "
+                "not be collected. Use `dvc repro` to get your pipeline up to "
                 "date.".format(self, self.stage)
             )
             return NamedCache()

dvc/output/local.py

@@ -18,6 +18,12 @@ class OutputLOCAL(OutputBase):
     REMOTE = RemoteLOCAL
     sep = os.sep
 
+    def __init__(self, stage, path, *args, **kwargs):
+        if stage and path_isin(path, stage.repo.root_dir):
+            path = relpath(path, stage.wdir)
+
+        super().__init__(stage, path, *args, **kwargs)
+
     def _parse_path(self, remote, path):
         parsed = urlparse(path)
         if parsed.scheme == "remote":

dvc/progress.py

@@ -1,10 +1,9 @@
-"""Manages progress bars for dvc repo."""
+"""Manages progress bars for DVC repo."""
 
 import logging
 import sys
 from threading import RLock
 
-from funcy import merge
 from tqdm import tqdm
 
 from dvc.utils import env2bool
@@ -31,7 +30,10 @@ class Tqdm(tqdm):
     )
     BAR_FMT_NOTOTAL = (
         "{desc:{ncols_desc}.{ncols_desc}}{n_fmt}"
-        " [{elapsed}<??:??, {rate_fmt:>11}{postfix}]"
+        " [{elapsed}, {rate_fmt:>11}{postfix}]"
+    )
+    BYTES_DEFAULTS = dict(
+        unit="B", unit_scale=True, unit_divisor=1024, miniters=1
     )
 
     def __init__(
@@ -44,6 +46,7 @@ class Tqdm(tqdm):
         bar_format=None,
         bytes=False,  # pylint: disable=W0622
         file=None,
+        total=None,
         **kwargs
     ):
         """
@@ -52,23 +55,22 @@ class Tqdm(tqdm):
         desc  : persists after `close()`
         level  : effective logging level for determining `disable`;
             used only if `disable` is unspecified
-        disable  : If (default: None), will be determined by logging level.
+        disable  : If (default: None) or False,
+            will be determined by logging level.
             May be overridden to `True` due to non-TTY status.
             Skip override by specifying env var `DVC_IGNORE_ISATTY`.
         kwargs  : anything accepted by `tqdm.tqdm()`
         """
         kwargs = kwargs.copy()
-        kwargs.setdefault("unit_scale", True)
         if bytes:
-            bytes_defaults = dict(
-                unit="B", unit_scale=True, unit_divisor=1024, miniters=1
-            )
-            kwargs = merge(bytes_defaults, kwargs)
+            kwargs = {**self.BYTES_DEFAULTS, **kwargs}
+        else:
+            kwargs.setdefault("unit_scale", total > 999 if total else True)
         if file is None:
             file = sys.stderr
         self.desc_persist = desc
         # auto-disable based on `logger.level`
-        if disable is None:
+        if not disable:
             disable = logger.getEffectiveLevel() > level
         # auto-disable based on TTY
         if (
@@ -84,6 +86,7 @@ class Tqdm(tqdm):
             desc=desc,
             bar_format="!",
             lock_args=(False,),
+            total=total,
             **kwargs
         )
         if bar_format is None:
@@ -114,6 +117,10 @@ class Tqdm(tqdm):
     def close(self):
         if self.desc_persist is not None:
             self.set_description_str(self.desc_persist, refresh=False)
+        # unknown/zero ETA
+        self.bar_format = self.bar_format.replace("<{remaining}", "")
+        # remove completed bar
+        self.bar_format = self.bar_format.replace("|{bar:10}|", " ")
         super().close()
 
     @property

dvc/remote/__init__.py

@@ -1,4 +1,6 @@
-from .config import RemoteConfig
+import posixpath
+from urllib.parse import urlparse
+
 from dvc.remote.azure import RemoteAZURE
 from dvc.remote.gdrive import RemoteGDrive
 from dvc.remote.gs import RemoteGS
@@ -25,9 +27,9 @@ REMOTES = [
 ]
 
 
-def _get(config):
+def _get(remote_conf):
     for remote in REMOTES:
-        if remote.supported(config):
+        if remote.supported(remote_conf):
             return remote
     return RemoteLOCAL
 
@@ -35,8 +37,39 @@ def _get(config):
 def Remote(repo, **kwargs):
     name = kwargs.get("name")
     if name:
-        remote_config = RemoteConfig(repo.config)
-        settings = remote_config.get_settings(name)
+        remote_conf = repo.config["remote"][name.lower()]
     else:
-        settings = kwargs
-    return _get(settings)(repo, settings)
+        remote_conf = kwargs
+    remote_conf = _resolve_remote_refs(repo.config, remote_conf)
+    return _get(remote_conf)(repo, remote_conf)
+
+
+def _resolve_remote_refs(config, remote_conf):
+    # Support for cross referenced remotes.
+    # This will merge the settings, shadowing base ref with remote_conf.
+    # For example, having:
+    #
+    #       dvc remote add server ssh://localhost
+    #       dvc remote modify server user root
+    #       dvc remote modify server ask_password true
+    #
+    #       dvc remote add images remote://server/tmp/pictures
+    #       dvc remote modify images user alice
+    #       dvc remote modify images ask_password false
+    #       dvc remote modify images password asdf1234
+    #
+    # Results on a config dictionary like:
+    #
+    #       {
+    #           "url": "ssh://localhost/tmp/pictures",
+    #           "user": "alice",
+    #           "password": "asdf1234",
+    #           "ask_password": False,
+    #       }
+    parsed = urlparse(remote_conf["url"])
+    if parsed.scheme != "remote":
+        return remote_conf
+
+    base = config["remote"][parsed.netloc]
+    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
+    return {**base, **remote_conf, "url": url}

dvc/remote/azure.py

@@ -7,7 +7,6 @@ import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.config import Config
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
@@ -33,7 +32,7 @@ class RemoteAZURE(RemoteBASE):
     def __init__(self, repo, config):
         super().__init__(repo, config)
 
-        url = config.get(Config.SECTION_REMOTE_URL, "azure://")
+        url = config.get("url", "azure://")
 
         match = re.match(self.REGEX, url)  # backward compatibility
         path = match.group("path")
@@ -44,7 +43,7 @@ class RemoteAZURE(RemoteBASE):
         )
 
         self.connection_string = (
-            config.get(Config.SECTION_AZURE_CONNECTION_STRING)
+            config.get("connection_string")
             or match.group("connection_string")  # backward compatibility
             or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
         )

dvc/remote/base.py

@@ -13,11 +13,11 @@ from operator import itemgetter
 from shortuuid import uuid
 
 import dvc.prompt as prompt
-from dvc.config import Config
 from dvc.exceptions import (
     DvcException,
     ConfirmRemoveError,
     DvcIgnoreInCollectedDirError,
+    RemoteCacheRequiredError,
 )
 from dvc.ignore import DvcIgnore
 from dvc.path_info import PathInfo, URLInfo
@@ -65,7 +65,7 @@ class RemoteMissingDepsError(DvcException):
 class DirCacheError(DvcException):
     def __init__(self, checksum):
         super().__init__(
-            "Failed to load dir cache for checksum: '{}'.".format(checksum)
+            "Failed to load dir cache for hash value: '{}'.".format(checksum)
         )
 
 
@@ -79,6 +79,8 @@ class RemoteBASE(object):
     CHECKSUM_DIR_SUFFIX = ".dir"
     CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
     DEFAULT_CACHE_TYPES = ["copy"]
+    DEFAULT_NO_TRAVERSE = True
+    DEFAULT_VERIFY = False
 
     state = StateNoop()
 
@@ -86,19 +88,18 @@ class RemoteBASE(object):
         self.repo = repo
 
         self._check_requires(config)
-        self.checksum_jobs = self._get_checksum_jobs(config)
 
+        self.checksum_jobs = (
+            config.get("checksum_jobs")
+            or (self.repo and self.repo.config["core"].get("checksum_jobs"))
+            or self.CHECKSUM_JOBS
+        )
         self.protected = False
-        self.no_traverse = config.get(Config.SECTION_REMOTE_NO_TRAVERSE, True)
+        self.no_traverse = config.get("no_traverse", self.DEFAULT_NO_TRAVERSE)
+        self.verify = config.get("verify", self.DEFAULT_VERIFY)
         self._dir_info = {}
 
-        types = config.get(Config.SECTION_CACHE_TYPE, None)
-        if types:
-            if isinstance(types, str):
-                types = [t.strip() for t in types.split(",")]
-            self.cache_types = types
-        else:
-            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
+        self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES)
         self.cache_type_confirmed = False
 
     def _check_requires(self, config):
@@ -115,9 +116,7 @@ class RemoteBASE(object):
         if not missing:
             return
 
-        url = config.get(
-            Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
-        )
+        url = config.get("url", "{}://".format(self.scheme))
         msg = (
             "URL '{}' is supported but requires these missing "
             "dependencies: {}. If you have installed dvc using pip, "
@@ -138,19 +137,6 @@ class RemoteBASE(object):
         ).format(url, missing, " ".join(missing), self.scheme)
         raise RemoteMissingDepsError(msg)
 
-    def _get_checksum_jobs(self, config):
-        checksum_jobs = config.get(Config.SECTION_REMOTE_CHECKSUM_JOBS)
-        if checksum_jobs:
-            return checksum_jobs
-
-        if self.repo:
-            core = self.repo.config.config.get(Config.SECTION_CORE, {})
-            return core.get(
-                Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
-            )
-
-        return self.CHECKSUM_JOBS
-
     def __repr__(self):
         return "{class_name}: '{path_info}'".format(
             class_name=type(self).__name__,
@@ -162,7 +148,7 @@ class RemoteBASE(object):
         if isinstance(config, (str, bytes)):
             url = config
         else:
-            url = config[Config.SECTION_REMOTE_URL]
+            url = config["url"]
 
         # NOTE: silently skipping remote, calling code should handle that
         parsed = urlparse(url)
@@ -183,7 +169,7 @@ class RemoteBASE(object):
                 tasks,
                 total=len(file_infos),
                 unit="md5",
-                desc="Computing hashes (only done once)",
+                desc="Computing file/dir hashes (only done once)",
             ) as tasks:
                 checksums = dict(zip(file_infos, tasks))
         return checksums
@@ -226,6 +212,9 @@ class RemoteBASE(object):
         return sorted(result, key=itemgetter(self.PARAM_RELPATH))
 
     def get_dir_checksum(self, path_info):
+        if not self.cache:
+            raise RemoteCacheRequiredError(path_info)
+
         dir_info = self._collect_dir(path_info)
         checksum, tmp_info = self._get_dir_info_checksum(dir_info)
         new_info = self.cache.checksum_to_path_info(checksum)
@@ -331,13 +320,13 @@ class RemoteBASE(object):
 
         A file is considered changed if:
             - It doesn't exist on the working directory (was unlinked)
-            - Checksum is not computed (saving a new file)
-            - The checkusm stored in the State is different from the given one
+            - Hash value is not computed (saving a new file)
+            - The hash value stored is different from the given one
             - There's no file in the cache
 
         Args:
             path_info: dict with path information.
-            checksum: expected checksum for this data.
+            checksum: expected hash value for this data.
 
         Returns:
             bool: True if data has changed, False otherwise.
@@ -353,7 +342,7 @@ class RemoteBASE(object):
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
         if checksum is None:
-            logger.debug("checksum for '{}' is missing.", path_info)
+            logger.debug("hash value for '{}' is missing.", path_info)
             return True
 
         if self.changed_cache(checksum):
@@ -365,7 +354,7 @@ class RemoteBASE(object):
         actual = self.get_checksum(path_info)
         if checksum != actual:
             logger.debug(
-                "checksum '{}'(actual '{}') for '{}' has changed.",
+                "hash value '{}' for '{}' has changed (actual '{}').",
                 checksum,
                 actual,
                 path_info,
@@ -385,13 +374,24 @@ class RemoteBASE(object):
 
         self._try_links(from_info, to_info, link_types)
 
+    def _verify_link(self, path_info, link_type):
+        if self.cache_type_confirmed:
+            return
+
+        is_link = getattr(self, "is_{}".format(link_type), None)
+        if is_link and not is_link(path_info):
+            self.remove(path_info)
+            raise DvcException("failed to verify {}".format(link_type))
+
+        self.cache_type_confirmed = True
+
     @slow_link_guard
     def _try_links(self, from_info, to_info, link_types):
         while link_types:
             link_method = getattr(self, link_types[0])
             try:
                 self._do_link(from_info, to_info, link_method)
-                self.cache_type_confirmed = True
+                self._verify_link(to_info, link_types[0])
                 return
 
             except DvcException as exc:
@@ -471,7 +471,9 @@ class RemoteBASE(object):
         cache_info = self.checksum_to_path_info(checksum)
         dir_info = self.get_dir_cache(checksum)
 
-        for entry in dir_info:
+        for entry in Tqdm(
+            dir_info, desc="Saving " + path_info.name, unit="file"
+        ):
             entry_info = path_info / entry[self.PARAM_RELPATH]
             entry_checksum = entry[self.PARAM_CHECKSUM]
             self._save_file(entry_info, entry_checksum, save_link=False)
@@ -854,9 +856,7 @@ class RemoteBASE(object):
         """The file is changed we need to checkout a new copy"""
         cache_info = self.checksum_to_path_info(checksum)
         if self.exists(path_info):
-            logger.warning(
-                "data '{}' exists. Removing before checkout.", path_info
-            )
+            logger.debug("data '{}' will be replaced.", path_info)
             self.safe_remove(path_info, force=force)
 
         self.link(cache_info, path_info)
@@ -869,7 +869,6 @@ class RemoteBASE(object):
         """Optional: Implement only if the remote needs to create
         directories before copying/linking/moving data
         """
-        pass
 
     def _checkout_dir(
         self,
@@ -938,7 +937,7 @@ class RemoteBASE(object):
         skip = False
         if not checksum:
             logger.warning(
-                "No checksum info found for '{}'. " "It won't be created.",
+                "No file hash info found for '{}'. " "It won't be created.",
                 path_info,
             )
             self.safe_remove(path_info, force=force)

dvc/remote/config.py

@@ -1,159 +0,0 @@
-import logging
-import os
-import posixpath
-from urllib.parse import urlparse
-
-from dvc.config import Config, ConfigError
-from dvc.utils import relpath
-
-
-logger = logging.getLogger(__name__)
-
-
-class RemoteConfig(object):
-    def __init__(self, config):
-        self.config = config
-
-    def get_settings(self, name):
-        """
-        Args:
-            name (str): The name of the remote that we want to retrieve
-
-        Returns:
-            dict: The content beneath the given remote name.
-
-        Example:
-            >>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
-            >>> get_settings("server")
-            {'url': 'ssh://localhost/'}
-        """
-        settings = self.config.config.get(
-            Config.SECTION_REMOTE_FMT.format(name.lower())
-        )
-
-        if settings is None:
-            raise ConfigError(
-                "unable to find remote section '{}'".format(name)
-            )
-
-        parsed = urlparse(settings["url"])
-
-        # Support for cross referenced remotes.
-        # This will merge the settings, giving priority to the outer reference.
-        # For example, having:
-        #
-        #       dvc remote add server ssh://localhost
-        #       dvc remote modify server user root
-        #       dvc remote modify server ask_password true
-        #
-        #       dvc remote add images remote://server/tmp/pictures
-        #       dvc remote modify images user alice
-        #       dvc remote modify images ask_password false
-        #       dvc remote modify images password asdf1234
-        #
-        # Results on a config dictionary like:
-        #
-        #       {
-        #           "url": "ssh://localhost/tmp/pictures",
-        #           "user": "alice",
-        #           "password": "asdf1234",
-        #           "ask_password": False,
-        #       }
-        #
-        if parsed.scheme == "remote":
-            reference = self.get_settings(parsed.netloc)
-            url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
-            merged = reference.copy()
-            merged.update(settings)
-            merged["url"] = url
-            return merged
-
-        return settings
-
-    @staticmethod
-    def resolve_path(path, config_file):
-        """Resolve path relative to config file location.
-
-        Args:
-            path: Path to be resolved.
-            config_file: Path to config file, which `path` is specified
-                relative to.
-
-        Returns:
-            Path relative to the `config_file` location. If `path` is an
-            absolute path then it will be returned without change.
-
-        """
-        if os.path.isabs(path):
-            return path
-        return relpath(path, os.path.dirname(config_file))
-
-    def add(self, name, url, default=False, force=False, level=None):
-        from dvc.remote import _get, RemoteLOCAL
-
-        configobj = self.config.get_configobj(level)
-        remote = _get({Config.SECTION_REMOTE_URL: url})
-        if remote == RemoteLOCAL and not url.startswith("remote://"):
-            url = self.resolve_path(url, configobj.filename)
-
-        self.config.set(
-            Config.SECTION_REMOTE_FMT.format(name),
-            Config.SECTION_REMOTE_URL,
-            url,
-            force=force,
-            level=level,
-        )
-        if default:
-            self.config.set(
-                Config.SECTION_CORE,
-                Config.SECTION_CORE_REMOTE,
-                name,
-                level=level,
-            )
-
-    def remove(self, name, level=None):
-        self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level)
-
-        if level is None:
-            level = Config.LEVEL_REPO
-
-        for lev in [
-            Config.LEVEL_LOCAL,
-            Config.LEVEL_REPO,
-            Config.LEVEL_GLOBAL,
-            Config.LEVEL_SYSTEM,
-        ]:
-            self.config.unset(
-                Config.SECTION_CORE,
-                Config.SECTION_CORE_REMOTE,
-                level=lev,
-                force=True,
-            )
-            if lev == level:
-                break
-
-    def modify(self, name, option, value, level=None):
-        # try to get remote settings to verify that section exists
-        self.get_settings(name)
-
-        self.config.set(
-            Config.SECTION_REMOTE_FMT.format(name), option, value, level=level
-        )
-
-    def list(self, level=None):
-        return self.config.list_options(
-            Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level
-        )
-
-    def set_default(self, name, unset=False, level=None):
-        if unset:
-            self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE)
-            return
-        self.config.set(
-            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level
-        )
-
-    def get_default(self, level=None):
-        return self.config.get(
-            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level
-        )

dvc/remote/gdrive.py

@@ -1,8 +1,9 @@
 import os
 import posixpath
 import logging
-import threading
 import re
+import threading
+from urllib.parse import urlparse
 
 from funcy import retry, compose, decorator, wrap_with
 from funcy.py3 import cat
@@ -11,9 +12,8 @@ from dvc.progress import Tqdm
 from dvc.scheme import Schemes
 from dvc.path_info import CloudURLInfo
 from dvc.remote.base import RemoteBASE
-from dvc.config import Config
 from dvc.exceptions import DvcException
-from dvc.utils import tmp_fname
+from dvc.utils import tmp_fname, format_link
 
 logger = logging.getLogger(__name__)
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
@@ -23,22 +23,31 @@ class GDriveRetriableError(DvcException):
     pass
 
 
+class GDrivePathNotFound(DvcException):
+    def __init__(self, path_info):
+        super().__init__("Google Drive path '{}' not found.".format(path_info))
+
+
 class GDriveAccessTokenRefreshError(DvcException):
-    pass
+    def __init__(self):
+        super().__init__("Google Drive access token refreshment is failed.")
 
 
 class GDriveMissedCredentialKeyError(DvcException):
-    pass
+    def __init__(self, path):
+        super().__init__(
+            "Google Drive user credentials file '{}' "
+            "misses value for key.".format(path)
+        )
 
 
 @decorator
 def _wrap_pydrive_retriable(call):
-    from apiclient import errors
     from pydrive2.files import ApiRequestError
 
     try:
         result = call()
-    except (ApiRequestError, errors.HttpError) as exception:
+    except ApiRequestError as exception:
         retry_codes = ["403", "500", "502", "503", "504"]
         if any(
             "HttpError {}".format(code) in str(exception)
@@ -50,66 +59,199 @@ def _wrap_pydrive_retriable(call):
 
 
 gdrive_retry = compose(
-    # 8 tries, start at 0.5s, multiply by golden ratio, cap at 10s
+    # 15 tries, start at 0.5s, multiply by golden ratio, cap at 20s
     retry(
-        8, GDriveRetriableError, timeout=lambda a: min(0.5 * 1.618 ** a, 10)
+        15, GDriveRetriableError, timeout=lambda a: min(0.5 * 1.618 ** a, 20)
     ),
     _wrap_pydrive_retriable,
 )
 
 
+class GDriveURLInfo(CloudURLInfo):
+    def __init__(self, url):
+        super().__init__(url)
+
+        # GDrive URL host part is case sensitive,
+        # we are restoring it here.
+        p = urlparse(url)
+        self.host = p.netloc
+        assert self.netloc == self.host
+
+        # Normalize path. Important since we have a cache (path to ID)
+        # and don't want to deal with different variations of path in it.
+        self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
+
+
 class RemoteGDrive(RemoteBASE):
     scheme = Schemes.GDRIVE
-    path_cls = CloudURLInfo
+    path_cls = GDriveURLInfo
     REQUIRES = {"pydrive2": "pydrive2"}
+    DEFAULT_NO_TRAVERSE = False
+    DEFAULT_VERIFY = True
+
     GDRIVE_USER_CREDENTIALS_DATA = "GDRIVE_USER_CREDENTIALS_DATA"
     DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
-        self.no_traverse = False
-        self.path_info = self.path_cls(config[Config.SECTION_REMOTE_URL])
+        self.path_info = self.path_cls(config["url"])
 
-        bucket = re.search(
-            "{}://(.*)".format(self.scheme),
-            config[Config.SECTION_REMOTE_URL],
-            re.IGNORECASE,
-        )
-        self.bucket = (
-            bucket.group(1).split("/")[0] if bucket else self.path_info.bucket
-        )
+        if not self.path_info.bucket:
+            raise DvcException(
+                "Empty Google Drive URL '{}'. Learn more at "
+                "{}.".format(
+                    config["url"],
+                    format_link("https://man.dvc.org/remote/add"),
+                )
+            )
 
-        self.config = config
-        self.init_drive()
-
-    def init_drive(self):
-        self.client_id = self.config.get(Config.SECTION_GDRIVE_CLIENT_ID, None)
-        self.client_secret = self.config.get(
-            Config.SECTION_GDRIVE_CLIENT_SECRET, None
-        )
-        if not self.client_id or not self.client_secret:
+        self._bucket = self.path_info.bucket
+        self._client_id = config.get("gdrive_client_id")
+        self._client_secret = config.get("gdrive_client_secret")
+        if not self._client_id or not self._client_secret:
             raise DvcException(
                 "Please specify Google Drive's client id and "
-                "secret in DVC's config. Learn more at "
-                "https://man.dvc.org/remote/add."
+                "secret in DVC config. Learn more at "
+                "{}.".format(format_link("https://man.dvc.org/remote/add"))
             )
-        self.gdrive_user_credentials_path = (
+        self._gdrive_user_credentials_path = (
             tmp_fname(os.path.join(self.repo.tmp_dir, ""))
             if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
-            else self.config.get(
-                Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE,
+            else config.get(
+                "gdrive_user_credentials_file",
                 os.path.join(
                     self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE
                 ),
             )
         )
 
+        self._list_params = None
+        self._gdrive = None
+
+        self._cache_initialized = False
+        self._remote_root_id = None
+        self._cached_dirs = None
+        self._cached_ids = None
+
+    @property
+    @wrap_with(threading.RLock())
+    def drive(self):
+        from pydrive2.auth import RefreshError
+
+        if not self._gdrive:
+            from pydrive2.auth import GoogleAuth
+            from pydrive2.drive import GoogleDrive
+
+            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                with open(
+                    self._gdrive_user_credentials_path, "w"
+                ) as credentials_file:
+                    credentials_file.write(
+                        os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
+                    )
+
+            GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
+            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
+                "client_id": self._client_id,
+                "client_secret": self._client_secret,
+                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                "token_uri": "https://oauth2.googleapis.com/token",
+                "revoke_uri": "https://oauth2.googleapis.com/revoke",
+                "redirect_uri": "",
+            }
+            GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
+            GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
+            GoogleAuth.DEFAULT_SETTINGS[
+                "save_credentials_file"
+            ] = self._gdrive_user_credentials_path
+            GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
+            GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
+                "https://www.googleapis.com/auth/drive",
+                "https://www.googleapis.com/auth/drive.appdata",
+            ]
+
+            # Pass non existent settings path to force DEFAULT_SETTINGS loading
+            gauth = GoogleAuth(settings_file="")
+
+            try:
+                gauth.CommandLineAuth()
+            except RefreshError as exc:
+                raise GDriveAccessTokenRefreshError from exc
+            except KeyError as exc:
+                raise GDriveMissedCredentialKeyError(
+                    self._gdrive_user_credentials_path
+                ) from exc
+            # Handle pydrive2.auth.AuthenticationError and other auth failures
+            except Exception as exc:
+                raise DvcException(
+                    "Google Drive authentication failed"
+                ) from exc
+            finally:
+                if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
+                    os.remove(self._gdrive_user_credentials_path)
+
+            self._gdrive = GoogleDrive(gauth)
+
+        return self._gdrive
+
+    @wrap_with(threading.RLock())
+    def _initialize_cache(self):
+        if self._cache_initialized:
+            return
+
+        cached_dirs = {}
+        cached_ids = {}
+        self._remote_root_id = self._get_remote_id(self.path_info)
+        for dir1 in self.gdrive_list_item(
+            "'{}' in parents and trashed=false".format(self._remote_root_id)
+        ):
+            remote_path = posixpath.join(self.path_info.path, dir1["title"])
+            cached_dirs.setdefault(remote_path, []).append(dir1["id"])
+            cached_ids[dir1["id"]] = dir1["title"]
+
+        self._cached_dirs = cached_dirs
+        self._cached_ids = cached_ids
+        self._cache_initialized = True
+
+    @property
+    def cached_dirs(self):
+        if not self._cache_initialized:
+            self._initialize_cache()
+        return self._cached_dirs
+
+    @property
+    def cached_ids(self):
+        if not self._cache_initialized:
+            self._initialize_cache()
+        return self._cached_ids
+
+    @property
+    def remote_root_id(self):
+        if not self._cache_initialized:
+            self._initialize_cache()
+        return self._remote_root_id
+
+    @property
+    def list_params(self):
+        if not self._list_params:
+            params = {"corpora": "default"}
+            if self._bucket != "root" and self._bucket != "appDataFolder":
+                params["driveId"] = self._get_remote_drive_id(self._bucket)
+                params["corpora"] = "drive"
+            self._list_params = params
+        return self._list_params
+
     @gdrive_retry
     def gdrive_upload_file(
-        self, args, no_progress_bar=True, from_file="", progress_name=""
+        self,
+        parent_id,
+        title,
+        no_progress_bar=True,
+        from_file="",
+        progress_name="",
     ):
         item = self.drive.CreateFile(
-            {"title": args["title"], "parents": [{"id": args["parent_id"]}]}
+            {"title": title, "parents": [{"id": parent_id}]}
         )
 
         with open(from_file, "rb") as fobj:
@@ -132,9 +274,11 @@ class RemoteGDrive(RemoteBASE):
     def gdrive_download_file(
         self, file_id, to_file, progress_name, no_progress_bar
     ):
-        gdrive_file = self.drive.CreateFile({"id": file_id})
+        param = {"id": file_id}
+        # it does not create a file on the remote
+        gdrive_file = self.drive.CreateFile(param)
         bar_format = (
-            "Donwloading {desc:{ncols_desc}.{ncols_desc}}... "
+            "Downloading {desc:{ncols_desc}.{ncols_desc}}... "
             + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
         )
         with Tqdm(
@@ -143,7 +287,10 @@ class RemoteGDrive(RemoteBASE):
             gdrive_file.GetContentFile(to_file)
 
     def gdrive_list_item(self, query):
-        file_list = self.drive.ListFile({"q": query, "maxResults": 1000})
+        param = {"q": query, "maxResults": 1000}
+        param.update(self.list_params)
+
+        file_list = self.drive.ListFile(param)
 
         # Isolate and decorate fetching of remote drive items in pages
         get_list = gdrive_retry(lambda: next(file_list, None))
@@ -151,114 +298,39 @@ class RemoteGDrive(RemoteBASE):
         # Fetch pages until None is received, lazily flatten the thing
         return cat(iter(get_list, None))
 
-    def cache_root_dirs(self):
-        cached_dirs = {}
-        cached_ids = {}
-        for dir1 in self.gdrive_list_item(
-            "'{}' in parents and trashed=false".format(self.remote_root_id)
-        ):
-            remote_path = posixpath.join(self.path_info.path, dir1["title"])
-            cached_dirs.setdefault(remote_path, []).append(dir1["id"])
-            cached_ids[dir1["id"]] = dir1["title"]
-        return cached_dirs, cached_ids
-
-    @property
-    def cached_dirs(self):
-        if not hasattr(self, "_cached_dirs"):
-            self.drive
-        return self._cached_dirs
-
-    @property
-    def cached_ids(self):
-        if not hasattr(self, "_cached_ids"):
-            self.drive
-        return self._cached_ids
-
-    @property
     @wrap_with(threading.RLock())
-    def drive(self):
-        from pydrive2.auth import RefreshError
+    def gdrive_create_dir(self, parent_id, title, remote_path):
+        if parent_id == self.remote_root_id:
+            cached = self.cached_dirs.get(remote_path, [])
+            if cached:
+                return cached[0]
 
-        if not hasattr(self, "_gdrive"):
-            from pydrive2.auth import GoogleAuth
-            from pydrive2.drive import GoogleDrive
+        item = self._create_remote_dir(parent_id, title)
 
-            if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
-                with open(
-                    self.gdrive_user_credentials_path, "w"
-                ) as credentials_file:
-                    credentials_file.write(
-                        os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
-                    )
+        if parent_id == self.remote_root_id:
+            self.cached_dirs.setdefault(remote_path, []).append(item["id"])
+            self.cached_ids[item["id"]] = item["title"]
 
-            GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
-            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
-                "client_id": self.client_id,
-                "client_secret": self.client_secret,
-                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-                "token_uri": "https://oauth2.googleapis.com/token",
-                "revoke_uri": "https://oauth2.googleapis.com/revoke",
-                "redirect_uri": "",
-            }
-            GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
-            GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
-            GoogleAuth.DEFAULT_SETTINGS[
-                "save_credentials_file"
-            ] = self.gdrive_user_credentials_path
-            GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
-            GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
-                "https://www.googleapis.com/auth/drive",
-                "https://www.googleapis.com/auth/drive.appdata",
-            ]
-
-            # Pass non existent settings path to force DEFAULT_SETTINGS loading
-            gauth = GoogleAuth(settings_file="")
-
-            try:
-                gauth.CommandLineAuth()
-            except RefreshError as exc:
-                raise GDriveAccessTokenRefreshError(
-                    "Google Drive's access token refreshment is failed"
-                ) from exc
-            except KeyError as exc:
-                raise GDriveMissedCredentialKeyError(
-                    "Google Drive's user credentials file '{}' "
-                    "misses value for key '{}'".format(
-                        self.gdrive_user_credentials_path, str(exc)
-                    )
-                )
-            # Handle pydrive2.auth.AuthenticationError and others auth failures
-            except Exception as exc:
-                raise DvcException(
-                    "Google Drive authentication failed"
-                ) from exc
-            finally:
-                if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
-                    os.remove(self.gdrive_user_credentials_path)
-
-            self._gdrive = GoogleDrive(gauth)
-
-            self.remote_root_id = self.get_remote_id(
-                self.path_info, create=True
-            )
-            self._cached_dirs, self._cached_ids = self.cache_root_dirs()
-
-        return self._gdrive
+        return item["id"]
 
     @gdrive_retry
-    def create_remote_dir(self, parent_id, title):
+    def _create_remote_dir(self, parent_id, title):
+        parent = {"id": parent_id}
         item = self.drive.CreateFile(
-            {
-                "title": title,
-                "parents": [{"id": parent_id}],
-                "mimeType": FOLDER_MIME_TYPE,
-            }
+            {"title": title, "parents": [parent], "mimeType": FOLDER_MIME_TYPE}
         )
         item.Upload()
         return item
 
     @gdrive_retry
-    def get_remote_item(self, name, parents_ids):
+    def _delete_remote_file(self, remote_id):
+        param = {"id": remote_id}
+        # it does not create a file on the remote
+        item = self.drive.CreateFile(param)
+        item.Delete()
+
+    @gdrive_retry
+    def _get_remote_item(self, name, parents_ids):
         if not parents_ids:
             return None
         query = "({})".format(
@@ -270,64 +342,78 @@ class RemoteGDrive(RemoteBASE):
 
         query += " and trashed=false and title='{}'".format(name)
 
+        # Remote might contain items with duplicated path (titles).
+        # We thus limit number of items.
+        param = {"q": query, "maxResults": 1}
+        param.update(self.list_params)
+
         # Limit found remote items count to 1 in response
-        item_list = self.drive.ListFile(
-            {"q": query, "maxResults": 1}
-        ).GetList()
+        item_list = self.drive.ListFile(param).GetList()
         return next(iter(item_list), None)
 
-    def resolve_remote_item_from_path(self, path_parts, create):
-        parents_ids = [self.bucket]
-        current_path = ""
-        for path_part in path_parts:
-            current_path = posixpath.join(current_path, path_part)
-            remote_ids = self.get_remote_id_from_cache(current_path)
-            if remote_ids:
-                parents_ids = remote_ids
-                continue
-            item = self.get_remote_item(path_part, parents_ids)
-            if not item and create:
-                item = self.create_remote_dir(parents_ids[0], path_part)
-            elif not item:
-                return None
-            parents_ids = [item["id"]]
-        return item
+    @gdrive_retry
+    def _get_remote_drive_id(self, remote_id):
+        param = {"id": remote_id}
+        # it does not create a file on the remote
+        item = self.drive.CreateFile(param)
+        item.FetchMetadata("driveId")
+        return item.get("driveId", None)
 
-    def get_remote_id_from_cache(self, remote_path):
-        if hasattr(self, "_cached_dirs"):
-            return self.cached_dirs.get(remote_path, [])
+    def _get_cached_remote_ids(self, path):
+        if not path:
+            return [self._bucket]
+        if self._cache_initialized:
+            if path == self.path_info.path:
+                return [self.remote_root_id]
+            return self.cached_dirs.get(path, [])
         return []
 
-    def get_remote_id(self, path_info, create=False):
-        remote_ids = self.get_remote_id_from_cache(path_info.path)
-
+    def _path_to_remote_ids(self, path, create):
+        remote_ids = self._get_cached_remote_ids(path)
         if remote_ids:
-            return remote_ids[0]
+            return remote_ids
 
-        file1 = self.resolve_remote_item_from_path(
-            path_info.path.split("/"), create
-        )
-        return file1["id"] if file1 else ""
+        parent_path, part = posixpath.split(path)
+        parent_ids = self._path_to_remote_ids(parent_path, create)
+        item = self._get_remote_item(part, parent_ids)
+
+        if not item:
+            return (
+                [self.gdrive_create_dir(parent_ids[0], part, path)]
+                if create
+                else []
+            )
+
+        return [item["id"]]
+
+    def _get_remote_id(self, path_info, create=False):
+        assert path_info.bucket == self._bucket
+
+        remote_ids = self._path_to_remote_ids(path_info.path, create)
+        if not remote_ids:
+            raise GDrivePathNotFound(path_info)
+
+        return remote_ids[0]
 
     def exists(self, path_info):
-        return self.get_remote_id(path_info) != ""
+        try:
+            self._get_remote_id(path_info)
+        except GDrivePathNotFound:
+            return False
+        else:
+            return True
 
     def _upload(self, from_file, to_info, name, no_progress_bar):
         dirname = to_info.parent
-        if dirname:
-            parent_id = self.get_remote_id(dirname, True)
-        else:
-            parent_id = to_info.bucket
+        assert dirname
+        parent_id = self._get_remote_id(dirname, True)
 
         self.gdrive_upload_file(
-            {"title": to_info.name, "parent_id": parent_id},
-            no_progress_bar,
-            from_file,
-            name,
+            parent_id, to_info.name, no_progress_bar, from_file, name
         )
 
     def _download(self, from_info, to_file, name, no_progress_bar):
-        file_id = self.get_remote_id(from_info)
+        file_id = self._get_remote_id(from_info)
         self.gdrive_download_file(file_id, to_file, name, no_progress_bar)
 
     def all(self):
@@ -349,3 +435,7 @@ class RemoteGDrive(RemoteBASE):
             except ValueError:
                 # We ignore all the non-cache looking files
                 logger.debug('Ignoring path as "non-cache looking"')
+
+    def remove(self, path_info):
+        remote_id = self._get_remote_id(path_info)
+        self._delete_remote_file(remote_id)

dvc/remote/gs.py

@@ -7,7 +7,6 @@ import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
@@ -75,11 +74,11 @@ class RemoteGS(RemoteBASE):
     def __init__(self, repo, config):
         super().__init__(repo, config)
 
-        url = config.get(Config.SECTION_REMOTE_URL, "gs:///")
+        url = config.get("url", "gs:///")
         self.path_info = self.path_cls(url)
 
-        self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
-        self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
+        self.projectname = config.get("projectname", None)
+        self.credentialpath = config.get("credentialpath")
 
     @wrap_prop(threading.Lock())
     @cached_property

dvc/remote/hdfs.py

@@ -10,7 +10,6 @@ from urllib.parse import urlparse
 
 from .base import RemoteBASE, RemoteCmdError
 from .pool import get_connection
-from dvc.config import Config
 from dvc.scheme import Schemes
 from dvc.utils import fix_env, tmp_fname
 
@@ -26,13 +25,12 @@ class RemoteHDFS(RemoteBASE):
     def __init__(self, repo, config):
         super().__init__(repo, config)
         self.path_info = None
-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
         if not url:
             return
 
         parsed = urlparse(url)
-
-        user = parsed.username or config.get(Config.SECTION_REMOTE_USER)
+        user = parsed.username or config.get("user")
 
         self.path_info = self.path_cls.from_parts(
             scheme=self.scheme,

dvc/remote/http.py

@@ -3,7 +3,6 @@ import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.config import Config
 from dvc.config import ConfigError
 from dvc.exceptions import DvcException, HTTPError
 from dvc.progress import Tqdm
@@ -24,7 +23,7 @@ class RemoteHTTP(RemoteBASE):
     def __init__(self, repo, config):
         super().__init__(repo, config)
 
-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
         self.path_info = self.path_cls(url) if url else None
 
         if not self.no_traverse:

dvc/remote/local.py

@@ -7,27 +7,17 @@ from functools import partial
 
 from shortuuid import uuid
 
-from dvc.config import Config
-from dvc.exceptions import DownloadError
-from dvc.exceptions import DvcException
-from dvc.exceptions import UploadError
+from dvc.compat import fspath_py35
+from dvc.exceptions import DvcException, DownloadError, UploadError
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
-from dvc.remote.base import STATUS_DELETED
-from dvc.remote.base import STATUS_MAP
-from dvc.remote.base import STATUS_MISSING
-from dvc.remote.base import STATUS_NEW
+from dvc.remote.base import RemoteBASE, STATUS_MAP
+from dvc.remote.base import STATUS_DELETED, STATUS_MISSING, STATUS_NEW
 from dvc.scheme import Schemes
 from dvc.scm.tree import is_working_tree
 from dvc.system import System
-from dvc.utils.fs import copyfile
-from dvc.utils import file_md5
-from dvc.utils import relpath
-from dvc.utils import tmp_fname
-from dvc.compat import fspath_py35
-from dvc.utils.fs import move, makedirs, remove
-from dvc.utils.fs import walk_files
+from dvc.utils import file_md5, relpath, tmp_fname
+from dvc.utils.fs import copyfile, move, makedirs, remove, walk_files
 
 logger = logging.getLogger(__name__)
 
@@ -46,22 +36,16 @@ class RemoteLOCAL(RemoteBASE):
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
-        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
+        self.protected = config.get("protected", False)
 
-        shared = config.get(Config.SECTION_CACHE_SHARED)
+        shared = config.get("shared")
         self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
 
         if self.protected:
             # cache files are set to be read-only for everyone
             self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
 
-        cache_dir = config.get(Config.SECTION_REMOTE_URL)
-
-        if cache_dir is not None and not os.path.isabs(cache_dir):
-            cwd = config[Config.PRIVATE_CWD]
-            cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
-
-        self.path_info = PathInfo(cache_dir) if cache_dir else None
+        self.cache_dir = config.get("url")
         self._dir_info = {}
 
     @property
@@ -72,6 +56,10 @@ class RemoteLOCAL(RemoteBASE):
     def cache_dir(self):
         return self.path_info.fspath if self.path_info else None
 
+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.path_info = PathInfo(value) if value else None
+
     @classmethod
     def supported(cls, config):
         return True
@@ -174,6 +162,10 @@ class RemoteLOCAL(RemoteBASE):
     def symlink(from_info, to_info):
         System.symlink(from_info, to_info)
 
+    @staticmethod
+    def is_symlink(path_info):
+        return System.is_symlink(path_info)
+
     def hardlink(self, from_info, to_info):
         # If there are a lot of empty files (which happens a lot in datasets),
         # and the cache type is `hardlink`, we might reach link limits and
@@ -200,6 +192,10 @@ class RemoteLOCAL(RemoteBASE):
 
         System.hardlink(from_info, to_info)
 
+    @staticmethod
+    def is_hardlink(path_info):
+        return System.is_hardlink(path_info)
+
     @staticmethod
     def reflink(from_info, to_info):
         System.reflink(from_info, to_info)

dvc/remote/oss.py

@@ -4,7 +4,6 @@ import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.config import Config
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
@@ -42,21 +41,19 @@ class RemoteOSS(RemoteBASE):
     def __init__(self, repo, config):
         super().__init__(repo, config)
 
-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
         self.path_info = self.path_cls(url) if url else None
 
-        self.endpoint = config.get(Config.SECTION_OSS_ENDPOINT) or os.getenv(
-            "OSS_ENDPOINT"
-        )
+        self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")
 
         self.key_id = (
-            config.get(Config.SECTION_OSS_ACCESS_KEY_ID)
+            config.get("oss_key_id")
             or os.getenv("OSS_ACCESS_KEY_ID")
             or "defaultId"
         )
 
         self.key_secret = (
-            config.get(Config.SECTION_OSS_ACCESS_KEY_SECRET)
+            config.get("oss_key_secret")
             or os.getenv("OSS_ACCESS_KEY_SECRET")
             or "defaultSecret"
         )

dvc/remote/s3.py

@@ -6,7 +6,7 @@ import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.config import Config, ConfigError
+from dvc.config import ConfigError
 from dvc.exceptions import DvcException
 from dvc.exceptions import ETagMismatchError
 from dvc.path_info import CloudURLInfo
@@ -26,35 +26,33 @@ class RemoteS3(RemoteBASE):
     def __init__(self, repo, config):
         super().__init__(repo, config)
 
-        url = config.get(Config.SECTION_REMOTE_URL, "s3://")
+        url = config.get("url", "s3://")
         self.path_info = self.path_cls(url)
 
-        self.region = config.get(Config.SECTION_AWS_REGION)
+        self.region = config.get("region")
+        self.profile = config.get("profile")
+        self.endpoint_url = config.get("endpointurl")
 
-        self.profile = config.get(Config.SECTION_AWS_PROFILE)
-
-        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
-
-        if config.get(Config.SECTION_AWS_LIST_OBJECTS):
+        if config.get("listobjects"):
             self.list_objects_api = "list_objects"
         else:
             self.list_objects_api = "list_objects_v2"
 
-        self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
+        self.use_ssl = config.get("use_ssl", True)
 
         self.extra_args = {}
 
-        self.sse = config.get(Config.SECTION_AWS_SSE, "")
+        self.sse = config.get("sse")
         if self.sse:
             self.extra_args["ServerSideEncryption"] = self.sse
 
-        self.acl = config.get(Config.SECTION_AWS_ACL, "")
+        self.acl = config.get("acl")
         if self.acl:
             self.extra_args["ACL"] = self.acl
 
         self._append_aws_grants_to_extra_args(config)
 
-        shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
+        shared_creds = config.get("credentialpath")
         if shared_creds:
             os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
 
@@ -322,14 +320,14 @@ class RemoteS3(RemoteBASE):
         """
 
         grants = {
-            Config.SECTION_AWS_GRANT_FULL_CONTROL: "GrantFullControl",
-            Config.SECTION_AWS_GRANT_READ: "GrantRead",
-            Config.SECTION_AWS_GRANT_READ_ACP: "GrantReadACP",
-            Config.SECTION_AWS_GRANT_WRITE_ACP: "GrantWriteACP",
+            "grant_full_control": "GrantFullControl",
+            "grant_read": "GrantRead",
+            "grant_read_acp": "GrantReadACP",
+            "grant_write_acp": "GrantWriteACP",
         }
 
         for grant_option, extra_args_key in grants.items():
-            if config.get(grant_option, ""):
+            if config.get(grant_option):
                 if self.acl:
                     raise ConfigError(
                         "`acl` and `grant_*` AWS S3 config options "

dvc/remote/slow_link_detection.py

@@ -1,12 +1,10 @@
 import logging
 import sys
 import time
+
 from functools import wraps
-
 import colorama
 
-from dvc.config import Config
-
 
 logger = logging.getLogger(__name__)
 this = sys.modules[__name__]
@@ -31,11 +29,9 @@ def slow_link_guard(f):
         if this.already_displayed:
             return f(remote, *args, **kwargs)
 
-        config = remote.repo.config.config.get(Config.SECTION_CACHE, {})
-        cache_type = config.get(Config.SECTION_CACHE_TYPE)
-        should_warn = config.get(Config.SECTION_CACHE_SLOW_LINK_WARNING, True)
-
-        if not should_warn or cache_type:
+        cache_conf = remote.repo.config["cache"]
+        slow_link_warning = cache_conf.get("slow_link_warning", True)
+        if not slow_link_warning or cache_conf.get("type"):
             return f(remote, *args, **kwargs)
 
         start = time.time()

dvc/remote/ssh/__init__.py

@@ -9,10 +9,9 @@ from concurrent.futures import ThreadPoolExecutor
 from contextlib import closing, contextmanager
 from urllib.parse import urlparse
 
-from funcy import memoize, wrap_with
+from funcy import memoize, wrap_with, silent, first
 
 import dvc.prompt as prompt
-from dvc.config import Config
 from dvc.progress import Tqdm
 from dvc.remote.base import RemoteBASE
 from dvc.remote.pool import get_connection
@@ -50,20 +49,20 @@ class RemoteSSH(RemoteBASE):
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
-        url = config.get(Config.SECTION_REMOTE_URL)
+        url = config.get("url")
         if url:
             parsed = urlparse(url)
             user_ssh_config = self._load_user_ssh_config(parsed.hostname)
 
             host = user_ssh_config.get("hostname", parsed.hostname)
             user = (
-                config.get(Config.SECTION_REMOTE_USER)
+                config.get("user")
                 or parsed.username
                 or user_ssh_config.get("user")
                 or getpass.getuser()
             )
             port = (
-                config.get(Config.SECTION_REMOTE_PORT)
+                config.get("port")
                 or parsed.port
                 or self._try_get_ssh_config_port(user_ssh_config)
                 or self.DEFAULT_PORT
@@ -80,14 +79,12 @@ class RemoteSSH(RemoteBASE):
             user_ssh_config = {}
 
         self.keyfile = config.get(
-            Config.SECTION_REMOTE_KEY_FILE
+            "keyfile"
         ) or self._try_get_ssh_config_keyfile(user_ssh_config)
-        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
-        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
-        self.ask_password = config.get(
-            Config.SECTION_REMOTE_ASK_PASSWORD, False
-        )
-        self.gss_auth = config.get(Config.SECTION_REMOTE_GSS_AUTH, False)
+        self.timeout = config.get("timeout", self.TIMEOUT)
+        self.password = config.get("password", None)
+        self.ask_password = config.get("ask_password", False)
+        self.gss_auth = config.get("gss_auth", False)
 
     @staticmethod
     def ssh_config_filename():
@@ -110,17 +107,11 @@ class RemoteSSH(RemoteBASE):
 
     @staticmethod
     def _try_get_ssh_config_port(user_ssh_config):
-        try:
-            return int(user_ssh_config.get("port"))
-        except (ValueError, TypeError):
-            return None
+        return silent(int)(user_ssh_config.get("port"))
 
     @staticmethod
     def _try_get_ssh_config_keyfile(user_ssh_config):
-        identity_file = user_ssh_config.get("identityfile")
-        if identity_file and len(identity_file) > 0:
-            return identity_file[0]
-        return None
+        return first(user_ssh_config.get("identityfile") or ())
 
     def ensure_credentials(self, path_info=None):
         if path_info is None:

dvc/repo/__init__.py

@@ -1,4 +1,3 @@
-import logging
 import os
 from contextlib import contextmanager
 from functools import wraps
@@ -17,14 +16,10 @@ from dvc.exceptions import (
 )
 from dvc.path_info import PathInfo
 from dvc.remote.base import RemoteActionNotImplemented
-from dvc.utils import relpath
 from dvc.utils.fs import path_isin
 from .graph import check_acyclic, get_pipeline, get_pipelines
 
 
-logger = logging.getLogger(__name__)
-
-
 def locked(f):
     @wraps(f)
     def wrapper(repo, *args, **kwargs):
@@ -44,6 +39,7 @@ class Repo(object):
     from dvc.repo.install import install
     from dvc.repo.add import add
     from dvc.repo.remove import remove
+    from dvc.repo.ls import ls
     from dvc.repo.lock import lock as lock_stage
     from dvc.repo.move import move
     from dvc.repo.run import run
@@ -88,7 +84,7 @@ class Repo(object):
         self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
         makedirs(self.tmp_dir, exist_ok=True)
 
-        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
+        hardlink_lock = self.config["core"].get("hardlink_lock", False)
         self.lock = make_lock(
             os.path.join(self.dvc_dir, "lock"),
             tmp_dir=os.path.join(self.dvc_dir, "tmp"),
@@ -98,13 +94,7 @@ class Repo(object):
 
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
-        self.state = State(self, self.config.config)
-
-        core = self.config.config[Config.SECTION_CORE]
-
-        level = core.get(Config.SECTION_CORE_LOGLEVEL)
-        if level:
-            logger.setLevel(level.upper())
+        self.state = State(self)
 
         self.cache = Cache(self)
         self.cloud = DataCloud(self)
@@ -126,23 +116,28 @@ class Repo(object):
         self._reset()
 
     def __repr__(self):
-        return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
+        return "{}: '{}'".format(self.__class__.__name__, self.root_dir)
 
     @classmethod
     def find_root(cls, root=None):
-        if root is None:
-            root = os.getcwd()
-        else:
-            root = os.path.abspath(os.path.realpath(root))
+        root_dir = os.path.realpath(root or os.curdir)
+
+        if not os.path.isdir(root_dir):
+            raise NotDvcRepoError("directory '{}' does not exist".format(root))
 
         while True:
-            dvc_dir = os.path.join(root, cls.DVC_DIR)
+            dvc_dir = os.path.join(root_dir, cls.DVC_DIR)
             if os.path.isdir(dvc_dir):
-                return root
-            if os.path.ismount(root):
+                return root_dir
+            if os.path.ismount(root_dir):
                 break
-            root = os.path.dirname(root)
-        raise NotDvcRepoError(root)
+            root_dir = os.path.dirname(root_dir)
+
+        message = (
+            "you are not inside of a DVC repository "
+            "(checked up to mount point '{}')"
+        ).format(root_dir)
+        raise NotDvcRepoError(message)
 
     @classmethod
     def find_dvc_dir(cls, root=None):
@@ -165,7 +160,7 @@ class Repo(object):
         updater = Updater(self.dvc_dir)
 
         flist = (
-            [self.config.config_local_file, updater.updater_file]
+            [self.config.files["local"], updater.updater_file]
             + [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir]
             + self.state.files
         )
@@ -206,7 +201,7 @@ class Repo(object):
             return [(stage, None) for stage in self.stages]
 
         try:
-            out, = self.find_outs_by_path(target, strict=False)
+            (out,) = self.find_outs_by_path(target, strict=False)
             filter_info = PathInfo(os.path.abspath(target))
             return [(out.stage, filter_info)]
         except OutputNotFoundError:
@@ -330,7 +325,17 @@ class Repo(object):
             for out in stage.outs:
                 for p in out.path_info.parents:
                     if p in outs:
-                        raise OverlappingOutputPathsError(outs[p], out)
+                        msg = (
+                            "Paths for outs:\n'{}'('{}')\n'{}'('{}')\n"
+                            "overlap. To avoid unpredictable behaviour, "
+                            "rerun command with non overlapping outs paths."
+                        ).format(
+                            str(outs[p]),
+                            outs[p].stage.relpath,
+                            str(out),
+                            out.stage.relpath,
+                        )
+                        raise OverlappingOutputPathsError(outs[p], out, msg)
 
         for stage in stages:
             stage_path_info = PathInfo(stage.path)
@@ -400,14 +405,13 @@ class Repo(object):
 
         abs_path = os.path.abspath(path)
         path_info = PathInfo(abs_path)
-        is_dir = self.tree.isdir(abs_path)
         match = path_info.__eq__ if strict else path_info.isin_or_eq
 
         def func(out):
             if out.scheme == "local" and match(out.path_info):
                 return True
 
-            if is_dir and recursive and out.path_info.isin(path_info):
+            if recursive and out.path_info.isin(path_info):
                 return True
 
             return False
@@ -420,7 +424,7 @@ class Repo(object):
 
     def find_out_by_relpath(self, relpath):
         path = os.path.join(self.root_dir, relpath)
-        out, = self.find_outs_by_path(path)
+        (out,) = self.find_outs_by_path(path)
         return out
 
     def is_dvc_internal(self, path):
@@ -428,11 +432,11 @@ class Repo(object):
         return self.DVC_DIR in path_parts
 
     @contextmanager
-    def open(self, path, remote=None, mode="r", encoding=None):
+    def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
         cause = None
         try:
-            out, = self.find_outs_by_path(path)
+            out = self.find_out_by_relpath(path)
         except OutputNotFoundError as exc:
             out = None
             cause = exc
@@ -443,14 +447,15 @@ class Repo(object):
                     yield fd
                 return
             except FileNotFoundError as exc:
-                raise FileMissingError(relpath(path, self.root_dir)) from exc
+                raise FileMissingError(path) from exc
 
-        if self.tree.exists(path):
-            with self.tree.open(path, mode, encoding) as fd:
+        abs_path = os.path.join(self.root_dir, path)
+        if os.path.exists(abs_path):
+            with open(abs_path, mode=mode, encoding=encoding) as fd:
                 yield fd
             return
 
-        raise FileMissingError(relpath(path, self.root_dir)) from cause
+        raise FileMissingError(path) from cause
 
     def _open_cached(self, out, remote=None, mode="r", encoding=None):
         if out.isdir():

dvc/repo/add.py

@@ -2,14 +2,17 @@ import logging
 import os
 
 import colorama
-from six import string_types
 
 from . import locked
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
-from dvc.progress import Tqdm
-from dvc.repo.scm_context import scm_context
-from dvc.stage import Stage
-from dvc.utils import LARGE_DIR_SIZE
+from ..exceptions import (
+    RecursiveAddingWhileUsingFilename,
+    OverlappingOutputPathsError,
+)
+from ..output.base import OutputDoesNotExistError
+from ..progress import Tqdm
+from ..repo.scm_context import scm_context
+from ..stage import Stage
+from ..utils import LARGE_DIR_SIZE
 
 logger = logging.getLogger(__name__)
 
@@ -20,11 +23,16 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
     if recursive and fname:
         raise RecursiveAddingWhileUsingFilename()
 
-    if isinstance(targets, string_types):
+    if isinstance(targets, str):
         targets = [targets]
 
     stages_list = []
-    with Tqdm(total=len(targets), desc="Add", unit="file", leave=True) as pbar:
+    num_targets = len(targets)
+    with Tqdm(total=num_targets, desc="Add", unit="file", leave=True) as pbar:
+        if num_targets == 1:
+            # clear unneeded top-level progress bar for single target
+            pbar.bar_format = "Adding..."
+            pbar.refresh()
         for target in targets:
             sub_targets = _find_all_targets(repo, target, recursive)
             pbar.total += len(sub_targets) - 1
@@ -44,19 +52,46 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
 
             stages = _create_stages(repo, sub_targets, fname, pbar=pbar)
 
-            repo.check_modified_graph(stages)
+            try:
+                repo.check_modified_graph(stages)
+            except OverlappingOutputPathsError as exc:
+                msg = (
+                    "Cannot add '{out}', because it is overlapping with other "
+                    "DVC tracked output: '{parent}'.\n"
+                    "To include '{out}' in '{parent}', run "
+                    "'dvc commit {parent_stage}'"
+                ).format(
+                    out=exc.overlapping_out.path_info,
+                    parent=exc.parent.path_info,
+                    parent_stage=exc.parent.stage.relpath,
+                )
+                raise OverlappingOutputPathsError(
+                    exc.parent, exc.overlapping_out, msg
+                )
 
-            for stage in stages:
-                stage.save()
+            with Tqdm(
+                total=len(stages),
+                desc="Processing",
+                unit="file",
+                disable=len(stages) == 1,
+            ) as pbar_stages:
+                for stage in stages:
+                    try:
+                        stage.save()
+                    except OutputDoesNotExistError:
+                        pbar.n -= 1
+                        raise
 
-                if not no_commit:
-                    stage.commit()
+                    if not no_commit:
+                        stage.commit()
 
-                stage.dump()
+                    stage.dump()
+                    pbar_stages.update()
 
             stages_list += stages
-        # remove filled bar bit of progress, leaving stats
-        pbar.bar_format = pbar.BAR_FMT_DEFAULT.replace("|{bar:10}|", " ")
+
+        if num_targets == 1:  # restore bar format for stats
+            pbar.bar_format = pbar.BAR_FMT_DEFAULT
 
     return stages_list
 
@@ -65,7 +100,12 @@ def _find_all_targets(repo, target, recursive):
     if os.path.isdir(target) and recursive:
         return [
             fname
-            for fname in repo.tree.walk_files(target)
+            for fname in Tqdm(
+                repo.tree.walk_files(target),
+                desc="Searching " + target,
+                bar_format=Tqdm.BAR_FMT_NOTOTAL,
+                unit="file",
+            )
             if not repo.is_dvc_internal(fname)
             if not Stage.is_stage_file(fname)
             if not repo.scm.belongs_to_scm(fname)
@@ -77,8 +117,15 @@ def _find_all_targets(repo, target, recursive):
 def _create_stages(repo, targets, fname, pbar=None):
     stages = []
 
-    for out in targets:
-        stage = Stage.create(repo, outs=[out], add=True, fname=fname)
+    for out in Tqdm(
+        targets,
+        desc="Creating DVC-files",
+        disable=len(targets) < LARGE_DIR_SIZE,
+        unit="file",
+    ):
+        stage = Stage.create(
+            repo, outs=[out], accompany_outs=True, fname=fname
+        )
 
         if not stage:
             if pbar is not None:

dvc/repo/brancher.py

@@ -42,8 +42,9 @@ def brancher(  # noqa: E302
             revs.extend(scm.list_tags())
 
     try:
-        for sha, names in group_by(scm.resolve_rev, revs).items():
-            self.tree = scm.get_tree(sha)
-            yield ", ".join(names)
+        if revs:
+            for sha, names in group_by(scm.resolve_rev, revs).items():
+                self.tree = scm.get_tree(sha)
+                yield ", ".join(names)
     finally:
         self.tree = saved_tree

dvc/repo/diff.py

@@ -1,246 +1,76 @@
 import os
-from errno import ENOENT
 
-from dvc import logger
-from dvc.scm.base import FileNotInCommitError
-from dvc.scm.git import DIFF_A_REF, DIFF_B_REF, DIFF_A_TREE, DIFF_B_TREE
-from dvc.scm.git import DIFF_EQUAL
-from . import locked
-
-
-DIFF_TARGET = "target"
-DIFF_IS_DIR = "is_dir"
-DIFF_OLD_FILE = "old_file"
-DIFF_OLD_CHECKSUM = "old_checksum"
-DIFF_NEW_FILE = "new_file"
-DIFF_NEW_CHECKSUM = "new_checksum"
-DIFF_SIZE = "size_diff"
-DIFF_DEL = "del"
-DIFF_IDENT = "ident"
-DIFF_CHANGE = "changes"
-DIFF_NEW = "new"
-DIFF_MOVE = "moves"
-DIFF_LIST = "diffs"
-DIFF_SIZE_UNKNOWN = "?"
-DIFF_A_OUTPUT = "a_output"
-DIFF_B_OUTPUT = "b_output"
-DIFF_DELETED = "deleted_file"
-DIFF_IS_NEW = "created_file"
-
-
-def _file_not_exists(error, result):
-    if error.errno == ENOENT:
-        result.update({DIFF_SIZE: DIFF_SIZE_UNKNOWN})
-    else:
-        raise error
-
-
-def _extract_dir(self, dir_not_exists, output):
-    """Extract the content of dvc tree file
-    Args:
-        self(object) - Repo class instance
-        dir_not_exists(bool) - flag for directory existence
-        output(object) - OutputLOCAL class instance
-    Returns:
-        dict - dictionary with keys - paths to file in .dvc/cache
-                               values -checksums for that files
-    """
-    if not dir_not_exists:
-        lst = output.dir_cache
-        return {i["relpath"]: i["md5"] for i in lst}
-    return {}
-
-
-def _get_dir_info(dir_not_exists, a_output):
-    if not dir_not_exists:
-        return str(a_output), a_output.checksum
-    return "", ""
-
-
-def _ident_files(a_entries, b_entries):
-    keys = [
-        key for key in a_entries.keys() if b_entries.get(key) == a_entries[key]
-    ]
-    return len(keys)
-
-
-def _modified_files(self, a_entries, b_entries):
-    keys = [key for key in a_entries.keys() if key in b_entries]
-    diff_size = 0
-    modified_count = 0
-    for key in keys:
-        if a_entries[key] != b_entries[key]:
-            modified_count += 1
-            diff_size += os.path.getsize(
-                self.cache.local.get(b_entries[key])
-            ) - os.path.getsize(self.cache.local.get(a_entries[key]))
-    return modified_count, diff_size
-
-
-def _deleted_files(self, a_entries, b_entries):
-    diff_size = 0
-    deleted_count = 0
-    for key, value in a_entries.items():
-        if key not in b_entries:
-            deleted_count += 1
-            diff_size -= os.path.getsize(self.cache.local.get(a_entries[key]))
-    return deleted_count, diff_size
-
-
-def _new_files(self, a_entries, b_entries):
-    diff_size = 0
-    new_count = 0
-    for key, value in b_entries.items():
-        if key not in a_entries:
-            new_count += 1
-            diff_size += os.path.getsize(self.cache.local.get(b_entries[key]))
-    return new_count, diff_size
-
-
-def _get_tree_changes(self, a_entries, b_entries):
-    result = {
-        DIFF_DEL: 0,
-        DIFF_IDENT: 0,
-        DIFF_CHANGE: 0,
-        DIFF_NEW: 0,
-        DIFF_MOVE: 0,
-    }
-    result[DIFF_IDENT] = _ident_files(a_entries, b_entries)
-    result[DIFF_CHANGE], diff_size = _modified_files(
-        self, a_entries, b_entries
-    )
-    result[DIFF_SIZE] = diff_size
-    result[DIFF_DEL], diff_size = _deleted_files(self, a_entries, b_entries)
-    result[DIFF_SIZE] += diff_size
-    result[DIFF_NEW], diff_size = _new_files(self, a_entries, b_entries)
-    result[DIFF_SIZE] += diff_size
-    return result
-
-
-def _check_local_cache(a_out, is_checked):
-    if a_out is not None and a_out.scheme != "local":
-        is_checked.append(str(a_out))
-        return True
-    return False
-
-
-def _is_dir(path, a_outs, b_outs):
-    if a_outs.get(path):
-        return a_outs[path].is_dir_checksum
-    else:
-        return b_outs[path].is_dir_checksum
-
-
-def _get_diff_outs(self, diff_dct):
-    self.tree = diff_dct[DIFF_A_TREE]
-    a_outs = {str(out): out for st in self.stages for out in st.outs}
-    self.tree = diff_dct[DIFF_B_TREE]
-    b_outs = {str(out): out for st in self.stages for out in st.outs}
-    outs_paths = set(a_outs.keys())
-    outs_paths.update(b_outs.keys())
-    results = {}
-    non_local_cache = []
-    for path in outs_paths:
-        check1 = _check_local_cache(a_outs.get(path), non_local_cache)
-        check2 = _check_local_cache(b_outs.get(path), non_local_cache)
-        # skip files/directories with non-local cache for now
-        if check1 or check2:
-            continue
-        results[path] = {}
-        results[path][DIFF_A_OUTPUT] = a_outs.get(path)
-        results[path][DIFF_B_OUTPUT] = b_outs.get(path)
-        results[path][DIFF_IS_NEW] = path not in a_outs
-        results[path][DIFF_DELETED] = path not in b_outs
-        results[path][DIFF_IS_DIR] = _is_dir(path, a_outs, b_outs)
-    if non_local_cache:
-        logger.warning(
-            "Diff is not supported for non-local outputs. Ignoring: {}".format(
-                non_local_cache
-            )
-        )
-
-    return results
-
-
-def _diff_dir(self, target, diff_dct):
-    result = {DIFF_TARGET: target}
-    result[DIFF_IS_DIR] = True
-    a_entries, b_entries = {}, {}
-    try:
-        a_entries = _extract_dir(
-            self, diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
-        )
-        b_entries = _extract_dir(
-            self, diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
-        )
-        result[DIFF_OLD_FILE], result[DIFF_OLD_CHECKSUM] = _get_dir_info(
-            diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
-        )
-        result[DIFF_NEW_FILE], result[DIFF_NEW_CHECKSUM] = _get_dir_info(
-            diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
-        )
-        result.update(_get_tree_changes(self, a_entries, b_entries))
-    except IOError as e:
-        _file_not_exists(e, result)
-    return result
-
-
-def _diff_file(self, target, diff_dct):
-    result = {DIFF_TARGET: target}
-    size = 0
-    try:
-        if not diff_dct[DIFF_IS_NEW]:
-            result[DIFF_OLD_FILE] = target
-            result[DIFF_OLD_CHECKSUM] = diff_dct[DIFF_A_OUTPUT].checksum
-            size -= os.path.getsize(
-                self.cache.local.get(diff_dct[DIFF_A_OUTPUT].checksum)
-            )
-        if not diff_dct[DIFF_DELETED]:
-            result[DIFF_NEW_FILE] = target
-            result[DIFF_NEW_CHECKSUM] = diff_dct[DIFF_B_OUTPUT].checksum
-            size += os.path.getsize(
-                self.cache.local.get(diff_dct[DIFF_B_OUTPUT].checksum)
-            )
-        result[DIFF_SIZE] = size
-    except IOError as e:
-        _file_not_exists(e, result)
-    return result
-
-
-def _diff_royal(self, target, diff_dct):
-    if diff_dct[DIFF_IS_DIR]:
-        return _diff_dir(self, target, diff_dct)
-    return _diff_file(self, target, diff_dct)
+from dvc.exceptions import DvcException
+from dvc.repo import locked
+from dvc.scm.git import Git
 
 
 @locked
-def diff(self, a_ref, target=None, b_ref=None):
-    """Generates diff message string output
-
-    Args:
-        target(str) - file/directory to check diff of
-        a_ref(str) - first tag
-        (optional) b_ref(str) - second git tag
-
-    Returns:
-        string: string of output message with diff info
+def diff(self, a_rev="HEAD", b_rev=None):
     """
-    result = {}
-    diff_dct = self.scm.get_diff_trees(a_ref, b_ref=b_ref)
-    result[DIFF_A_REF] = diff_dct[DIFF_A_REF]
-    result[DIFF_B_REF] = diff_dct[DIFF_B_REF]
-    if diff_dct[DIFF_EQUAL]:
-        result[DIFF_EQUAL] = True
+    By default, it compares the working tree with the last commit's tree.
+
+    This implementation differs from `git diff` since DVC doesn't have
+    the concept of `index`, but it keeps the same interface, thus,
+    `dvc diff` would be the same as `dvc diff HEAD`.
+    """
+    if type(self.scm) is not Git:
+        raise DvcException("only supported for Git repositories")
+
+    def _paths_checksums():
+        """
+        A dictionary of checksums addressed by relpaths collected from
+        the current tree outputs.
+
+        Unpack directories to include their entries
+
+        To help distinguish between a directory and a file output,
+        the former one will come with a trailing slash in the path:
+
+            directory: "data/"
+            file:      "data"
+        """
+        result = {}
+
+        for stage in self.stages:
+            for output in stage.outs:
+                if not output.is_dir_checksum:
+                    result.update({str(output): output.checksum})
+                    continue
+
+                result.update({os.path.join(str(output), ""): output.checksum})
+
+                for entry in output.dir_cache:
+                    path = str(output.path_info / entry["relpath"])
+                    result.update({path: entry["md5"]})
+
         return result
-    result[DIFF_LIST] = []
-    diff_outs = _get_diff_outs(self, diff_dct)
-    if target is None:
-        result[DIFF_LIST] = [
-            _diff_royal(self, path, diff_outs[path]) for path in diff_outs
-        ]
-    elif target in diff_outs:
-        result[DIFF_LIST] = [_diff_royal(self, target, diff_outs[target])]
-    else:
-        msg = "Have not found file/directory '{}' in the commits"
-        raise FileNotInCommitError(msg.format(target))
-    return result
+
+    working_tree = self.tree
+    a_tree = self.scm.get_tree(a_rev)
+    b_tree = self.scm.get_tree(b_rev) if b_rev else working_tree
+
+    try:
+        self.tree = a_tree
+        old = _paths_checksums()
+
+        self.tree = b_tree
+        new = _paths_checksums()
+    finally:
+        self.tree = working_tree
+
+    # Compare paths between the old and new tree.
+    # set() efficiently converts dict keys to a set
+    added = sorted(set(new) - set(old))
+    deleted = sorted(set(old) - set(new))
+    modified = sorted(set(old) & set(new))
+
+    return {
+        "added": [{"path": path, "checksum": new[path]} for path in added],
+        "deleted": [{"path": path, "checksum": old[path]} for path in deleted],
+        "modified": [
+            {"path": path, "checksum": {"old": old[path], "new": new[path]}}
+            for path in modified
+            if old[path] != new[path]
+        ],
+    }

dvc/repo/fetch.py

@@ -4,7 +4,6 @@ from dvc.cache import NamedCache
 from dvc.config import NoRemoteError
 from dvc.exceptions import DownloadError
 from dvc.exceptions import OutputNotFoundError
-from dvc.external_repo import external_repo
 from dvc.scm.base import CloneError
 
 
@@ -70,11 +69,13 @@ def _fetch(
 
 
 def _fetch_external(self, repo_url, repo_rev, files):
-    failed = 0
+    from dvc.external_repo import external_repo
 
-    cache_dir = self.cache.local.cache_dir
+    failed = 0
     try:
-        with external_repo(repo_url, repo_rev, cache_dir=cache_dir) as repo:
+        with external_repo(repo_url, repo_rev) as repo:
+            repo.cache.local.cache_dir = self.cache.local.cache_dir
+
             with repo.state:
                 cache = NamedCache()
                 for name in files:

dvc/repo/get.py

@@ -3,22 +3,11 @@ import os
 
 import shortuuid
 
-from dvc.exceptions import (
-    DvcException,
-    NotDvcRepoError,
-    OutputNotFoundError,
-    PathMissingError,
-)
-from dvc.external_repo import (
-    external_repo,
-    cached_clone,
-    NoOutputInExternalRepoError,
-)
-from dvc.path_info import PathInfo
+from dvc.exceptions import DvcException
 from dvc.stage import Stage
 from dvc.utils import resolve_output
 from dvc.utils.fs import remove
-from dvc.utils.fs import fs_copy
+from dvc.path_info import PathInfo
 
 logger = logging.getLogger(__name__)
 
@@ -33,6 +22,8 @@ class GetDVCFileError(DvcException):
 
 @staticmethod
 def get(url, path, out=None, rev=None):
+    from dvc.external_repo import external_repo
+
     out = resolve_output(path, out)
 
     if Stage.is_valid_filename(out):
@@ -46,8 +37,10 @@ def get(url, path, out=None, rev=None):
     dpath = os.path.dirname(os.path.abspath(out))
     tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid()))
     try:
-        try:
-            with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo:
+        with external_repo(url=url, rev=rev) as repo:
+            if hasattr(repo, "cache"):
+                repo.cache.local.cache_dir = tmp_dir
+
                 # Try any links possible to avoid data duplication.
                 #
                 # Not using symlink, because we need to remove cache after we
@@ -58,33 +51,7 @@ def get(url, path, out=None, rev=None):
                 # Also, we can't use theoretical "move" link type here, because
                 # the same cache file might be used a few times in a directory.
                 repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]
-                output = repo.find_out_by_relpath(path)
-                if output.use_cache:
-                    _get_cached(repo, output, out)
-                    return
-                # Non-cached output, fall through and try to copy from git.
-        except (NotDvcRepoError, NoOutputInExternalRepoError):
-            # Not a DVC repository or, possibly, path is not tracked by DVC.
-            # Fall through and try to copy from git.
-            pass
 
-        if os.path.isabs(path):
-            raise FileNotFoundError
-
-        repo_dir = cached_clone(url, rev=rev)
-
-        fs_copy(os.path.join(repo_dir, path), out)
-    except (OutputNotFoundError, FileNotFoundError):
-        raise PathMissingError(path, url)
+            repo.pull_to(path, PathInfo(out))
     finally:
         remove(tmp_dir)
-
-
-def _get_cached(repo, output, out):
-    with repo.state:
-        repo.cloud.pull(output.get_used_cache())
-        output.path_info = PathInfo(os.path.abspath(out))
-        failed = output.checkout()
-        # This might happen when pull haven't really pulled all the files
-        if failed:
-            raise FileNotFoundError

dvc/repo/get_url.py

@@ -14,6 +14,6 @@ def get_url(url, out=None):
 
     out = os.path.abspath(out)
 
-    dep, = dependency.loads_from(None, [url])
-    out, = output.loads_from(None, [out], use_cache=False)
+    (dep,) = dependency.loads_from(None, [url])
+    (out,) = output.loads_from(None, [out], use_cache=False)
     dep.download(out)

dvc/repo/imp_url.py

@@ -11,7 +11,13 @@ def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
     out = resolve_output(url, out)
 
     stage = Stage.create(
-        self, cmd=None, deps=[url], outs=[out], fname=fname, erepo=erepo
+        self,
+        cmd=None,
+        deps=[url],
+        outs=[out],
+        fname=fname,
+        erepo=erepo,
+        accompany_outs=True,
     )
 
     if stage is None:

dvc/repo/init.py

@@ -89,7 +89,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
     config = Config.init(dvc_dir)
     proj = Repo(root_dir)
 
-    scm.add([config.config_file])
+    scm.add([config.files["repo"]])
 
     if scm.ignore_file:
         scm.add([os.path.join(dvc_dir, scm.ignore_file)])

dvc/repo/ls.py

@@ -0,0 +1,84 @@
+import os
+
+from dvc.exceptions import PathMissingError, OutputNotFoundError
+
+
+@staticmethod
+def ls(url, target=None, rev=None, recursive=None, outs_only=False):
+    from dvc.external_repo import external_repo
+    from dvc.repo import Repo
+    from dvc.utils import relpath
+
+    with external_repo(url, rev) as repo:
+        target_path_info = _get_target_path_info(repo, target)
+        result = []
+        if isinstance(repo, Repo):
+            result.extend(_ls_outs_repo(repo, target_path_info, recursive))
+
+        if not outs_only:
+            result.extend(_ls_files_repo(target_path_info, recursive))
+
+        if target and not result:
+            raise PathMissingError(target, repo, output_only=outs_only)
+
+        def prettify(path_info):
+            if path_info == target_path_info:
+                return path_info.name
+            return relpath(path_info, target_path_info)
+
+        result = list(set(map(prettify, result)))
+        result.sort()
+    return result
+
+
+def _ls_files_repo(target_path_info, recursive=None):
+    from dvc.compat import fspath
+    from dvc.ignore import CleanTree
+    from dvc.path_info import PathInfo
+    from dvc.scm.tree import WorkingTree
+
+    if not os.path.exists(fspath(target_path_info)):
+        return []
+
+    files = []
+    tree = CleanTree(WorkingTree(target_path_info))
+    try:
+        for dirpath, dirnames, filenames in tree.walk(target_path_info):
+            files.extend(map(lambda f: PathInfo(dirpath, f), filenames))
+            if not recursive:
+                files.extend(map(lambda d: PathInfo(dirpath, d), dirnames))
+                break
+    except NotADirectoryError:
+        if os.path.isfile(fspath(target_path_info)):
+            return [target_path_info]
+
+    return files
+
+
+def _ls_outs_repo(repo, target_path_info, recursive=None):
+    from dvc.compat import fspath
+    from dvc.path_info import PathInfo
+
+    try:
+        outs = repo.find_outs_by_path(fspath(target_path_info), recursive=True)
+    except OutputNotFoundError:
+        return []
+
+    if recursive:
+        return [out.path_info for out in outs]
+
+    def get_top_part(path_info):
+        relpath = path_info.relpath(target_path_info)
+        if relpath.parts:
+            return PathInfo(target_path_info, relpath.parts[0])
+        return path_info
+
+    return list({get_top_part(out.path_info) for out in outs})
+
+
+def _get_target_path_info(repo, target=None):
+    from dvc.path_info import PathInfo
+
+    if not target:
+        return PathInfo(repo.root_dir)
+    return PathInfo(repo.root_dir, target)

dvc/repo/metrics/diff.py

@@ -1,7 +1,7 @@
 import json
 from collections import defaultdict
 
-from flatten_dict import flatten
+from flatten_json import flatten
 
 from dvc.exceptions import NoMetricsError
 
@@ -34,25 +34,18 @@ def _diff_vals(old, new):
     return res
 
 
-# dot_reducer is not released yet (flatten-dict > 0.2.0)
-def _dot(k1, k2):
-    if k1 is None:
-        return k2
-    return "{0}.{1}".format(k1, k2)
-
-
 def _diff_dicts(old_dict, new_dict):
     old_default = None
     new_default = None
 
     if isinstance(new_dict, dict):
-        new = flatten(new_dict, reducer=_dot)
+        new = flatten(new_dict, ".")
     else:
         new = defaultdict(lambda: "not a dict")
         new_default = "unable to parse"
 
     if isinstance(old_dict, dict):
-        old = flatten(old_dict, reducer=_dot)
+        old = flatten(old_dict, ".")
     else:
         old = defaultdict(lambda: "not a dict")
         old_default = "unable to parse"
@@ -90,9 +83,9 @@ def _get_metrics(repo, *args, rev=None, **kwargs):
         return {}
 
 
-def diff(repo, *args, a_ref=None, b_ref=None, **kwargs):
-    old = _get_metrics(repo, *args, **kwargs, rev=(a_ref or "HEAD"))
-    new = _get_metrics(repo, *args, **kwargs, rev=b_ref)
+def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
+    old = _get_metrics(repo, *args, **kwargs, rev=(a_rev or "HEAD"))
+    new = _get_metrics(repo, *args, **kwargs, rev=b_rev)
 
     paths = set(old.keys())
     paths.update(set(new.keys()))

dvc/repo/reproduce.py

@@ -33,11 +33,18 @@ def _get_active_graph(G):
     for stage in G:
         if not stage.locked:
             continue
-        for st in nx.dfs_postorder_nodes(G, stage):
-            if st == stage:
-                continue
-            if st in active:
-                active.remove_node(st)
+        active.remove_edges_from(G.out_edges(stage))
+        for edge in G.out_edges(stage):
+            _, to_stage = edge
+            for node in nx.dfs_preorder_nodes(G, to_stage):
+                # NOTE: `in_degree` will return InDegreeView({}) if stage
+                # no longer exists in the `active` DAG.
+                if not active.in_degree(node):
+                    # NOTE: if some edge no longer exists `remove_edges_from`
+                    # will ignore it without error.
+                    active.remove_edges_from(G.out_edges(node))
+                    active.remove_node(node)
+
     return active
 
 
@@ -58,11 +65,7 @@ def reproduce(
 
     interactive = kwargs.get("interactive", False)
     if not interactive:
-        config = self.config
-        core = config.config[config.SECTION_CORE]
-        kwargs["interactive"] = core.get(
-            config.SECTION_CORE_INTERACTIVE, False
-        )
+        kwargs["interactive"] = self.config["core"].get("interactive", False)
 
     active_graph = _get_active_graph(self.graph)
     active_pipelines = get_pipelines(active_graph)

dvc/repo/update.py

@@ -2,10 +2,12 @@ from . import locked
 
 
 @locked
-def update(self, target):
+def update(self, target, rev=None):
     from dvc.stage import Stage
 
     stage = Stage.load(self, target)
-    stage.update()
+    stage.update(rev)
 
     stage.dump()
+
+    return stage

dvc/rwlock.py

@@ -4,16 +4,20 @@ import json
 from collections import defaultdict
 from contextlib import contextmanager
 
-from voluptuous import Schema, Invalid
-from funcy.py3 import lmap, lfilter, lmapcat
+from voluptuous import Schema, Invalid, Required, Optional
 
 from .exceptions import DvcException
 from .lock import LockError
 from .utils import relpath
 
-INFO_SCHEMA = {"pid": int, "cmd": str}
+INFO_SCHEMA = {Required("pid"): int, Required("cmd"): str}
 
-SCHEMA = Schema({"write": {str: INFO_SCHEMA}, "read": {str: [INFO_SCHEMA]}})
+SCHEMA = Schema(
+    {
+        Optional("write", default={}): {str: INFO_SCHEMA},
+        Optional("read", default={}): {str: [INFO_SCHEMA]},
+    }
+)
 
 
 class RWLockFileCorruptedError(DvcException):
@@ -36,15 +40,13 @@ def _edit_rwlock(lock_dir):
     path = os.path.join(lock_dir, "rwlock")
     try:
         with open(path, "r") as fobj:
-            lock = json.load(fobj)
-        lock = SCHEMA(lock)
+            lock = SCHEMA(json.load(fobj))
     except FileNotFoundError:
         lock = SCHEMA({})
     except json.JSONDecodeError as exc:
         raise RWLockFileCorruptedError(path) from exc
     except Invalid as exc:
         raise RWLockFileFormatError(path) from exc
-    lock = defaultdict(dict, lock)
     lock["read"] = defaultdict(list, lock["read"])
     lock["write"] = defaultdict(dict, lock["write"])
     yield lock
@@ -61,38 +63,26 @@ def _infos_to_str(infos):
     )
 
 
-def _check_no_writers(lock, info, path_infos):
-    for path_info in path_infos:
-        blocking_urls = lfilter(path_info.overlaps, lock["write"])
-        if not blocking_urls:
-            continue
+def _check_blockers(lock, info, *, mode, waiters):
+    for path_info in waiters:
+        blockers = [
+            blocker
+            for path, infos in lock[mode].items()
+            if path_info.overlaps(path)
+            for blocker in (infos if isinstance(infos, list) else [infos])
+            if blocker != info
+        ]
 
-        writers = lmap(lock["write"].get, blocking_urls)
-        writers = lfilter(lambda i: info != i, writers)
-        if not writers:
+        if not blockers:
             continue
 
         raise LockError(
-            "'{}' is busy, it is being written to by:\n{}".format(
-                str(path_info), _infos_to_str(writers)
-            )
-        )
-
-
-def _check_no_readers(lock, info, path_infos):
-    for path_info in path_infos:
-        blocking_urls = lfilter(path_info.overlaps, lock["read"])
-        if not blocking_urls:
-            continue
-
-        readers = lmapcat(lock["read"].get, blocking_urls)
-        readers = lfilter(lambda i: info != i, readers)
-        if not readers:
-            continue
-
-        raise LockError(
-            "'{}' is busy, it is being read by:\n{}".format(
-                str(path_info), _infos_to_str(readers)
+            "'{path}' is busy, it is being blocked by:\n"
+            "{blockers}\n"
+            "\n"
+            "If there are no processes with such PIDs, you can manually remove"
+            "'.dvc/tmp/rwlock' and try again.".format(
+                path=str(path_info), blockers=_infos_to_str(blockers)
             )
         )
 
@@ -171,8 +161,9 @@ def rwlock(tmp_dir, cmd, read, write):
     info = {"pid": os.getpid(), "cmd": cmd}
 
     with _edit_rwlock(tmp_dir) as lock:
-        _check_no_writers(lock, info, read + write)
-        _check_no_readers(lock, info, write)
+
+        _check_blockers(lock, info, mode="write", waiters=read + write)
+        _check_blockers(lock, info, mode="read", waiters=write)
 
         rchanges = _acquire_read(lock, info, read)
         wchanges = _acquire_write(lock, info, write)

dvc/scm/__init__.py

@@ -16,7 +16,7 @@ def SCM(root_dir):  # pylint: disable=invalid-name
 
     Args:
         root_dir (str): path to a root directory of the repo.
-        repo (dvc.repo.Repo): dvc repo instance that root_dir belongs to.
+        repo (dvc.repo.Repo): DVC repo instance that root_dir belongs to.
 
     Returns:
         dvc.scm.base.Base: SCM instance.

dvc/scm/base.py

@@ -15,22 +15,13 @@ class FileNotInRepoError(SCMError):
     """
 
 
-class FileNotInCommitError(SCMError):
-    """Thrown when trying to find a file/directory that is not
-    in the specified commit in the repository.
-    """
-
-
 class CloneError(SCMError):
     def __init__(self, url, path):
         super().__init__("Failed to clone repo '{}' to '{}'".format(url, path))
 
 
 class RevError(SCMError):
-    def __init__(self, url, rev):
-        super().__init__(
-            "Failed to access revision '{}' for repo '{}'".format(rev, url)
-        )
+    pass
 
 
 class Base(object):

dvc/scm/git/__init__.py

@@ -8,27 +8,14 @@ from pathspec.patterns import GitWildMatchPattern
 
 from dvc.exceptions import GitHookAlreadyExistsError
 from dvc.scm.base import Base
-from dvc.scm.base import CloneError
-from dvc.scm.base import FileNotInRepoError
-from dvc.scm.base import RevError
-from dvc.scm.base import SCMError
+from dvc.scm.base import CloneError, FileNotInRepoError, RevError, SCMError
 from dvc.scm.git.tree import GitTree
-from dvc.utils import fix_env
-from dvc.utils import is_binary
-from dvc.utils import relpath
+from dvc.utils import fix_env, is_binary, relpath
 from dvc.utils.fs import path_isin
 
-
 logger = logging.getLogger(__name__)
 
 
-DIFF_A_TREE = "a_tree"
-DIFF_B_TREE = "b_tree"
-DIFF_A_REF = "a_ref"
-DIFF_B_REF = "b_ref"
-DIFF_EQUAL = "equal"
-
-
 class Git(Base):
     """Class for managing Git."""
 
@@ -94,10 +81,20 @@ class Git(Base):
             try:
                 repo.checkout(rev)
             except git.exc.GitCommandError as exc:
-                raise RevError(url, rev) from exc
+                raise RevError(
+                    "failed to access revision '{}' for repo '{}'".format(
+                        rev, url
+                    )
+                ) from exc
 
         return repo
 
+    @staticmethod
+    def is_sha(rev):
+        import git
+
+        return rev and git.Repo.re_hexsha_shortened.search(rev)
+
     @staticmethod
     def is_repo(root_dir):
         return os.path.isdir(Git._get_git_dir(root_dir))
@@ -136,20 +133,19 @@ class Git(Base):
 
         return entry, gitignore
 
-    @staticmethod
-    def _ignored(entry, gitignore_path):
-        if os.path.exists(gitignore_path):
-            with open(gitignore_path, "r") as fobj:
-                ignore_list = fobj.readlines()
-            return any(
-                filter(lambda x: x.strip() == entry.strip(), ignore_list)
-            )
-        return False
+    def _ignored(self, path):
+        from git.exc import GitCommandError
+
+        try:
+            self.repo.git.check_ignore(path)
+            return True
+        except GitCommandError:
+            return False
 
     def ignore(self, path):
         entry, gitignore = self._get_gitignore(path)
 
-        if self._ignored(entry, gitignore):
+        if self._ignored(path):
             return
 
         msg = "Adding '{}' to '{}'.".format(relpath(path), relpath(gitignore))
@@ -215,6 +211,18 @@ class Git(Base):
         else:
             self.repo.git.checkout(branch)
 
+    def pull(self):
+        infos = self.repo.remote().pull()
+        for info in infos:
+            if info.flags & info.ERROR:
+                raise SCMError("pull failed: {}".format(info.note))
+
+    def push(self):
+        infos = self.repo.remote().push()
+        for info in infos:
+            if info.flags & info.ERROR:
+                raise SCMError("push failed: {}".format(info.summary))
+
     def branch(self, branch):
         self.repo.git.branch(branch)
 
@@ -247,11 +255,12 @@ class Git(Base):
     def list_all_commits(self):
         return [c.hexsha for c in self.repo.iter_commits("--all")]
 
-    def _install_hook(self, name, cmd):
-        command = (
-            '[ "$3" = "0" ]'
-            ' || [ -z "$(git ls-files .dvc)" ]'
-            " || exec dvc {}".format(cmd)
+    def _install_hook(self, name, preconditions, cmd):
+        # only run in dvc repo
+        in_dvc_repo = '[ -n "$(git ls-files --full-name .dvc)" ]'
+
+        command = "if {}; then exec dvc {}; fi".format(
+            " && ".join([in_dvc_repo] + preconditions), cmd
         )
 
         hook = self._hook_path(name)
@@ -269,9 +278,20 @@ class Git(Base):
     def install(self):
         self._verify_dvc_hooks()
 
-        self._install_hook("post-checkout", "checkout")
-        self._install_hook("pre-commit", "status")
-        self._install_hook("pre-push", "push")
+        self._install_hook(
+            "post-checkout",
+            [
+                # checking out some reference and not specific file.
+                '[ "$3" = "1" ]',
+                # make sure we are not in the middle of a rebase/merge, so we
+                # don't accidentally break it with an unsuccessful checkout.
+                # Note that git hooks are always running in repo root.
+                "[ ! -d .git/rebase-merge ]",
+            ],
+            "checkout",
+        )
+        self._install_hook("pre-commit", [], "status")
+        self._install_hook("pre-push", [], "push")
 
     def cleanup_ignores(self):
         for path in self.ignored_paths:
@@ -304,66 +324,48 @@ class Git(Base):
         return basename == self.ignore_file or Git.GIT_DIR in path_parts
 
     def get_tree(self, rev):
-        return GitTree(self.repo, rev)
-
-    def _get_diff_trees(self, a_ref, b_ref):
-        """Private method for getting the trees and commit hashes of 2 git
-        references. Requires `gitdb` module (from gitpython package).
-
-        Args:
-            a_ref (str): git reference
-            b_ref (str): second git reference. If None, uses HEAD
-
-        Returns:
-            tuple: tuple with elements: (trees, commits)
-        """
-        from gitdb.exc import BadObject, BadName
-
-        trees = {DIFF_A_TREE: None, DIFF_B_TREE: None}
-        commits = []
-        if b_ref is None:
-            b_ref = self.repo.head.commit
-        try:
-            a_commit = self.repo.git.rev_parse(a_ref, short=True)
-            b_commit = self.repo.git.rev_parse(b_ref, short=True)
-            # See https://gitpython.readthedocs.io
-            # /en/2.1.11/reference.html#git.objects.base.Object.__str__
-            commits.append(a_commit)
-            commits.append(b_commit)
-            trees[DIFF_A_TREE] = self.get_tree(commits[0])
-            trees[DIFF_B_TREE] = self.get_tree(commits[1])
-        except (BadName, BadObject) as exc:
-            raise SCMError("git problem") from exc
-        return trees, commits
-
-    def get_diff_trees(self, a_ref, b_ref=None):
-        """Method for getting two repo trees between two git tag commits.
-        Returns the dvc hash names of changed file/directory
-
-        Args:
-            a_ref (str): git reference
-            b_ref (str): optional second git reference, default None
-
-        Returns:
-            dict: dictionary with keys: {a_ref, b_ref, equal}
-                or {a_ref, b_ref, a_tree, b_tree}
-        """
-        diff_dct = {DIFF_EQUAL: False}
-        trees, commits = self._get_diff_trees(a_ref, b_ref)
-        diff_dct[DIFF_A_REF] = commits[0]
-        diff_dct[DIFF_B_REF] = commits[1]
-        if commits[0] == commits[1]:
-            diff_dct[DIFF_EQUAL] = True
-            return diff_dct
-        diff_dct[DIFF_A_TREE] = trees[DIFF_A_TREE]
-        diff_dct[DIFF_B_TREE] = trees[DIFF_B_TREE]
-        return diff_dct
+        return GitTree(self.repo, self.resolve_rev(rev))
 
     def get_rev(self):
-        return self.repo.git.rev_parse("HEAD")
+        return self.repo.rev_parse("HEAD").hexsha
 
     def resolve_rev(self, rev):
-        return self.repo.git.rev_parse(rev)
+        from git.exc import BadName, GitCommandError
+        from contextlib import suppress
+
+        def _resolve_rev(name):
+            with suppress(BadName, GitCommandError):
+                try:
+                    # Try python implementation of rev-parse first, it's faster
+                    return self.repo.rev_parse(name).hexsha
+                except NotImplementedError:
+                    # Fall back to `git rev-parse` for advanced features
+                    return self.repo.git.rev_parse(name)
+
+        # Resolve across local names
+        sha = _resolve_rev(rev)
+        if sha:
+            return sha
+
+        # Try all the remotes and if it resolves unambiguously then take it
+        if not Git.is_sha(rev):
+            shas = {
+                _resolve_rev("{}/{}".format(remote.name, rev))
+                for remote in self.repo.remotes
+            } - {None}
+            if len(shas) > 1:
+                raise RevError("ambiguous Git revision '{}'".format(rev))
+            if len(shas) == 1:
+                return shas.pop()
+
+        raise RevError("unknown Git revision '{}'".format(rev))
+
+    def has_rev(self, rev):
+        try:
+            self.resolve_rev(rev)
+            return True
+        except RevError:
+            return False
 
     def close(self):
         self.repo.close()

dvc/scm/git/tree.py

@@ -102,7 +102,7 @@ class GitTree(BaseTree):
             tree = self.git.tree(self.rev)
         except git.exc.BadName as exc:
             raise DvcException(
-                "revision '{}' not found in git '{}'".format(
+                "revision '{}' not found in Git '{}'".format(
                     self.rev, os.path.relpath(self.git.working_dir)
                 )
             ) from exc

dvc/stage.py

@@ -76,7 +76,7 @@ class StageFileBadNameError(DvcException):
 
 class StagePathOutsideError(DvcException):
     def __init__(self, path):
-        msg = "stage working or file path '{}' is outside of dvc repo"
+        msg = "stage working or file path '{}' is outside of DVC repo"
         super().__init__(msg.format(path))
 
 
@@ -191,7 +191,7 @@ class Stage(object):
     PARAM_ALWAYS_CHANGED = "always_changed"
 
     SCHEMA = {
-        PARAM_MD5: Any(str, None),
+        PARAM_MD5: output.CHECKSUM_SCHEMA,
         PARAM_CMD: Any(str, None),
         PARAM_WDIR: Any(str, None),
         PARAM_DEPS: Any([dependency.SCHEMA], None),
@@ -405,11 +405,11 @@ class Stage(object):
 
         return self
 
-    def update(self):
+    def update(self, rev=None):
         if not self.is_repo_import and not self.is_import:
             raise StageUpdateError(self.relpath)
 
-        self.deps[0].update()
+        self.deps[0].update(rev=rev)
         locked = self.locked
         self.locked = False
         try:
@@ -502,12 +502,11 @@ class Stage(object):
         return True
 
     @staticmethod
-    def create(repo, **kwargs):
+    def create(repo, accompany_outs=False, **kwargs):
 
         wdir = kwargs.get("wdir", None)
         cwd = kwargs.get("cwd", None)
         fname = kwargs.get("fname", None)
-        add = kwargs.get("add", False)
 
         # Backward compatibility for `cwd` option
         if wdir is None and cwd is not None:
@@ -539,12 +538,12 @@ class Stage(object):
         stage._check_duplicated_arguments()
 
         if not fname:
-            fname = Stage._stage_fname(stage.outs, add)
+            fname = Stage._stage_fname(stage.outs, accompany_outs)
         stage._check_dvc_filename(fname)
 
         # Autodetecting wdir for add, we need to create outs first to do that,
         # so we start with wdir = . and remap out paths later.
-        if add and kwargs.get("wdir") is None and cwd is None:
+        if accompany_outs and kwargs.get("wdir") is None and cwd is None:
             wdir = os.path.dirname(fname)
 
             for out in chain(stage.outs, stage.deps):
@@ -809,7 +808,7 @@ class Stage(object):
             msg += " and " if (changed_deps and changed_outs) else ""
             msg += "outputs {}".format(changed_outs) if changed_outs else ""
             msg += "md5" if not (changed_deps or changed_outs) else ""
-            msg += " of '{}' changed.".format(self.relpath)
+            msg += " of '{}' changed. ".format(self.relpath)
             msg += "Are you sure you want to commit it?"
             if not force and not prompt.confirm(msg):
                 raise StageCommitError(

dvc/state.py

@@ -6,7 +6,6 @@ import re
 import sqlite3
 from urllib.parse import urlunparse, urlencode
 
-from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.utils import current_timestamp
 from dvc.utils import relpath
@@ -23,13 +22,13 @@ logger = logging.getLogger(__name__)
 
 
 class StateVersionTooNewError(DvcException):
-    """Thrown when dvc version is older than the state database version."""
+    """Thrown when DVC version is older than the state database version."""
 
     def __init__(self, dvc_version, expected, actual):
         super().__init__(
-            "you are using an old version '{dvc_version}' of dvc that is "
-            "using state file version '{expected}' which is not compatible "
-            "with the state file version '{actual}' that is used in this "
+            "you are using an old version '{dvc_version}' of DVC that is "
+            "using state file version '{expected}', which is not compatible "
+            "with the state file version '{actual}', that is used in this "
             "repo. Please upgrade right now!".format(
                 dvc_version=dvc_version, expected=expected, actual=actual
             )
@@ -95,18 +94,15 @@ class State(object):  # pylint: disable=too-many-instance-attributes
     MAX_INT = 2 ** 63 - 1
     MAX_UINT = 2 ** 64 - 2
 
-    def __init__(self, repo, config):
+    def __init__(self, repo):
         self.repo = repo
         self.dvc_dir = repo.dvc_dir
         self.root_dir = repo.root_dir
 
-        state_config = config.get(Config.SECTION_STATE, {})
-        self.row_limit = state_config.get(
-            Config.SECTION_STATE_ROW_LIMIT, self.STATE_ROW_LIMIT
-        )
+        state_config = repo.config.get("state", {})
+        self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
         self.row_cleanup_quota = state_config.get(
-            Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
-            self.STATE_ROW_CLEANUP_QUOTA,
+            "row_cleanup_quota", self.STATE_ROW_CLEANUP_QUOTA
         )
 
         if not self.dvc_dir:

dvc/utils/fs.py

@@ -97,8 +97,8 @@ def move(src, dst, mode=None):
     of data is happening.
     """
 
-    src = fspath_py35(src)
-    dst = fspath_py35(dst)
+    src = fspath(src)
+    dst = fspath(dst)
 
     dst = os.path.abspath(dst)
     tmp = "{}.{}".format(dst, uuid())

dvc/version.py

@@ -7,11 +7,11 @@ import os
 import subprocess
 
 
-_BASE_VERSION = "0.82.0"
+_BASE_VERSION = "0.86.2"
 
 
 def _generate_version(base_version):
-    """Generate a version with information about the git repository"""
+    """Generate a version with information about the Git repository."""
     pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
     if not _is_git_repo(pkg_dir) or not _have_git():
@@ -28,7 +28,7 @@ def _generate_version(base_version):
 
 
 def _is_git_repo(dir_path):
-    """Is the given directory version-controlled with git?"""
+    """Is the given directory version-controlled with Git?"""
     return os.path.exists(os.path.join(dir_path, ".git"))
 
 
@@ -57,7 +57,7 @@ def _is_release(dir_path, base_version):
 
 
 def _git_revision(dir_path):
-    """Get the SHA-1 of the HEAD of a git repository."""
+    """Get SHA of the HEAD of a Git repository."""
     return subprocess.check_output(
         ["git", "rev-parse", "HEAD"], cwd=dir_path
     ).strip()

scripts/build_package.sh

@@ -4,26 +4,26 @@ set -e
 set -x
 
 if [ ! -d "dvc" ]; then
-    echo "Please run this script from repository root"
-    exit 1
+  echo "Please run this script from repository root" >&2
+  exit 1
 fi
 
-echo 'PKG = "pip"' > dvc/utils/build.py
+echo 'PKG = "pip"' >dvc/utils/build.py
 
 python setup.py sdist
 python setup.py bdist_wheel --universal
 
 # Make sure we have a correct version
 if [[ -n "$TRAVIS_TAG" ]]; then
-    pip uninstall -y dvc
-    if which dvc; then
-        echo "ERROR: dvc command still exists! Unable to verify dvc version."
-	exit 1
-    fi
-    pip install dist/dvc-*.whl
-    if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then
-        echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'"
-        exit 1
-    fi
-    pip uninstall -y dvc
+  pip uninstall -y dvc
+  if which dvc; then
+    echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2
+    exit 1
+  fi
+  pip install dist/dvc-*.whl
+  if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then
+    echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2
+    exit 1
+  fi
+  pip uninstall -y dvc
 fi

scripts/build_posix.sh

@@ -4,13 +4,13 @@ set -e
 set -x
 
 if [[ "$(uname)" == 'Linux' ]]; then
-	INSTALL_DIR=usr
-	FPM_FLAGS=
+  INSTALL_DIR=usr
+  FPM_FLAGS=
 else
-	INSTALL_DIR=usr/local
-	FPM_FLAGS='--osxpkg-identifier-prefix com.iterative'
-	FPM_FLAGS+=' --after-install scripts/fpm/after-install.sh'
-	FPM_FLAGS+=' --after-remove scripts/fpm/after-remove.sh'
+  INSTALL_DIR=usr/local
+  FPM_FLAGS='--osxpkg-identifier-prefix com.iterative'
+  FPM_FLAGS+=' --after-install scripts/fpm/after-install.sh'
+  FPM_FLAGS+=' --after-remove scripts/fpm/after-remove.sh'
 fi
 
 BUILD_DIR=build
@@ -21,132 +21,121 @@ LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib
 FPM_PACKAGE_DIRS="usr"
 ZSH_CMPLT_DIR=usr/share/zsh/site-functions/_dvc
 if [[ "$(uname)" == 'Linux' ]]; then
-	BASH_CMPLT_DIR=etc/bash_completion.d
-	FPM_PACKAGE_DIRS="$FPM_PACKAGE_DIRS etc"
+  BASH_CMPLT_DIR=etc/bash_completion.d
+  FPM_PACKAGE_DIRS="$FPM_PACKAGE_DIRS etc"
 else
-	BASH_CMPLT_DIR=usr/local/etc/bash_completion.d
+  BASH_CMPLT_DIR=usr/local/etc/bash_completion.d
 fi
 
-print_error()
-{
-	echo -e "\e[31m$1\e[0m"
+print_error() {
+  echo -e "\e[31m$1\e[0m" >&2
 }
 
 if [ ! -d "dvc" ]; then
-	print_error "Please run this script from repository root"
-	exit 1
+  print_error "Please run this script from repository root"
+  exit 1
 fi
 
 trap 'print_error "FAIL"; exit 1' ERR
 
-print_info()
-{
-	echo -e "\e[32m$1\e[0m"
+print_info() {
+  echo -e "\e[32m$1\e[0m"
 }
 
-command_exists()
-{
-	command -v $1 > /dev/null 2>&1
+command_exists() {
+  command -v $1 >/dev/null 2>&1
 }
 
-fpm_build()
-{
-	print_info "Building $1..."
-	VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))")
-	fpm -s dir \
-	    -f \
-	    -t $1 \
-            --description "$DESC" \
-	    $FPM_FLAGS \
-	    -n dvc \
-	    -v $VERSION \
-	    -C $BUILD_DIR \
-	    $FPM_PACKAGE_DIRS
+fpm_build() {
+  print_info "Building $1..."
+  VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))")
+  fpm -s dir \
+    -f \
+    -t $1 \
+    --description "$DESC" \
+    $FPM_FLAGS \
+    -n dvc \
+    -v $VERSION \
+    -C $BUILD_DIR \
+    $FPM_PACKAGE_DIRS
 }
 
-cleanup()
-{
-	print_info "Cleaning up..."
-	rm -rf build
+cleanup() {
+  print_info "Cleaning up..."
+  rm -rf build
 }
 
-install_dependencies()
-{
-	print_info "Installing fpm..."
-	if command_exists dnf; then
-		sudo dnf install ruby-devel gcc make rpm-build
-	elif command_exists yum; then
-		sudo yum install ruby-devel gcc make rpm-build
-	elif command_exists apt-get; then
-		sudo apt-get update -y
-		sudo apt-get install ruby-dev build-essential rpm python-pip python-dev
-	elif command_exists brew; then
-		brew install ruby
-	else
-		echo "Unable to install fpm dependencies" && exit 1
-	fi
+install_dependencies() {
+  print_info "Installing fpm..."
+  if command_exists dnf; then
+    sudo dnf install ruby-devel gcc make rpm-build
+  elif command_exists yum; then
+    sudo yum install ruby-devel gcc make rpm-build
+  elif command_exists apt-get; then
+    sudo apt-get update -y
+    sudo apt-get install ruby-dev build-essential rpm python-pip python-dev
+  elif command_exists brew; then
+    brew install ruby
+  else
+    echo "Unable to install fpm dependencies" && exit 1
+  fi
 
-	gem install --no-document fpm
-	
-	# https://github.com/iterative/dvc/issues/2970
-	gem uninstall -i /Users/travis/.rvm/gems/ruby-2.4.3@global rubygems-bundler
+  gem install --no-document fpm
 
-	print_info "Upgrading pip..."
-	pip install --upgrade pip
+  # https://github.com/iterative/dvc/issues/2970
+  gem uninstall -i /Users/travis/.rvm/gems/ruby-2.4.3@global rubygems-bundler
 
-	print_info "Installing requirements..."
-	pip install .[all]
-	pip install -r scripts/build-requirements.txt
+  print_info "Installing requirements..."
+  pip install .[all]
+  pip install -r scripts/build-requirements.txt
 }
 
-build_dvc()
-{
-	print_info "Building dvc binary..."
-	pyinstaller \
-            --additional-hooks-dir $(pwd)/scripts/hooks dvc/__main__.py \
-            --name dvc \
-            --distpath $LIB_DIR \
-            --specpath $BUILD_DIR
+build_dvc() {
+  print_info "Building dvc binary..."
+  pyinstaller \
+    --additional-hooks-dir $(pwd)/scripts/hooks dvc/__main__.py \
+    --name dvc \
+    --distpath $LIB_DIR \
+    --specpath $BUILD_DIR
 
-	$LIB_DIR/dvc/dvc --help
+  $LIB_DIR/dvc/dvc --help
 
-	# NOTE: in osxpkg fpm replaces symlinks with actual file that it
-	# points to, so we need to use after-install hook. See FPM_FLAGS
-	# above.
-	if [[ "$(uname)" == 'Linux' ]]; then
-		mkdir -p $BIN_DIR
-		pushd $BIN_DIR
-		ln -s ../lib/dvc/dvc dvc
-		popd
-		$BIN_DIR/dvc --help
-	fi
+  # NOTE: in osxpkg fpm replaces symlinks with actual file that it
+  # points to, so we need to use after-install hook. See FPM_FLAGS
+  # above.
+  if [[ "$(uname)" == 'Linux' ]]; then
+    mkdir -p $BIN_DIR
+    pushd $BIN_DIR
+    ln -s ../lib/dvc/dvc dvc
+    popd
+    $BIN_DIR/dvc --help
+  fi
 
-        # NOTE: temporarily not adding scripts to mac package. See [1]
-        # [1] https://github.com/iterative/dvc/issues/2585
-        if [[ "$(uname)" == 'Linux' ]]; then
-            mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
-            cp scripts/completion/dvc.bash $BUILD_DIR/$BASH_CMPLT_DIR/dvc
+  # NOTE: temporarily not adding scripts to mac package. See [1]
+  # [1] https://github.com/iterative/dvc/issues/2585
+  if [[ "$(uname)" == 'Linux' ]]; then
+    mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
+    cp scripts/completion/dvc.bash $BUILD_DIR/$BASH_CMPLT_DIR/dvc
 
-            mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
-            cp scripts/completion/dvc.zsh $BUILD_DIR/$ZSH_CMPLT_DIR
-        fi
+    mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
+    cp scripts/completion/dvc.zsh $BUILD_DIR/$ZSH_CMPLT_DIR
+  fi
 }
 
-build()
-{
-	cleanup
-	echo "PKG = \"$1\"" > dvc/utils/build.py
-	build_dvc
-	fpm_build $1
+build() {
+  cleanup
+  echo "PKG = \"$1\"" >dvc/utils/build.py
+  build_dvc
+  fpm_build $1
 }
 
 install_dependencies
 
 if [[ "$(uname)" == 'Linux' ]]; then
-	build rpm
-	build deb
+  build rpm
+  build deb
 else
-	build osxpkg
+  build osxpkg
 fi
 
 cleanup

scripts/build_snap.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+set -x
+
+if [ ! -d "dvc" ]; then
+  echo "Please run this script from repository root"
+  exit 1
+fi
+
+sudo snapcraft --use-lxd
+
+pip uninstall -y dvc
+if which dvc; then
+  echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2
+  exit 1
+fi
+sudo snap install --dangerous --classic dvc_*.snap
+if [[ -n "$TRAVIS_TAG" ]]; then
+  # Make sure we have a correct version
+  if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then
+    echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2
+    exit 1
+  fi
+fi
+# ensure basic commands can run
+# N.B.: cannot run `dvc get` on travis (#2956)
+dvc version
+dvc.git status
+dvc.git fetch --all
+sudo snap remove dvc

scripts/ci/before_install.sh

@@ -6,47 +6,53 @@ set -e
 scriptdir="$(dirname $0)"
 
 if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    ulimit -a
-    sudo sysctl -w kern.maxproc=2048
-    sudo sysctl -w kern.maxprocperuid=2048
-    echo '\nulimit -u 2048' >> ~/.bash_profile
-    ulimit -a
+  ulimit -a
+  sudo sysctl -w kern.maxproc=2048
+  sudo sysctl -w kern.maxprocperuid=2048
+  echo '\nulimit -u 2048' >>~/.bash_profile
+  ulimit -a
 fi
 
-echo > env.sh
+echo >env.sh
 
 if [[ "$TRAVIS_BUILD_STAGE_NAME" == "Test" ]]; then
-    if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then
-        # NOTE: ssh keys for ssh test to be able to ssh to the localhost
-        ssh-keygen -t rsa -N "" -f mykey
-        mkdir -p ~/.ssh
-        cp mykey ~/.ssh/id_rsa
-        cp mykey.pub ~/.ssh/id_rsa.pub
-        cat mykey.pub >> ~/.ssh/authorized_keys
-        ssh-keyscan localhost >> ~/.ssh/known_hosts
-        ssh localhost ls &> /dev/null
-        ssh-keyscan 127.0.0.1 >> ~/.ssh/known_hosts
-        ssh 127.0.0.1 ls &> /dev/null
-        ssh-keyscan 0.0.0.0 >> ~/.ssh/known_hosts
-        ssh 0.0.0.0 ls &> /dev/null
-    fi
+  if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then
+    # NOTE: ssh keys for ssh test to be able to ssh to the localhost
+    ssh-keygen -t rsa -N "" -f mykey
+    mkdir -p ~/.ssh
+    cp mykey ~/.ssh/id_rsa
+    cp mykey.pub ~/.ssh/id_rsa.pub
+    cat mykey.pub >>~/.ssh/authorized_keys
+    ssh-keyscan localhost >>~/.ssh/known_hosts
+    ssh localhost ls &>/dev/null
+    ssh-keyscan 127.0.0.1 >>~/.ssh/known_hosts
+    ssh 127.0.0.1 ls &>/dev/null
+    ssh-keyscan 0.0.0.0 >>~/.ssh/known_hosts
+    ssh 0.0.0.0 ls &>/dev/null
+  fi
 
-    if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-      bash "$scriptdir/install_azurite.sh"
-      bash "$scriptdir/install_oss.sh"
-      bash "$scriptdir/install_hadoop.sh"
-    fi
+  if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+    bash "$scriptdir/install_azurite.sh"
+    bash "$scriptdir/install_oss.sh"
+    bash "$scriptdir/install_hadoop.sh"
+  fi
 
-    if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
-        brew install openssl
-        $scriptdir/retry.sh brew cask install google-cloud-sdk
-    fi
+  if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
+    brew install openssl
+    $scriptdir/retry.sh brew cask install google-cloud-sdk
+  fi
 fi
 
 if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
-    $scriptdir/retry.sh choco install python --version 3.7.5
-    echo 'PATH="/c/Python37:/c/Python37/Scripts:$PATH"' >> env.sh
+  $scriptdir/retry.sh choco install python --version 3.7.5
+  echo 'PATH="/c/Python37:/c/Python37/Scripts:$PATH"' >>env.sh
 elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    ln -s -f /usr/local/bin/python3 /usr/local/bin/python
-    ln -s -f /usr/local/bin/pip3 /usr/local/bin/pip
+  ln -s -f /usr/local/bin/python3 /usr/local/bin/python
+  ln -s -f /usr/local/bin/pip3 /usr/local/bin/pip
+fi
+
+if [[ -n "$TRAVIS_TAG" ]]; then
+  echo "export SNAP_CHANNEL=stable" >>env.sh
+else
+  echo "export SNAP_CHANNEL=edge" >>env.sh
 fi

scripts/ci/check_patch.sh

@@ -3,7 +3,7 @@
 set -x
 set -e
 
-pip install Pygments collective.checkdocs pre-commit 
+pip install Pygments collective.checkdocs pre-commit
 
 # stop the build if there are any readme formatting errors
 python setup.py checkdocs

scripts/ci/check_python.sh

@@ -4,11 +4,11 @@ set -e
 set -x
 
 if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
-    where python
-    where pip
+  where python
+  where pip
 else
-    which python
-    which pip
+  which python
+  which pip
 fi
 
 py_ver="$(python -c 'import sys; print(sys.version[0])')"

scripts/ci/deploy_condition.sh

@@ -3,7 +3,7 @@
 set -e
 
 if [[ $TRAVIS_EVENT_TYPE = pull_request || $TRAVIS_EVENT_TPYE = cron ]]; then
-    exit 2
+  exit 2
 fi
 
 # positional args are assumed to be file glob patterns to deploy

scripts/ci/install.sh

@@ -8,16 +8,15 @@ scriptdir="$(dirname $0)"
 # NOTE: it is not uncommon for pip to hang on travis for what seems to be
 # networking issues. Thus, let's retry a few times to see if it will eventually
 # work or not.
-$scriptdir/retry.sh pip install --upgrade pip setuptools wheel
 $scriptdir/retry.sh pip install .[all,tests]
 
 git config --global user.email "[email protected]"
 git config --global user.name "DVC Tester"
 
 if [[ "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
-	aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
-	aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
-	aws configure set region us-east-2
+  aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
+  aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+  aws configure set region us-east-2
 
-	openssl enc -d -aes-256-cbc -md md5 -k $GCP_CREDS -in scripts/ci/gcp-creds.json.enc -out scripts/ci/gcp-creds.json
+  openssl enc -d -aes-256-cbc -md md5 -k $GCP_CREDS -in scripts/ci/gcp-creds.json.enc -out scripts/ci/gcp-creds.json
 fi

scripts/ci/install_azurite.sh

@@ -4,18 +4,18 @@ set -e
 set -x
 
 # install docker
- export DEBIAN_FRONTEND=noninteractive
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
- sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
- sudo apt-get update
- sudo apt-get install -y docker-ce
+export DEBIAN_FRONTEND=noninteractive
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+sudo apt-get update
+sudo apt-get install -y docker-ce
 
 # run azurite
 sudo docker run -d --restart always -e executable=blob -p 10000:10000 \
-	--tmpfs /opt/azurite/folder \
-	mcr.microsoft.com/azure-storage/azurite:3.3.0-preview \
-	azurite -l /data --blobHost 0.0.0.0
+  --tmpfs /opt/azurite/folder \
+  mcr.microsoft.com/azure-storage/azurite:3.3.0-preview \
+  azurite -l /data --blobHost 0.0.0.0
 
 # save secrets
-echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >> env.sh
-echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >> env.sh
+echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >>env.sh
+echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >>env.sh

scripts/ci/install_hadoop.sh

@@ -17,14 +17,14 @@ sudo mkdir hadoop
 sudo mv hadoop-2.6.5/* hadoop/
 popd
 
-echo "export HADOOP_HOME=/usr/local/hadoop" >> env.sh
-echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >> env.sh
-echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >> env.sh
-echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >> env.sh
-echo "export YARN_HOME=/usr/local/hadoop" >> env.sh
-echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >> env.sh
-echo "export JAVA_HOME=/usr/" >> env.sh
-echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> env.sh
+echo "export HADOOP_HOME=/usr/local/hadoop" >>env.sh
+echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >>env.sh
+echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >>env.sh
+echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >>env.sh
+echo "export YARN_HOME=/usr/local/hadoop" >>env.sh
+echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >>env.sh
+echo "export JAVA_HOME=/usr/" >>env.sh
+echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >>env.sh
 
 source env.sh
 

scripts/ci/install_oss.sh

@@ -5,6 +5,6 @@ set -euo pipefail
 git clone https://github.com/iterative/oss-emulator.git
 sudo docker image build -t oss:1.0 oss-emulator
 sudo docker run --detach --restart always -p 8880:8880 --name oss-emulator oss:1.0
-echo "export OSS_ENDPOINT='localhost:8880'" >> env.sh
-echo "export OSS_ACCESS_KEY_ID='AccessKeyID'" >> env.sh
-echo "export OSS_ACCESS_KEY_SECRET='AccessKeySecret'" >> env.sh
+echo "export OSS_ENDPOINT='localhost:8880'" >>env.sh
+echo "export OSS_ACCESS_KEY_ID='AccessKeyID'" >>env.sh
+echo "export OSS_ACCESS_KEY_SECRET='AccessKeySecret'" >>env.sh

scripts/ci/retry.sh

@@ -4,8 +4,8 @@ set -e
 
 N_RETRIES=3
 for i in $(seq $N_RETRIES); do
-    echo "Attempt #$i"
-    $@ && exit $?
+  echo "Attempt #$i"
+  $@ && exit $?
 done
 
 exit $?

scripts/ci/script.sh

@@ -5,8 +5,7 @@ set -e
 
 python -mtests
 
-if [[ "$TRAVIS_PULL_REQUEST" == "false" && \
-      "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
-	pip install codecov
-	codecov
+if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
+  pip install codecov
+  codecov
 fi

scripts/completion/dvc.bash

@@ -9,8 +9,8 @@
 #----------------------------------------------------------
 
 _dvc_commands='add cache checkout commit config destroy diff fetch get-url get gc \
-              import-url import init install lock metrics move pipeline pull push \
-              remote remove repro root run status unlock unprotect update version'
+  import-url import init install lock list metrics move pipeline pull push \
+  remote remove repro root run status unlock unprotect update version'
 
 _dvc_options='-h --help -V --version'
 _dvc_global_options='-h --help -q --quiet -v --verbose'
@@ -22,7 +22,7 @@ _dvc_checkout='-d --with-deps -R --recursive -f --force --relink $(compgen -G *.
 _dvc_commit='-f --force -d --with-deps -R --recursive $(compgen -G *.dvc)'
 _dvc_config='-u --unset --local --system --global'
 _dvc_destroy='-f --force'
-_dvc_diff='-t --target'
+_dvc_diff='-t --show-json --checksums'
 _dvc_fetch='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive $(compgen -G *.dvc)'
 _dvc_get_url=''
 _dvc_get='-o --out --rev --show-url'
@@ -31,12 +31,14 @@ _dvc_import_url='-f --file'
 _dvc_import='-o --out --rev'
 _dvc_init='--no-scm -f --force'
 _dvc_install=''
+_dvc_list='-R --recursive --outs-only --rev $(compgen -G *)'
 _dvc_lock='$(compgen -G *.dvc)'
 _dvc_metrics='add modify rmeove show'
 _dvc_metrics_add='-t --type -x --xpath $(compgen -G *)'
+_dvc_metrics_show='$(-t --type -x --xpath -a --all-branches -T --all-tags -R --recursive $(compgen -G *)'
+_dvc_metrics_diff='--targets -t --type -x --xpath -R --show-json'
 _dvc_metrics_modify='-t --type -x --xpath $(compgen -G *)'
 _dvc_metrics_remove='$(compgen -G *)'
-_dvc_metrics_show='$(-t --type -x --xpath -a --all-branches -T --all-tags -R --recursive $(compgen -G *)'
 _dvc_move='$(compgen -G *)'
 _dvc_pipeline='list show'
 _dvc_pipeline_list=''
@@ -56,9 +58,29 @@ _dvc_run='--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache --
 _dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud $(compgen -G *.dvc)'
 _dvc_unlock='$(compgen -G *.dvc)'
 _dvc_unprotect='$(compgen -G *)'
-_dvc_update='$(compgen -G *.dvc)'
+_dvc_update='--rev $(compgen -G *.dvc)'
 _dvc_version=''
 
+# Params
+# $1 - COMP_WORDS[1]
+comp_command() {
+  local options_list="_dvc_$(replace_hyphen $1)"
+
+  COMPREPLY=( $(compgen -W "$_dvc_global_options ${!options_list}" -- "$word") )
+}
+
+# Params
+# $1 - COMP_WORDS[1]
+# $1 - COMP_WORDS[2]
+comp_subcommand() {
+  local options_list="_dvc_$(replace_hyphen $1)_$(replace_hyphen $2)"
+  if [ -z "${!options_list}" ]; then
+    comp_command $1
+  else
+    COMPREPLY=( $(compgen -W "$_dvc_global_options ${!options_list}" -- "$word") )
+  fi
+}
+
 # Notes:
 #
 # `COMPREPLY` contains what will be rendered after completion is triggered
@@ -71,11 +93,10 @@ _dvc_version=''
 #       x="hello"
 #       ${!x} ->  ${hello} ->  "world"
 #
-_dvc () {
-  replace_hyphen () {
+_dvc() {
+  replace_hyphen() {
     echo $(echo $1 | sed 's/-/_/g')
   }
-
   local word="${COMP_WORDS[COMP_CWORD]}"
 
   COMPREPLY=()
@@ -83,16 +104,12 @@ _dvc () {
   if [ "${COMP_CWORD}" -eq 1 ]; then
     case "$word" in
       -*) COMPREPLY=($(compgen -W "$_dvc_options" -- "$word")) ;;
-      *)  COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
+      *) COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
     esac
   elif [ "${COMP_CWORD}" -eq 2 ]; then
-    local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})"
-
-    COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
+    comp_command ${COMP_WORDS[1]}
   elif [ "${COMP_CWORD}" -eq 3 ]; then
-    local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})_$(replace_hyphen ${COMP_WORDS[2]})"
-
-    COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
+    comp_subcommand ${COMP_WORDS[1]} ${COMP_WORDS[2]}
   fi
 
   return 0

scripts/completion/dvc.zsh

@@ -12,21 +12,22 @@
 
 _dvc_commands() {
   local _commands=(
-    "add:Take data files or directories under DVC control."
+    "add:Track data files or directories with DVC."
     "cache:Manage cache settings."
     "checkout:Checkout data files from cache."
     "commit:Save changed data to cache and update DVC-files."
     "config:Get or set config settings."
     "destroy:Remove DVC-files, local DVC config and data cache."
-    "diff:Show a diff of a DVC controlled data file or a directory."
-    "fetch:Fetch data files from a DVC remote storage."
+    "diff:Show changes between commits in the DVC repository, or between a commit and the workspace."
+    "fetch:Get files or directories tracked by DVC from remote storage into the cache."
     "get-url:Download or copy files from URL."
     "get:Download data from DVC repository."
-    "gc:Collect unused data from DVC cache or a remote storage."
+    "gc:Garbage collect unused objects from cache or remote storage."
     "import-url:Download or copy file from URL and take it under DVC control."
     "import:Download data from DVC repository and take it under DVC control."
     "init:Initialize DVC in the current directory."
     "install:Install DVC git hooks into the repository."
+    "list:List files."
     "lock:Lock DVC-file."
     "metrics:Commands to add, manage, collect and display metrics."
     "move:Rename or move a DVC controlled data file or a directory."
@@ -97,7 +98,10 @@ _dvc_destroy=(
 )
 
 _dvc_diff=(
-  {-t,--target}"[Source path to a data file or directory.]:Target files:"
+  "--show-json[Format the output into a JSON]"
+  "--checksums[Display checksums for each entry]"
+  "1:Old Git commit to compare (defaults to HEAD):"
+  "2:New Git commit to compare (defaults to the current workspace):"
 )
 
 _dvc_fetch=(
@@ -117,7 +121,7 @@ _dvc_geturl=(
 
 _dvc_get=(
   {-o,--out}"[Destination path to put data to.]:OUT:_files -/"
-  "--rev[Git revision (e.g. branch, tag, SHA)]:Revision:"
+  "--rev[Git revision (e.g. SHA, branch, tag)]:Revision:"
   "--show-url[Returns path/url to the location in remote for given path]"
   "1:URL:"
   "2:Path:"
@@ -141,7 +145,7 @@ _dvc_importurl=(
 
 _dvc_import=(
   {-o,--out}"[Destination path to put data to.]:OUT:_files -/"
-  "--rev[DVC repository git revision.]:Commit hash:"
+  "--rev[Git revision (e.g. SHA, branch, tag)]:Commit hash:"
   "1:URL:"
   "2:Path:"
 )
@@ -157,8 +161,16 @@ _dvc_lock=(
   "*:Stages:_files -g '(*.dvc|Dvcfile)'"
 )
 
+_dvc_list=(
+  "--rev[Git revision (e.g. branch, tag, SHA)]:Revision:"
+  {-R,--recursive}"[Recursively add each file under the directory.]"
+  "--outs-only[Only outputs DVC-outs.]"
+  "1:URL:"
+  "2:Target:"
+)
+
 _dvc_metrics=(
-  "1:Sub command:(show add modify remove)"
+  "1:Sub command:(add show diff modify remove)"
 )
 
 _dvc_move=(
@@ -261,6 +273,7 @@ _dvc_unprotect=(
 )
 
 _dvc_update=(
+  "--rev[Git revision (e.g. SHA, branch, tag)]:Revision:"
   "*:Stages:_files -g '(*.dvc|Dvcfile)'"
 )
 
@@ -273,34 +286,35 @@ _arguments \
   '*::args:->args'
 
 case $words[1] in
-  add)        _arguments $_dvc_global_options   $_dvc_add       ;;
-  cache)      _arguments $_dvc_global_options   $_dvc_cache     ;;
-  checkout)   _arguments $_dvc_global_options   $_dvc_checkout  ;;
-  commit)     _arguments $_dvc_global_options   $_dvc_commit    ;;
-  config)     _arguments $_dvc_global_options   $_dvc_config    ;;
-  destroy)    _arguments $_dvc_global_options   $_dvc_destroy   ;;
-  diff)       _arguments $_dvc_global_options   $_dvc_diff      ;;
-  fetch)      _arguments $_dvc_global_options   $_dvc_fetch     ;;
-  get-url)    _arguments $_dvc_global_options   $_dvc_geturl    ;;
-  get)        _arguments $_dvc_global_options   $_dvc_get       ;;
-  gc)         _arguments $_dvc_global_options   $_dvc_gc        ;;
-  import-url) _arguments $_dvc_global_options   $_dvc_importurl ;;
-  import)     _arguments $_dvc_global_options   $_dvc_import    ;;
-  init)       _arguments $_dvc_global_options   $_dvc_init      ;;
-  install)    _arguments $_dvc_global_options   $_dvc_install   ;;
-  lock)       _arguments $_dvc_global_options   $_dvc_lock      ;;
-  metrics)    _arguments $_dvc_global_options   $_dvc_metrics   ;;
-  move)       _arguments $_dvc_global_options   $_dvc_move      ;;
-  pipeline)   _arguments $_dvc_global_options   $_dvc_pipeline  ;;
-  pull)       _arguments $_dvc_global_options   $_dvc_pull      ;;
-  push)       _arguments $_dvc_global_options   $_dvc_push      ;;
-  remote)     _arguments $_dvc_global_options   $_dvc_remote    ;;
-  remove)     _arguments $_dvc_global_options   $_dvc_remove    ;;
-  repro)      _arguments $_dvc_global_options   $_dvc_repro     ;;
-  root)       _arguments $_dvc_global_options   $_dvc_root      ;;
-  run)        _arguments $_dvc_global_options   $_dvc_run       ;;
-  status)     _arguments $_dvc_global_options   $_dvc_status    ;;
-  unlock)     _arguments $_dvc_global_options   $_dvc_unlock    ;;
-  unprotect)  _arguments $_dvc_global_options   $_dvc_unprotect ;;
-  update)     _arguments $_dvc_global_options   $_dvc_update    ;;
+  add) _arguments $_dvc_global_options $_dvc_add ;;
+  cache) _arguments $_dvc_global_options $_dvc_cache ;;
+  checkout) _arguments $_dvc_global_options $_dvc_checkout ;;
+  commit) _arguments $_dvc_global_options $_dvc_commit ;;
+  config) _arguments $_dvc_global_options $_dvc_config ;;
+  destroy) _arguments $_dvc_global_options $_dvc_destroy ;;
+  diff) _arguments $_dvc_global_options $_dvc_diff ;;
+  fetch) _arguments $_dvc_global_options $_dvc_fetch ;;
+  get-url) _arguments $_dvc_global_options $_dvc_geturl ;;
+  get) _arguments $_dvc_global_options $_dvc_get ;;
+  gc) _arguments $_dvc_global_options $_dvc_gc ;;
+  import-url) _arguments $_dvc_global_options $_dvc_importurl ;;
+  import) _arguments $_dvc_global_options $_dvc_import ;;
+  init) _arguments $_dvc_global_options $_dvc_init ;;
+  install) _arguments $_dvc_global_options $_dvc_install ;;
+  lock) _arguments $_dvc_global_options $_dvc_lock ;;
+  list) _arguments $_dvc_global_options $_dvc_list ;;
+  metrics) _arguments $_dvc_global_options $_dvc_metrics ;;
+  move) _arguments $_dvc_global_options $_dvc_move ;;
+  pipeline) _arguments $_dvc_global_options $_dvc_pipeline ;;
+  pull) _arguments $_dvc_global_options $_dvc_pull ;;
+  push) _arguments $_dvc_global_options $_dvc_push ;;
+  remote) _arguments $_dvc_global_options $_dvc_remote ;;
+  remove) _arguments $_dvc_global_options $_dvc_remove ;;
+  repro) _arguments $_dvc_global_options $_dvc_repro ;;
+  root) _arguments $_dvc_global_options $_dvc_root ;;
+  run) _arguments $_dvc_global_options $_dvc_run ;;
+  status) _arguments $_dvc_global_options $_dvc_status ;;
+  unlock) _arguments $_dvc_global_options $_dvc_unlock ;;
+  unprotect) _arguments $_dvc_global_options $_dvc_unprotect ;;
+  update) _arguments $_dvc_global_options $_dvc_update ;;
 esac

scripts/pypi_upload.sh

@@ -3,8 +3,8 @@
 set -e
 
 if [ ! -d "dvc" ]; then
-	echo "Please run this script from repository root"
-	exit 1
+  echo "Please run this script from repository root"
+  exit 1
 fi
 
 rm -rf dist/

setup.py

@@ -48,8 +48,7 @@ install_requires = [
     "ply>=3.9",  # See https://github.com/pyinstaller/pyinstaller/issues/1945
     "colorama>=0.3.9",
     "configobj>=5.0.6",
-    "gitpython>=2.1.8",
-    "gitdb2>=2.0.6",  # See https://github.com/iterative/dvc/issues/1880
+    "gitpython>3",
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
@@ -74,9 +73,10 @@ install_requires = [
     "win-unicode-console>=0.5; sys_platform == 'win32'",
     "pywin32>=225; sys_platform == 'win32'",
     "networkx>=2.1,<2.4",
+    "pydot>=1.2.4",
     "speedcopy>=2.0.1",
     "pyfastcopy>=1.0.3",
-    "flatten-dict>=0.2.0",
+    "flatten_json>=0.1.6",
     "texttable>=0.5.2",
 ]
 
@@ -84,10 +84,7 @@ install_requires = [
 # Extra dependencies for remote integrations
 
 gs = ["google-cloud-storage==1.19.0"]
-# google-api-python-client is internal dependency of pydrive. After merge of
-# https://github.com/gsuitedevs/PyDrive/pull/180 into pydrive's master,
-# usage of google-api-python-client can be removed from DVC.
-gdrive = ["pydrive2>=1.4.0", "google-api-python-client>=1.2"]
+gdrive = ["pydrive2>=1.4.5"]
 s3 = ["boto3>=1.9.201"]
 azure = ["azure-storage-blob==2.1.0"]
 oss = ["oss2==2.6.1"]
@@ -103,7 +100,6 @@ all_remotes = gs + s3 + azure + ssh + oss + gdrive + hdfs
 # Extra dependecies to run tests
 tests_requirements = [
     "wheel>=0.31.1",
-    "pydot>=1.2.4",
     # Test requirements:
     "pytest>=4.6.0",
     "pytest-timeout>=1.3.3",
@@ -128,7 +124,7 @@ tests_requirements = [
 ]
 
 if (sys.version_info) >= (3, 6):
-    tests_requirements.append("black==19.3b0")
+    tests_requirements.append("black==19.10b0")
 
 setup(
     name="dvc",
@@ -136,7 +132,7 @@ setup(
     description="Git for data scientists - manage your code and data together",
     long_description=open("README.rst", "r").read(),
     author="Dmitry Petrov",
-    author_email="dmitry@dataversioncontrol.com",
+    author_email="dmitry@dvc.org",
     download_url="https://github.com/iterative/dvc",
     license="Apache License 2.0",
     install_requires=install_requires,

snap/snapcraft.yaml

@@ -3,7 +3,7 @@ summary: Data Version Control
 description: Git for Data & Models https://dvc.org
 adopt-info: dvc
 grade: stable
-confinement: classic  # for same reasons as snap `git-ubuntu`
+confinement: classic  # for `dvc run`ning external binaries
 base: core18
 license: Apache-2.0
 layout:
@@ -20,6 +20,8 @@ parts:
     plugin: python
     python-packages: [wheel]
     source: .
+    build-snaps:
+    - snapd
     build-packages:
     - git
     - libc-dev
@@ -27,10 +29,13 @@ parts:
     - git
     override-pull: |
         snapcraftctl pull
-        echo 'PKG = "snap"' > $SNAPCRAFT_PART_SRC/dvc/utils/build.py
         snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe --tags`
+        git diff --quiet || error_dirty_build
+        echo 'PKG = "snap"' > $SNAPCRAFT_PART_SRC/dvc/utils/build.py
         # install all optional extras
         sed -ri 's/(=install_requires)/\1+all_remotes+hdfs/' $SNAPCRAFT_PART_SRC/setup.py
+        # ensure dvc knows the state isn't really dirty
+        sed -rin 's/.*git.*diff.*--quiet.*//' $SNAPCRAFT_PART_SRC/dvc/version.py
     override-build: |
         snapcraftctl build
         cp $SNAPCRAFT_PART_BUILD/scripts/completion/dvc.bash $SNAPCRAFT_PART_INSTALL/completion.sh