dvc: bump to 1.0.0a4

* erepo: load external repos directly from git tree

* erepo: add fetch_external

* remote/RepoTree: add callback for keeping track of download counts when
saving tree to cache

* repo: use erepo.fetch_external() in repo.fetch()

* erepo: return cache save_infos in fetch_external

* DependencyRepo: use erepo.fetch_external and cache.checkout instead of
pull_to

* erepo: remove pull_to

* DvcTree: support granularity for isdir()

* DvcTree: fix indentation bug

* RepoTree: add copytree() method

* LocalRemote: use working tree for local cache walk_files

* repo: state should be noop for GitTree based erepos

* erepo: add get_external()

* test_get: remove unneeded caplog check

* erepo: add get_rev()

* tests: fix erepo test bug

* ls: use RepoTree

* remote: support trees in _collect_dir

* remote: support trees in get_dir_checksum

* RepoTree: fix dir_cache relpaths in windows

* remote: fix duplicated fetch bug

* utils: don't skip md5 dos2unix check for git blobs

* otherwise md5 checksums will not match when comparing git version to
  cached filesystem version on windows

.flake8

@@ -1,7 +0,0 @@
-[flake8]
-ignore =
-    E203, # Whitespace before ':'
-    E266, # Too many leading '#' for block comment
-    W503, # Line break occurred before a binary operator
-max-line-length = 79
-select = B,C,E,F,W,T4,B9

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+patreon: DVCorg

.pre-commit-config.yaml

@@ -4,11 +4,20 @@ repos:
     language_version: python3
   repo: https://github.com/ambv/black
   rev: 19.10b0
+-   repo: https://github.com/asottile/seed-isort-config
+    rev: v2.1.1
+    hooks:
+    -   id: seed-isort-config
+- hooks:
+  - id: isort
+    language_version: python3
+  repo: https://github.com/timothycrosley/isort
+  rev: 4.3.21
 - hooks:
   - id: flake8
     language_version: python3
   repo: https://gitlab.com/pycqa/flake8
-  rev: master
+  rev: 3.7.9
 - hooks:
   - args:
     - -i

.pre-commit-hooks.yaml

@@ -8,6 +8,7 @@
   name: DVC pre-commit
   stages:
   - commit
+  verbose: true
 - args:
   - git-hook
   - pre-push

.restyled.yaml

@@ -9,3 +9,12 @@ restylers:
     - "**/*.py"
     interpreters:
     - python
+  - name: isort
+    image: restyled/restyler-isort:v4.3.21
+    command:
+    - isort
+    arguments: []
+    include:
+    - "**/*.py"
+    interpreters:
+    - python

.travis.yml

@@ -46,9 +46,6 @@ jobs:
     env:
       - PYTHON_VERSION=3.8.0
       - PATH=/c/Python38:/c/Python38/Scripts:$PATH
-  - name: "3.5 on Linux"
-    language: python
-    python: 3.5
   - name: "3.6 on Linux"
     language: python
     python: 3.6
@@ -106,6 +103,7 @@ jobs:
     - sudo /snap/bin/lxd.migrate -yes
     - sudo /snap/bin/lxd waitready
     - sudo /snap/bin/lxd init --auto
+    - git fetch --tags
     script:
     - ./scripts/build_snap.sh
     after_failure:

dvc/__init__.py

@@ -3,17 +3,7 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-import warnings
-
 import dvc.logger
 from dvc.version import __version__  # noqa: F401
 
-
 dvc.logger.setup()
-
-# Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
-# We don't directly import numpy, but our dependency networkx does, causing
-# these warnings in some environments. Luckily these warnings are benign and
-# we can simply ignore them so that they don't show up when you are using dvc.
-warnings.filterwarnings("ignore", message="numpy.dtype size changed")
-warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

dvc/analytics.py

@@ -2,12 +2,12 @@ import json
 import logging
 import os
 import platform
-import requests
 import sys
 import tempfile
 import uuid
 
 import distro
+import requests
 
 from dvc import __version__
 from dvc.config import Config, to_bool
@@ -20,7 +20,6 @@ from dvc.scm.base import SCMError
 from dvc.utils import env2bool, is_binary
 from dvc.utils.fs import makedirs
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -36,7 +35,7 @@ def collect_and_send_report(args=None, return_code=None):
     report as a JSON, where the _collector_ generates it and the _sender_
     removes it after sending it.
     """
-    report = _runtime_info()
+    report = {}
 
     # Include command execution information on the report only when available.
     if args and hasattr(args, "func"):
@@ -62,7 +61,7 @@ def is_enabled():
     return enabled
 
 
-def send(report):
+def send(path):
     """
     Side effect: Removes the report after sending it.
 
@@ -73,13 +72,17 @@ def send(report):
     url = "https://analytics.dvc.org"
     headers = {"content-type": "application/json"}
 
-    with open(report, "rb") as fobj:
-        try:
-            requests.post(url, data=fobj, headers=headers, timeout=5)
-        except requests.exceptions.RequestException:
-            logger.debug("failed to send analytics report", exc_info=True)
+    with open(path) as fobj:
+        report = json.load(fobj)
 
-    os.remove(report)
+    report.update(_runtime_info())
+
+    try:
+        requests.post(url, json=report, headers=headers, timeout=5)
+    except requests.exceptions.RequestException:
+        logger.debug("failed to send analytics report", exc_info=True)
+
+    os.remove(path)
 
 
 def _scm_in_use():
@@ -155,7 +158,7 @@ def _find_or_create_user_id():
     try:
         with Lock(lockfile):
             try:
-                with open(fname, "r") as fobj:
+                with open(fname) as fobj:
                     user_id = json.load(fobj)["user_id"]
 
             except (FileNotFoundError, ValueError, KeyError):
@@ -167,6 +170,4 @@ def _find_or_create_user_id():
             return user_id
 
     except LockError:
-        logger.debug(
-            "Failed to acquire '{lockfile}'".format(lockfile=lockfile)
-        )
+        logger.debug(f"Failed to acquire '{lockfile}'")

dvc/api.py

@@ -1,16 +1,17 @@
 import os
-from contextlib import contextmanager, _GeneratorContextManager as GCM
+from contextlib import _GeneratorContextManager as GCM
+from contextlib import contextmanager
 
-from dvc.repo import Repo
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.external_repo import external_repo
+from dvc.repo import Repo
 
 
 class UrlNotDvcRepoError(DvcException):
     """Thrown if the given URL is not a DVC repository."""
 
     def __init__(self, url):
-        super().__init__("'{}' is not a DVC repository.".format(url))
+        super().__init__(f"'{url}' is not a DVC repository.")
 
 
 def get_url(path, repo=None, rev=None, remote=None):

dvc/cache.py

@@ -40,7 +40,7 @@ def _make_remote_property(name):
     return cached_property(getter)
 
 
-class Cache(object):
+class Cache:
     """Class that manages cache locations of a DVC repo.
 
     Args:
@@ -71,9 +71,39 @@ class Cache(object):
     azure = _make_remote_property("azure")
 
 
-class NamedCache(object):
+class NamedCacheItem:
     def __init__(self):
-        self._items = defaultdict(lambda: defaultdict(set))
+        self.names = set()
+        self.children = defaultdict(NamedCacheItem)
+
+    def __eq__(self, other):
+        return self.names == other.names and self.children == other.children
+
+    def child_keys(self):
+        for key, child in self.children.items():
+            yield key
+            yield from child.child_keys()
+
+    def child_names(self):
+        for key, child in self.children.items():
+            yield key, child.names
+            yield from child.child_names()
+
+    def add(self, checksum, item):
+        self.children[checksum].update(item)
+
+    def update(self, item, suffix=""):
+        if suffix:
+            self.names.update(n + suffix for n in item.names)
+        else:
+            self.names.update(item.names)
+        for checksum, child_item in item.children.items():
+            self.children[checksum].update(child_item)
+
+
+class NamedCache:
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(NamedCacheItem))
         self.external = defaultdict(set)
 
     @classmethod
@@ -86,7 +116,18 @@ class NamedCache(object):
         return self._items[key]
 
     def add(self, scheme, checksum, name):
-        self._items[scheme][checksum].add(name)
+        """Add a mapped name for the specified checksum."""
+        self._items[scheme][checksum].names.add(name)
+
+    def add_child_cache(self, checksum, cache, suffix=""):
+        """Add/update child cache for the specified checksum."""
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme][checksum].children
+            for child_checksum, item in src.items():
+                dst[child_checksum].update(item, suffix=suffix)
+
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)
 
     def add_external(self, url, rev, path):
         self.external[url, rev].add(path)
@@ -94,11 +135,32 @@ class NamedCache(object):
     def update(self, cache, suffix=""):
         for scheme, src in cache._items.items():
             dst = self._items[scheme]
-            for checksum, names in src.items():
-                if suffix:
-                    dst[checksum].update(n + suffix for n in names)
-                else:
-                    dst[checksum].update(names)
+            for checksum, item in src.items():
+                dst[checksum].update(item, suffix=suffix)
 
         for repo_pair, files in cache.external.items():
             self.external[repo_pair].update(files)
+
+    def scheme_keys(self, scheme):
+        """Iterate over a flat list of all keys for the specified scheme,
+        including children.
+        """
+        for key, item in self._items[scheme].items():
+            yield key
+            yield from item.child_keys()
+
+    def scheme_names(self, scheme):
+        """Iterate over a flat list of checksum, names items for the specified
+        scheme, including children.
+        """
+        for key, item in self._items[scheme].items():
+            yield key, item.names
+            yield from item.child_names()
+
+    def dir_keys(self, scheme):
+        return (
+            key for key, item in self._items[scheme].items() if item.children
+        )
+
+    def child_keys(self, scheme, checksum):
+        return self._items[scheme][checksum].child_keys()

dvc/cli.py

@@ -16,30 +16,30 @@ from .command import (
     gc,
     get,
     get_url,
+    git_hook,
     imp,
     imp_url,
     init,
     install,
-    ls,
     lock,
+    ls,
     metrics,
     move,
+    params,
     pipeline,
+    plots,
     remote,
     remove,
     repro,
     root,
     run,
-    tag,
     unprotect,
     update,
     version,
-    git_hook,
 )
 from .command.base import fix_subparsers
 from .exceptions import DvcParserError
 
-
 logger = logging.getLogger(__name__)
 
 COMMANDS = [
@@ -62,6 +62,7 @@ COMMANDS = [
     remote,
     cache,
     metrics,
+    params,
     install,
     root,
     ls,
@@ -69,11 +70,11 @@ COMMANDS = [
     pipeline,
     daemon,
     commit,
-    tag,
     diff,
     version,
     update,
     git_hook,
+    plots,
 ]
 
 
@@ -115,14 +116,10 @@ def get_parent_parser():
 
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
-        "-q", "--quiet", action="store_true", default=False, help="Be quiet."
+        "-q", "--quiet", action="count", default=0, help="Be quiet."
     )
     log_level_group.add_argument(
-        "-v",
-        "--verbose",
-        action="store_true",
-        default=False,
-        help="Be verbose.",
+        "-v", "--verbose", action="count", default=0, help="Be verbose."
     )
 
     return parent_parser

dvc/command/add.py

@@ -1,7 +1,7 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link, CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
 logger = logging.getLogger(__name__)
@@ -53,6 +53,7 @@ def add_parser(subparsers, parent_parser):
         "-f",
         "--file",
         help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
     )
     parser.add_argument(
         "targets", nargs="+", help="Input files/directories to add."

dvc/command/base.py

@@ -1,6 +1,5 @@
 import logging
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -27,7 +26,7 @@ def append_doc_link(help_message, path):
     )
 
 
-class CmdBase(object):
+class CmdBase:
     def __init__(self, args):
         from dvc.repo import Repo
         from dvc.updater import Updater
@@ -36,17 +35,16 @@ class CmdBase(object):
         self.config = self.repo.config
         self.args = args
         hardlink_lock = self.config["core"].get("hardlink_lock", False)
-        updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
+        updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
         updater.check()
 
     @property
     def default_targets(self):
-        """Default targets for `dvc repro` and `dvc pipeline`."""
-        from dvc.stage import Stage
+        """Default targets for `dvc repro`."""
+        from dvc.dvcfile import PIPELINE_FILE
 
-        msg = "assuming default target '{}'.".format(Stage.STAGE_FILE)
-        logger.warning(msg)
-        return [Stage.STAGE_FILE]
+        logger.debug(f"assuming default target '{PIPELINE_FILE}'.")
+        return [PIPELINE_FILE]
 
     # Abstract methods that have to be implemented by any inheritance class
     def run(self):

dvc/command/checkout.py

@@ -1,39 +1,16 @@
 import argparse
 import logging
+import operator
 
 import colorama
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import CheckoutError
+from dvc.utils.humanize import get_summary
 
 logger = logging.getLogger(__name__)
 
 
-def _human_join(words):
-    words = list(words)
-    if not words:
-        return ""
-
-    return (
-        "{before} and {after}".format(
-            before=", ".join(words[:-1]), after=words[-1],
-        )
-        if len(words) > 1
-        else words[0]
-    )
-
-
-def log_summary(stats):
-    states = ("added", "deleted", "modified")
-    summary = (
-        "{} {}".format(len(stats[state]), state)
-        for state in states
-        if stats.get(state)
-    )
-    logger.info(_human_join(summary) or "No changes.")
-
-
 def log_changes(stats):
     colors = [
         ("modified", colorama.Fore.YELLOW,),
@@ -75,7 +52,11 @@ class CmdCheckout(CmdBase):
             stats = exc.stats
 
         if self.args.summary:
-            log_summary(stats)
+            default_message = "No changes."
+            msg = get_summary(
+                sorted(stats.items(), key=operator.itemgetter(0))
+            )
+            logger.info(msg or default_message)
         else:
             log_changes(stats)
 

dvc/command/commit.py

@@ -1,11 +1,9 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 

dvc/command/config.py

@@ -1,7 +1,7 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link, CmdBaseNoRepo
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.config import Config, ConfigError
 
 logger = logging.getLogger(__name__)

dvc/command/daemon.py

@@ -1,5 +1,4 @@
-from dvc.command.base import CmdBaseNoRepo
-from dvc.command.base import fix_subparsers
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
 
 
 class CmdDaemonBase(CmdBaseNoRepo):
@@ -15,9 +14,10 @@ class CmdDaemonUpdater(CmdDaemonBase):
 
         root_dir = Repo.find_root()
         dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
+        tmp_dir = os.path.join(dvc_dir, "tmp")
         config = Config(dvc_dir, validate=False)
         hardlink_lock = config.get("core", {}).get("hardlink_lock", False)
-        updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
+        updater = Updater(tmp_dir, hardlink_lock=hardlink_lock)
         updater.fetch(detach=False)
 
         return 0

dvc/command/data_sync.py

@@ -1,23 +1,25 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.command.checkout import log_summary
-from dvc.exceptions import DvcException, CheckoutError
-
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.checkout import log_changes
+from dvc.exceptions import CheckoutError, DvcException
+from dvc.utils.humanize import get_summary
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
-    @classmethod
-    def check_up_to_date(cls, processed_files_count):
-        if processed_files_count == 0:
-            logger.info("Everything is up to date.")
+    def log_summary(self, stats):
+        default_msg = "Everything is up to date."
+        logger.info(get_summary(stats.items()) or default_msg)
 
 
 class CmdDataPull(CmdDataBase):
+    def log_summary(self, stats):
+        log_changes(stats)
+        super().log_summary(stats)
+
     def run(self):
         try:
             stats = self.repo.pull(
@@ -26,17 +28,18 @@ class CmdDataPull(CmdDataBase):
                 remote=self.args.remote,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
                 force=self.args.force,
                 recursive=self.args.recursive,
+                run_cache=self.args.run_cache,
             )
-            log_summary(stats)
+            self.log_summary(stats)
         except (CheckoutError, DvcException) as exc:
-            log_summary(getattr(exc, "stats", {}))
+            self.log_summary(getattr(exc, "stats", {}))
             logger.exception("failed to pull data from the cloud")
             return 1
 
-        self.check_up_to_date(stats["downloaded"])
         return 0
 
 
@@ -49,13 +52,15 @@ class CmdDataPush(CmdDataBase):
                 remote=self.args.remote,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
+                run_cache=self.args.run_cache,
             )
+            self.log_summary({"pushed": processed_files_count})
         except DvcException:
             logger.exception("failed to push data to the cloud")
             return 1
-        self.check_up_to_date(processed_files_count)
         return 0
 
 
@@ -68,13 +73,15 @@ class CmdDataFetch(CmdDataBase):
                 remote=self.args.remote,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
+                run_cache=self.args.run_cache,
             )
+            self.log_summary({"fetched": processed_files_count})
         except DvcException:
             logger.exception("failed to fetch data from the cloud")
             return 1
-        self.check_up_to_date(processed_files_count)
         return 0
 
 
@@ -82,20 +89,24 @@ def shared_parent_parser():
     from dvc.cli import get_parent_parser
 
     # Parent parser used in pull/push/status
-    shared_parent_parser = argparse.ArgumentParser(
+    parent_parser = argparse.ArgumentParser(
         add_help=False, parents=[get_parent_parser()]
     )
-    shared_parent_parser.add_argument(
-        "-j", "--jobs", type=int, help="Number of jobs to run simultaneously."
+    parent_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help="Number of jobs to run simultaneously.",
+        metavar="<number>",
     )
-    shared_parent_parser.add_argument(
+    parent_parser.add_argument(
         "targets",
         nargs="*",
         help="Limit command scope to these DVC-files. "
         "Using -R, directories to search DVC-files in can also be given.",
     )
 
-    return shared_parent_parser
+    return parent_parser
 
 
 def add_parser(subparsers, _parent_parser):
@@ -112,7 +123,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pull_parser.add_argument(
-        "-r", "--remote", help="Remote repository to pull from."
+        "-r", "--remote", help="Remote storage to pull from", metavar="<name>",
     )
     pull_parser.add_argument(
         "-a",
@@ -128,6 +139,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch cache for all tags.",
     )
+    pull_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all commits.",
+    )
     pull_parser.add_argument(
         "-f",
         "--force",
@@ -149,6 +166,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Pull cache for subdirectories of the specified directory.",
     )
+    pull_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
@@ -162,7 +185,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     push_parser.add_argument(
-        "-r", "--remote", help="Remote repository to push to."
+        "-r", "--remote", help="Remote storage to push to", metavar="<name>",
     )
     push_parser.add_argument(
         "-a",
@@ -178,6 +201,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Push cache for all tags.",
     )
+    push_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Push cache for all commits.",
+    )
     push_parser.add_argument(
         "-d",
         "--with-deps",
@@ -192,6 +221,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Push cache for subdirectories of specified directory.",
     )
+    push_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
@@ -208,7 +243,10 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     fetch_parser.add_argument(
-        "-r", "--remote", help="Remote repository to fetch from."
+        "-r",
+        "--remote",
+        help="Remote storage to fetch from",
+        metavar="<name>",
     )
     fetch_parser.add_argument(
         "-a",
@@ -224,6 +262,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch cache for all tags.",
     )
+    fetch_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all commits.",
+    )
     fetch_parser.add_argument(
         "-d",
         "--with-deps",
@@ -238,6 +282,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch cache for subdirectories of specified directory.",
     )
+    fetch_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
@@ -271,7 +321,10 @@ def add_parser(subparsers, _parent_parser):
         help="Show status of a local cache compared to a remote repository.",
     )
     status_parser.add_argument(
-        "-r", "--remote", help="Remote repository to compare local cache to."
+        "-r",
+        "--remote",
+        help="Remote storage to compare local cache to",
+        metavar="<name>",
     )
     status_parser.add_argument(
         "-a",
@@ -289,6 +342,13 @@ def add_parser(subparsers, _parent_parser):
         help="Show status of a local cache compared to a remote repository "
         "for all tags.",
     )
+    status_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show status of a local cache compared to a remote repository "
+        "for all commits.",
+    )
     status_parser.add_argument(
         "-d",
         "--with-deps",
@@ -296,4 +356,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Show status for all dependencies of the specified target.",
     )
+    status_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Show status of all stages in the specified directory.",
+    )
+
     status_parser.set_defaults(func=CmdDataStatus)

dvc/command/destroy.py

@@ -2,11 +2,9 @@ import argparse
 import logging
 
 import dvc.prompt as prompt
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 

dvc/command/diff.py

@@ -8,10 +8,24 @@ import colorama
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
+def _show_md(diff):
+    from dvc.utils.diff import table
+
+    rows = []
+    for status in ["added", "deleted", "modified"]:
+        entries = diff.get(status, [])
+        if not entries:
+            continue
+        paths = sorted([entry["path"] for entry in entries])
+        for path in paths:
+            rows.append([status, path])
+
+    return table(["Status", "Path"], rows, True)
+
+
 class CmdDiff(CmdBase):
     @staticmethod
     def _format(diff):
@@ -97,20 +111,17 @@ class CmdDiff(CmdBase):
         try:
             diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
 
-            if not any(diff.values()):
-                return 0
-
             if not self.args.show_hash:
                 for _, entries in diff.items():
                     for entry in entries:
                         del entry["hash"]
 
             if self.args.show_json:
-                res = json.dumps(diff)
-            else:
-                res = self._format(diff)
-
-            logger.info(res)
+                logger.info(json.dumps(diff))
+            elif self.args.show_md:
+                logger.info(_show_md(diff))
+            elif diff:
+                logger.info(self._format(diff))
 
         except DvcException:
             logger.exception("failed to get diff")
@@ -120,9 +131,8 @@ class CmdDiff(CmdBase):
 
 def add_parser(subparsers, parent_parser):
     DIFF_DESCRIPTION = (
-        "Show changes between commits in the DVC repository,"
-        " or between a commit and the workspace. "
-        "Prints the list of paths added, modified, or deleted."
+        "Show added, modified, or deleted data between commits in the DVC"
+        " repository, or between a commit and the workspace."
     )
     diff_parser = subparsers.add_parser(
         "diff",
@@ -154,4 +164,10 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
     )
+    diff_parser.add_argument(
+        "--show-md",
+        help="Show tabulated output in the Markdown format (GFM).",
+        action="store_true",
+        default=False,
+    )
     diff_parser.set_defaults(func=CmdDiff)

dvc/command/gc.py

@@ -3,9 +3,7 @@ import logging
 import os
 
 import dvc.prompt as prompt
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-
+from dvc.command.base import CmdBase, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -109,7 +107,10 @@ def add_parser(subparsers, parent_parser):
         help="Collect garbage in remote repository.",
     )
     gc_parser.add_argument(
-        "-r", "--remote", help="Remote storage to collect garbage in."
+        "-r",
+        "--remote",
+        help="Remote storage to collect garbage in",
+        metavar="<name>",
     )
     gc_parser.add_argument(
         "-f",
@@ -119,7 +120,11 @@ def add_parser(subparsers, parent_parser):
         help="Force garbage collection - automatically agree to all prompts.",
     )
     gc_parser.add_argument(
-        "-j", "--jobs", type=int, help="Number of jobs to run simultaneously."
+        "-j",
+        "--jobs",
+        type=int,
+        help="Number of jobs to run simultaneously.",
+        metavar="<number>",
     )
     gc_parser.add_argument(
         "-p",
@@ -130,5 +135,6 @@ def add_parser(subparsers, parent_parser):
         help="Keep data files required by these projects "
         "in addition to the current one. "
         "Useful if you share a single cache across repos.",
+        metavar="<paths>",
     )
     gc_parser.set_defaults(func=CmdGC)

dvc/command/get.py

@@ -1,10 +1,9 @@
 import argparse
 import logging
 
-from .base import append_doc_link
-from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
+from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -66,10 +65,17 @@ def add_parser(subparsers, parent_parser):
         "path", help="Path to a file or directory within the repository"
     )
     get_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to download files to"
+        "-o",
+        "--out",
+        nargs="?",
+        help="Destination path to download files to",
+        metavar="<path>",
     )
     get_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
     )
     get_parser.add_argument(
         "--show-url",

dvc/command/get_url.py

@@ -1,10 +1,9 @@
 import argparse
 import logging
 
-from .base import append_doc_link
-from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
+from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -17,7 +16,7 @@ class CmdGetUrl(CmdBaseNoRepo):
             Repo.get_url(self.args.url, out=self.args.out)
             return 0
         except DvcException:
-            logger.exception("failed to get '{}'".format(self.args.url))
+            logger.exception(f"failed to get '{self.args.url}'")
             return 1
 
 

dvc/command/imp.py

@@ -1,11 +1,9 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -48,9 +46,16 @@ def add_parser(subparsers, parent_parser):
         "path", help="Path to a file or directory within the repository"
     )
     import_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to download files to"
+        "-o",
+        "--out",
+        nargs="?",
+        help="Destination path to download files to",
+        metavar="<path>",
     )
     import_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
     )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -1,11 +1,9 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -59,6 +57,9 @@ def add_parser(subparsers, parent_parser):
         "out", nargs="?", help="Destination path to put files to."
     )
     import_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC-file it generates."
+        "-f",
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
     )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/command/init.py

@@ -1,9 +1,7 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
-
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -47,7 +45,7 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
         help="Initiate DVC in directory that is "
-        "not tracked by any scm tool (e.g. git).",
+        "not tracked by any SCM tool (e.g. Git).",
     )
     init_parser.add_argument(
         "-f",

dvc/command/install.py

@@ -1,9 +1,7 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-
+from dvc.command.base import CmdBase, append_doc_link
 
 logger = logging.getLogger(__name__)
 

dvc/command/lock.py

@@ -1,11 +1,9 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 

dvc/command/ls/__init__.py

@@ -2,12 +2,10 @@ import argparse
 import logging
 import sys
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.command.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -33,14 +31,14 @@ class CmdList(CmdBaseNoRepo):
                 self.args.path,
                 rev=self.args.rev,
                 recursive=self.args.recursive,
-                outs_only=self.args.outs_only,
+                dvc_only=self.args.dvc_only,
             )
             if entries:
                 entries = _prettify(entries, sys.stdout.isatty())
                 logger.info("\n".join(entries))
             return 0
         except DvcException:
-            logger.exception("failed to list '{}'".format(self.args.url))
+            logger.exception(f"failed to list '{self.args.url}'")
             return 1
 
 
@@ -64,10 +62,13 @@ def add_parser(subparsers, parent_parser):
         help="Recursively list files.",
     )
     list_parser.add_argument(
-        "--outs-only", action="store_true", help="Show only DVC outputs."
+        "--dvc-only", action="store_true", help="Show only DVC outputs."
     )
     list_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
     )
     list_parser.add_argument(
         "path",

dvc/command/ls/ls_colors.py

@@ -1,7 +1,7 @@
 import os
 
 
-class LsColors(object):
+class LsColors:
     default = "rs=0:di=01;34:ex=01;32"
 
     def __init__(self, lscolors=None):
@@ -45,4 +45,4 @@ class LsColors(object):
         if not val:
             return text
         rs = self._codes.get("rs", 0)
-        return "\033[{}m{}\033[{}m".format(val, text, rs)
+        return f"\033[{val}m{text}\033[{rs}m"

dvc/command/metrics.py

@@ -1,46 +1,34 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import BadMetricError
-from dvc.exceptions import DvcException
-
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import BadMetricError, DvcException
 
 logger = logging.getLogger(__name__)
 
 
-def show_metrics(metrics, all_branches=False, all_tags=False):
-    """
-    Args:
-        metrics (list): Where each element is either a `list`
-            if an xpath was specified, otherwise a `str`
-    """
+def show_metrics(
+    metrics, all_branches=False, all_tags=False, all_commits=False
+):
+    from flatten_json import flatten
+    from dvc.utils.diff import format_dict
+
     # When `metrics` contains a `None` key, it means that some files
     # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
     missing = metrics.pop(None, None)
 
     for branch, val in metrics.items():
-        if all_branches or all_tags:
-            logger.info("{branch}:".format(branch=branch))
+        if all_branches or all_tags or all_commits:
+            logger.info(f"{branch}:")
 
         for fname, metric in val.items():
-            if isinstance(metric, dict):
-                lines = list(metric.values())
-            elif isinstance(metric, list):
-                lines = metric
-            else:
-                lines = metric.splitlines()
+            if not isinstance(metric, dict):
+                logger.info("\t{}: {}".format(fname, str(metric)))
+                continue
 
-            if len(lines) > 1:
-                logger.info("\t{fname}:".format(fname=fname))
-
-                for line in lines:
-                    logger.info("\t\t{content}".format(content=line))
-
-            else:
-                logger.info("\t{}: {}".format(fname, metric))
+            logger.info(f"\t{fname}:")
+            for key, value in flatten(format_dict(metric), ".").items():
+                logger.info(f"\t\t{key}: {value}")
 
     if missing:
         raise BadMetricError(missing)
@@ -51,14 +39,23 @@ class CmdMetricsShow(CmdBase):
         try:
             metrics = self.repo.metrics.show(
                 self.args.targets,
-                typ=self.args.type,
-                xpath=self.args.xpath,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 recursive=self.args.recursive,
             )
 
-            show_metrics(metrics, self.args.all_branches, self.args.all_tags)
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(metrics))
+            else:
+                show_metrics(
+                    metrics,
+                    self.args.all_branches,
+                    self.args.all_tags,
+                    self.args.all_commits,
+                )
         except DvcException:
             logger.exception("failed to show metrics")
             return 1
@@ -66,27 +63,12 @@ class CmdMetricsShow(CmdBase):
         return 0
 
 
-class CmdMetricsModify(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.modify(
-                self.args.path, typ=self.args.type, xpath=self.args.xpath
-            )
-        except DvcException:
-            logger.exception("failed to modify metric file settings")
-            return 1
-
-        return 0
-
-
 class CmdMetricsAdd(CmdBase):
     def run(self):
         try:
-            self.repo.metrics.add(
-                self.args.path, self.args.type, self.args.xpath
-            )
+            self.repo.metrics.add(self.args.path)
         except DvcException:
-            msg = "failed to add metric file '{}'".format(self.args.path)
+            msg = f"failed to add metric file '{self.args.path}'"
             logger.exception(msg)
             return 1
 
@@ -98,38 +80,39 @@ class CmdMetricsRemove(CmdBase):
         try:
             self.repo.metrics.remove(self.args.path)
         except DvcException:
-            msg = "failed to remove metric file '{}'".format(self.args.path)
+            msg = f"failed to remove metric file '{self.args.path}'"
             logger.exception(msg)
             return 1
 
         return 0
 
 
-def _show_diff(diff):
-    from texttable import Texttable
+def _show_diff(diff, markdown=False, no_path=False, old=False):
+    from collections import OrderedDict
 
-    if not diff:
-        return "No changes."
+    from dvc.utils.diff import table
 
-    table = Texttable()
-
-    # remove borders to make it easier for users to copy stuff
-    table.set_chars(("", "", "", ""))
-    table.set_deco(0)
-
-    rows = [["Path", "Metric", "Value", "Change"]]
+    rows = []
     for fname, mdiff in diff.items():
-        for metric, change in mdiff.items():
-            rows.append(
-                [
-                    fname,
-                    metric,
-                    change["new"],
-                    change.get("diff", "diff not supported"),
-                ]
-            )
-    table.add_rows(rows)
-    return table.draw()
+        sorted_mdiff = OrderedDict(sorted(mdiff.items()))
+        for metric, change in sorted_mdiff.items():
+            row = [] if no_path else [fname]
+            row.append(metric)
+            if old:
+                row.append(change.get("old"))
+            row.append(change["new"])
+            row.append(change.get("diff", "diff not supported"))
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append("Metric")
+    if old:
+        header.extend(["Old", "New"])
+    else:
+        header.append("Value")
+    header.append("Change")
+
+    return table(header, rows, markdown)
 
 
 class CmdMetricsDiff(CmdBase):
@@ -139,9 +122,8 @@ class CmdMetricsDiff(CmdBase):
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
                 targets=self.args.targets,
-                typ=self.args.type,
-                xpath=self.args.xpath,
                 recursive=self.args.recursive,
+                all=self.args.all,
             )
 
             if self.args.show_json:
@@ -149,7 +131,11 @@ class CmdMetricsDiff(CmdBase):
 
                 logger.info(json.dumps(diff))
             else:
-                logger.info(_show_diff(diff))
+                table = _show_diff(
+                    diff, self.args.show_md, self.args.no_path, self.args.old
+                )
+                if table:
+                    logger.info(table)
 
         except DvcException:
             logger.exception("failed to show metrics diff")
@@ -176,6 +162,17 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(metrics_subparsers)
 
+    METRICS_ADD_HELP = "Mark a DVC-tracked file as a metric."
+    metrics_add_parser = metrics_subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
+        help=METRICS_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_add_parser.add_argument("path", help="Path to a metric file.")
+    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
+
     METRICS_SHOW_HELP = "Print metrics, with optional formatting."
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
@@ -189,18 +186,6 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="Metric files or directories (see -R) to show",
     )
-    metrics_show_parser.add_argument(
-        "-t",
-        "--type",
-        help=(
-            "Type of metrics (json/tsv/htsv/csv/hcsv). "
-            "It can be detected by the file extension automatically. "
-            "Unsupported types will be treated as raw."
-        ),
-    )
-    metrics_show_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
-    )
     metrics_show_parser.add_argument(
         "-a",
         "--all-branches",
@@ -215,6 +200,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show metrics for all tags.",
     )
+    metrics_show_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show metrics for all commits.",
+    )
     metrics_show_parser.add_argument(
         "-R",
         "--recursive",
@@ -225,53 +216,14 @@ def add_parser(subparsers, parent_parser):
             "metric files."
         ),
     )
+    metrics_show_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
-    METRICS_ADD_HELP = "Mark a DVC-tracked file as a metric."
-    metrics_add_parser = metrics_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
-        help=METRICS_ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_add_parser.add_argument(
-        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
-    )
-    metrics_add_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
-    )
-    metrics_add_parser.add_argument("path", help="Path to a metric file.")
-    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
-
-    METRICS_MODIFY_HELP = "Modify metric default formatting."
-    metrics_modify_parser = metrics_subparsers.add_parser(
-        "modify",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_MODIFY_HELP, "metrics/modify"),
-        help=METRICS_MODIFY_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_modify_parser.add_argument(
-        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
-    )
-    metrics_modify_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
-    )
-    metrics_modify_parser.add_argument("path", help="Path to a metric file.")
-    metrics_modify_parser.set_defaults(func=CmdMetricsModify)
-
-    METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
-    metrics_remove_parser = metrics_subparsers.add_parser(
-        "remove",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
-        help=METRICS_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
-    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
-
     METRICS_DIFF_HELP = "Show changes in metrics between commits"
     " in the DVC repository, or between a commit and the workspace."
     metrics_diff_parser = metrics_subparsers.add_parser(
@@ -296,18 +248,7 @@ def add_parser(subparsers, parent_parser):
             "Metric files or directories (see -R) to show diff for. "
             "Shows diff for all metric files by default."
         ),
-    )
-    metrics_diff_parser.add_argument(
-        "-t",
-        "--type",
-        help=(
-            "Type of metrics (json/tsv/htsv/csv/hcsv). "
-            "It can be detected by the file extension automatically. "
-            "Unsupported types will be treated as raw."
-        ),
-    )
-    metrics_diff_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
+        metavar="<paths>",
     )
     metrics_diff_parser.add_argument(
         "-R",
@@ -319,10 +260,45 @@ def add_parser(subparsers, parent_parser):
             "metric files."
         ),
     )
+    metrics_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged metrics as well.",
+    )
     metrics_diff_parser.add_argument(
         "--show-json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
     )
+    metrics_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    metrics_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show metric path.",
+    )
+    metrics_diff_parser.add_argument(
+        "--old",
+        action="store_true",
+        default=False,
+        help="Show old metric value.",
+    )
     metrics_diff_parser.set_defaults(func=CmdMetricsDiff)
+
+    METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
+    metrics_remove_parser = metrics_subparsers.add_parser(
+        "remove",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
+        help=METRICS_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
+    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)

dvc/command/move.py

@@ -1,11 +1,9 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 

dvc/command/params.py

@@ -0,0 +1,118 @@
+import argparse
+import logging
+from collections import OrderedDict
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _show_diff(diff, markdown=False, no_path=False):
+    from dvc.utils.diff import table
+
+    rows = []
+    for fname, pdiff in diff.items():
+        sorted_pdiff = OrderedDict(sorted(pdiff.items()))
+        for param, change in sorted_pdiff.items():
+            row = [] if no_path else [fname]
+            row.append(param)
+            row.append(change["old"])
+            row.append(change["new"])
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append("Param")
+    header.append("Old")
+    header.append("New")
+
+    return table(header, rows, markdown)
+
+
+class CmdParamsDiff(CmdBase):
+    def run(self):
+        try:
+            diff = self.repo.params.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                all=self.args.all,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                table = _show_diff(diff, self.args.show_md, self.args.no_path)
+                if table:
+                    logger.info(table)
+
+        except DvcException:
+            logger.exception("failed to show params diff")
+            return 1
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    PARAMS_HELP = "Commands to display params."
+
+    params_parser = subparsers.add_parser(
+        "params",
+        parents=[parent_parser],
+        description=append_doc_link(PARAMS_HELP, "params"),
+        help=PARAMS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    params_subparsers = params_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc params CMD --help` to display command-specific help.",
+    )
+
+    fix_subparsers(params_subparsers)
+
+    PARAMS_DIFF_HELP = (
+        "Show changes in params between commits in the DVC repository, or "
+        "between a commit and the workspace."
+    )
+    params_diff_parser = params_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(PARAMS_DIFF_HELP, "params/diff"),
+        help=PARAMS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    params_diff_parser.add_argument(
+        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
+    )
+    params_diff_parser.add_argument(
+        "b_rev",
+        nargs="?",
+        help=("New Git commit to compare (defaults to the current workspace)"),
+    )
+    params_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged params as well.",
+    )
+    params_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    params_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    params_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show params path.",
+    )
+    params_diff_parser.set_defaults(func=CmdParamsDiff)

dvc/command/pipeline.py

@@ -4,19 +4,19 @@ import logging
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
 class CmdPipelineShow(CmdBase):
     def _show(self, target, commands, outs, locked):
         import networkx
-        from dvc.stage import Stage
+        from dvc import dvcfile
+        from dvc.utils import parse_target
 
-        stage = Stage.load(self.repo, target)
+        path, name = parse_target(target)
+        stage = dvcfile.Dvcfile(self.repo, path).stages[name]
         G = self.repo.graph
         stages = networkx.dfs_postorder_nodes(G, stage)
-
         if locked:
             stages = [s for s in stages if s.locked]
 
@@ -29,14 +29,41 @@ class CmdPipelineShow(CmdBase):
                 for out in stage.outs:
                     logger.info(str(out))
             else:
-                logger.info(stage.path_in_repo)
+                logger.info(stage.addressing)
 
-    def _build_graph(self, target, commands, outs):
+    @staticmethod
+    def _build_output_graph(G, target_stage):
         import networkx
-        from dvc.stage import Stage
-        from dvc.repo.graph import get_pipeline
+        from itertools import product
 
-        target_stage = Stage.load(self.repo, target)
+        nodes = {str(out) for out in target_stage.outs}
+        edges = []
+
+        for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
+            from_stage_deps = {dep.path_info.parts for dep in from_stage.deps}
+            to_outs = {
+                to_out
+                for to_out in to_stage.outs
+                if to_out.path_info.parts in from_stage_deps
+            }
+            from_outs = {
+                from_out
+                for from_out in from_stage.outs
+                if str(from_out) in nodes
+            }
+            nodes |= {str(to_out) for to_out in to_outs}
+            for from_out, to_out in product(from_outs, to_outs):
+                edges.append((str(from_out), str(to_out)))
+        return nodes, edges
+
+    def _build_graph(self, target, commands=False, outs=False):
+        import networkx
+        from dvc import dvcfile
+        from dvc.repo.graph import get_pipeline
+        from dvc.utils import parse_target
+
+        path, name = parse_target(target)
+        target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
         G = get_pipeline(self.repo.pipelines, target_stage)
 
         nodes = set()
@@ -45,11 +72,8 @@ class CmdPipelineShow(CmdBase):
                 if stage.cmd is None:
                     continue
                 nodes.add(stage.cmd)
-            elif outs:
-                for out in stage.outs:
-                    nodes.add(str(out))
-            else:
-                nodes.add(stage.relpath)
+            elif not outs:
+                nodes.add(stage.addressing)
 
         edges = []
         for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
@@ -57,12 +81,11 @@ class CmdPipelineShow(CmdBase):
                 if to_stage.cmd is None:
                     continue
                 edges.append((from_stage.cmd, to_stage.cmd))
-            elif outs:
-                for from_out in from_stage.outs:
-                    for to_out in to_stage.outs:
-                        edges.append((str(from_out), str(to_out)))
-            else:
-                edges.append((from_stage.relpath, to_stage.relpath))
+            elif not outs:
+                edges.append((from_stage.addressing, to_stage.addressing))
+
+        if outs:
+            nodes, edges = self._build_output_graph(G, target_stage)
 
         return list(nodes), edges, networkx.is_tree(G)
 
@@ -116,8 +139,10 @@ class CmdPipelineShow(CmdBase):
         logger.info(dot_file.getvalue())
 
     def run(self):
+        from dvc.dvcfile import DVC_FILE
+
         if not self.args.targets:
-            self.args.targets = self.default_targets
+            self.args.targets = [DVC_FILE]
 
         for target in self.args.targets:
             try:
@@ -139,7 +164,7 @@ class CmdPipelineShow(CmdBase):
                         self.args.locked,
                     )
             except DvcException:
-                msg = "failed to show pipeline for '{}'".format(target)
+                msg = f"failed to show pipeline for '{target}'"
                 logger.exception(msg)
                 return 1
         return 0
@@ -150,7 +175,7 @@ class CmdPipelineList(CmdBase):
         pipelines = self.repo.pipelines
         for pipeline in pipelines:
             for stage in pipeline:
-                logger.info(stage.relpath)
+                logger.info(stage.addressing)
             if len(pipeline) != 0:
                 logger.info("=" * 80)
         logger.info("{} pipelines total".format(len(pipelines)))
@@ -175,6 +200,16 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(pipeline_subparsers)
 
+    PIPELINE_LIST_HELP = "List connected groups of stages (pipelines)."
+    pipeline_list_parser = pipeline_subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
+        help=PIPELINE_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    pipeline_list_parser.set_defaults(func=CmdPipelineList)
+
     PIPELINE_SHOW_HELP = "Show stages in a pipeline."
     pipeline_show_parser = pipeline_subparsers.add_parser(
         "show",
@@ -230,13 +265,3 @@ def add_parser(subparsers, parent_parser):
         "(Finds all DVC-files in the workspace by default.)",
     )
     pipeline_show_parser.set_defaults(func=CmdPipelineShow)
-
-    PIPELINE_LIST_HELP = "List connected groups of stages (pipelines)."
-    pipeline_list_parser = pipeline_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
-        help=PIPELINE_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pipeline_list_parser.set_defaults(func=CmdPipelineList)

dvc/command/plots.py

@@ -0,0 +1,219 @@
+import argparse
+import logging
+import os
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import DvcException
+from dvc.utils import format_link
+
+logger = logging.getLogger(__name__)
+
+PAGE_HTML = """<!DOCTYPE html>
+<html>
+<head>
+    <title>DVC Plot</title>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+</head>
+<body>
+    {divs}
+</body>
+</html>"""
+
+DIV_HTML = """<div id = "{id}"></div>
+<script type = "text/javascript">
+    var spec = {vega_json};
+    vegaEmbed('#{id}', spec);
+</script>"""
+
+
+class CmdPlots(CmdBase):
+    def _func(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def run(self):
+        try:
+            plots = self._func(
+                targets=self.args.targets,
+                template=self.args.template,
+                x_field=self.args.x,
+                y_field=self.args.y,
+                csv_header=not self.args.no_csv_header,
+                title=self.args.title,
+                x_title=self.args.xlab,
+                y_title=self.args.ylab,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(plots))
+                return 0
+
+            divs = [
+                DIV_HTML.format(id=f"plot{i}", vega_json=plot)
+                for i, plot in enumerate(plots.values())
+            ]
+            html = PAGE_HTML.format(divs="\n".join(divs))
+            path = self.args.out or "plots.html"
+
+            with open(path, "w") as fobj:
+                fobj.write(html)
+
+            logger.info(
+                "file://{}".format(os.path.join(self.repo.root_dir, path))
+            )
+
+        except DvcException:
+            logger.exception("")
+            return 1
+
+        return 0
+
+
+class CmdPlotsShow(CmdPlots):
+    def _func(self, *args, **kwargs):
+        return self.repo.plots.show(*args, **kwargs)
+
+
+class CmdPlotsDiff(CmdPlots):
+    def _func(self, *args, **kwargs):
+        return self.repo.plots.diff(*args, revs=self.args.revisions, **kwargs)
+
+
+def add_parser(subparsers, parent_parser):
+    PLOTS_HELP = (
+        "Generating plots for metrics stored in structured files "
+        "(JSON, CSV, TSV)."
+    )
+
+    plots_parser = subparsers.add_parser(
+        "plots",
+        parents=[parent_parser],
+        description=append_doc_link(PLOTS_HELP, "plots"),
+        help=PLOTS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_subparsers = plots_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc plots CMD --help` to display command-specific help.",
+    )
+
+    fix_subparsers(plots_subparsers)
+
+    SHOW_HELP = "Generate a plots image file from a metrics file."
+    plots_show_parser = plots_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(SHOW_HELP, "plots/show"),
+        help=SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_show_parser.add_argument(
+        "-t",
+        "--template",
+        nargs="?",
+        default=None,
+        help=(
+            "Special JSON or HTML schema file to inject with the data. "
+            "See {}".format(
+                format_link("https://man.dvc.org/plots#plot-templates")
+            )
+        ),
+    )
+    plots_show_parser.add_argument(
+        "-o", "--out", default=None, help="Destination path to save plots to.",
+    )
+    plots_show_parser.add_argument(
+        "-x", default=None, help="Field name for x axis."
+    )
+    plots_show_parser.add_argument(
+        "-y", default=None, help="Field name for y axis."
+    )
+    plots_show_parser.add_argument(
+        "--no-csv-header",
+        action="store_true",
+        default=False,
+        help="Required when CSV or TSV datafile does not have a header.",
+    )
+    plots_show_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    plots_show_parser.add_argument("--title", default=None, help="Plot title.")
+    plots_show_parser.add_argument(
+        "--xlab", default=None, help="X axis title."
+    )
+    plots_show_parser.add_argument(
+        "--ylab", default=None, help="Y axis title."
+    )
+    plots_show_parser.add_argument(
+        "targets",
+        nargs="*",
+        help="Metrics files to visualize. Shows all plots by default.",
+    )
+    plots_show_parser.set_defaults(func=CmdPlotsShow)
+
+    PLOTS_DIFF_HELP = (
+        "Plot differences in metrics between commits in the DVC "
+        "repository, or between the last commit and the workspace."
+    )
+    plots_diff_parser = plots_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(PLOTS_DIFF_HELP, "plots/diff"),
+        help=PLOTS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_diff_parser.add_argument(
+        "-t",
+        "--template",
+        nargs="?",
+        default=None,
+        help=(
+            "Special JSON or HTML schema file to inject with the data. "
+            "See {}".format(
+                format_link("https://man.dvc.org/plots#plot-templates")
+            )
+        ),
+    )
+    plots_diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help="Metrics file to visualize. Shows all plots by default.",
+    )
+    plots_diff_parser.add_argument(
+        "-o", "--out", default=None, help="Destination path to save plots to.",
+    )
+    plots_diff_parser.add_argument(
+        "-x", default=None, help="Field name for x axis."
+    )
+    plots_diff_parser.add_argument(
+        "-y", default=None, help="Field name for y axis."
+    )
+    plots_diff_parser.add_argument(
+        "--no-csv-header",
+        action="store_true",
+        default=False,
+        help="Provided CSV ot TSV datafile does not have a header.",
+    )
+    plots_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    plots_diff_parser.add_argument("--title", default=None, help="Plot title.")
+    plots_diff_parser.add_argument(
+        "--xlab", default=None, help="X axis title."
+    )
+    plots_diff_parser.add_argument(
+        "--ylab", default=None, help="Y axis title."
+    )
+    plots_diff_parser.add_argument(
+        "revisions", nargs="*", default=None, help="Git commits to plot from",
+    )
+    plots_diff_parser.set_defaults(func=CmdPlotsDiff)

dvc/command/remote.py

@@ -1,9 +1,9 @@
 import argparse
 import logging
 
-from dvc.config import ConfigError
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
+from dvc.config import ConfigError
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
@@ -18,17 +18,13 @@ class CmdRemote(CmdConfig):
 
     def _check_exists(self, conf):
         if self.args.name not in conf["remote"]:
-            raise ConfigError(
-                "remote '{}' doesn't exists.".format(self.args.name)
-            )
+            raise ConfigError(f"remote '{self.args.name}' doesn't exists.")
 
 
 class CmdRemoteAdd(CmdRemote):
     def run(self):
         if self.args.default:
-            logger.info(
-                "Setting '{}' as a default remote.".format(self.args.name)
-            )
+            logger.info(f"Setting '{self.args.name}' as a default remote.")
 
         with self.config.edit(self.args.level) as conf:
             if self.args.name in conf["remote"] and not self.args.force:
@@ -85,7 +81,18 @@ class CmdRemoteDefault(CmdRemote):
                 if self.args.unset:
                     conf["core"].pop("remote", None)
                 else:
-                    conf["core"]["remote"] = self.args.name
+                    merged_conf = self.config.load_config_to_level(
+                        self.args.level
+                    )
+                    if (
+                        self.args.name in conf["remote"]
+                        or self.args.name in merged_conf["remote"]
+                    ):
+                        conf["core"]["remote"] = self.args.name
+                    else:
+                        raise ConfigError(
+                            "default remote must be present in remote list."
+                        )
         return 0
 
 
@@ -97,6 +104,35 @@ class CmdRemoteList(CmdRemote):
         return 0
 
 
+class CmdRemoteRename(CmdRemote):
+    def _rename_default(self, conf):
+        if conf["core"].get("remote") == self.args.name:
+            conf["core"]["remote"] = self.args.new
+
+    def run(self):
+        all_config = self.config.load_config_to_level(None)
+        if self.args.new in all_config.get("remote", {}):
+            raise ConfigError(
+                "Rename failed. Remote name '{}' already exists.".format(
+                    {self.args.new}
+                )
+            )
+
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            conf["remote"][self.args.new] = conf["remote"][self.args.name]
+            del conf["remote"][self.args.name]
+            self._rename_default(conf)
+
+        for level in reversed(self.config.LEVELS):
+            if level == self.args.level:
+                break
+            with self.config.edit(level) as level_conf:
+                self._rename_default(level_conf)
+
+        return 0
+
+
 def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
@@ -167,19 +203,6 @@ def add_parser(subparsers, parent_parser):
     )
     remote_default_parser.set_defaults(func=CmdRemoteDefault)
 
-    REMOTE_REMOVE_HELP = "Remove a data remote."
-    remote_remove_parser = remote_subparsers.add_parser(
-        "remove",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
-        help=REMOTE_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    remote_remove_parser.add_argument(
-        "name", help="Name of the remote to remove."
-    )
-    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
-
     REMOTE_MODIFY_HELP = "Modify the configuration of a data remote."
     remote_modify_parser = remote_subparsers.add_parser(
         "modify",
@@ -213,3 +236,27 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_list_parser.set_defaults(func=CmdRemoteList)
+
+    REMOTE_REMOVE_HELP = "Remove a data remote."
+    remote_remove_parser = remote_subparsers.add_parser(
+        "remove",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
+        help=REMOTE_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_remove_parser.add_argument(
+        "name", help="Name of the remote to remove."
+    )
+    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
+    REMOTE_RENAME_HELP = "Rename a DVC remote"
+    remote_rename_parser = remote_subparsers.add_parser(
+        "rename",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_RENAME_HELP, "remote/rename"),
+        help=REMOTE_RENAME_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_rename_parser.add_argument("name", help="Remote to be renamed")
+    remote_rename_parser.add_argument("new", help="New name of the remote")
+    remote_rename_parser.set_defaults(func=CmdRemoteRename)

dvc/command/remove.py

@@ -2,16 +2,14 @@ import argparse
 import logging
 
 import dvc.prompt as prompt
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
 class CmdRemove(CmdBase):
-    def _is_outs_only(self, target):
+    def _is_dvc_only(self, target):
         if not self.args.purge:
             return True
 
@@ -33,10 +31,10 @@ class CmdRemove(CmdBase):
     def run(self):
         for target in self.args.targets:
             try:
-                outs_only = self._is_outs_only(target)
-                self.repo.remove(target, outs_only=outs_only)
+                dvc_only = self._is_dvc_only(target)
+                self.repo.remove(target, dvc_only=dvc_only)
             except DvcException:
-                logger.exception("failed to remove '{}'".format(target))
+                logger.exception(f"failed to remove '{target}'")
                 return 1
         return 0
 

dvc/command/repro.py

@@ -2,13 +2,12 @@ import argparse
 import logging
 import os
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
+from dvc.dvcfile import PIPELINE_FILE
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -35,10 +34,11 @@ class CmdRepro(CmdBase):
                     interactive=self.args.interactive,
                     pipeline=self.args.pipeline,
                     all_pipelines=self.args.all_pipelines,
-                    ignore_build_cache=self.args.ignore_build_cache,
+                    run_cache=not self.args.no_run_cache,
                     no_commit=self.args.no_commit,
                     downstream=self.args.downstream,
                     recursive=self.args.recursive,
+                    force_downstream=self.args.force_downstream,
                 )
 
                 if len(stages) == 0:
@@ -70,7 +70,7 @@ def add_parser(subparsers, parent_parser):
     repro_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC-file to reproduce. 'Dvcfile' by default.",
+        help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
     )
     repro_parser.add_argument(
         "-f",
@@ -92,6 +92,7 @@ def add_parser(subparsers, parent_parser):
         "--cwd",
         default=os.path.curdir,
         help="Directory within your repo to reproduce from.",
+        metavar="<path>",
     )
     repro_parser.add_argument(
         "-m",
@@ -137,7 +138,16 @@ def add_parser(subparsers, parent_parser):
         help="Reproduce all stages in the specified directory.",
     )
     repro_parser.add_argument(
-        "--ignore-build-cache",
+        "--no-run-cache",
+        action="store_true",
+        default=False,
+        help=(
+            "Execute stage commands even if they have already been run with "
+            "the same command/dependencies/outputs/etc before."
+        ),
+    )
+    repro_parser.add_argument(
+        "--force-downstream",
         action="store_true",
         default=False,
         help="Reproduce all descendants of a changed stage even if their "

dvc/command/root.py

@@ -1,8 +1,7 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)

dvc/command/run.py

@@ -1,18 +1,14 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
 class CmdRun(CmdBase):
     def run(self):
-        overwrite = self.args.yes or self.args.overwrite_dvcfile
-
         if not any(
             [
                 self.args.deps,
@@ -20,6 +16,8 @@ class CmdRun(CmdBase):
                 self.args.outs_no_cache,
                 self.args.metrics,
                 self.args.metrics_no_cache,
+                self.args.plots,
+                self.args.plots_no_cache,
                 self.args.outs_persist,
                 self.args.outs_persist_no_cache,
                 self.args.params,
@@ -28,8 +26,8 @@ class CmdRun(CmdBase):
         ):  # pragma: no cover
             logger.error(
                 "too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
-                "`-m`, `-M`, `-p`, `--outs-persist`, "
-                "`--outs-persist-no-cache`, `command`."
+                "`-m`, `-M`, `-p`, `--plots`, `--plots-no-cache`, "
+                "`--outs-persist`, `--outs-persist-no-cache`, `command`."
             )
             return 1
 
@@ -40,22 +38,24 @@ class CmdRun(CmdBase):
                 outs_no_cache=self.args.outs_no_cache,
                 metrics=self.args.metrics,
                 metrics_no_cache=self.args.metrics_no_cache,
+                plots=self.args.plots,
+                plots_no_cache=self.args.plots_no_cache,
                 deps=self.args.deps,
                 params=self.args.params,
                 fname=self.args.file,
-                cwd=self.args.cwd,
                 wdir=self.args.wdir,
                 no_exec=self.args.no_exec,
-                overwrite=overwrite,
-                ignore_build_cache=self.args.ignore_build_cache,
-                remove_outs=self.args.remove_outs,
+                overwrite=self.args.overwrite_dvcfile,
+                run_cache=not self.args.no_run_cache,
                 no_commit=self.args.no_commit,
                 outs_persist=self.args.outs_persist,
                 outs_persist_no_cache=self.args.outs_persist_no_cache,
                 always_changed=self.args.always_changed,
+                name=self.args.name,
+                single_stage=self.args.single_stage,
             )
         except DvcException:
-            logger.exception("failed to run command")
+            logger.exception("")
             return 1
 
         return 0
@@ -79,7 +79,7 @@ class CmdRun(CmdBase):
         if " " not in argument or '"' in argument:
             return argument
 
-        return '"{}"'.format(argument)
+        return f'"{argument}"'
 
 
 def add_parser(subparsers, parent_parser):
@@ -97,6 +97,10 @@ def add_parser(subparsers, parent_parser):
         action="append",
         default=[],
         help="Declare dependencies for reproducible cmd.",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "-n", "--name", help="Stage name.",
     )
     run_parser.add_argument(
         "-o",
@@ -104,6 +108,7 @@ def add_parser(subparsers, parent_parser):
         action="append",
         default=[],
         help="Declare output file or directory.",
+        metavar="<filename>",
     )
     run_parser.add_argument(
         "-O",
@@ -112,6 +117,7 @@ def add_parser(subparsers, parent_parser):
         default=[],
         help="Declare output file or directory "
         "(do not put into DVC cache).",
+        metavar="<filename>",
     )
     run_parser.add_argument(
         "-p",
@@ -119,6 +125,7 @@ def add_parser(subparsers, parent_parser):
         action="append",
         default=[],
         help="Declare parameter to use as additional dependency.",
+        metavar="[<filename>:]<params_list>",
     )
     run_parser.add_argument(
         "-m",
@@ -126,6 +133,7 @@ def add_parser(subparsers, parent_parser):
         action="append",
         default=[],
         help="Declare output metric file or directory.",
+        metavar="<path>",
     )
     run_parser.add_argument(
         "-M",
@@ -134,17 +142,33 @@ def add_parser(subparsers, parent_parser):
         default=[],
         help="Declare output metric file or directory "
         "(do not put into DVC cache).",
+        metavar="<path>",
     )
     run_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC-file it generates."
+        "--plots",
+        action="append",
+        default=[],
+        help="Declare output plot file.",
+        metavar="<path>",
     )
     run_parser.add_argument(
-        "-c", "--cwd", help="Deprecated, use -w and -f instead."
+        "--plots-no-cache",
+        action="append",
+        default=[],
+        help="Declare output plot file (do not put into DVC cache).",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
     )
     run_parser.add_argument(
         "-w",
         "--wdir",
         help="Directory within your repo to run your command in.",
+        metavar="<path>",
     )
     run_parser.add_argument(
         "--no-exec",
@@ -152,13 +176,6 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Only create stage file without actually running it.",
     )
-    run_parser.add_argument(
-        "-y",
-        "--yes",
-        action="store_true",
-        default=False,
-        help="Deprecated, use --overwrite-dvcfile instead",
-    )
     run_parser.add_argument(
         "--overwrite-dvcfile",
         action="store_true",
@@ -166,17 +183,13 @@ def add_parser(subparsers, parent_parser):
         help="Overwrite existing DVC-file without asking for confirmation.",
     )
     run_parser.add_argument(
-        "--ignore-build-cache",
+        "--no-run-cache",
         action="store_true",
         default=False,
-        help="Run this stage even if it has been already ran with the same "
-        "command/dependencies/outputs/etc before.",
-    )
-    run_parser.add_argument(
-        "--remove-outs",
-        action="store_true",
-        default=False,
-        help="Deprecated, this is now the default behavior",
+        help=(
+            "Execute the command even if this stage has already been run "
+            "with the same command/dependencies/outputs/etc before."
+        ),
     )
     run_parser.add_argument(
         "--no-commit",
@@ -190,6 +203,7 @@ def add_parser(subparsers, parent_parser):
         default=[],
         help="Declare output file or directory that will not be "
         "removed upon repro.",
+        metavar="<filename>",
     )
     run_parser.add_argument(
         "--outs-persist-no-cache",
@@ -197,6 +211,7 @@ def add_parser(subparsers, parent_parser):
         default=[],
         help="Declare output file or directory that will not be "
         "removed upon repro (do not put into DVC cache).",
+        metavar="<filename>",
     )
     run_parser.add_argument(
         "--always-changed",
@@ -204,6 +219,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Always consider this DVC-file as changed.",
     )
+    run_parser.add_argument(
+        "--single-stage",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."
     )

dvc/command/status.py

@@ -2,7 +2,6 @@ import logging
 
 from dvc.command.data_sync import CmdDataBase
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -20,7 +19,7 @@ class CmdDataStatus(CmdDataBase):
         ind = indent * self.STATUS_INDENT
 
         if isinstance(status, str):
-            logger.info("{}{}".format(ind, status))
+            logger.info(f"{ind}{status}")
             return
 
         if isinstance(status, list):
@@ -34,7 +33,7 @@ class CmdDataStatus(CmdDataBase):
             if isinstance(value, str):
                 logger.info("{}{}{}".format(ind, self._normalize(value), key))
             elif value:
-                logger.info("{}{}:".format(ind, key))
+                logger.info(f"{ind}{key}:")
                 self._show(value, indent + 1)
 
     def run(self):
@@ -47,7 +46,9 @@ class CmdDataStatus(CmdDataBase):
                 remote=self.args.remote,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
+                recursive=self.args.recursive,
             )
             if st:
                 if self.args.quiet:

dvc/command/tag.py

@@ -1,156 +0,0 @@
-import logging
-
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import DvcException
-from dvc.utils import to_yaml_string
-
-
-logger = logging.getLogger(__name__)
-
-
-class CmdTagAdd(CmdBase):
-    def run(self):
-        for target in self.args.targets:
-            try:
-                self.repo.tag.add(
-                    self.args.tag,
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    recursive=self.args.recursive,
-                )
-            except DvcException:
-                logger.exception("failed to add tag")
-                return 1
-        return 0
-
-
-class CmdTagRemove(CmdBase):
-    def run(self):
-        for target in self.args.targets:
-            try:
-                self.repo.tag.remove(
-                    self.args.tag,
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    recursive=self.args.recursive,
-                )
-            except DvcException:
-                logger.exception("failed to remove tag")
-                return 1
-        return 0
-
-
-class CmdTagList(CmdBase):
-    def run(self):
-        for target in self.args.targets:
-            try:
-                tags = self.repo.tag.list(
-                    target,
-                    with_deps=self.args.with_deps,
-                    recursive=self.args.recursive,
-                )
-                if tags:
-                    logger.info(to_yaml_string(tags))
-            except DvcException:
-                logger.exception("failed list tags")
-                return 1
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    TAG_HELP = "A set of commands to manage DVC tags."
-    tag_parser = subparsers.add_parser(
-        "tag",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_HELP, "tag"),
-        add_help=False,
-    )
-
-    tag_subparsers = tag_parser.add_subparsers(
-        dest="cmd",
-        help="Use DVC tag CMD --help to display command-specific help.",
-    )
-
-    fix_subparsers(tag_subparsers)
-
-    TAG_ADD_HELP = "Add DVC tag."
-    tag_add_parser = tag_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_ADD_HELP, "tag-add"),
-        help=TAG_ADD_HELP,
-    )
-    tag_add_parser.add_argument("tag", help="Dvc tag.")
-    tag_add_parser.add_argument(
-        "targets", nargs="*", default=[None], help="DVC-files."
-    )
-    tag_add_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="Add tag for all dependencies of the specified DVC-file.",
-    )
-    tag_add_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="Add tag for subdirectories of the specified directory.",
-    )
-    tag_add_parser.set_defaults(func=CmdTagAdd)
-
-    TAG_REMOVE_HELP = "Remove DVC tag."
-    tag_remove_parser = tag_subparsers.add_parser(
-        "remove",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_REMOVE_HELP, "tag-remove"),
-        help=TAG_REMOVE_HELP,
-    )
-    tag_remove_parser.add_argument("tag", help="Dvc tag.")
-    tag_remove_parser.add_argument(
-        "targets", nargs="*", default=[None], help="DVC-files."
-    )
-    tag_remove_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="Remove tag for all dependencies of the specified DVC-file.",
-    )
-    tag_remove_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="Remove tag for subdirectories of the specified directory.",
-    )
-    tag_remove_parser.set_defaults(func=CmdTagRemove)
-
-    TAG_LIST_HELP = "List DVC tags."
-    tag_list_parser = tag_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_LIST_HELP, "tag-list"),
-        help=TAG_LIST_HELP,
-    )
-    tag_list_parser.add_argument(
-        "targets", nargs="*", default=[None], help="DVC-files."
-    )
-    tag_list_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="List tags for all dependencies of the specified DVC-file.",
-    )
-    tag_list_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="List tags for subdirectories of the specified directory.",
-    )
-    tag_list_parser.set_defaults(func=CmdTagList)

dvc/command/unprotect.py

@@ -1,11 +1,9 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -15,7 +13,7 @@ class CmdUnprotect(CmdBase):
             try:
                 self.repo.unprotect(target)
             except DvcException:
-                msg = "failed to unprotect '{}'".format(target)
+                msg = f"failed to unprotect '{target}'"
                 logger.exception(msg)
                 return 1
         return 0

dvc/command/update.py

@@ -1,23 +1,24 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
 class CmdUpdate(CmdBase):
     def run(self):
         ret = 0
-        for target in self.args.targets:
-            try:
-                self.repo.update(target, self.args.rev)
-            except DvcException:
-                logger.exception("failed to update '{}'.".format(target))
-                ret = 1
+        try:
+            self.repo.update(
+                targets=self.args.targets,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+            )
+        except DvcException:
+            logger.exception("failed update data")
+            ret = 1
         return ret
 
 
@@ -34,6 +35,16 @@ def add_parser(subparsers, parent_parser):
         "targets", nargs="+", help="DVC-files to update."
     )
     update_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    update_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Update all stages in the specified directory.",
     )
     update_parser.set_defaults(func=CmdUpdate)

dvc/command/version.py

@@ -2,22 +2,23 @@ import argparse
 import itertools
 import logging
 import os
+import pathlib
 import platform
 import uuid
-import pathlib
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.scm.base import SCMError
+from dvc.system import System
+from dvc.utils import is_binary, relpath
+from dvc.utils.pkg import PKG
+from dvc.version import __version__
 
 try:
     import psutil
 except ImportError:
     psutil = None
 
-from dvc.utils import is_binary, relpath
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.version import __version__
-from dvc.exceptions import DvcException, NotDvcRepoError
-from dvc.scm.base import SCMError
-from dvc.system import System
-from dvc.utils.pkg import PKG
 
 logger = logging.getLogger(__name__)
 
@@ -27,12 +28,12 @@ class CmdVersion(CmdBaseNoRepo):
         from dvc.repo import Repo
 
         info = [
-            "DVC version: {}".format(__version__),
-            "Python version: {}".format(platform.python_version()),
-            "Platform: {}".format(platform.platform()),
-            "Binary: {}".format(is_binary()),
-            "Package: {}".format(PKG),
-            "Supported remotes: {}".format(self.get_supported_remotes()),
+            f"DVC version: {__version__}",
+            f"Python version: {platform.python_version()}",
+            f"Platform: {platform.platform()}",
+            f"Binary: {is_binary()}",
+            f"Package: {PKG}",
+            f"Supported remotes: {self.get_supported_remotes()}",
         ]
 
         try:
@@ -50,7 +51,7 @@ class CmdVersion(CmdBaseNoRepo):
                 if psutil:
                     fs_type = self.get_fs_type(repo.cache.local.cache_dir)
                     info.append(
-                        "Filesystem type (cache directory): {}".format(fs_type)
+                        f"Filesystem type (cache directory): {fs_type}"
                     )
             else:
                 logger.warning(
@@ -71,7 +72,7 @@ class CmdVersion(CmdBaseNoRepo):
 
         if psutil:
             fs_root = self.get_fs_type(os.path.abspath(root_directory))
-            info.append("Filesystem type (workspace): {}".format(fs_root))
+            info.append(f"Filesystem type (workspace): {fs_root}")
 
         logger.info("\n".join(info))
         return 0
@@ -114,7 +115,7 @@ class CmdVersion(CmdBaseNoRepo):
                 os.unlink(dst)
             except DvcException:
                 status = "not supported"
-            cache.append("{name} - {status}".format(name=name, status=status))
+            cache.append(f"{name} - {status}")
         os.remove(src)
 
         return ", ".join(cache)

dvc/compat.py

@@ -1,42 +0,0 @@
-"""Python version compatibility code"""
-
-# Backport os.fspath() from Python 3.6
-try:
-    from os import fspath  # noqa: F821
-
-    fspath_py35 = lambda s: s  # noqa: E731
-except ImportError:
-
-    def fspath(path):
-        """Return the path representation of a path-like object.
-
-        If str or bytes is passed in, it is returned unchanged. Otherwise the
-        os.PathLike interface is used to get the path representation. If the
-        path representation is not str or bytes, TypeError is raised. If the
-        provided path is not str, bytes, or os.PathLike, TypeError is raised.
-        """
-        if isinstance(path, (str, bytes)):
-            return path
-
-        # Work from the object's type to match method resolution of other magic
-        # methods.
-        path_type = type(path)
-        try:
-            path_repr = path_type.__fspath__(path)
-        except AttributeError:
-            if hasattr(path_type, "__fspath__"):
-                raise
-            else:
-                raise TypeError(
-                    "expected str, bytes or os.PathLike object, "
-                    "not " + path_type.__name__
-                )
-        if isinstance(path_repr, (str, bytes)):
-            return path_repr
-        else:
-            raise TypeError(
-                "expected {}.__fspath__() to return str or bytes, "
-                "not {}".format(path_type.__name__, type(path_repr).__name__)
-            )
-
-    fspath_py35 = fspath

dvc/config.py

@@ -1,16 +1,27 @@
 """DVC config objects."""
-from contextlib import contextmanager
 import logging
 import os
 import re
+from contextlib import contextmanager
+from functools import partial
 from urllib.parse import urlparse
 
-from funcy import cached_property, re_find, walk_values, compact
 import configobj
-from voluptuous import Schema, Optional, Invalid, ALLOW_EXTRA
-from voluptuous import All, Any, Lower, Range, Coerce
+from funcy import cached_property, compact, re_find, walk_values
+from voluptuous import (
+    ALLOW_EXTRA,
+    All,
+    Any,
+    Coerce,
+    Invalid,
+    Lower,
+    Optional,
+    Range,
+    Schema,
+)
 
 from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.path_info import PathInfo
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
@@ -20,7 +31,7 @@ class ConfigError(DvcException):
     """DVC config exception."""
 
     def __init__(self, msg):
-        super().__init__("config file error: {}".format(msg))
+        super().__init__(f"config file error: {msg}")
 
 
 class NoRemoteError(ConfigError):
@@ -78,7 +89,7 @@ def ByUrl(mapping):
         if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
             return schemas[""](data)
         if parsed.scheme not in schemas:
-            raise Invalid("Unsupported URL type {}://".format(parsed.scheme))
+            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
 
         return schemas[parsed.scheme](data)
 
@@ -140,6 +151,7 @@ SCHEMA = {
                     Optional("listobjects", default=False): Bool,
                     Optional("use_ssl", default=True): Bool,
                     "sse": str,
+                    "sse_kms_key_id": str,
                     "acl": str,
                     "grant_read": str,
                     "grant_read_acp": str,
@@ -278,10 +290,7 @@ class Config(dict):
         Raises:
             ConfigError: thrown if config has an invalid format.
         """
-        conf = {}
-        for level in self.LEVELS:
-            if level in self.files:
-                _merge(conf, self.load_one(level))
+        conf = self.load_config_to_level()
 
         if validate:
             conf = self.validate(conf)
@@ -297,7 +306,7 @@ class Config(dict):
         conf = _load_config(self.files[level])
         conf = self._load_paths(conf, self.files[level])
 
-        # Autovivify sections
+        # Auto-verify sections
         for key in COMPILED_SCHEMA.schema:
             conf.setdefault(key, {})
 
@@ -314,25 +323,42 @@ class Config(dict):
 
         return Config._map_dirs(conf, resolve)
 
+    @staticmethod
+    def _to_relpath(conf_dir, path):
+        if re.match(r"\w+://", path):
+            return path
+
+        if isinstance(path, RelPath) or not os.path.isabs(path):
+            path = relpath(path, conf_dir)
+
+        return PathInfo(path).as_posix()
+
     @staticmethod
     def _save_paths(conf, filename):
         conf_dir = os.path.dirname(filename)
-
-        def rel(path):
-            if re.match(r"\w+://", path):
-                return path
-
-            if isinstance(path, RelPath) or not os.path.isabs(path):
-                return relpath(path, conf_dir)
-            return path
+        rel = partial(Config._to_relpath, conf_dir)
 
         return Config._map_dirs(conf, rel)
 
     @staticmethod
     def _map_dirs(conf, func):
-        dirs_schema = {"cache": {"dir": func}, "remote": {str: {"url": func}}}
+        dirs_schema = {
+            "cache": {"dir": func},
+            "remote": {
+                str: {"url": func, "gdrive_user_credentials_file": func}
+            },
+        }
         return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
 
+    def load_config_to_level(self, level=None):
+        merged_conf = {}
+        for merge_level in self.LEVELS:
+            if merge_level == level:
+                break
+            if merge_level in self.files:
+                _merge(merged_conf, self.load_one(merge_level))
+        return merged_conf
+
     @contextmanager
     def edit(self, level="repo"):
         if level in {"repo", "local"} and self.dvc_dir is None:
@@ -342,6 +368,11 @@ class Config(dict):
         yield conf
 
         conf = self._save_paths(conf, self.files[level])
+
+        merged_conf = self.load_config_to_level(level)
+        _merge(merged_conf, conf)
+        self.validate(merged_conf)
+
         _save_config(self.files[level], conf)
         self.load()
 
@@ -359,7 +390,7 @@ def _load_config(filename):
 
 
 def _save_config(filename, conf_dict):
-    logger.debug("Writing '{}'.".format(filename))
+    logger.debug(f"Writing '{filename}'.")
     os.makedirs(os.path.dirname(filename), exist_ok=True)
 
     config = configobj.ConfigObj(_pack_remotes(conf_dict))
@@ -386,7 +417,7 @@ def _pack_remotes(conf):
 
     # Transform remote.name -> 'remote "name"'
     for name, val in conf["remote"].items():
-        result['remote "{}"'.format(name)] = val
+        result[f'remote "{name}"'] = val
     result.pop("remote", None)
 
     return result

dvc/daemon.py

@@ -7,9 +7,7 @@ import sys
 from subprocess import Popen
 
 from dvc.env import DVC_DAEMON
-from dvc.utils import fix_env
-from dvc.utils import is_binary
-
+from dvc.utils import fix_env, is_binary
 
 logger = logging.getLogger(__name__)
 
@@ -68,7 +66,7 @@ def _spawn_posix(cmd, env):
 
 
 def _spawn(cmd, env):
-    logger.debug("Trying to spawn '{}'".format(cmd))
+    logger.debug(f"Trying to spawn '{cmd}'")
 
     if os.name == "nt":
         _spawn_windows(cmd, env)
@@ -77,7 +75,7 @@ def _spawn(cmd, env):
     else:
         raise NotImplementedError
 
-    logger.debug("Spawned '{}'".format(cmd))
+    logger.debug(f"Spawned '{cmd}'")
 
 
 def daemon(args):

dvc/dagascii.py

@@ -6,12 +6,9 @@ import os
 import pydoc
 import sys
 
-from grandalf.graphs import Edge
-from grandalf.graphs import Graph
-from grandalf.graphs import Vertex
+from grandalf.graphs import Edge, Graph, Vertex
 from grandalf.layouts import SugiyamaLayout
-from grandalf.routing import EdgeViewer
-from grandalf.routing import route_with_lines
+from grandalf.routing import EdgeViewer, route_with_lines
 
 from dvc.env import DVC_PAGER
 from dvc.utils import format_link
@@ -40,7 +37,7 @@ def find_pager():
     if env_pager:
         return make_pager(env_pager)
 
-    if os.system("({}) 2>{}".format(DEFAULT_PAGER, os.devnull)) == 0:
+    if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
         return make_pager(DEFAULT_PAGER_FORMATTED)
 
     logger.warning(
@@ -52,7 +49,7 @@ def find_pager():
     return pydoc.plainpager
 
 
-class VertexViewer(object):
+class VertexViewer:
     """Class to define vertex box boundaries that will be accounted for during
     graph building by grandalf.
 
@@ -78,7 +75,7 @@ class VertexViewer(object):
         return self._w
 
 
-class AsciiCanvas(object):
+class AsciiCanvas:
     """Class for drawing in ASCII.
 
     Args:
@@ -95,7 +92,7 @@ class AsciiCanvas(object):
         self.cols = cols
         self.lines = lines
 
-        self.canvas = [[" "] * cols for l in range(lines)]
+        self.canvas = [[" "] * cols for line in range(lines)]
 
     def draw(self):
         """Draws ASCII canvas on the screen."""
@@ -148,21 +145,21 @@ class AsciiCanvas(object):
                 if dx == 0:
                     y = y0
                 else:
-                    y = y0 + int(round((x - x0) * dy / float((dx))))
+                    y = y0 + int(round((x - x0) * dy / float(dx)))
                 self.point(x, y, char)
         elif y0 < y1:
             for y in range(y0, y1 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x0 + int(round((y - y0) * dx / float((dy))))
+                    x = x0 + int(round((y - y0) * dx / float(dy)))
                 self.point(x, y, char)
         else:
             for y in range(y1, y0 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x1 + int(round((y - y1) * dx / float((dy))))
+                    x = x1 + int(round((y - y1) * dx / float(dy)))
                 self.point(x, y, char)
 
     def text(self, x, y, text):
@@ -216,7 +213,7 @@ def _build_sugiyama_layout(vertexes, edges):
     # Y
     #
 
-    vertexes = {v: Vertex(" {} ".format(v)) for v in vertexes}
+    vertexes = {v: Vertex(f" {v} ") for v in vertexes}
     # NOTE: reverting edges to correctly orientate the graph
     edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
     vertexes = vertexes.values()

dvc/data_cloud.py

@@ -5,11 +5,10 @@ import logging
 from dvc.config import NoRemoteError
 from dvc.remote import Remote
 
-
 logger = logging.getLogger(__name__)
 
 
-class DataCloud(object):
+class DataCloud:
     """Class that manages dvc remotes.
 
     Args:
@@ -23,12 +22,12 @@ class DataCloud(object):
     def __init__(self, repo):
         self.repo = repo
 
-    def get_remote(self, remote=None, command="<command>"):
-        if not remote:
-            remote = self.repo.config["core"].get("remote")
+    def get_remote(self, name=None, command="<command>"):
+        if not name:
+            name = self.repo.config["core"].get("remote")
 
-        if remote:
-            return self._init_remote(remote)
+        if name:
+            return self._init_remote(name)
 
         if bool(self.repo.config["remote"]):
             error_msg = (
@@ -45,39 +44,43 @@ class DataCloud(object):
 
         raise NoRemoteError(error_msg)
 
-    def _init_remote(self, remote):
-        return Remote(self.repo, name=remote)
+    def _init_remote(self, name):
+        return Remote(self.repo, name=name)
 
-    def push(self, cache, jobs=None, remote=None, show_checksums=False):
+    def push(
+        self, cache, jobs=None, remote=None, show_checksums=False,
+    ):
         """Push data items in a cloud-agnostic way.
 
         Args:
             cache (NamedCache): named checksums to push to the cloud.
             jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.RemoteBASE): optional remote to push to.
+            remote (dvc.remote.base.BaseRemote): optional remote to push to.
                 By default remote from core.remote config option is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
+        remote = self.get_remote(remote, "push")
+
         return self.repo.cache.local.push(
-            cache,
-            jobs=jobs,
-            remote=self.get_remote(remote, "push"),
-            show_checksums=show_checksums,
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums,
         )
 
-    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
+    def pull(
+        self, cache, jobs=None, remote=None, show_checksums=False,
+    ):
         """Pull data items in a cloud-agnostic way.
 
         Args:
             cache (NamedCache): named checksums to pull from the cloud.
             jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
+            remote (dvc.remote.base.BaseRemote): optional remote to pull from.
                 By default remote from core.remote config option is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
         remote = self.get_remote(remote, "pull")
+
         downloaded_items_num = self.repo.cache.local.pull(
             cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )
@@ -88,12 +91,13 @@ class DataCloud(object):
         return downloaded_items_num
 
     def _save_pulled_checksums(self, cache):
-        for checksum in cache["local"].keys():
+        for checksum in cache.scheme_keys("local"):
             cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
             if self.repo.cache.local.exists(cache_file):
-                # We can safely save here, as existing corrupted files will be
-                # removed upon status, while files corrupted during download
-                # will not be moved from tmp_file (see `RemoteBASE.download()`)
+                # We can safely save here, as existing corrupted files will
+                # be removed upon status, while files corrupted during
+                # download will not be moved from tmp_file
+                # (see `RemoteBASE.download()`)
                 self.repo.state.save(cache_file, checksum)
 
     def status(self, cache, jobs=None, remote=None, show_checksums=False):
@@ -102,7 +106,7 @@ class DataCloud(object):
         Args:
             cache (NamedCache): named checksums to check status for.
             jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.RemoteBASE): optional remote to compare
+            remote (dvc.remote.base.BaseRemote): optional remote to compare
                 cache to. By default remote from core.remote config option
                 is used.
             show_checksums (bool): show checksums instead of file names in

dvc/dependency/__init__.py

@@ -1,39 +1,39 @@
-from urllib.parse import urlparse
 from collections import defaultdict
+from urllib.parse import urlparse
 
 import dvc.output as output
-from dvc.dependency.gs import DependencyGS
-from dvc.dependency.hdfs import DependencyHDFS
-from dvc.dependency.http import DependencyHTTP
-from dvc.dependency.https import DependencyHTTPS
-from dvc.dependency.local import DependencyLOCAL
-from dvc.dependency.s3 import DependencyS3
-from dvc.dependency.ssh import DependencySSH
-from dvc.dependency.param import DependencyPARAMS
-from dvc.output.base import OutputBase
+from dvc.dependency.gs import GSDependency
+from dvc.dependency.hdfs import HDFSDependency
+from dvc.dependency.http import HTTPDependency
+from dvc.dependency.https import HTTPSDependency
+from dvc.dependency.local import LocalDependency
+from dvc.dependency.param import ParamsDependency
+from dvc.dependency.s3 import S3Dependency
+from dvc.dependency.ssh import SSHDependency
+from dvc.output.base import BaseOutput
 from dvc.remote import Remote
 from dvc.scheme import Schemes
-from .repo import DependencyREPO
 
+from .repo import RepoDependency
 
 DEPS = [
-    DependencyGS,
-    DependencyHDFS,
-    DependencyHTTP,
-    DependencyHTTPS,
-    DependencyS3,
-    DependencySSH,
-    # NOTE: DependencyLOCAL is the default choice
+    GSDependency,
+    HDFSDependency,
+    HTTPDependency,
+    HTTPSDependency,
+    S3Dependency,
+    SSHDependency,
+    # NOTE: LocalDependency is the default choice
 ]
 
 DEP_MAP = {
-    Schemes.LOCAL: DependencyLOCAL,
-    Schemes.SSH: DependencySSH,
-    Schemes.S3: DependencyS3,
-    Schemes.GS: DependencyGS,
-    Schemes.HDFS: DependencyHDFS,
-    Schemes.HTTP: DependencyHTTP,
-    Schemes.HTTPS: DependencyHTTPS,
+    Schemes.LOCAL: LocalDependency,
+    Schemes.SSH: SSHDependency,
+    Schemes.S3: S3Dependency,
+    Schemes.GS: GSDependency,
+    Schemes.HDFS: HDFSDependency,
+    Schemes.HTTP: HTTPDependency,
+    Schemes.HTTPS: HTTPSDependency,
 }
 
 
@@ -42,10 +42,10 @@ DEP_MAP = {
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
 # output is a metric file and how to parse it, see `-M` flag for `dvc run`).
 SCHEMA = output.SCHEMA.copy()
-del SCHEMA[OutputBase.PARAM_CACHE]
-del SCHEMA[OutputBase.PARAM_METRIC]
-SCHEMA.update(DependencyREPO.REPO_SCHEMA)
-SCHEMA.update(DependencyPARAMS.PARAM_SCHEMA)
+del SCHEMA[BaseOutput.PARAM_CACHE]
+del SCHEMA[BaseOutput.PARAM_METRIC]
+SCHEMA.update(RepoDependency.REPO_SCHEMA)
+SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
 
 
 def _get(stage, p, info):
@@ -54,24 +54,24 @@ def _get(stage, p, info):
         remote = Remote(stage.repo, name=parsed.netloc)
         return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
 
-    if info and info.get(DependencyREPO.PARAM_REPO):
-        repo = info.pop(DependencyREPO.PARAM_REPO)
-        return DependencyREPO(repo, stage, p, info)
+    if info and info.get(RepoDependency.PARAM_REPO):
+        repo = info.pop(RepoDependency.PARAM_REPO)
+        return RepoDependency(repo, stage, p, info)
 
-    if info and info.get(DependencyPARAMS.PARAM_PARAMS):
-        params = info.pop(DependencyPARAMS.PARAM_PARAMS)
-        return DependencyPARAMS(stage, p, params)
+    if info and info.get(ParamsDependency.PARAM_PARAMS):
+        params = info.pop(ParamsDependency.PARAM_PARAMS)
+        return ParamsDependency(stage, p, params)
 
     for d in DEPS:
         if d.supported(p):
             return d(stage, p, info)
-    return DependencyLOCAL(stage, p, info)
+    return LocalDependency(stage, p, info)
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(OutputBase.PARAM_PATH, None)
+        p = d.pop(BaseOutput.PARAM_PATH, None)
         ret.append(_get(stage, p, d))
     return ret
 
@@ -79,7 +79,7 @@ def loadd_from(stage, d_list):
 def loads_from(stage, s_list, erepo=None):
     ret = []
     for s in s_list:
-        info = {DependencyREPO.PARAM_REPO: erepo} if erepo else {}
+        info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
         ret.append(_get(stage, s, info))
     return ret
 
@@ -101,8 +101,8 @@ def loads_params(stage, s_list):
     for path, params in params_by_path.items():
         d_list.append(
             {
-                OutputBase.PARAM_PATH: path,
-                DependencyPARAMS.PARAM_PARAMS: params,
+                BaseOutput.PARAM_PATH: path,
+                ParamsDependency.PARAM_PARAMS: params,
             }
         )
 

dvc/dependency/base.py

@@ -3,24 +3,22 @@ from dvc.exceptions import DvcException
 
 class DependencyDoesNotExistError(DvcException):
     def __init__(self, path):
-        msg = "dependency '{}' does not exist".format(path)
+        msg = f"dependency '{path}' does not exist"
         super().__init__(msg)
 
 
 class DependencyIsNotFileOrDirError(DvcException):
     def __init__(self, path):
-        msg = "dependency '{}' is not a file or directory".format(path)
+        msg = f"dependency '{path}' is not a file or directory"
         super().__init__(msg)
 
 
 class DependencyIsStageFileError(DvcException):
     def __init__(self, path):
-        super().__init__(
-            "Stage file '{}' cannot be a dependency.".format(path)
-        )
+        super().__init__(f"Stage file '{path}' cannot be a dependency.")
 
 
-class DependencyBase(object):
+class BaseDependency:
     IS_DEPENDENCY = True
 
     DoesNotExistError = DependencyDoesNotExistError

dvc/dependency/gs.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.gs import OutputGS
+from dvc.dependency.base import BaseDependency
+from dvc.output.gs import GSOutput
 
 
-class DependencyGS(DependencyBase, OutputGS):
+class GSDependency(BaseDependency, GSOutput):
     pass

dvc/dependency/hdfs.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.hdfs import OutputHDFS
+from dvc.dependency.base import BaseDependency
+from dvc.output.hdfs import HDFSOutput
 
 
-class DependencyHDFS(DependencyBase, OutputHDFS):
+class HDFSDependency(BaseDependency, HDFSOutput):
     pass

dvc/dependency/http.py

@@ -1,7 +1,7 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.base import OutputBase
-from dvc.remote.http import RemoteHTTP
+from dvc.dependency.base import BaseDependency
+from dvc.output.base import BaseOutput
+from dvc.remote.http import HTTPRemote
 
 
-class DependencyHTTP(DependencyBase, OutputBase):
-    REMOTE = RemoteHTTP
+class HTTPDependency(BaseDependency, BaseOutput):
+    REMOTE = HTTPRemote

dvc/dependency/https.py

@@ -1,6 +1,7 @@
-from .http import DependencyHTTP
-from dvc.remote.https import RemoteHTTPS
+from dvc.remote.https import HTTPSRemote
+
+from .http import HTTPDependency
 
 
-class DependencyHTTPS(DependencyHTTP):
-    REMOTE = RemoteHTTPS
+class HTTPSDependency(HTTPDependency):
+    REMOTE = HTTPSRemote

dvc/dependency/local.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.local import OutputLOCAL
+from dvc.dependency.base import BaseDependency
+from dvc.output.local import LocalOutput
 
 
-class DependencyLOCAL(DependencyBase, OutputLOCAL):
+class LocalDependency(BaseDependency, LocalOutput):
     pass

dvc/dependency/param.py

@@ -1,12 +1,11 @@
 import os
-import yaml
 from collections import defaultdict
 
 import dpath.util
+import yaml
 from voluptuous import Any
 
-from dvc.compat import fspath_py35
-from dvc.dependency.local import DependencyLOCAL
+from dvc.dependency.local import LocalDependency
 from dvc.exceptions import DvcException
 
 
@@ -18,7 +17,7 @@ class BadParamFileError(DvcException):
     pass
 
 
-class DependencyPARAMS(DependencyLOCAL):
+class ParamsDependency(LocalDependency):
     PARAM_PARAMS = "params"
     PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
     DEFAULT_PARAMS_FILE = "params.yaml"
@@ -41,6 +40,15 @@ class DependencyPARAMS(DependencyLOCAL):
             info=info,
         )
 
+    def _dyn_load(self, values=None):
+        """Load params values dynamically."""
+        if not values:
+            return
+        for param in self.params:
+            value = values.get(param)
+            if value:
+                self.info[param] = value
+
     def save(self):
         super().save()
         self.info = self.save_info()
@@ -52,7 +60,7 @@ class DependencyPARAMS(DependencyLOCAL):
             return status
 
         status = defaultdict(dict)
-        info = self._get_info()
+        info = self.read_params()
         for param in self.params:
             if param not in info.keys():
                 st = "deleted"
@@ -74,25 +82,28 @@ class DependencyPARAMS(DependencyLOCAL):
             self.PARAM_PARAMS: self.info or self.params,
         }
 
-    def _get_info(self):
+    def read_params(self):
         if not self.exists:
             return {}
 
-        with open(fspath_py35(self.path_info), "r") as fobj:
+        with self.repo.tree.open(self.path_info, "r") as fobj:
             try:
                 config = yaml.safe_load(fobj)
             except yaml.YAMLError as exc:
                 raise BadParamFileError(
-                    "Unable to read parameters from '{}'".format(self)
+                    f"Unable to read parameters from '{self}'"
                 ) from exc
 
         ret = {}
         for param in self.params:
-            ret[param] = dpath.util.get(config, param, separator=".")
+            try:
+                ret[param] = dpath.util.get(config, param, separator=".")
+            except KeyError:
+                pass
         return ret
 
     def save_info(self):
-        info = self._get_info()
+        info = self.read_params()
 
         missing_params = set(self.params) - set(info.keys())
         if missing_params:

dvc/dependency/repo.py

@@ -2,12 +2,13 @@ import os
 
 from voluptuous import Required
 
-from .local import DependencyLOCAL
 from dvc.exceptions import OutputNotFoundError
 from dvc.path_info import PathInfo
 
+from .local import LocalDependency
 
-class DependencyREPO(DependencyLOCAL):
+
+class RepoDependency(LocalDependency):
     PARAM_REPO = "repo"
     PARAM_URL = "url"
     PARAM_REV = "rev"
@@ -49,13 +50,24 @@ class DependencyREPO(DependencyLOCAL):
         return external_repo(d["url"], rev=rev)
 
     def _get_checksum(self, locked=True):
+        from dvc.repo.tree import RepoTree
+
         with self._make_repo(locked=locked) as repo:
             try:
                 return repo.find_out_by_relpath(self.def_path).info["md5"]
             except OutputNotFoundError:
                 path = PathInfo(os.path.join(repo.root_dir, self.def_path))
+
+                # we want stream but not fetch, so DVC out directories are
+                # walked, but dir contents is not fetched
+                tree = RepoTree(repo, stream=True)
+
                 # We are polluting our repo cache with some dir listing here
-                return self.repo.cache.local.get_checksum(path)
+                if tree.isdir(path):
+                    return self.repo.cache.local.get_dir_checksum(
+                        path, tree=tree
+                    )
+                return tree.get_file_checksum(path)
 
     def status(self):
         current_checksum = self._get_checksum(locked=True)
@@ -75,16 +87,16 @@ class DependencyREPO(DependencyLOCAL):
     def download(self, to):
         with self._make_repo() as repo:
             if self.def_repo.get(self.PARAM_REV_LOCK) is None:
-                self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+                self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
 
-            if hasattr(repo, "cache"):
-                repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
-
-            repo.pull_to(self.def_path, to.path_info)
+            cache = self.repo.cache.local
+            with repo.use_cache(cache):
+                _, _, cache_infos = repo.fetch_external([self.def_path])
+            cache.checkout(to.path_info, cache_infos[0])
 
     def update(self, rev=None):
         if rev:
             self.def_repo[self.PARAM_REV] = rev
 
         with self._make_repo(locked=False) as repo:
-            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+            self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()

dvc/dependency/s3.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.s3 import OutputS3
+from dvc.dependency.base import BaseDependency
+from dvc.output.s3 import S3Output
 
 
-class DependencyS3(DependencyBase, OutputS3):
+class S3Dependency(BaseDependency, S3Output):
     pass

dvc/dependency/ssh.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.ssh import OutputSSH
+from dvc.dependency.base import BaseDependency
+from dvc.output.ssh import SSHOutput
 
 
-class DependencySSH(DependencyBase, OutputSSH):
+class SSHDependency(BaseDependency, SSHOutput):
     pass

dvc/dvcfile.py

@@ -0,0 +1,270 @@
+import contextlib
+import logging
+import os
+
+from voluptuous import MultipleInvalid
+
+import dvc.prompt as prompt
+from dvc import serialize
+from dvc.exceptions import DvcException
+from dvc.stage.exceptions import (
+    StageFileAlreadyExistsError,
+    StageFileBadNameError,
+    StageFileDoesNotExistError,
+    StageFileFormatError,
+    StageFileIsNotDvcFileError,
+)
+from dvc.stage.loader import SingleStageLoader, StageLoader
+from dvc.utils import relpath
+from dvc.utils.collections import apply_diff
+from dvc.utils.stage import (
+    dump_stage_file,
+    parse_stage,
+    parse_stage_for_update,
+)
+
+logger = logging.getLogger(__name__)
+
+DVC_FILE = "Dvcfile"
+DVC_FILE_SUFFIX = ".dvc"
+PIPELINE_FILE = "dvc.yaml"
+PIPELINE_LOCK = "dvc.lock"
+
+
+class LockfileCorruptedError(DvcException):
+    pass
+
+
+def is_valid_filename(path):
+    return path.endswith(DVC_FILE_SUFFIX) or os.path.basename(path) in [
+        DVC_FILE,
+        PIPELINE_FILE,
+    ]
+
+
+def is_dvc_file(path):
+    return os.path.isfile(path) and (
+        is_valid_filename(path) or os.path.basename(path) == PIPELINE_LOCK
+    )
+
+
+def check_dvc_filename(path):
+    if not is_valid_filename(path):
+        raise StageFileBadNameError(
+            "bad DVC-file name '{}'. DVC-files should be named "
+            "'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
+                relpath(path), os.path.basename(path)
+            )
+        )
+
+
+class FileMixin:
+    SCHEMA = None
+
+    def __init__(self, repo, path, **kwargs):
+        self.repo = repo
+        self.path = path
+
+    def __repr__(self):
+        return "{}: {}".format(
+            self.__class__.__name__, relpath(self.path, self.repo.root_dir)
+        )
+
+    def __hash__(self):
+        return hash(self.path)
+
+    def __eq__(self, other):
+        return self.repo == other.repo and os.path.abspath(
+            self.path
+        ) == os.path.abspath(other.path)
+
+    def __str__(self):
+        return f"{self.__class__.__name__}: {self.relpath}"
+
+    @property
+    def relpath(self):
+        return relpath(self.path)
+
+    def exists(self):
+        return self.repo.tree.exists(self.path)
+
+    def _load(self):
+        # it raises the proper exceptions by priority:
+        # 1. when the file doesn't exists
+        # 2. filename is not a DVC-file
+        # 3. path doesn't represent a regular file
+        if not self.exists():
+            raise StageFileDoesNotExistError(self.path)
+        check_dvc_filename(self.path)
+        if not self.repo.tree.isfile(self.path):
+            raise StageFileIsNotDvcFileError(self.path)
+
+        with self.repo.tree.open(self.path) as fd:
+            stage_text = fd.read()
+        d = parse_stage(stage_text, self.path)
+        self.validate(d, self.relpath)
+        return d, stage_text
+
+    @classmethod
+    def validate(cls, d, fname=None):
+        assert cls.SCHEMA
+        try:
+            cls.SCHEMA(d)
+        except MultipleInvalid as exc:
+            raise StageFileFormatError(f"'{fname}' format error: {exc}")
+
+    def remove_with_prompt(self, force=False):
+        raise NotImplementedError
+
+    def remove(self, force=False):
+        with contextlib.suppress(FileNotFoundError):
+            os.unlink(self.path)
+
+    def dump(self, stage, **kwargs):
+        raise NotImplementedError
+
+
+class SingleStageFile(FileMixin):
+    from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
+
+    def __init__(self, repo, path):
+        super().__init__(repo, path)
+
+    @property
+    def stage(self):
+        data, raw = self._load()
+        return SingleStageLoader.load_stage(self, data, raw)
+
+    @property
+    def stages(self):
+        data, raw = self._load()
+        return SingleStageLoader(self, data, raw)
+
+    def dump(self, stage, **kwargs):
+        """Dumps given stage appropriately in the dvcfile."""
+        from dvc.stage import PipelineStage
+
+        assert not isinstance(stage, PipelineStage)
+        check_dvc_filename(self.path)
+        logger.debug(
+            "Saving information to '{file}'.".format(file=relpath(self.path))
+        )
+        dump_stage_file(self.path, serialize.to_single_stage_file(stage))
+        self.repo.scm.track_file(relpath(self.path))
+
+    def remove_with_prompt(self, force=False):
+        if not self.exists():
+            return
+
+        msg = (
+            "'{}' already exists. Do you wish to run the command and "
+            "overwrite it?".format(relpath(self.path))
+        )
+        if not (force or prompt.confirm(msg)):
+            raise StageFileAlreadyExistsError(self.path)
+
+        self.remove()
+
+
+class PipelineFile(FileMixin):
+    """Abstraction for pipelines file, .yaml + .lock combined."""
+
+    from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA
+
+    @property
+    def _lockfile(self):
+        return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
+
+    def dump(self, stage, update_pipeline=False, **kwargs):
+        """Dumps given stage appropriately in the dvcfile."""
+        from dvc.stage import PipelineStage
+
+        assert isinstance(stage, PipelineStage)
+        check_dvc_filename(self.path)
+        self._dump_lockfile(stage)
+        if update_pipeline and not stage.is_data_source:
+            self._dump_pipeline_file(stage)
+
+    def _dump_lockfile(self, stage):
+        self._lockfile.dump(stage)
+
+    def _dump_pipeline_file(self, stage):
+        data = {}
+        if self.exists():
+            with open(self.path) as fd:
+                data = parse_stage_for_update(fd.read(), self.path)
+        else:
+            open(self.path, "w+").close()
+
+        data["stages"] = data.get("stages", {})
+        stage_data = serialize.to_pipeline_file(stage)
+        if data["stages"].get(stage.name):
+            orig_stage_data = data["stages"][stage.name]
+            apply_diff(stage_data[stage.name], orig_stage_data)
+        else:
+            data["stages"].update(stage_data)
+
+        dump_stage_file(self.path, data)
+        self.repo.scm.track_file(relpath(self.path))
+
+    @property
+    def stage(self):
+        raise DvcException(
+            "PipelineFile has multiple stages. Please specify it's name."
+        )
+
+    @property
+    def stages(self):
+        data, _ = self._load()
+        lockfile_data = self._lockfile.load()
+        return StageLoader(self, data.get("stages", {}), lockfile_data)
+
+    def remove(self, force=False):
+        if not force:
+            logger.warning("Cannot remove pipeline file.")
+            return
+
+        super().remove()
+        self._lockfile.remove()
+
+
+class Lockfile(FileMixin):
+    from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
+
+    def load(self):
+        if not self.exists():
+            return {}
+        with self.repo.tree.open(self.path) as fd:
+            data = parse_stage(fd.read(), self.path)
+        try:
+            self.validate(data, fname=self.relpath)
+        except StageFileFormatError:
+            raise LockfileCorruptedError(
+                f"Lockfile '{self.relpath}' is corrupted."
+            )
+        return data
+
+    def dump(self, stage, **kwargs):
+        stage_data = serialize.to_lockfile(stage)
+        if not self.exists():
+            data = stage_data
+            open(self.path, "w+").close()
+        else:
+            with self.repo.tree.open(self.path, "r") as fd:
+                data = parse_stage_for_update(fd.read(), self.path)
+            data.update(stage_data)
+
+        dump_stage_file(self.path, data)
+        self.repo.scm.track_file(relpath(self.path))
+
+
+class Dvcfile:
+    def __new__(cls, repo, path, **kwargs):
+        assert path
+        assert repo
+
+        _, ext = os.path.splitext(path)
+        if ext in [".yaml", ".yml"]:
+            return PipelineFile(repo, path, **kwargs)
+        # fallback to single stage file for better error messages
+        return SingleStageFile(repo, path, **kwargs)

dvc/exceptions.py

@@ -1,11 +1,16 @@
 """Exceptions raised by the dvc."""
+from funcy import first
 
-from dvc.utils import relpath, format_link
+from dvc.utils import format_link, relpath
 
 
 class DvcException(Exception):
     """Base class for all dvc exceptions."""
 
+    def __init__(self, msg, *args):
+        assert msg
+        super().__init__(msg, *args)
+
 
 class InvalidArgumentError(ValueError, DvcException):
     """Thrown if arguments are invalid."""
@@ -23,12 +28,17 @@ class OutputDuplicationError(DvcException):
     def __init__(self, output, stages):
         assert isinstance(output, str)
         assert all(hasattr(stage, "relpath") for stage in stages)
-        msg = (
-            "file/directory '{}' is specified as an output in more than one "
-            "stage: {}\n"
-            "This is not allowed. Consider using a different output name."
-        ).format(output, "\n    ".join(s.relpath for s in stages))
+        if len(stages) == 1:
+            msg = "output '{}' is already specified in {}.".format(
+                output, first(stages)
+            )
+        else:
+            msg = "output '{}' is already specified in stages:\n{}".format(
+                output, "\n".join(f"\t- {s.addressing}" for s in stages),
+            )
         super().__init__(msg)
+        self.stages = stages
+        self.output = output
 
 
 class OutputNotFoundError(DvcException):
@@ -60,8 +70,8 @@ class StagePathAsOutputError(DvcException):
     def __init__(self, stage, output):
         assert isinstance(output, str)
         super().__init__(
-            "'{stage}' is within an output '{output}' of another stage".format(
-                stage=stage.relpath, output=output
+            "{stage} is within an output '{output}' of another stage".format(
+                stage=stage, output=output
             )
         )
 
@@ -77,10 +87,7 @@ class CircularDependencyError(DvcException):
     def __init__(self, dependency):
         assert isinstance(dependency, str)
 
-        msg = (
-            "file/directory '{}' is specified as an output and as a "
-            "dependency."
-        )
+        msg = "'{}' is specified as an output and as a dependency."
         super().__init__(msg.format(dependency))
 
 
@@ -94,7 +101,7 @@ class ArgumentDuplicationError(DvcException):
 
     def __init__(self, path):
         assert isinstance(path, str)
-        super().__init__("file '{}' is specified more than once.".format(path))
+        super().__init__(f"file '{path}' is specified more than once.")
 
 
 class MoveNotDataSourceError(DvcException):
@@ -129,12 +136,8 @@ class DvcParserError(DvcException):
 class CyclicGraphError(DvcException):
     def __init__(self, stages):
         assert isinstance(stages, list)
-        stages = "\n".join("\t- {}".format(stage.relpath) for stage in stages)
-        msg = (
-            "you've introduced a cycle in your pipeline that involves "
-            "the following stages:"
-            "\n"
-            "{stages}".format(stages=stages)
+        msg = "Pipeline has a cycle involving: {}.".format(
+            ", ".join(s.addressing for s in stages)
         )
         super().__init__(msg)
 
@@ -142,7 +145,7 @@ class CyclicGraphError(DvcException):
 class ConfirmRemoveError(DvcException):
     def __init__(self, path):
         super().__init__(
-            "unable to remove '{}' without a confirmation from the user. Use "
+            "unable to remove '{}' without a confirmation. Use "
             "`-f` to force.".format(path)
         )
 
@@ -154,7 +157,7 @@ class InitError(DvcException):
 class ReproductionError(DvcException):
     def __init__(self, dvc_file_name):
         self.path = dvc_file_name
-        super().__init__("failed to reproduce '{}'".format(dvc_file_name))
+        super().__init__(f"failed to reproduce '{dvc_file_name}'")
 
 
 class BadMetricError(DvcException):
@@ -162,7 +165,7 @@ class BadMetricError(DvcException):
         super().__init__(
             "the following metrics do not exist, "
             "are not metric files or are malformed: {paths}".format(
-                paths=", ".join("'{}'".format(path) for path in paths)
+                paths=", ".join(f"'{path}'" for path in paths)
             )
         )
 
@@ -200,7 +203,7 @@ class OverlappingOutputPathsError(DvcException):
 
 class CheckoutErrorSuggestGit(DvcException):
     def __init__(self, target):
-        super().__init__("Did you mean `git checkout {}`?".format(target))
+        super().__init__(f"Did you mean `git checkout {target}`?")
 
 
 class ETagMismatchError(DvcException):
@@ -214,9 +217,7 @@ class ETagMismatchError(DvcException):
 class FileMissingError(DvcException):
     def __init__(self, path):
         self.path = path
-        super().__init__(
-            "Can't find '{}' neither locally nor on remote".format(path)
-        )
+        super().__init__(f"Can't find '{path}' neither locally nor on remote")
 
 
 class DvcIgnoreInCollectedDirError(DvcException):
@@ -241,18 +242,14 @@ class DownloadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super().__init__(
-            "{amount} files failed to download".format(amount=amount)
-        )
+        super().__init__(f"{amount} files failed to download")
 
 
 class UploadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super().__init__(
-            "{amount} files failed to upload".format(amount=amount)
-        )
+        super().__init__(f"{amount} files failed to upload")
 
 
 class CheckoutError(DvcException):
@@ -274,7 +271,7 @@ class CollectCacheError(DvcException):
 class NoRemoteInExternalRepoError(DvcException):
     def __init__(self, url):
         super().__init__(
-            "No DVC remote is specified in target repository '{}'.".format(url)
+            f"No DVC remote is specified in target repository '{url}'."
         )
 
 
@@ -289,7 +286,7 @@ class NoOutputInExternalRepoError(DvcException):
 
 class HTTPError(DvcException):
     def __init__(self, code, reason):
-        super().__init__("'{} {}'".format(code, reason))
+        super().__init__(f"'{code} {reason}'")
 
 
 class PathMissingError(DvcException):
@@ -297,19 +294,15 @@ class PathMissingError(DvcException):
         "The path '{}' does not exist in the target repository '{}'"
         " neither as a DVC output nor as a Git-tracked file."
     )
-    default_msg_output_only = (
+    default_msg_dvc_only = (
         "The path '{}' does not exist in the target repository '{}'"
         " as an DVC output."
     )
 
-    def __init__(self, path, repo, output_only=False):
-        msg = (
-            self.default_msg
-            if not output_only
-            else self.default_msg_output_only
-        )
+    def __init__(self, path, repo, dvc_only=False):
+        msg = self.default_msg if not dvc_only else self.default_msg_dvc_only
         super().__init__(msg.format(path, repo))
-        self.output_only = output_only
+        self.dvc_only = dvc_only
 
 
 class RemoteCacheRequiredError(DvcException):

dvc/external_repo.py

@@ -1,23 +1,27 @@
 import logging
 import os
 import tempfile
+import threading
 from contextlib import contextmanager
 from distutils.dir_util import copy_tree
-import threading
+from typing import Iterable
 
-from funcy import retry, suppress, wrap_with, cached_property
+from funcy import cached_property, retry, wrap_with
 
-from dvc.path_info import PathInfo
-from dvc.compat import fspath
-from dvc.repo import Repo
 from dvc.config import NoRemoteError, NotDvcRepoError
-from dvc.exceptions import NoRemoteInExternalRepoError, CheckoutError
-from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
-from dvc.exceptions import FileMissingError, PathMissingError
-from dvc.utils.fs import remove, fs_copy, move
-from dvc.utils import tmp_fname
+from dvc.exceptions import (
+    FileMissingError,
+    NoOutputInExternalRepoError,
+    NoRemoteInExternalRepoError,
+    OutputNotFoundError,
+    PathMissingError,
+)
+from dvc.path_info import PathInfo
+from dvc.repo import Repo
+from dvc.repo.tree import RepoTree
 from dvc.scm.git import Git
-
+from dvc.scm.tree import is_working_tree
+from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
@@ -26,10 +30,12 @@ logger = logging.getLogger(__name__)
 def external_repo(url, rev=None, for_write=False):
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
+    if not rev:
+        rev = "HEAD"
     try:
-        repo = ExternalRepo(path, url)
+        repo = ExternalRepo(path, url, rev, for_write=for_write)
     except NotDvcRepoError:
-        repo = ExternalGitRepo(path, url)
+        repo = ExternalGitRepo(path, url, rev)
 
     try:
         yield repo
@@ -61,59 +67,99 @@ def clean_repos():
         _remove(path)
 
 
-class ExternalRepo(Repo):
-    def __init__(self, root_dir, url):
-        super().__init__(root_dir)
+class BaseExternalRepo:
+
+    _local_cache = None
+
+    @property
+    def local_cache(self):
+        if hasattr(self, "cache"):
+            return self.cache.local
+        return self._local_cache
+
+    @contextmanager
+    def use_cache(self, cache):
+        """Use the specified cache in place of default tmpdir cache for
+        download operations.
+        """
+        if hasattr(self, "cache"):
+            save_cache = self.cache.local
+            self.cache.local = cache
+        self._local_cache = cache
+
+        yield
+
+        if hasattr(self, "cache"):
+            self.cache.local = save_cache
+        self._local_cache = None
+
+    @cached_property
+    def repo_tree(self):
+        return RepoTree(self, fetch=True)
+
+    def get_rev(self):
+        if is_working_tree(self.tree):
+            return self.scm.get_rev()
+        if hasattr(self.tree, "tree"):
+            return self.tree.tree.rev
+        return self.tree.rev
+
+    def fetch_external(self, paths: Iterable, **kwargs):
+        """Fetch specified external repo paths into cache.
+
+        Returns 3-tuple in the form
+            (downloaded, failed, list(cache_infos))
+        where cache_infos can be used as checkout targets for the
+        fetched paths.
+        """
+        download_results = []
+        failed = 0
+
+        paths = [PathInfo(self.root_dir) / path for path in paths]
+
+        def download_update(result):
+            download_results.append(result)
+
+        save_infos = []
+        for path in paths:
+            if not self.repo_tree.exists(path):
+                raise PathMissingError(path, self.url)
+            save_info = self.local_cache.save(
+                path,
+                None,
+                tree=self.repo_tree,
+                download_callback=download_update,
+            )
+            save_infos.append(save_info)
+
+        return sum(download_results), failed, save_infos
+
+    def get_external(self, path, dest):
+        """Convenience wrapper for fetch_external and checkout."""
+        if self.local_cache:
+            # fetch DVC and git files to tmpdir cache, then checkout
+            _, _, save_infos = self.fetch_external([path])
+            self.local_cache.checkout(PathInfo(dest), save_infos[0])
+        else:
+            # git-only erepo with no cache, just copy files directly
+            # to dest
+            path = PathInfo(self.root_dir) / path
+            if not self.repo_tree.exists(path):
+                raise PathMissingError(path, self.url)
+            self.repo_tree.copytree(path, dest)
+
+
+class ExternalRepo(Repo, BaseExternalRepo):
+    def __init__(self, root_dir, url, rev, for_write=False):
+        if for_write:
+            super().__init__(root_dir)
+        else:
+            root_dir = os.path.realpath(root_dir)
+            super().__init__(root_dir, scm=Git(root_dir), rev=rev)
         self.url = url
         self._set_cache_dir()
         self._fix_upstream()
 
-    def pull_to(self, path, to_info):
-        """
-        Pull the corresponding file or directory specified by `path` and
-        checkout it into `to_info`.
-
-        It works with files tracked by Git and DVC, and also local files
-        outside the repository.
-        """
-        out = None
-        path_info = PathInfo(self.root_dir) / path
-
-        with suppress(OutputNotFoundError):
-            (out,) = self.find_outs_by_path(fspath(path_info), strict=False)
-
-        try:
-            if out and out.use_cache:
-                self._pull_cached(out, path_info, to_info)
-                return
-
-            # Check if it is handled by Git (it can't have an absolute path)
-            if os.path.isabs(path):
-                raise FileNotFoundError
-
-            fs_copy(fspath(path_info), fspath(to_info))
-        except FileNotFoundError:
-            raise PathMissingError(path, self.url)
-
-    def _pull_cached(self, out, path_info, dest):
-        with self.state:
-            tmp = PathInfo(tmp_fname(dest))
-            src = tmp / path_info.relative_to(out.path_info)
-
-            out.path_info = tmp
-
-            # Only pull unless all needed cache is present
-            if out.changed_cache(filter_info=src):
-                self.cloud.pull(out.get_used_cache(filter_info=src))
-
-            try:
-                out.checkout(filter_info=src)
-            except CheckoutError:
-                raise FileNotFoundError
-
-            move(src, dest)
-            remove(tmp)
-
     @wrap_with(threading.Lock())
     def _set_cache_dir(self):
         try:
@@ -122,6 +168,7 @@ class ExternalRepo(Repo):
             cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
 
         self.cache.local.cache_dir = cache_dir
+        self._local_cache = self.cache.local
 
     def _fix_upstream(self):
         if not os.path.isdir(self.url):
@@ -162,10 +209,11 @@ class ExternalRepo(Repo):
         self.config["core"]["remote"] = "auto-generated-upstream"
 
 
-class ExternalGitRepo:
-    def __init__(self, root_dir, url):
-        self.root_dir = root_dir
+class ExternalGitRepo(BaseExternalRepo):
+    def __init__(self, root_dir, url, rev):
+        self.root_dir = os.path.realpath(root_dir)
         self.url = url
+        self.tree = self.scm.get_tree(rev)
 
     @cached_property
     def scm(self):
@@ -178,23 +226,15 @@ class ExternalGitRepo:
     def find_out_by_relpath(self, path):
         raise OutputNotFoundError(path, self)
 
-    def pull_to(self, path, to_info):
-        try:
-            # Git handled files can't have absolute path
-            if os.path.isabs(path):
-                raise FileNotFoundError
-
-            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
-        except FileNotFoundError:
-            raise PathMissingError(path, self.url)
-
     @contextmanager
     def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
         """Opens a specified resource as a file object."""
+        tree = RepoTree(self)
         try:
-            abs_path = os.path.join(self.root_dir, path)
-            with open(abs_path, mode, encoding=encoding) as fd:
-                yield fd
+            with tree.open(
+                path, mode=mode, encoding=encoding, **kwargs
+            ) as fobj:
+                yield fobj
         except FileNotFoundError:
             raise PathMissingError(path, self.url)
 
@@ -206,14 +246,12 @@ def _cached_clone(url, rev, for_write=False):
     revision checked out. If for_write is set prevents reusing this dir via
     cache.
     """
-    if not for_write and Git.is_sha(rev) and (url, rev) in CLONES:
-        return CLONES[url, rev]
-
+    # even if we have already cloned this repo, we may need to
+    # fetch/fast-forward to get specified rev
     clone_path = _clone_default_branch(url, rev)
-    rev_sha = Git(clone_path).resolve_rev(rev or "HEAD")
 
-    if not for_write and (url, rev_sha) in CLONES:
-        return CLONES[url, rev_sha]
+    if not for_write and (url) in CLONES:
+        return CLONES[url]
 
     # Copy to a new dir to keep the clone clean
     repo_path = tempfile.mkdtemp("dvc-erepo")
@@ -221,11 +259,10 @@ def _cached_clone(url, rev, for_write=False):
     copy_tree(clone_path, repo_path)
 
     # Check out the specified revision
-    if rev is not None:
+    if for_write:
         _git_checkout(repo_path, rev)
-
-    if not for_write:
-        CLONES[url, rev_sha] = repo_path
+    else:
+        CLONES[url] = repo_path
     return repo_path
 
 

dvc/ignore.py

@@ -11,7 +11,7 @@ from dvc.utils import relpath
 logger = logging.getLogger(__name__)
 
 
-class DvcIgnore(object):
+class DvcIgnore:
     DVCIGNORE_FILE = ".dvcignore"
 
     def __call__(self, root, dirs, files):
@@ -83,7 +83,7 @@ class DvcIgnoreRepo(DvcIgnore):
         return dirs, files
 
 
-class DvcIgnoreFilter(object):
+class DvcIgnoreFilter:
     def __init__(self, tree):
         self.tree = tree
         self.ignores = {
@@ -130,13 +130,14 @@ class CleanTree(BaseTree):
     def isfile(self, path):
         return self.tree.isfile(path)
 
+    def isexec(self, path):
+        return self.tree.isexec(path)
+
     def walk(self, top, topdown=True):
         for root, dirs, files in self.tree.walk(top, topdown):
             dirs[:], files[:] = self.dvcignore(root, dirs, files)
 
             yield root, dirs, files
 
-    def walk_files(self, top):
-        for root, _, files in self.walk(top):
-            for file in files:
-                yield os.path.join(root, file)
+    def stat(self, path):
+        return self.tree.stat(path)

dvc/istextfile.py

@@ -7,13 +7,17 @@
 TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
 
 
-def istextfile(fname, blocksize=512):
+def istextfile(fname, blocksize=512, tree=None):
     """ Uses heuristics to guess whether the given file is text or binary,
         by reading a single block of bytes from the file.
         If more than 30% of the chars in the block are non-text, or there
         are NUL ('\x00') bytes in the block, assume this is a binary file.
     """
-    with open(fname, "rb") as fobj:
+    if tree:
+        open_func = tree.open
+    else:
+        open_func = open
+    with open_func(fname, "rb") as fobj:
         block = fobj.read(blocksize)
 
     if not block:

dvc/lock.py

@@ -5,19 +5,19 @@ import os
 import time
 from datetime import timedelta
 
-import zc.lockfile
 import flufl.lock
+import zc.lockfile
 
 from dvc.exceptions import DvcException
-from dvc.utils import format_link
 from dvc.progress import Tqdm
+from dvc.utils import format_link
 
 DEFAULT_TIMEOUT = 5
 
 FAILED_TO_LOCK_MESSAGE = (
     "cannot perform the command because another DVC process seems to be "
     "running on this project. If that is not the case, manually remove "
-    "`.dvc/lock` and try again."
+    "`.dvc/tmp/lock` and try again."
 )
 
 
@@ -25,7 +25,7 @@ class LockError(DvcException):
     """Thrown when unable to acquire the lock for DVC repo."""
 
 
-class Lock(object):
+class Lock:
     """Class for DVC repo lock.
 
     Uses zc.lockfile as backend.
@@ -47,7 +47,7 @@ class Lock(object):
                 disable=not self._friendly,
                 desc=(
                     "If DVC froze, see `hardlink_lock` in {}".format(
-                        format_link("man.dvc.org/config#core")
+                        format_link("https://man.dvc.org/config#core")
                     )
                 ),
             ):

dvc/logger.py

@@ -7,7 +7,6 @@ import colorama
 
 from dvc.progress import Tqdm
 
-
 FOOTER = (
     "\n{yellow}Having any troubles?{nc}"
     " Hit us up at {blue}https://dvc.org/support{nc},"
@@ -19,20 +18,45 @@ FOOTER = (
 )
 
 
+def addLoggingLevel(levelName, levelNum, methodName=None):
+    """
+    Adds a new logging level to the `logging` module and the
+    currently configured logging class.
+
+    Based on https://stackoverflow.com/questions/2183233
+    """
+    if methodName is None:
+        methodName = levelName.lower()
+
+    assert not hasattr(logging, levelName)
+    assert not hasattr(logging, methodName)
+    assert not hasattr(logging.getLoggerClass(), methodName)
+
+    def logForLevel(self, message, *args, **kwargs):
+        if self.isEnabledFor(levelNum):
+            self._log(levelNum, message, args, **kwargs)
+
+    def logToRoot(message, *args, **kwargs):
+        logging.log(levelNum, message, *args, **kwargs)
+
+    logging.addLevelName(levelNum, levelName)
+    setattr(logging, levelName, levelNum)
+    setattr(logging.getLoggerClass(), methodName, logForLevel)
+    setattr(logging, methodName, logToRoot)
+
+
 class LoggingException(Exception):
     def __init__(self, record):
         msg = "failed to log {}".format(str(record))
         super().__init__(msg)
 
 
-class ExcludeErrorsFilter(logging.Filter):
-    def filter(self, record):
-        return record.levelno < logging.WARNING
+def excludeFilter(level):
+    class ExcludeLevelFilter(logging.Filter):
+        def filter(self, record):
+            return record.levelno < level
 
-
-class ExcludeInfoFilter(logging.Filter):
-    def filter(self, record):
-        return record.levelno < logging.INFO
+    return ExcludeLevelFilter
 
 
 class ColorFormatter(logging.Formatter):
@@ -48,6 +72,7 @@ class ColorFormatter(logging.Formatter):
     """
 
     color_code = {
+        "TRACE": colorama.Fore.GREEN,
         "DEBUG": colorama.Fore.BLUE,
         "WARNING": colorama.Fore.YELLOW,
         "ERROR": colorama.Fore.RED,
@@ -117,7 +142,11 @@ class LoggerHandler(logging.StreamHandler):
 
 
 def _is_verbose():
-    return logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG
+    return (
+        logging.NOTSET
+        < logging.getLogger("dvc").getEffectiveLevel()
+        <= logging.DEBUG
+    )
 
 
 def _iter_causes(exc):
@@ -143,6 +172,7 @@ def _stack_trace(exc_info):
 
 
 def disable_other_loggers():
+    logging.captureWarnings(True)
     root = logging.root
     for (logger_name, logger) in root.manager.loggerDict.items():
         if logger_name != "dvc" and not logger_name.startswith("dvc."):
@@ -152,12 +182,14 @@ def disable_other_loggers():
 def setup(level=logging.INFO):
     colorama.init()
 
+    addLoggingLevel("TRACE", logging.DEBUG - 5)
     logging.config.dictConfig(
         {
             "version": 1,
             "filters": {
-                "exclude_errors": {"()": ExcludeErrorsFilter},
-                "exclude_info": {"()": ExcludeInfoFilter},
+                "exclude_errors": {"()": excludeFilter(logging.WARNING)},
+                "exclude_info": {"()": excludeFilter(logging.INFO)},
+                "exclude_debug": {"()": excludeFilter(logging.DEBUG)},
             },
             "formatters": {"color": {"()": ColorFormatter}},
             "handlers": {
@@ -175,6 +207,13 @@ def setup(level=logging.INFO):
                     "stream": "ext://sys.stdout",
                     "filters": ["exclude_info"],
                 },
+                "console_trace": {
+                    "class": "dvc.logger.LoggerHandler",
+                    "level": "TRACE",
+                    "formatter": "color",
+                    "stream": "ext://sys.stdout",
+                    "filters": ["exclude_debug"],
+                },
                 "console_errors": {
                     "class": "dvc.logger.LoggerHandler",
                     "level": "WARNING",
@@ -188,6 +227,7 @@ def setup(level=logging.INFO):
                     "handlers": [
                         "console_info",
                         "console_debug",
+                        "console_trace",
                         "console_errors",
                     ],
                 },

dvc/main.py

@@ -6,12 +6,11 @@ import logging
 from dvc import analytics
 from dvc.cli import parse_args
 from dvc.config import ConfigError
-from dvc.exceptions import DvcParserError
-from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
 from dvc.external_repo import clean_repos
-from dvc.logger import disable_other_loggers, FOOTER
-from dvc.utils import format_link
+from dvc.logger import FOOTER, disable_other_loggers
 from dvc.remote.pool import close_pools
+from dvc.utils import format_link
 
 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
@@ -19,7 +18,6 @@ from dvc.remote.pool import close_pools
 
 "".encode("idna")
 
-
 logger = logging.getLogger("dvc")
 
 
@@ -33,18 +31,23 @@ def main(argv=None):
         int: command's return code.
     """
     args = None
-    cmd = None
     disable_other_loggers()
 
     outerLogLevel = logger.level
     try:
         args = parse_args(argv)
 
-        if args.quiet:
-            logger.setLevel(logging.CRITICAL)
-
-        elif args.verbose:
-            logger.setLevel(logging.DEBUG)
+        verbosity = args.verbose - args.quiet
+        if verbosity:
+            logger.setLevel(
+                {
+                    -2: logging.CRITICAL,
+                    -1: logging.ERROR,
+                    1: logging.DEBUG,
+                    2: logging.TRACE,
+                }[max(-2, min(verbosity, 2))]
+            )
+        logger.trace(args)
 
         cmd = args.func(args)
         ret = cmd.run()
@@ -59,6 +62,9 @@ def main(argv=None):
         ret = 253
     except DvcParserError:
         ret = 254
+    except DvcException:
+        ret = 255
+        logger.exception("")
     except Exception as exc:  # pylint: disable=broad-except
         if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
             logger.exception(

dvc/output/__init__.py

@@ -1,32 +1,33 @@
 from urllib.parse import urlparse
-from voluptuous import Any, Required, Lower, Length, Coerce, And, SetTo
 
-from dvc.output.base import OutputBase
-from dvc.output.gs import OutputGS
-from dvc.output.hdfs import OutputHDFS
-from dvc.output.local import OutputLOCAL
-from dvc.output.s3 import OutputS3
-from dvc.output.ssh import OutputSSH
+from voluptuous import And, Any, Coerce, Length, Lower, Required, SetTo
+
+from dvc.output.base import BaseOutput
+from dvc.output.gs import GSOutput
+from dvc.output.hdfs import HDFSOutput
+from dvc.output.local import LocalOutput
+from dvc.output.s3 import S3Output
+from dvc.output.ssh import SSHOutput
 from dvc.remote import Remote
-from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
+from dvc.remote.hdfs import HDFSRemote
+from dvc.remote.local import LocalRemote
+from dvc.remote.s3 import S3Remote
 from dvc.scheme import Schemes
 
 OUTS = [
-    OutputHDFS,
-    OutputS3,
-    OutputGS,
-    OutputSSH,
-    # NOTE: OutputLOCAL is the default choice
+    HDFSOutput,
+    S3Output,
+    GSOutput,
+    SSHOutput,
+    # NOTE: LocalOutput is the default choice
 ]
 
 OUTS_MAP = {
-    Schemes.HDFS: OutputHDFS,
-    Schemes.S3: OutputS3,
-    Schemes.GS: OutputGS,
-    Schemes.SSH: OutputSSH,
-    Schemes.LOCAL: OutputLOCAL,
+    Schemes.HDFS: HDFSOutput,
+    Schemes.S3: S3Output,
+    Schemes.GS: GSOutput,
+    Schemes.SSH: SSHOutput,
+    Schemes.LOCAL: LocalOutput,
 }
 
 CHECKSUM_SCHEMA = Any(
@@ -44,22 +45,20 @@ CHECKSUM_SCHEMA = Any(
 # so when a few types of outputs share the same name, we only need
 # specify it once.
 CHECKSUMS_SCHEMA = {
-    RemoteLOCAL.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    RemoteS3.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    RemoteHDFS.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    LocalRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    S3Remote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    HDFSRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
 }
 
-TAGS_SCHEMA = {str: CHECKSUMS_SCHEMA}
-
 SCHEMA = CHECKSUMS_SCHEMA.copy()
-SCHEMA[Required(OutputBase.PARAM_PATH)] = str
-SCHEMA[OutputBase.PARAM_CACHE] = bool
-SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA
-SCHEMA[OutputBase.PARAM_TAGS] = TAGS_SCHEMA
-SCHEMA[OutputBase.PARAM_PERSIST] = bool
+SCHEMA[Required(BaseOutput.PARAM_PATH)] = str
+SCHEMA[BaseOutput.PARAM_CACHE] = bool
+SCHEMA[BaseOutput.PARAM_METRIC] = BaseOutput.METRIC_SCHEMA
+SCHEMA[BaseOutput.PARAM_PLOT] = bool
+SCHEMA[BaseOutput.PARAM_PERSIST] = bool
 
 
-def _get(stage, p, info, cache, metric, persist=False, tags=None):
+def _get(stage, p, info, cache, metric, plot=False, persist=False):
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":
@@ -71,8 +70,8 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
             cache=cache,
             remote=remote,
             metric=metric,
+            plot=plot,
             persist=persist,
-            tags=tags,
         )
 
     for o in OUTS:
@@ -84,29 +83,29 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
                 cache=cache,
                 remote=None,
                 metric=metric,
+                plot=plot,
                 persist=persist,
-                tags=tags,
             )
-    return OutputLOCAL(
+    return LocalOutput(
         stage,
         p,
         info,
         cache=cache,
         remote=None,
         metric=metric,
+        plot=plot,
         persist=persist,
-        tags=tags,
     )
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(OutputBase.PARAM_PATH)
-        cache = d.pop(OutputBase.PARAM_CACHE, True)
-        metric = d.pop(OutputBase.PARAM_METRIC, False)
-        persist = d.pop(OutputBase.PARAM_PERSIST, False)
-        tags = d.pop(OutputBase.PARAM_TAGS, None)
+        p = d.pop(BaseOutput.PARAM_PATH)
+        cache = d.pop(BaseOutput.PARAM_CACHE, True)
+        metric = d.pop(BaseOutput.PARAM_METRIC, False)
+        plot = d.pop(BaseOutput.PARAM_PLOT, False)
+        persist = d.pop(BaseOutput.PARAM_PERSIST, False)
         ret.append(
             _get(
                 stage,
@@ -114,24 +113,25 @@ def loadd_from(stage, d_list):
                 info=d,
                 cache=cache,
                 metric=metric,
+                plot=plot,
                 persist=persist,
-                tags=tags,
             )
         )
     return ret
 
 
-def loads_from(stage, s_list, use_cache=True, metric=False, persist=False):
-    ret = []
-    for s in s_list:
-        ret.append(
-            _get(
-                stage,
-                s,
-                info={},
-                cache=use_cache,
-                metric=metric,
-                persist=persist,
-            )
+def loads_from(
+    stage, s_list, use_cache=True, metric=False, plot=False, persist=False
+):
+    return [
+        _get(
+            stage,
+            s,
+            info={},
+            cache=use_cache,
+            metric=metric,
+            plot=plot,
+            persist=persist,
         )
-    return ret
+        for s in s_list
+    ]

dvc/output/base.py

@@ -1,52 +1,61 @@
 import logging
-from urllib.parse import urlparse
+import os
 from copy import copy
+from urllib.parse import urlparse
 
 from voluptuous import Any
 
 import dvc.prompt as prompt
 from dvc.cache import NamedCache
-from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError
-from dvc.exceptions import DvcException
-from dvc.remote.base import RemoteBASE
-
+from dvc.exceptions import (
+    CollectCacheError,
+    DvcException,
+    RemoteCacheRequiredError,
+)
+from dvc.remote.base import BaseRemote
 
 logger = logging.getLogger(__name__)
 
 
 class OutputDoesNotExistError(DvcException):
     def __init__(self, path):
-        msg = "output '{}' does not exist".format(path)
+        msg = f"output '{path}' does not exist"
         super().__init__(msg)
 
 
 class OutputIsNotFileOrDirError(DvcException):
     def __init__(self, path):
-        msg = "output '{}' is not a file or directory".format(path)
+        msg = f"output '{path}' is not a file or directory"
         super().__init__(msg)
 
 
 class OutputAlreadyTrackedError(DvcException):
     def __init__(self, path):
-        msg = "output '{}' is already tracked by scm (e.g. git)".format(path)
+        msg = f""" output '{path}' is already tracked by SCM (e.g. Git).
+    You can remove it from Git, then add to DVC.
+        To stop tracking from Git:
+            git rm -r --cached '{path}'
+            git commit -m "stop tracking {path}" """
         super().__init__(msg)
 
 
 class OutputIsStageFileError(DvcException):
     def __init__(self, path):
-        super().__init__("Stage file '{}' cannot be an output.".format(path))
+        super().__init__(f"Stage file '{path}' cannot be an output.")
 
 
-class OutputBase(object):
+class BaseOutput:
     IS_DEPENDENCY = False
 
-    REMOTE = RemoteBASE
+    REMOTE = BaseRemote
 
     PARAM_PATH = "path"
     PARAM_CACHE = "cache"
     PARAM_METRIC = "metric"
     PARAM_METRIC_TYPE = "type"
     PARAM_METRIC_XPATH = "xpath"
+    PARAM_PLOT = "plot"
+    PARAM_PLOT_TEMPLATE = "template"
     PARAM_PERSIST = "persist"
 
     METRIC_SCHEMA = Any(
@@ -58,8 +67,6 @@ class OutputBase(object):
         },
     )
 
-    PARAM_TAGS = "tags"
-
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
     IsStageFileError = OutputIsStageFileError
@@ -74,8 +81,8 @@ class OutputBase(object):
         remote=None,
         cache=True,
         metric=False,
+        plot=False,
         persist=False,
-        tags=None,
     ):
         self._validate_output_path(path)
         # This output (and dependency) objects have too many paths/urls
@@ -95,8 +102,8 @@ class OutputBase(object):
         self.remote = remote or self.REMOTE(self.repo, {})
         self.use_cache = False if self.IS_DEPENDENCY else cache
         self.metric = False if self.IS_DEPENDENCY else metric
+        self.plot = False if self.IS_DEPENDENCY else plot
         self.persist = persist
-        self.tags = None if self.IS_DEPENDENCY else (tags or {})
 
         self.path_info = self._parse_path(remote, path)
         if self.use_cache and self.cache is None:
@@ -147,10 +154,21 @@ class OutputBase(object):
     def cache_path(self):
         return self.cache.checksum_to_path_info(self.checksum).url
 
+    @property
+    def checksum_type(self):
+        return self.remote.PARAM_CHECKSUM
+
     @property
     def checksum(self):
         return self.info.get(self.remote.PARAM_CHECKSUM)
 
+    @checksum.setter
+    def checksum(self, checksum):
+        self.info[self.remote.PARAM_CHECKSUM] = checksum
+
+    def get_checksum(self):
+        return self.remote.get_checksum(self.path_info)
+
     @property
     def is_dir_checksum(self):
         return self.remote.is_dir_checksum(self.checksum)
@@ -163,7 +181,7 @@ class OutputBase(object):
         return self.remote.save_info(self.path_info)
 
     def changed_checksum(self):
-        return self.checksum != self.remote.get_checksum(self.path_info)
+        return self.checksum != self.get_checksum()
 
     def changed_cache(self, filter_info=None):
         if not self.use_cache or not self.checksum:
@@ -201,6 +219,15 @@ class OutputBase(object):
     def isfile(self):
         return self.remote.isfile(self.path_info)
 
+    def ignore(self):
+        if not self.use_scm_ignore:
+            return
+
+        if self.repo.scm.is_tracked(self.fspath):
+            raise OutputAlreadyTrackedError(self)
+
+        self.repo.scm.ignore(self.fspath)
+
     def save(self):
         if not self.exists:
             raise self.DoesNotExistError(self)
@@ -209,17 +236,13 @@ class OutputBase(object):
             raise self.IsNotFileOrDirError(self)
 
         if self.is_empty:
-            logger.warning("'{}' is empty.".format(self))
+            logger.warning(f"'{self}' is empty.")
 
-        if self.use_scm_ignore:
-            if self.repo.scm.is_tracked(self.fspath):
-                raise OutputAlreadyTrackedError(self)
-
-            self.repo.scm.ignore(self.fspath)
+        self.ignore()
 
         if not self.use_cache:
             self.info = self.save_info()
-            if self.metric:
+            if self.metric or self.plot:
                 self.verify_metric()
             if not self.IS_DEPENDENCY:
                 logger.info(
@@ -232,9 +255,7 @@ class OutputBase(object):
         assert not self.IS_DEPENDENCY
 
         if not self.changed():
-            logger.info(
-                "Output '{}' didn't change. Skipping saving.".format(self)
-            )
+            logger.info(f"Output '{self}' didn't change. Skipping saving.")
             return
 
         self.info = self.save_info()
@@ -250,7 +271,8 @@ class OutputBase(object):
         if self.IS_DEPENDENCY:
             return ret
 
-        ret[self.PARAM_CACHE] = self.use_cache
+        if not self.use_cache:
+            ret[self.PARAM_CACHE] = self.use_cache
 
         if isinstance(self.metric, dict):
             if (
@@ -259,18 +281,19 @@ class OutputBase(object):
             ):
                 del self.metric[self.PARAM_METRIC_XPATH]
 
-        ret[self.PARAM_METRIC] = self.metric
-        ret[self.PARAM_PERSIST] = self.persist
+        if self.metric:
+            ret[self.PARAM_METRIC] = self.metric
 
-        if self.tags:
-            ret[self.PARAM_TAGS] = self.tags
+        if self.plot:
+            ret[self.PARAM_PLOT] = self.plot
+
+        if self.persist:
+            ret[self.PARAM_PERSIST] = self.persist
 
         return ret
 
     def verify_metric(self):
-        raise DvcException(
-            "verify metric is not supported for {}".format(self.scheme)
-        )
+        raise DvcException(f"verify metric is not supported for {self.scheme}")
 
     def download(self, to):
         self.remote.download(self.path_info, to.path_info)
@@ -279,7 +302,6 @@ class OutputBase(object):
         self,
         force=False,
         progress_callback=None,
-        tag=None,
         relink=False,
         filter_info=None,
     ):
@@ -290,14 +312,9 @@ class OutputBase(object):
                 )
             return None
 
-        if tag:
-            info = self.tags[tag]
-        else:
-            info = self.info
-
         return self.cache.checkout(
             self.path_info,
-            info,
+            self.info,
             force=force,
             progress_callback=progress_callback,
             relink=relink,
@@ -337,7 +354,18 @@ class OutputBase(object):
         if self.exists:
             self.remote.unprotect(self.path_info)
 
-    def _collect_used_dir_cache(
+    def get_dir_cache(self, **kwargs):
+        if not self.is_dir_checksum:
+            raise DvcException("cannot get dir cache for file checksum")
+        if self.cache.changed_cache_file(self.checksum):
+            self.repo.cloud.pull(
+                NamedCache.make("local", self.checksum, str(self)),
+                show_checksums=False,
+                **kwargs,
+            )
+        return self.dir_cache
+
+    def collect_used_dir_cache(
         self, remote=None, force=False, jobs=None, filter_info=None
     ):
         """Get a list of `info`s related to the given directory.
@@ -361,16 +389,10 @@ class OutputBase(object):
 
         cache = NamedCache()
 
-        if self.cache.changed_cache_file(self.checksum):
-            try:
-                self.repo.cloud.pull(
-                    NamedCache.make("local", self.checksum, str(self)),
-                    jobs=jobs,
-                    remote=remote,
-                    show_checksums=False,
-                )
-            except DvcException:
-                logger.debug("failed to pull cache for '{}'".format(self))
+        try:
+            self.get_dir_cache(jobs=jobs, remote=remote)
+        except DvcException:
+            logger.debug(f"failed to pull cache for '{self}'")
 
         if self.cache.changed_cache_file(self.checksum):
             msg = (
@@ -386,11 +408,21 @@ class OutputBase(object):
             else:
                 return cache
 
+        path = str(self.path_info)
+        filter_path = str(filter_info) if filter_info else None
+        is_win = os.name == "nt"
         for entry in self.dir_cache:
             checksum = entry[self.remote.PARAM_CHECKSUM]
-            info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            if not filter_info or info.isin_or_eq(filter_info):
-                cache.add(self.scheme, checksum, str(info))
+            entry_relpath = entry[self.remote.PARAM_RELPATH]
+            if is_win:
+                entry_relpath = entry_relpath.replace("/", os.sep)
+            entry_path = os.path.join(path, entry_relpath)
+            if (
+                not filter_path
+                or entry_path == filter_path
+                or entry_path.startswith(filter_path + os.sep)
+            ):
+                cache.add(self.scheme, checksum, entry_path)
 
         return cache
 
@@ -423,9 +455,9 @@ class OutputBase(object):
             if self.exists:
                 msg += (
                     "\n"
-                    "You can also use `dvc commit {stage}` to associate "
-                    "existing '{out}' with '{stage}'.".format(
-                        out=self, stage=self.stage.relpath
+                    "You can also use `dvc commit {stage.addressing}` "
+                    "to associate existing '{out}' with {stage}.".format(
+                        out=self, stage=self.stage
                     )
                 )
             logger.warning(msg)
@@ -436,13 +468,15 @@ class OutputBase(object):
         if not self.is_dir_checksum:
             return ret
 
-        ret.update(self._collect_used_dir_cache(**kwargs))
+        ret.add_child_cache(
+            self.checksum, self.collect_used_dir_cache(**kwargs),
+        )
 
         return ret
 
     @classmethod
     def _validate_output_path(cls, path):
-        from dvc.stage import Stage
+        from dvc.dvcfile import is_valid_filename
 
-        if Stage.is_valid_filename(path):
+        if is_valid_filename(path):
             raise cls.IsStageFileError(path)

dvc/output/gs.py

@@ -1,6 +1,6 @@
-from dvc.output.s3 import OutputS3
-from dvc.remote.gs import RemoteGS
+from dvc.output.s3 import S3Output
+from dvc.remote.gs import GSRemote
 
 
-class OutputGS(OutputS3):
-    REMOTE = RemoteGS
+class GSOutput(S3Output):
+    REMOTE = GSRemote

dvc/output/hdfs.py

@@ -1,6 +1,6 @@
-from dvc.output.base import OutputBase
-from dvc.remote.hdfs import RemoteHDFS
+from dvc.output.base import BaseOutput
+from dvc.remote.hdfs import HDFSRemote
 
 
-class OutputHDFS(OutputBase):
-    REMOTE = RemoteHDFS
+class HDFSOutput(BaseOutput):
+    REMOTE = HDFSRemote

dvc/output/local.py

@@ -4,18 +4,16 @@ from urllib.parse import urlparse
 
 from dvc.exceptions import DvcException
 from dvc.istextfile import istextfile
-from dvc.output.base import OutputBase
-from dvc.remote.local import RemoteLOCAL
+from dvc.output.base import BaseOutput
+from dvc.remote.local import LocalRemote
 from dvc.utils import relpath
-from dvc.compat import fspath_py35
 from dvc.utils.fs import path_isin
 
-
 logger = logging.getLogger(__name__)
 
 
-class OutputLOCAL(OutputBase):
-    REMOTE = RemoteLOCAL
+class LocalOutput(BaseOutput):
+    REMOTE = LocalRemote
     sep = os.sep
 
     def __init__(self, stage, path, *args, **kwargs):
@@ -39,7 +37,7 @@ class OutputLOCAL(OutputBase):
             if not p.is_absolute():
                 p = self.stage.wdir / p
 
-        abs_p = os.path.abspath(os.path.normpath(fspath_py35(p)))
+        abs_p = os.path.abspath(os.path.normpath(p))
         return self.REMOTE.path_cls(abs_p)
 
     def __str__(self):
@@ -73,17 +71,18 @@ class OutputLOCAL(OutputBase):
         return ret
 
     def verify_metric(self):
-        if not self.metric:
+        if not self.metric or self.plot:
             return
 
-        path = fspath_py35(self.path_info)
+        path = os.fspath(self.path_info)
         if not os.path.exists(path):
             return
 
+        name = "metrics" if self.metric else "plot"
         if os.path.isdir(path):
-            msg = "directory '{}' cannot be used as metrics."
-            raise DvcException(msg.format(self.path_info))
+            msg = "directory '{}' cannot be used as {}."
+            raise DvcException(msg.format(self.path_info, name))
 
         if not istextfile(path):
-            msg = "binary file '{}' cannot be used as metrics."
-            raise DvcException(msg.format(self.path_info))
+            msg = "binary file '{}' cannot be used as {}."
+            raise DvcException(msg.format(self.path_info, name))

dvc/output/s3.py

@@ -1,6 +1,6 @@
-from dvc.output.base import OutputBase
-from dvc.remote.s3 import RemoteS3
+from dvc.output.base import BaseOutput
+from dvc.remote.s3 import S3Remote
 
 
-class OutputS3(OutputBase):
-    REMOTE = RemoteS3
+class S3Output(BaseOutput):
+    REMOTE = S3Remote

dvc/output/ssh.py

@@ -1,6 +1,6 @@
-from dvc.output.base import OutputBase
-from dvc.remote.ssh import RemoteSSH
+from dvc.output.base import BaseOutput
+from dvc.remote.ssh import SSHRemote
 
 
-class OutputSSH(OutputBase):
-    REMOTE = RemoteSSH
+class SSHOutput(BaseOutput):
+    REMOTE = SSHRemote

dvc/path_info.py

@@ -1,6 +1,6 @@
 import os
-import posixpath
 import pathlib
+import posixpath
 from urllib.parse import urlparse
 
 from funcy import cached_property
@@ -8,7 +8,7 @@ from funcy import cached_property
 from dvc.utils import relpath
 
 
-class _BasePath(object):
+class _BasePath:
     def overlaps(self, other):
         if isinstance(other, (str, bytes)):
             other = self.__class__(other)
@@ -33,10 +33,6 @@ class PathInfo(pathlib.PurePath, _BasePath):
             cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo
         return cls._from_parts(args)
 
-    @classmethod
-    def from_posix(cls, s):
-        return cls(PosixPathInfo(s))
-
     def as_posix(self):
         f = self._flavour
         # Unlike original implementation [1] that uses `str()` we actually need
@@ -54,7 +50,6 @@ class PathInfo(pathlib.PurePath, _BasePath):
         return "{}: '{}'".format(type(self).__name__, self)
 
     # This permits passing it to file utils directly in Python 3.6+
-    # With Python 2.7, Python 3.5+ we are stuck with path_info.fspath for now
     def __fspath__(self):
         return pathlib.PurePath.__str__(self)
 
@@ -92,7 +87,7 @@ class _URLPathInfo(PosixPathInfo):
     __unicode__ = __str__
 
 
-class _URLPathParents(object):
+class _URLPathParents:
     def __init__(self, src):
         self.src = src
         self._parents = self.src._path.parents
@@ -104,7 +99,7 @@ class _URLPathParents(object):
         return self.src.replace(path=self._parents[idx])
 
     def __repr__(self):
-        return "<{}.parents>".format(self.src)
+        return f"<{self.src}.parents>"
 
 
 class URLInfo(_BasePath):
@@ -124,7 +119,7 @@ class URLInfo(_BasePath):
         assert bool(host) ^ bool(netloc)
 
         if netloc is not None:
-            return cls("{}://{}{}".format(scheme, netloc, path))
+            return cls(f"{scheme}://{netloc}{path}")
 
         obj = cls.__new__(cls)
         obj._fill_parts(scheme, host, user, port, path)
@@ -158,7 +153,7 @@ class URLInfo(_BasePath):
 
     @cached_property
     def url(self):
-        return "{}://{}{}".format(self.scheme, self.netloc, self._spath)
+        return f"{self.scheme}://{self.netloc}{self._spath}"
 
     def __str__(self):
         return self.url
@@ -220,10 +215,10 @@ class URLInfo(_BasePath):
         if isinstance(other, (str, bytes)):
             other = self.__class__(other)
         if self.__class__ != other.__class__:
-            msg = "'{}' has incompatible class with '{}'".format(self, other)
+            msg = f"'{self}' has incompatible class with '{other}'"
             raise ValueError(msg)
         if self._base_parts != other._base_parts:
-            msg = "'{}' does not start with '{}'".format(self, other)
+            msg = f"'{self}' does not start with '{other}'"
             raise ValueError(msg)
         return self._path.relative_to(other._path)
 

dvc/progress.py

@@ -19,18 +19,17 @@ class Tqdm(tqdm):
 
     BAR_FMT_DEFAULT = (
         "{percentage:3.0f}% {desc}|{bar}|"
-        "{n_fmt}/{total_fmt}"
-        " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
+        "{postfix[info]}{n_fmt}/{total_fmt}"
+        " [{elapsed}<{remaining}, {rate_fmt:>11}]"
     )
     # nested bars should have fixed bar widths to align nicely
     BAR_FMT_DEFAULT_NESTED = (
         "{percentage:3.0f}%|{bar:10}|{desc:{ncols_desc}.{ncols_desc}}"
-        "{n_fmt}/{total_fmt}"
-        " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
+        "{postfix[info]}{n_fmt}/{total_fmt}"
+        " [{elapsed}<{remaining}, {rate_fmt:>11}]"
     )
     BAR_FMT_NOTOTAL = (
-        "{desc:{ncols_desc}.{ncols_desc}}{n_fmt}"
-        " [{elapsed}, {rate_fmt:>11}{postfix}]"
+        "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
     )
     BYTES_DEFAULTS = dict(
         unit="B", unit_scale=True, unit_divisor=1024, miniters=1
@@ -47,6 +46,7 @@ class Tqdm(tqdm):
         bytes=False,  # pylint: disable=W0622
         file=None,
         total=None,
+        postfix=None,
         **kwargs
     ):
         """
@@ -68,7 +68,6 @@ class Tqdm(tqdm):
             kwargs.setdefault("unit_scale", total > 999 if total else True)
         if file is None:
             file = sys.stderr
-        self.desc_persist = desc
         # auto-disable based on `logger.level`
         if not disable:
             disable = logger.getEffectiveLevel() > level
@@ -89,6 +88,7 @@ class Tqdm(tqdm):
             total=total,
             **kwargs
         )
+        self.postfix = postfix or {"info": ""}
         if bar_format is None:
             if self.__len__():
                 self.bar_format = (
@@ -102,11 +102,11 @@ class Tqdm(tqdm):
             self.bar_format = bar_format
         self.refresh()
 
-    def update_desc(self, desc, n=1):
+    def update_msg(self, msg, n=1):
         """
-        Calls `set_description_str(desc)` and `update(n)`
+        Sets `msg` as a postfix and calls `update(n)`.
         """
-        self.set_description_str(desc, refresh=False)
+        self.postfix["info"] = " %s |" % msg
         self.update(n)
 
     def update_to(self, current, total=None):
@@ -130,12 +130,11 @@ class Tqdm(tqdm):
         return wrapped
 
     def close(self):
-        if self.desc_persist is not None:
-            self.set_description_str(self.desc_persist, refresh=False)
-        # unknown/zero ETA
-        self.bar_format = self.bar_format.replace("<{remaining}", "")
-        # remove completed bar
-        self.bar_format = self.bar_format.replace("|{bar:10}|", " ")
+        self.postfix["info"] = ""
+        # remove ETA (either unknown or zero); remove completed bar
+        self.bar_format = self.bar_format.replace("<{remaining}", "").replace(
+            "|{bar:10}|", " "
+        )
         super().close()
 
     @property
@@ -143,12 +142,15 @@ class Tqdm(tqdm):
         """inject `ncols_desc` to fill the display width (`ncols`)"""
         d = super().format_dict
         ncols = d["ncols"] or 80
-        ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1
-        ncols_desc = max(ncols_desc, 0)
-        if ncols_desc:
-            d["ncols_desc"] = ncols_desc
+        # assumes `bar_format` has max one of ("ncols_desc" & "ncols_info")
+        ncols_left = (
+            ncols - len(self.format_meter(ncols_desc=1, ncols_info=1, **d)) + 1
+        )
+        ncols_left = max(ncols_left, 0)
+        if ncols_left:
+            d["ncols_desc"] = d["ncols_info"] = ncols_left
         else:
             # work-around for zero-width description
-            d["ncols_desc"] = 1
+            d["ncols_desc"] = d["ncols_info"] = 1
             d["prefix"] = ""
         return d

dvc/prompt.py

@@ -38,7 +38,7 @@ def confirm(statement):
     Returns:
         bool: whether or not specified statement was confirmed.
     """
-    prompt = "{statement} [y/n]".format(statement=statement)
+    prompt = f"{statement} [y/n]"
     answer = _ask(prompt, limited_to=["yes", "no", "y", "n"])
     return answer and answer.startswith("y")
 
@@ -52,5 +52,5 @@ def password(statement):
     Returns:
         str: password entered by the user.
     """
-    logger.info("{statement}: ".format(statement=statement))
+    logger.info(f"{statement}: ")
     return getpass("")

dvc/remote/__init__.py

@@ -1,29 +1,28 @@
 import posixpath
 from urllib.parse import urlparse
 
-from dvc.remote.azure import RemoteAZURE
-from dvc.remote.gdrive import RemoteGDrive
-from dvc.remote.gs import RemoteGS
-from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.http import RemoteHTTP
-from dvc.remote.https import RemoteHTTPS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.oss import RemoteOSS
-from dvc.remote.s3 import RemoteS3
-from dvc.remote.ssh import RemoteSSH
-
+from dvc.remote.azure import AzureRemote
+from dvc.remote.gdrive import GDriveRemote
+from dvc.remote.gs import GSRemote
+from dvc.remote.hdfs import HDFSRemote
+from dvc.remote.http import HTTPRemote
+from dvc.remote.https import HTTPSRemote
+from dvc.remote.local import LocalRemote
+from dvc.remote.oss import OSSRemote
+from dvc.remote.s3 import S3Remote
+from dvc.remote.ssh import SSHRemote
 
 REMOTES = [
-    RemoteAZURE,
-    RemoteGDrive,
-    RemoteGS,
-    RemoteHDFS,
-    RemoteHTTP,
-    RemoteHTTPS,
-    RemoteS3,
-    RemoteSSH,
-    RemoteOSS,
-    # NOTE: RemoteLOCAL is the default
+    AzureRemote,
+    GDriveRemote,
+    GSRemote,
+    HDFSRemote,
+    HTTPRemote,
+    HTTPSRemote,
+    S3Remote,
+    SSHRemote,
+    OSSRemote,
+    # NOTE: LocalRemote is the default
 ]
 
 
@@ -31,7 +30,7 @@ def _get(remote_conf):
     for remote in REMOTES:
         if remote.supported(remote_conf):
             return remote
-    return RemoteLOCAL
+    return LocalRemote
 
 
 def Remote(repo, **kwargs):

dvc/remote/azure.py

@@ -1,31 +1,22 @@
 import logging
 import os
 import posixpath
-import re
-from datetime import datetime, timedelta
-from urllib.parse import urlparse
 import threading
+from datetime import datetime, timedelta
 
 from funcy import cached_property, wrap_prop
 
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
+from dvc.remote.base import BaseRemote
 from dvc.scheme import Schemes
 
-
 logger = logging.getLogger(__name__)
 
 
-class RemoteAZURE(RemoteBASE):
+class AzureRemote(BaseRemote):
     scheme = Schemes.AZURE
     path_cls = CloudURLInfo
-    REGEX = (
-        r"azure://((?P<path>[^=;]*)?|("
-        # backward compatibility
-        r"(ContainerName=(?P<container_name>[^;]+);?)?"
-        r"(?P<connection_string>.+)?)?)$"
-    )
     REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
@@ -35,31 +26,16 @@ class RemoteAZURE(RemoteBASE):
         super().__init__(repo, config)
 
         url = config.get("url", "azure://")
+        self.path_info = self.path_cls(url)
 
-        match = re.match(self.REGEX, url)  # backward compatibility
-        path = match.group("path")
-        bucket = (
-            urlparse(url if path else "").netloc
-            or match.group("container_name")  # backward compatibility
-            or os.getenv("AZURE_STORAGE_CONTAINER_NAME")
-        )
+        if not self.path_info.bucket:
+            self.path_info = self.path_cls.from_parts(
+                scheme=self.scheme,
+                netloc=os.getenv("AZURE_STORAGE_CONTAINER_NAME"),
+            )
 
-        self.connection_string = (
-            config.get("connection_string")
-            or match.group("connection_string")  # backward compatibility
-            or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
-        )
-
-        if not bucket:
-            raise ValueError("azure storage container name missing")
-
-        if not self.connection_string:
-            raise ValueError("azure storage connection string missing")
-
-        self.path_info = (
-            self.path_cls(url)
-            if path
-            else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
+        self.connection_string = config.get("connection_string") or os.getenv(
+            "AZURE_STORAGE_CONNECTION_STRING"
         )
 
     @wrap_prop(threading.Lock())
@@ -68,12 +44,12 @@ class RemoteAZURE(RemoteBASE):
         from azure.storage.blob import BlockBlobService
         from azure.common import AzureMissingResourceHttpError
 
-        logger.debug("URL {}".format(self.path_info))
-        logger.debug("Connection string {}".format(self.connection_string))
+        logger.debug(f"URL {self.path_info}")
+        logger.debug(f"Connection string {self.connection_string}")
         blob_service = BlockBlobService(
             connection_string=self.connection_string
         )
-        logger.debug("Container name {}".format(self.path_info.bucket))
+        logger.debug(f"Container name {self.path_info.bucket}")
         try:  # verify that container exists
             blob_service.list_blobs(
                 self.path_info.bucket, delimiter="/", num_results=1
@@ -86,7 +62,7 @@ class RemoteAZURE(RemoteBASE):
         if path_info.scheme != self.scheme:
             raise NotImplementedError
 
-        logger.debug("Removing {}".format(path_info))
+        logger.debug(f"Removing {path_info}")
         self.blob_service.delete_blob(path_info.bucket, path_info.path)
 
     def _list_paths(self, bucket, prefix, progress_callback=None):

dvc/remote/base.py

@@ -1,32 +1,35 @@
 import errno
+import hashlib
 import itertools
 import json
 import logging
 import tempfile
-from urllib.parse import urlparse
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
-from functools import partial
+from functools import partial, wraps
 from multiprocessing import cpu_count
 from operator import itemgetter
+from urllib.parse import urlparse
 
 from shortuuid import uuid
 
 import dvc.prompt as prompt
 from dvc.exceptions import (
     CheckoutError,
-    DvcException,
     ConfirmRemoveError,
+    DvcException,
     DvcIgnoreInCollectedDirError,
     RemoteCacheRequiredError,
 )
 from dvc.ignore import DvcIgnore
-from dvc.path_info import PathInfo, URLInfo
+from dvc.path_info import PathInfo, URLInfo, WindowsPathInfo
 from dvc.progress import Tqdm
+from dvc.remote.index import RemoteIndex, RemoteIndexNoop
 from dvc.remote.slow_link_detection import slow_link_guard
+from dvc.scm.tree import is_working_tree
 from dvc.state import StateNoop
 from dvc.utils import tmp_fname
-from dvc.utils.fs import move, makedirs
+from dvc.utils.fs import makedirs, move
 from dvc.utils.http import open_url
 
 logger = logging.getLogger(__name__)
@@ -55,7 +58,7 @@ class RemoteCmdError(DvcException):
 
 class RemoteActionNotImplemented(DvcException):
     def __init__(self, action, scheme):
-        m = "{} is not supported for {} remotes".format(action, scheme)
+        m = f"{action} is not supported for {scheme} remotes"
         super().__init__(m)
 
 
@@ -66,15 +69,26 @@ class RemoteMissingDepsError(DvcException):
 class DirCacheError(DvcException):
     def __init__(self, checksum):
         super().__init__(
-            "Failed to load dir cache for hash value: '{}'.".format(checksum)
+            f"Failed to load dir cache for hash value: '{checksum}'."
         )
 
 
-class RemoteBASE(object):
+def index_locked(f):
+    @wraps(f)
+    def wrapper(remote_obj, *args, **kwargs):
+        remote = kwargs.get("remote", remote_obj)
+        with remote.index:
+            return f(remote_obj, *args, **kwargs)
+
+    return wrapper
+
+
+class BaseRemote:
     scheme = "base"
     path_cls = URLInfo
     REQUIRES = {}
     JOBS = 4 * cpu_count()
+    INDEX_CLS = RemoteIndex
 
     PARAM_RELPATH = "relpath"
     CHECKSUM_DIR_SUFFIX = ".dir"
@@ -82,7 +96,7 @@ class RemoteBASE(object):
     DEFAULT_CACHE_TYPES = ["copy"]
     DEFAULT_VERIFY = False
     LIST_OBJECT_PAGE_SIZE = 1000
-    TRAVERSE_WEIGHT_MULTIPLIER = 20
+    TRAVERSE_WEIGHT_MULTIPLIER = 5
     TRAVERSE_PREFIX_LEN = 3
     TRAVERSE_THRESHOLD_SIZE = 500000
     CAN_TRAVERSE = True
@@ -111,6 +125,15 @@ class RemoteBASE(object):
         self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES)
         self.cache_type_confirmed = False
 
+        url = config.get("url")
+        if url:
+            index_name = hashlib.sha256(url.encode("utf-8")).hexdigest()
+            self.index = self.INDEX_CLS(
+                self.repo, index_name, dir_suffix=self.CHECKSUM_DIR_SUFFIX
+            )
+        else:
+            self.index = RemoteIndexNoop()
+
     @classmethod
     def get_missing_deps(cls):
         import importlib
@@ -129,7 +152,7 @@ class RemoteBASE(object):
         if not missing:
             return
 
-        url = config.get("url", "{}://".format(self.scheme))
+        url = config.get("url", f"{self.scheme}://")
         msg = (
             "URL '{}' is supported but requires these missing "
             "dependencies: {}. If you have installed dvc using pip, "
@@ -189,23 +212,33 @@ class RemoteBASE(object):
                 checksums = dict(zip(file_infos, tasks))
         return checksums
 
-    def _collect_dir(self, path_info):
+    def _collect_dir(self, path_info, tree=None, save_tree=False, **kwargs):
         file_infos = set()
 
-        for fname in self.walk_files(path_info):
+        if tree:
+            walk_files = tree.walk_files
+        else:
+            walk_files = self.walk_files
+
+        for fname in walk_files(path_info, **kwargs):
             if DvcIgnore.DVCIGNORE_FILE == fname.name:
                 raise DvcIgnoreInCollectedDirError(fname.parent)
 
             file_infos.add(fname)
 
-        checksums = {fi: self.state.get(fi) for fi in file_infos}
-        not_in_state = {
-            fi for fi, checksum in checksums.items() if checksum is None
-        }
+        if tree:
+            checksums = {fi: tree.get_file_checksum(fi) for fi in file_infos}
+            if save_tree:
+                for fi, checksum in checksums.items():
+                    self._save_file(fi, checksum, tree=tree, **kwargs)
+        else:
+            checksums = {fi: self.state.get(fi) for fi in file_infos}
+            not_in_state = {
+                fi for fi, checksum in checksums.items() if checksum is None
+            }
 
-        new_checksums = self._calculate_checksums(not_in_state)
-
-        checksums.update(new_checksums)
+            new_checksums = self._calculate_checksums(not_in_state)
+            checksums.update(new_checksums)
 
         result = [
             {
@@ -226,18 +259,25 @@ class RemoteBASE(object):
         # Sorting the list by path to ensure reproducibility
         return sorted(result, key=itemgetter(self.PARAM_RELPATH))
 
-    def get_dir_checksum(self, path_info):
+    def get_dir_checksum(self, path_info, tree=None):
         if not self.cache:
             raise RemoteCacheRequiredError(path_info)
 
-        dir_info = self._collect_dir(path_info)
+        dir_info = self._collect_dir(path_info, tree=None)
+        if tree:
+            # don't save state entry for path_info if it is a tree path
+            path_info = None
+        return self._save_dir_info(dir_info, path_info)
+
+    def _save_dir_info(self, dir_info, path_info=None):
         checksum, tmp_info = self._get_dir_info_checksum(dir_info)
         new_info = self.cache.checksum_to_path_info(checksum)
         if self.cache.changed_cache_file(checksum):
             self.cache.makedirs(new_info.parent)
             self.cache.move(tmp_info, new_info, mode=self.CACHE_MODE)
 
-        self.state.save(path_info, checksum)
+        if path_info:
+            self.state.save(path_info, checksum)
         self.state.save(new_info, checksum)
 
         return checksum
@@ -285,19 +325,24 @@ class RemoteBASE(object):
             )
             return []
 
-        for info in d:
-            # NOTE: here is a BUG, see comment to .as_posix() below
-            relative_path = PathInfo.from_posix(info[self.PARAM_RELPATH])
-            info[self.PARAM_RELPATH] = relative_path.fspath
+        if self.path_cls == WindowsPathInfo:
+            # only need to convert it for Windows
+            for info in d:
+                # NOTE: here is a BUG, see comment to .as_posix() below
+                info[self.PARAM_RELPATH] = info[self.PARAM_RELPATH].replace(
+                    "/", self.path_cls.sep
+                )
 
         return d
 
     @classmethod
     def is_dir_checksum(cls, checksum):
+        if not checksum:
+            return False
         return checksum.endswith(cls.CHECKSUM_DIR_SUFFIX)
 
     def get_checksum(self, path_info):
-        assert path_info.scheme == self.scheme
+        assert isinstance(path_info, str) or path_info.scheme == self.scheme
 
         if not self.exists(path_info):
             return None
@@ -393,10 +438,10 @@ class RemoteBASE(object):
         if self.cache_type_confirmed:
             return
 
-        is_link = getattr(self, "is_{}".format(link_type), None)
+        is_link = getattr(self, f"is_{link_type}", None)
         if is_link and not is_link(path_info):
             self.remove(path_info)
-            raise DvcException("failed to verify {}".format(link_type))
+            raise DvcException(f"failed to verify {link_type}")
 
         self.cache_type_confirmed = True
 
@@ -419,7 +464,7 @@ class RemoteBASE(object):
 
     def _do_link(self, from_info, to_info, link_method):
         if self.exists(to_info):
-            raise DvcException("Link '{}' already exists!".format(to_info))
+            raise DvcException(f"Link '{to_info}' already exists!")
 
         link_method(from_info, to_info)
 
@@ -427,28 +472,45 @@ class RemoteBASE(object):
             "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
         )
 
-    def _save_file(self, path_info, checksum, save_link=True):
+    def _save_file(
+        self, path_info, checksum, save_link=True, tree=None, **kwargs
+    ):
         assert checksum
 
         cache_info = self.checksum_to_path_info(checksum)
-        if self.changed_cache(checksum):
-            self.move(path_info, cache_info, mode=self.CACHE_MODE)
-            self.link(cache_info, path_info)
-        elif self.iscopy(path_info) and self._cache_is_copy(path_info):
-            # Default relink procedure involves unneeded copy
-            self.unprotect(path_info)
+        if tree:
+            if self.changed_cache(checksum):
+                with tree.open(path_info, mode="rb") as fobj:
+                    # if tree has fetch enabled, DVC out will be fetched on
+                    # open and we do not need to read/copy any data
+                    if not (
+                        tree.isdvc(path_info, strict=False) and tree.fetch
+                    ):
+                        self.copy_fobj(fobj, cache_info)
+                callback = kwargs.get("download_callback")
+                if callback:
+                    callback(1)
         else:
-            self.remove(path_info)
-            self.link(cache_info, path_info)
+            if self.changed_cache(checksum):
+                self.move(path_info, cache_info, mode=self.CACHE_MODE)
+                self.link(cache_info, path_info)
+            elif self.iscopy(path_info) and self._cache_is_copy(path_info):
+                # Default relink procedure involves unneeded copy
+                self.unprotect(path_info)
+            else:
+                self.remove(path_info)
+                self.link(cache_info, path_info)
 
-        if save_link:
-            self.state.save_link(path_info)
+            if save_link:
+                self.state.save_link(path_info)
 
         # we need to update path and cache, since in case of reflink,
         # or copy cache type moving original file results in updates on
         # next executed command, which causes md5 recalculation
-        self.state.save(path_info, checksum)
+        if not tree or is_working_tree(tree):
+            self.state.save(path_info, checksum)
         self.state.save(cache_info, checksum)
+        return {self.PARAM_CHECKSUM: checksum}
 
     def _cache_is_copy(self, path_info):
         """Checks whether cache uses copies."""
@@ -472,22 +534,32 @@ class RemoteBASE(object):
         self.cache_type_confirmed = True
         return self.cache_types[0] == "copy"
 
-    def _save_dir(self, path_info, checksum, save_link=True):
+    def _save_dir(
+        self, path_info, checksum, save_link=True, tree=None, **kwargs
+    ):
+        if tree:
+            dir_info = self._collect_dir(
+                path_info, tree=tree, save_tree=True, **kwargs
+            )
+            checksum = self._save_dir_info(dir_info)
+        else:
+            dir_info = self.get_dir_cache(checksum)
+
+            for entry in Tqdm(
+                dir_info, desc="Saving " + path_info.name, unit="file"
+            ):
+                entry_info = path_info / entry[self.PARAM_RELPATH]
+                entry_checksum = entry[self.PARAM_CHECKSUM]
+                self._save_file(entry_info, entry_checksum, save_link=False)
+
+            if save_link:
+                self.state.save_link(path_info)
+
         cache_info = self.checksum_to_path_info(checksum)
-        dir_info = self.get_dir_cache(checksum)
-
-        for entry in Tqdm(
-            dir_info, desc="Saving " + path_info.name, unit="file"
-        ):
-            entry_info = path_info / entry[self.PARAM_RELPATH]
-            entry_checksum = entry[self.PARAM_CHECKSUM]
-            self._save_file(entry_info, entry_checksum, save_link=False)
-
-        if save_link:
-            self.state.save_link(path_info)
-
         self.state.save(cache_info, checksum)
-        self.state.save(path_info, checksum)
+        if not tree or is_working_tree(tree):
+            self.state.save(path_info, checksum)
+        return {self.PARAM_CHECKSUM: checksum}
 
     def is_empty(self, path_info):
         return False
@@ -516,23 +588,42 @@ class RemoteBASE(object):
     def protect(path_info):
         pass
 
-    def save(self, path_info, checksum_info, save_link=True):
+    def save(
+        self, path_info, checksum_info, save_link=True, tree=None, **kwargs
+    ):
         if path_info.scheme != self.scheme:
             raise RemoteActionNotImplemented(
-                "save {} -> {}".format(path_info.scheme, self.scheme),
-                self.scheme,
+                f"save {path_info.scheme} -> {self.scheme}", self.scheme,
             )
 
-        checksum = checksum_info[self.PARAM_CHECKSUM]
-        self._save(path_info, checksum, save_link)
+        if tree:
+            if tree.isdir(path_info):
+                # save checksum will be computed during tree walk
+                checksum = None
+            else:
+                checksum = tree.get_file_checksum(path_info)
+        else:
+            checksum = checksum_info[self.PARAM_CHECKSUM]
+        return self._save(path_info, checksum, save_link, tree, **kwargs)
 
-    def _save(self, path_info, checksum, save_link=True):
-        to_info = self.checksum_to_path_info(checksum)
-        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
-        if self.isdir(path_info):
-            self._save_dir(path_info, checksum, save_link)
-            return
-        self._save_file(path_info, checksum, save_link)
+    def _save(self, path_info, checksum, save_link=True, tree=None, **kwargs):
+        if tree:
+            logger.debug("Saving tree path '%s' to cache.", path_info)
+        else:
+            to_info = self.checksum_to_path_info(checksum)
+            logger.debug("Saving '%s' to '%s'.", path_info, to_info)
+
+        if tree:
+            isdir = tree.isdir
+            save_link = False
+        else:
+            isdir = self.isdir
+
+        if isdir(path_info):
+            return self._save_dir(
+                path_info, checksum, save_link, tree, **kwargs
+            )
+        return self._save_file(path_info, checksum, save_link, tree, **kwargs)
 
     def _handle_transfer_exception(
         self, from_info, to_info, exception, operation
@@ -671,6 +762,9 @@ class RemoteBASE(object):
     def copy(self, from_info, to_info):
         raise RemoteActionNotImplemented("copy", self.scheme)
 
+    def copy_fobj(self, fobj, to_info):
+        raise RemoteActionNotImplemented("copy_fobj", self.scheme)
+
     def symlink(self, from_info, to_info):
         raise RemoteActionNotImplemented("symlink", self.scheme)
 
@@ -687,22 +781,27 @@ class RemoteBASE(object):
         parts = self.path_cls(path).parts[-2:]
 
         if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
-            raise ValueError("Bad cache file path '{}'".format(path))
+            raise ValueError(f"Bad cache file path '{path}'")
 
         return "".join(parts)
 
     def checksum_to_path_info(self, checksum):
         return self.path_info / checksum[0:2] / checksum[2:]
 
+    # Return path as a string instead of PathInfo for remotes which support
+    # string paths (see local)
+    checksum_to_path = checksum_to_path_info
+
     def list_cache_paths(self, prefix=None, progress_callback=None):
         raise NotImplementedError
 
-    def all(self, *args, **kwargs):
-        # NOTE: The list might be way too big(e.g. 100M entries, md5 for each
-        # is 32 bytes, so ~3200Mb list) and we don't really need all of it at
-        # the same time, so it makes sense to use a generator to gradually
-        # iterate over it, without keeping all of it in memory.
-        for path in self.list_cache_paths(*args, **kwargs):
+    def cache_checksums(self, prefix=None, progress_callback=None):
+        """Iterate over remote cache checksums.
+
+        If `prefix` is specified, only checksums which begin with `prefix`
+        will be returned.
+        """
+        for path in self.list_cache_paths(prefix, progress_callback):
             try:
                 yield self.path_to_checksum(path)
             except ValueError:
@@ -710,14 +809,40 @@ class RemoteBASE(object):
                     "'%s' doesn't look like a cache file, skipping", path
                 )
 
-    def gc(self, named_cache):
+    def all(self, jobs=None, name=None):
+        """Iterate over all checksums in the remote cache.
+
+        Checksums will be fetched in parallel threads according to prefix
+        (except for small remotes) and a progress bar will be displayed.
+        """
+        logger.debug(
+            "Fetching all checksums from '{}'".format(
+                name if name else "remote cache"
+            )
+        )
+
+        if not self.CAN_TRAVERSE:
+            return self.cache_checksums()
+
+        remote_size, remote_checksums = self._estimate_cache_size(name=name)
+        return self._cache_checksums_traverse(
+            remote_size, remote_checksums, jobs, name
+        )
+
+    @index_locked
+    def gc(self, named_cache, jobs=None):
         used = self.extract_used_local_checksums(named_cache)
 
         if self.scheme != "":
-            used.update(named_cache[self.scheme])
+            used.update(named_cache.scheme_keys(self.scheme))
 
         removed = False
-        for checksum in self.all():
+        # checksums must be sorted to ensure we always remove .dir files first
+        for checksum in sorted(
+            self.all(jobs, str(self.path_info)),
+            key=self.is_dir_checksum,
+            reverse=True,
+        ):
             if checksum in used:
                 continue
             path_info = self.checksum_to_path_info(checksum)
@@ -725,6 +850,8 @@ class RemoteBASE(object):
                 self._remove_unpacked_dir(checksum)
             self.remove(path_info)
             removed = True
+        if removed:
+            self.index.clear()
         return removed
 
     def is_protected(self, path_info):
@@ -742,8 +869,8 @@ class RemoteBASE(object):
 
         - Remove the file from cache if it doesn't match the actual checksum
         """
-
-        cache_info = self.checksum_to_path_info(checksum)
+        # Prefer string path over PathInfo when possible due to performance
+        cache_info = self.checksum_to_path(checksum)
         if self.is_protected(cache_info):
             logger.debug(
                 "Assuming '%s' is unchanged since it is read-only", cache_info
@@ -843,14 +970,22 @@ class RemoteBASE(object):
         # cache_exists() (see ssh, local)
         assert self.TRAVERSE_PREFIX_LEN >= 2
 
-        if len(checksums) == 1 or not self.CAN_TRAVERSE:
-            return self._cache_object_exists(checksums, jobs, name)
+        checksums = set(checksums)
+        indexed_checksums = set(self.index.intersection(checksums))
+        checksums -= indexed_checksums
+        logger.debug(
+            "Matched '{}' indexed checksums".format(len(indexed_checksums))
+        )
+        if not checksums:
+            return indexed_checksums
 
-        checksums = frozenset(checksums)
+        if len(checksums) == 1 or not self.CAN_TRAVERSE:
+            remote_checksums = self._cache_object_exists(checksums, jobs, name)
+            return list(indexed_checksums) + remote_checksums
 
         # Max remote size allowed for us to use traverse method
         remote_size, remote_checksums = self._estimate_cache_size(
-            checksums, name=name
+            checksums, name
         )
 
         traverse_pages = remote_size / self.LIST_OBJECT_PAGE_SIZE
@@ -869,38 +1004,37 @@ class RemoteBASE(object):
                     len(checksums), traverse_weight
                 )
             )
-            return list(
-                checksums & remote_checksums
-            ) + self._cache_object_exists(
-                checksums - remote_checksums, jobs, name
-            )
-
-        if traverse_pages < 256 / self.JOBS:
-            # Threaded traverse will require making at least 255 more requests
-            # to the remote, so for small enough remotes, fetching the entire
-            # list at once will require fewer requests (but also take into
-            # account that this must be done sequentially rather than in
-            # parallel)
-            logger.debug(
-                "Querying {} checksums via default traverse".format(
-                    len(checksums)
+            return (
+                list(indexed_checksums)
+                + list(checksums & remote_checksums)
+                + self._cache_object_exists(
+                    checksums - remote_checksums, jobs, name
                 )
             )
-            return list(checksums & set(self.all()))
 
-        return self._cache_exists_traverse(
-            checksums, remote_checksums, remote_size, jobs, name
+        logger.debug(
+            "Querying '{}' checksums via traverse".format(len(checksums))
+        )
+        remote_checksums = set(
+            self._cache_checksums_traverse(
+                remote_size, remote_checksums, jobs, name
+            )
+        )
+        return list(indexed_checksums) + list(
+            checksums & set(remote_checksums)
         )
 
-    def _all_with_limit(self, max_paths, prefix=None, progress_callback=None):
+    def _checksums_with_limit(
+        self, limit, prefix=None, progress_callback=None
+    ):
         count = 0
-        for checksum in self.all(prefix, progress_callback):
+        for checksum in self.cache_checksums(prefix, progress_callback):
             yield checksum
             count += 1
-            if count > max_paths:
+            if count > limit:
                 logger.debug(
-                    "`all()` returned max '{}' checksums, "
-                    "skipping remaining results".format(max_paths)
+                    "`cache_checksums()` returned max '{}' checksums, "
+                    "skipping remaining results".format(limit)
                 )
                 return
 
@@ -913,74 +1047,95 @@ class RemoteBASE(object):
             * self.LIST_OBJECT_PAGE_SIZE,
         )
 
-    def _estimate_cache_size(self, checksums, short_circuit=True, name=None):
+    def _estimate_cache_size(self, checksums=None, name=None):
         """Estimate remote cache size based on number of entries beginning with
         "00..." prefix.
         """
         prefix = "0" * self.TRAVERSE_PREFIX_LEN
         total_prefixes = pow(16, self.TRAVERSE_PREFIX_LEN)
-        if short_circuit:
-            max_remote_size = self._max_estimation_size(checksums)
+        if checksums:
+            max_checksums = self._max_estimation_size(checksums)
         else:
-            max_remote_size = None
+            max_checksums = None
 
         with Tqdm(
             desc="Estimating size of "
-            + ("cache in '{}'".format(name) if name else "remote cache"),
+            + (f"cache in '{name}'" if name else "remote cache"),
             unit="file",
-            total=max_remote_size,
         ) as pbar:
 
             def update(n=1):
                 pbar.update(n * total_prefixes)
 
-            if max_remote_size:
-                checksums = self._all_with_limit(
-                    max_remote_size / total_prefixes, prefix, update
+            if max_checksums:
+                checksums = self._checksums_with_limit(
+                    max_checksums / total_prefixes, prefix, update
                 )
             else:
-                checksums = self.all(prefix, update)
+                checksums = self.cache_checksums(prefix, update)
 
             remote_checksums = set(checksums)
             if remote_checksums:
                 remote_size = total_prefixes * len(remote_checksums)
             else:
                 remote_size = total_prefixes
-            logger.debug("Estimated remote size: {} files".format(remote_size))
+            logger.debug(f"Estimated remote size: {remote_size} files")
         return remote_size, remote_checksums
 
-    def _cache_exists_traverse(
-        self, checksums, remote_checksums, remote_size, jobs=None, name=None
+    def _cache_checksums_traverse(
+        self, remote_size, remote_checksums, jobs=None, name=None
     ):
-        logger.debug(
-            "Querying {} checksums via threaded traverse".format(
-                len(checksums)
-            )
-        )
+        """Iterate over all checksums in the remote cache.
+        Checksums are fetched in parallel according to prefix, except in
+        cases where the remote size is very small.
 
-        traverse_prefixes = ["{:02x}".format(i) for i in range(1, 256)]
-        if self.TRAVERSE_PREFIX_LEN > 2:
-            traverse_prefixes += [
-                "{0:0{1}x}".format(i, self.TRAVERSE_PREFIX_LEN)
-                for i in range(1, pow(16, self.TRAVERSE_PREFIX_LEN - 2))
-            ]
+        All checksums from the remote (including any from the size
+        estimation step passed via the `remote_checksums` argument) will be
+        returned.
+
+        NOTE: For large remotes the list of checksums will be very
+        big(e.g. 100M entries, md5 for each is 32 bytes, so ~3200Mb list)
+        and we don't really need all of it at the same time, so it makes
+        sense to use a generator to gradually iterate over it, without
+        keeping all of it in memory.
+        """
+        num_pages = remote_size / self.LIST_OBJECT_PAGE_SIZE
+        if num_pages < 256 / self.JOBS:
+            # Fetching prefixes in parallel requires at least 255 more
+            # requests, for small enough remotes it will be faster to fetch
+            # entire cache without splitting it into prefixes.
+            #
+            # NOTE: this ends up re-fetching checksums that were already
+            # fetched during remote size estimation
+            traverse_prefixes = [None]
+            initial = 0
+        else:
+            yield from remote_checksums
+            initial = len(remote_checksums)
+            traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
+            if self.TRAVERSE_PREFIX_LEN > 2:
+                traverse_prefixes += [
+                    "{0:0{1}x}".format(i, self.TRAVERSE_PREFIX_LEN)
+                    for i in range(1, pow(16, self.TRAVERSE_PREFIX_LEN - 2))
+                ]
         with Tqdm(
             desc="Querying "
-            + ("cache in '{}'".format(name) if name else "remote cache"),
+            + (f"cache in '{name}'" if name else "remote cache"),
             total=remote_size,
-            initial=len(remote_checksums),
-            unit="objects",
+            initial=initial,
+            unit="file",
         ) as pbar:
 
             def list_with_update(prefix):
-                return self.all(prefix=prefix, progress_callback=pbar.update)
+                return list(
+                    self.cache_checksums(
+                        prefix=prefix, progress_callback=pbar.update
+                    )
+                )
 
             with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
                 in_remote = executor.map(list_with_update, traverse_prefixes,)
-                remote_checksums.update(
-                    itertools.chain.from_iterable(in_remote)
-                )
-            return list(checksums & remote_checksums)
+                yield from itertools.chain.from_iterable(in_remote)
 
     def _cache_object_exists(self, checksums, jobs=None, name=None):
         logger.debug(
@@ -995,7 +1150,7 @@ class RemoteBASE(object):
 
             def exists_with_progress(path_info):
                 ret = self.exists(path_info)
-                pbar.update_desc(str(path_info))
+                pbar.update_msg(str(path_info))
                 return ret
 
             with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
@@ -1211,7 +1366,7 @@ class RemoteBASE(object):
         return set()
 
     def extract_used_local_checksums(self, named_cache):
-        used = set(named_cache["local"])
+        used = set(named_cache.scheme_keys("local"))
         unpacked = self._get_unpacked_dir_names(used)
         return used | unpacked
 

dvc/remote/gdrive.py

@@ -1,41 +1,45 @@
-from collections import defaultdict
+import logging
 import os
 import posixpath
-import logging
 import re
 import threading
+from collections import defaultdict
 from urllib.parse import urlparse
 
-from funcy import retry, wrap_with, wrap_prop, cached_property
+from funcy import cached_property, retry, wrap_prop, wrap_with
 from funcy.py3 import cat
 
-from dvc.progress import Tqdm
-from dvc.scheme import Schemes
-from dvc.path_info import CloudURLInfo
-from dvc.remote.base import RemoteBASE
 from dvc.exceptions import DvcException
-from dvc.utils import tmp_fname, format_link
+from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.remote.base import BaseRemote
+from dvc.scheme import Schemes
+from dvc.utils import format_link, tmp_fname
 
 logger = logging.getLogger(__name__)
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
 
 
 class GDrivePathNotFound(DvcException):
-    def __init__(self, path_info):
-        super().__init__("Google Drive path '{}' not found.".format(path_info))
+    def __init__(self, path_info, hint):
+        hint = "" if hint is None else f" {hint}"
+        super().__init__(f"GDrive path '{path_info}' not found.{hint}")
 
 
-class GDriveAccessTokenRefreshError(DvcException):
-    def __init__(self):
-        super().__init__("Google Drive access token refreshment is failed.")
+class GDriveAuthError(DvcException):
+    def __init__(self, cred_location):
 
+        if cred_location:
+            message = (
+                "GDrive remote auth failed with credentials in '{}'.\n"
+                "Backup first, remove of fix them, and run DVC again.\n"
+                "It should do auth again and refresh the credentials.\n\n"
+                "Details:".format(cred_location)
+            )
+        else:
+            message = "Failed to authenticate GDrive remote"
 
-class GDriveMissedCredentialKeyError(DvcException):
-    def __init__(self, path):
-        super().__init__(
-            "Google Drive user credentials file '{}' "
-            "misses value for key.".format(path)
-        )
+        super().__init__(message)
 
 
 def _gdrive_retry(func):
@@ -45,10 +49,19 @@ def _gdrive_retry(func):
         if not isinstance(exc, ApiRequestError):
             return False
 
-        retry_codes = [403, 500, 502, 503, 504]
-        result = exc.error.get("code", 0) in retry_codes
+        error_code = exc.error.get("code", 0)
+        result = False
+        if 500 <= error_code < 600:
+            result = True
+
+        if error_code == 403:
+            result = exc.GetField("reason") in [
+                "userRateLimitExceeded",
+                "rateLimitExceeded",
+            ]
         if result:
-            logger.debug("Retrying GDrive API call, error: {}.".format(exc))
+            logger.debug(f"Retrying GDrive API call, error: {exc}.")
+
         return result
 
     # 16 tries, start at 0.5s, multiply by golden ratio, cap at 20s
@@ -59,19 +72,6 @@ def _gdrive_retry(func):
     )(func)
 
 
-def _location(exc):
-    from pydrive2.files import ApiRequestError
-
-    assert isinstance(exc, ApiRequestError)
-
-    # https://cloud.google.com/storage/docs/json_api/v1/status-codes#errorformat
-    return (
-        exc.error["errors"][0].get("location", "")
-        if exc.error.get("errors", [])
-        else ""
-    )
-
-
 class GDriveURLInfo(CloudURLInfo):
     def __init__(self, url):
         super().__init__(url)
@@ -87,7 +87,7 @@ class GDriveURLInfo(CloudURLInfo):
         self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
 
 
-class RemoteGDrive(RemoteBASE):
+class GDriveRemote(BaseRemote):
     scheme = Schemes.GDRIVE
     path_cls = GDriveURLInfo
     REQUIRES = {"pydrive2": "pydrive2"}
@@ -108,8 +108,7 @@ class RemoteGDrive(RemoteBASE):
 
         if not self.path_info.bucket:
             raise DvcException(
-                "Empty Google Drive URL '{}'. Learn more at "
-                "{}.".format(
+                "Empty GDrive URL '{}'. Learn more at {}".format(
                     config["url"],
                     format_link("https://man.dvc.org/remote/add"),
                 )
@@ -132,7 +131,7 @@ class RemoteGDrive(RemoteBASE):
         self._validate_config()
         self._gdrive_user_credentials_path = (
             tmp_fname(os.path.join(self.repo.tmp_dir, ""))
-            if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA)
+            if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA)
             else config.get(
                 "gdrive_user_credentials_file",
                 os.path.join(
@@ -148,13 +147,11 @@ class RemoteGDrive(RemoteBASE):
             or not self._service_account_p12_file_path
         ):
             raise DvcException(
-                "To use service account please specify {}, {} and "
-                "{} in DVC config. Learn more at "
-                "{}.".format(
-                    "gdrive_service_account_email",
-                    "gdrive_service_account_p12_file_path",
-                    "gdrive_service_account_user_email (optional)",
-                    format_link("https://man.dvc.org/remote/modify"),
+                "To use service account, set `gdrive_service_account_email`,\n"
+                "`gdrive_service_account_p12_file_path`, and optionally "
+                "`gdrive_service_account_user_email`\nin DVC config. "
+                "Learn more at {}".format(
+                    format_link("https://man.dvc.org/remote/modify")
                 )
             )
 
@@ -162,9 +159,48 @@ class RemoteGDrive(RemoteBASE):
         if not self._use_service_account:
             if bool(self._client_id) != bool(self._client_secret):
                 raise DvcException(
-                    "Please specify Google Drive's client id and secret in "
-                    "DVC config or omit both to use defaults. Learn more at "
-                    "{}.".format(
+                    "Please specify GDrive's client ID and secret in "
+                    "DVC config or omit both to use the defaults.\n"
+                    "Learn more at {}".format(
+                        format_link("https://man.dvc.org/remote/modify")
+                    )
+                )
+
+    @cached_property
+    def credentials_location(self):
+        """
+        Helper to determine where will GDrive remote read credentials from.
+        Useful for tests, exception messages, etc. Returns either env variable
+        name if it's set or actual path to the credentials file.
+        """
+        if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
+            return GDriveRemote.GDRIVE_CREDENTIALS_DATA
+        if os.path.exists(self._gdrive_user_credentials_path):
+            return self._gdrive_user_credentials_path
+        return None
+
+    @staticmethod
+    def _validate_credentials(auth, settings):
+        """
+        Detects discrepancy in DVC config and cached credentials file.
+        Usually happens when a second remote is added and it is using
+        the same credentials default file. Or when someones decides to change
+        DVC config client id or secret but forgets to remove the cached
+        credentials file.
+        """
+        if not os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
+            if (
+                settings["client_config"]["client_id"]
+                != auth.credentials.client_id
+                or settings["client_config"]["client_secret"]
+                != auth.credentials.client_secret
+            ):
+                logger.warning(
+                    "Client ID and secret configured do not match the "
+                    "actual ones used\nto access the remote. Do you "
+                    "use multiple GDrive remotes and forgot to\nset "
+                    "`gdrive_user_credentials_file` for one or more of them? "
+                    "Learn more at\n{}.\n".format(
                         format_link("https://man.dvc.org/remote/modify")
                     )
                 )
@@ -172,27 +208,35 @@ class RemoteGDrive(RemoteBASE):
     @wrap_prop(threading.RLock())
     @cached_property
     def _drive(self):
-        from pydrive2.auth import RefreshError
         from pydrive2.auth import GoogleAuth
         from pydrive2.drive import GoogleDrive
 
-        if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA):
-            with open(
-                self._gdrive_user_credentials_path, "w"
-            ) as credentials_file:
-                credentials_file.write(
-                    os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA)
+        if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
+            with open(self._gdrive_user_credentials_path, "w") as cred_file:
+                cred_file.write(
+                    os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA)
                 )
 
-        GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
+        auth_settings = {
+            "client_config_backend": "settings",
+            "save_credentials": True,
+            "save_credentials_backend": "file",
+            "save_credentials_file": self._gdrive_user_credentials_path,
+            "get_refresh_token": True,
+            "oauth_scope": [
+                "https://www.googleapis.com/auth/drive",
+                "https://www.googleapis.com/auth/drive.appdata",
+            ],
+        }
+
         if self._use_service_account:
-            GoogleAuth.DEFAULT_SETTINGS["service_config"] = {
+            auth_settings["service_config"] = {
                 "client_service_email": self._service_account_email,
                 "client_user_email": self._service_account_user_email,
                 "client_pkcs12_file_path": self._service_account_p12_file_path,
             }
         else:
-            GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
+            auth_settings["client_config"] = {
                 "client_id": self._client_id or self.DEFAULT_GDRIVE_CLIENT_ID,
                 "client_secret": self._client_secret
                 or self.DEFAULT_GDRIVE_CLIENT_SECRET,
@@ -201,36 +245,32 @@ class RemoteGDrive(RemoteBASE):
                 "revoke_uri": "https://oauth2.googleapis.com/revoke",
                 "redirect_uri": "",
             }
-        GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
-        GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
-        GoogleAuth.DEFAULT_SETTINGS[
-            "save_credentials_file"
-        ] = self._gdrive_user_credentials_path
-        GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
-        GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
-            "https://www.googleapis.com/auth/drive",
-            "https://www.googleapis.com/auth/drive.appdata",
-        ]
 
-        # Pass non existent settings path to force DEFAULT_SETTINGS loading
+        GoogleAuth.DEFAULT_SETTINGS.update(auth_settings)
+
+        # Pass non existent settings path to force DEFAULT_SETTINGS loadings
         gauth = GoogleAuth(settings_file="")
 
         try:
+            logger.debug(
+                "GDrive remote auth with config '{}'.".format(
+                    GoogleAuth.DEFAULT_SETTINGS
+                )
+            )
             if self._use_service_account:
                 gauth.ServiceAuth()
             else:
                 gauth.CommandLineAuth()
-        except RefreshError as exc:
-            raise GDriveAccessTokenRefreshError from exc
-        except KeyError as exc:
-            raise GDriveMissedCredentialKeyError(
-                self._gdrive_user_credentials_path
-            ) from exc
-        # Handle pydrive2.auth.AuthenticationError and other auth failures
+                GDriveRemote._validate_credentials(gauth, auth_settings)
+
+        # Handle AuthenticationError, RefreshError and other auth failures
+        # It's hard to come up with a narrow exception, since PyDrive throws
+        # a lot of different errors - broken credentials file, refresh token
+        # expired, flow failed, etc.
         except Exception as exc:
-            raise DvcException("Google Drive authentication failed") from exc
+            raise GDriveAuthError(self.credentials_location) from exc
         finally:
-            if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA):
+            if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
                 os.remove(self._gdrive_user_credentials_path)
 
         return GoogleDrive(gauth)
@@ -241,7 +281,11 @@ class RemoteGDrive(RemoteBASE):
         cache = {
             "dirs": defaultdict(list),
             "ids": {},
-            "root_id": self._get_item_id(self.path_info, use_cache=False),
+            "root_id": self._get_item_id(
+                self.path_info,
+                use_cache=False,
+                hint="Confirm the directory exists and you can access it.",
+            ),
         }
 
         self._cache_path_id(self.path_info.path, cache["root_id"], cache)
@@ -265,18 +309,42 @@ class RemoteGDrive(RemoteBASE):
         if self._bucket != "root" and self._bucket != "appDataFolder":
             drive_id = self._gdrive_shared_drive_id(self._bucket)
             if drive_id:
+                logger.debug(
+                    "GDrive remote '{}' is using shared drive id '{}'.".format(
+                        self.path_info, drive_id
+                    )
+                )
                 params["driveId"] = drive_id
                 params["corpora"] = "drive"
         return params
 
     @_gdrive_retry
     def _gdrive_shared_drive_id(self, item_id):
+        from pydrive2.files import ApiRequestError
+
         param = {"id": item_id}
         # it does not create a file on the remote
         item = self._drive.CreateFile(param)
         # ID of the shared drive the item resides in.
         # Only populated for items in shared drives.
-        item.FetchMetadata("driveId")
+        try:
+            item.FetchMetadata("driveId")
+        except ApiRequestError as exc:
+            error_code = exc.error.get("code", 0)
+            if error_code == 404:
+                raise DvcException(
+                    "'{}' for '{}':\n\n"
+                    "1. Confirm the directory exists and you can access it.\n"
+                    "2. Make sure that credentials in '{}'\n"
+                    "   are correct for this remote e.g. "
+                    "use the `gdrive_user_credentials_file` config\n"
+                    "   option if you use multiple GDrive remotes with "
+                    "different email accounts.\n\nDetails".format(
+                        item_id, self.path_info, self._credentials_location
+                    )
+                ) from exc
+            raise
+
         return item.get("driveId", None)
 
     @_gdrive_retry
@@ -315,14 +383,15 @@ class RemoteGDrive(RemoteBASE):
         param = {"id": item_id}
         # it does not create a file on the remote
         gdrive_file = self._drive.CreateFile(param)
-        bar_format = (
-            "Downloading {desc:{ncols_desc}.{ncols_desc}}... "
-            + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
-        )
+
         with Tqdm(
-            bar_format=bar_format, desc=progress_desc, disable=no_progress_bar
-        ):
-            gdrive_file.GetContentFile(to_file)
+            desc=progress_desc,
+            disable=no_progress_bar,
+            bytes=True,
+            # explicit `bar_format` as `total` will be set by `update_to`
+            bar_format=Tqdm.BAR_FMT_DEFAULT,
+        ) as pbar:
+            gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
 
     @_gdrive_retry
     def _gdrive_delete_file(self, item_id):
@@ -339,7 +408,7 @@ class RemoteGDrive(RemoteBASE):
             if (
                 http_error_code == 403
                 and self._list_params["corpora"] == "drive"
-                and _location(exc) == "file.permissions"
+                and exc.GetField("location") == "file.permissions"
             ):
                 raise DvcException(
                     "Insufficient permissions to {}. You should have {} "
@@ -394,7 +463,7 @@ class RemoteGDrive(RemoteBASE):
             return None
         query = "trashed=false and ({})".format(
             " or ".join(
-                "'{}' in parents".format(parent_id) for parent_id in parent_ids
+                f"'{parent_id}' in parents" for parent_id in parent_ids
             )
         )
         query += " and title='{}'".format(title.replace("'", "\\'"))
@@ -429,14 +498,15 @@ class RemoteGDrive(RemoteBASE):
             [self._create_dir(min(parent_ids), title, path)] if create else []
         )
 
-    def _get_item_id(self, path_info, create=False, use_cache=True):
+    def _get_item_id(self, path_info, create=False, use_cache=True, hint=None):
         assert path_info.bucket == self._bucket
 
         item_ids = self._path_to_item_ids(path_info.path, create, use_cache)
         if item_ids:
             return min(item_ids)
 
-        raise GDrivePathNotFound(path_info)
+        assert not create
+        raise GDrivePathNotFound(path_info, hint)
 
     def exists(self, path_info):
         try:
@@ -470,9 +540,9 @@ class RemoteGDrive(RemoteBASE):
         else:
             dir_ids = self._ids_cache["ids"]
         parents_query = " or ".join(
-            "'{}' in parents".format(dir_id) for dir_id in dir_ids
+            f"'{dir_id}' in parents" for dir_id in dir_ids
         )
-        query = "({}) and trashed=false".format(parents_query)
+        query = f"({parents_query}) and trashed=false"
 
         for item in self._gdrive_list(query):
             if progress_callback:
@@ -485,3 +555,9 @@ class RemoteGDrive(RemoteBASE):
     def remove(self, path_info):
         item_id = self._get_item_id(path_info)
         self._gdrive_delete_file(item_id)
+
+    def get_file_checksum(self, path_info):
+        raise NotImplementedError
+
+    def walk_files(self, path_info):
+        raise NotImplementedError

dvc/remote/gs.py

@@ -1,17 +1,16 @@
 import logging
-from datetime import timedelta
-from functools import wraps
-import io
 import os.path
 import posixpath
 import threading
+from datetime import timedelta
+from functools import wraps
 
 from funcy import cached_property, wrap_prop
 
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
+from dvc.remote.base import BaseRemote
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
@@ -55,7 +54,7 @@ def _upload_to_bucket(
     no_progress_bar=False,
 ):
     blob = bucket.blob(to_info.path, chunk_size=chunk_size)
-    with io.open(from_file, mode="rb") as fobj:
+    with open(from_file, mode="rb") as fobj:
         with Tqdm.wrapattr(
             fobj,
             "read",
@@ -66,7 +65,7 @@ def _upload_to_bucket(
             blob.upload_from_file(wrapped)
 
 
-class RemoteGS(RemoteBASE):
+class GSRemote(BaseRemote):
     scheme = Schemes.GS
     path_cls = CloudURLInfo
     REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
@@ -110,7 +109,7 @@ class RemoteGS(RemoteBASE):
         from_bucket = self.gs.bucket(from_info.bucket)
         blob = from_bucket.get_blob(from_info.path)
         if not blob:
-            msg = "'{}' doesn't exist in the cloud".format(from_info.path)
+            msg = f"'{from_info.path}' doesn't exist in the cloud"
             raise DvcException(msg)
 
         to_bucket = self.gs.bucket(to_info.bucket)
@@ -120,7 +119,7 @@ class RemoteGS(RemoteBASE):
         if path_info.scheme != "gs":
             raise NotImplementedError
 
-        logger.debug("Removing gs://{}".format(path_info))
+        logger.debug(f"Removing gs://{path_info}")
         blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
         if not blob:
             return
@@ -193,7 +192,7 @@ class RemoteGS(RemoteBASE):
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
         bucket = self.gs.bucket(from_info.bucket)
         blob = bucket.get_blob(from_info.path)
-        with io.open(to_file, mode="wb") as fobj:
+        with open(to_file, mode="wb") as fobj:
             with Tqdm.wrapattr(
                 fobj,
                 "write",

dvc/remote/hdfs.py

@@ -8,15 +8,16 @@ from collections import deque
 from contextlib import closing, contextmanager
 from urllib.parse import urlparse
 
-from .base import RemoteBASE, RemoteCmdError
-from .pool import get_connection
 from dvc.scheme import Schemes
 from dvc.utils import fix_env, tmp_fname
 
+from .base import BaseRemote, RemoteCmdError
+from .pool import get_connection
+
 logger = logging.getLogger(__name__)
 
 
-class RemoteHDFS(RemoteBASE):
+class HDFSRemote(BaseRemote):
     scheme = Schemes.HDFS
     REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
     PARAM_CHECKSUM = "checksum"
@@ -54,7 +55,7 @@ class RemoteHDFS(RemoteBASE):
     def hadoop_fs(self, cmd, user=None):
         cmd = "hadoop fs -" + cmd
         if user:
-            cmd = "HADOOP_USER_NAME={} ".format(user) + cmd
+            cmd = f"HADOOP_USER_NAME={user} " + cmd
 
         # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
         # See https://github.com/iterative/dvc/issues/1197.
@@ -86,7 +87,7 @@ class RemoteHDFS(RemoteBASE):
         # NOTE: pyarrow doesn't support checksum, so we need to use hadoop
         regex = r".*\t.*\t(?P<checksum>.*)"
         stdout = self.hadoop_fs(
-            "checksum {}".format(path_info.path), user=path_info.user
+            f"checksum {path_info.path}", user=path_info.user
         )
         return self._group(regex, stdout, "checksum")
 
@@ -111,7 +112,7 @@ class RemoteHDFS(RemoteBASE):
             raise NotImplementedError
 
         if self.exists(path_info):
-            logger.debug("Removing {}".format(path_info.path))
+            logger.debug(f"Removing {path_info.path}")
             with self.hdfs(path_info) as hdfs:
                 hdfs.rm(path_info.path)
 
@@ -146,7 +147,7 @@ class RemoteHDFS(RemoteBASE):
                     yield fd
                 else:
                     yield io.TextIOWrapper(fd, encoding=encoding)
-        except IOError as e:
+        except OSError as e:
             # Empty .errno and not specific enough error class in pyarrow,
             # see https://issues.apache.org/jira/browse/ARROW-6248
             if "file does not exist" in str(e):
@@ -174,7 +175,7 @@ class RemoteHDFS(RemoteBASE):
                             if progress_callback:
                                 progress_callback()
                             yield urlparse(entry["name"]).path
-                except IOError as e:
+                except OSError as e:
                     # When searching for a specific prefix pyarrow raises an
                     # exception if the specified cache dir does not exist
                     if not prefix:

dvc/remote/http.py

@@ -4,11 +4,11 @@ import threading
 
 from funcy import cached_property, memoize, wrap_prop, wrap_with
 
-from dvc.path_info import HTTPURLInfo
 import dvc.prompt as prompt
 from dvc.exceptions import DvcException, HTTPError
+from dvc.path_info import HTTPURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
+from dvc.remote.base import BaseRemote
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
@@ -23,7 +23,7 @@ def ask_password(host, user):
     )
 
 
-class RemoteHTTP(RemoteBASE):
+class HTTPRemote(BaseRemote):
     scheme = Schemes.HTTP
     path_cls = HTTPURLInfo
     SESSION_RETRIES = 5
@@ -185,7 +185,7 @@ class RemoteHTTP(RemoteBASE):
             return res
 
         except requests.exceptions.RequestException:
-            raise DvcException("could not perform a {} request".format(method))
+            raise DvcException(f"could not perform a {method} request")
 
     def gc(self):
         raise NotImplementedError

dvc/remote/https.py

@@ -1,6 +1,7 @@
-from .http import RemoteHTTP
 from dvc.scheme import Schemes
 
+from .http import HTTPRemote
 
-class RemoteHTTPS(RemoteHTTP):
+
+class HTTPSRemote(HTTPRemote):
     scheme = Schemes.HTTPS

dvc/remote/index.py

@@ -0,0 +1,212 @@
+import logging
+import os
+import sqlite3
+import threading
+
+from funcy import lchunks
+
+from dvc.state import _connect_sqlite
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteIndexNoop:
+    """No-op class for remotes which are not indexed (i.e. local)."""
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, typ, value, tbck):
+        pass
+
+    def __iter__(self):
+        return iter([])
+
+    def __contains__(self, checksum):
+        return False
+
+    @staticmethod
+    def checksums():
+        return []
+
+    @staticmethod
+    def dir_checksums():
+        return []
+
+    def load(self):
+        pass
+
+    def dump(self):
+        pass
+
+    def clear(self):
+        pass
+
+    def update(self, *args):
+        pass
+
+    @staticmethod
+    def intersection(*args):
+        return []
+
+
+class RemoteIndex:
+    """Class for indexing remote checksums in a sqlite3 database.
+
+    Args:
+        repo: repo for this remote index.
+        name: name for this index. Index db will be loaded from and saved to
+            ``.dvc/tmp/index/{name}.idx``.
+        dir_suffix: suffix used for naming directory checksums
+    """
+
+    INDEX_SUFFIX = ".idx"
+    VERSION = 1
+    INDEX_TABLE = "remote_index"
+    INDEX_TABLE_LAYOUT = "checksum TEXT PRIMARY KEY, " "dir INTEGER NOT NULL"
+
+    def __init__(self, repo, name, dir_suffix=".dir"):
+        self.path = os.path.join(repo.index_dir, f"{name}{self.INDEX_SUFFIX}")
+
+        self.dir_suffix = dir_suffix
+        self.database = None
+        self.cursor = None
+        self.modified = False
+        self.lock = threading.Lock()
+
+    def __iter__(self):
+        cmd = f"SELECT checksum FROM {self.INDEX_TABLE}"
+        for (checksum,) in self._execute(cmd):
+            yield checksum
+
+    def __enter__(self):
+        self.lock.acquire()
+        self.load()
+
+    def __exit__(self, typ, value, tbck):
+        self.dump()
+        self.lock.release()
+
+    def __contains__(self, checksum):
+        cmd = "SELECT checksum FROM {} WHERE checksum = (?)".format(
+            self.INDEX_TABLE
+        )
+        self._execute(cmd, (checksum,))
+        return self.cursor.fetchone() is not None
+
+    def checksums(self):
+        """Iterate over checksums stored in the index."""
+        return iter(self)
+
+    def dir_checksums(self):
+        """Iterate over .dir checksums stored in the index."""
+        cmd = f"SELECT checksum FROM {self.INDEX_TABLE} WHERE dir = 1"
+        for (checksum,) in self._execute(cmd):
+            yield checksum
+
+    def is_dir_checksum(self, checksum):
+        return checksum.endswith(self.dir_suffix)
+
+    def _execute(self, cmd, parameters=()):
+        return self.cursor.execute(cmd, parameters)
+
+    def _executemany(self, cmd, seq_of_parameters):
+        return self.cursor.executemany(cmd, seq_of_parameters)
+
+    def _prepare_db(self, empty=False):
+        if not empty:
+            cmd = "PRAGMA user_version;"
+            self._execute(cmd)
+            ret = self.cursor.fetchall()
+            assert len(ret) == 1
+            assert len(ret[0]) == 1
+            assert isinstance(ret[0][0], int)
+            version = ret[0][0]
+
+            if version != self.VERSION:
+                logger.error(
+                    "Index file version '{}' will be reformatted "
+                    "to the current version '{}'.".format(
+                        version, self.VERSION,
+                    )
+                )
+                cmd = "DROP TABLE IF EXISTS {};"
+                self._execute(cmd.format(self.INDEX_TABLE))
+
+        cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
+        self._execute(cmd.format(self.INDEX_TABLE, self.INDEX_TABLE_LAYOUT))
+
+        cmd = "PRAGMA user_version = {};"
+        self._execute(cmd.format(self.VERSION))
+
+    def load(self):
+        """(Re)load this index database."""
+        retries = 1
+        while True:
+            assert self.database is None
+            assert self.cursor is None
+
+            empty = not os.path.isfile(self.path)
+            self.database = _connect_sqlite(self.path, {"nolock": 1})
+            self.cursor = self.database.cursor()
+
+            try:
+                self._prepare_db(empty=empty)
+                return
+            except sqlite3.DatabaseError:
+                self.cursor.close()
+                self.database.close()
+                self.database = None
+                self.cursor = None
+                if retries > 0:
+                    os.unlink(self.path)
+                    retries -= 1
+                else:
+                    raise
+
+    def dump(self):
+        """Save this index database."""
+        assert self.database is not None
+
+        self.database.commit()
+        self.cursor.close()
+        self.database.close()
+        self.database = None
+        self.cursor = None
+
+    def clear(self):
+        """Clear this index (to force re-indexing later).
+
+        Changes to the index will not committed until dump() is called.
+        """
+        cmd = f"DELETE FROM {self.INDEX_TABLE}"
+        self._execute(cmd)
+
+    def update(self, dir_checksums, file_checksums):
+        """Update this index, adding the specified checksums.
+
+        Changes to the index will not committed until dump() is called.
+        """
+        cmd = "INSERT OR IGNORE INTO {} (checksum, dir) VALUES (?, ?)".format(
+            self.INDEX_TABLE
+        )
+        self._executemany(
+            cmd, ((checksum, True) for checksum in dir_checksums)
+        )
+        self._executemany(
+            cmd, ((checksum, False) for checksum in file_checksums)
+        )
+
+    def intersection(self, checksums):
+        """Iterate over values from `checksums` which exist in the index."""
+        # sqlite has a compile time limit of 999, see:
+        # https://www.sqlite.org/c3ref/c_limit_attached.html#sqlitelimitvariablenumber
+        for chunk in lchunks(999, checksums):
+            cmd = "SELECT checksum FROM {} WHERE checksum IN ({})".format(
+                self.INDEX_TABLE, ",".join("?" for checksum in chunk)
+            )
+            for (checksum,) in self._execute(cmd, chunk):
+                yield checksum

dvc/remote/local.py

@@ -2,31 +2,47 @@ import errno
 import logging
 import os
 import stat
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from functools import partial
 
+from funcy import cached_property, concat
 from shortuuid import uuid
 
-from dvc.compat import fspath_py35
-from dvc.exceptions import DvcException, DownloadError, UploadError
+from dvc.exceptions import DownloadError, DvcException, UploadError
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE, STATUS_MAP
-from dvc.remote.base import STATUS_DELETED, STATUS_MISSING, STATUS_NEW
+from dvc.remote.base import (
+    STATUS_DELETED,
+    STATUS_MAP,
+    STATUS_MISSING,
+    STATUS_NEW,
+    BaseRemote,
+    index_locked,
+)
+from dvc.remote.index import RemoteIndexNoop
 from dvc.scheme import Schemes
-from dvc.scm.tree import is_working_tree
+from dvc.scm.tree import WorkingTree, is_working_tree
 from dvc.system import System
 from dvc.utils import file_md5, relpath, tmp_fname
-from dvc.utils.fs import copyfile, move, makedirs, remove, walk_files
+from dvc.utils.fs import (
+    copy_fobj_to_file,
+    copyfile,
+    makedirs,
+    move,
+    remove,
+    walk_files,
+)
 
 logger = logging.getLogger(__name__)
 
 
-class RemoteLOCAL(RemoteBASE):
+class LocalRemote(BaseRemote):
     scheme = Schemes.LOCAL
     path_cls = PathInfo
     PARAM_CHECKSUM = "md5"
     PARAM_PATH = "path"
+    TRAVERSE_PREFIX_LEN = 2
+    INDEX_CLS = RemoteIndexNoop
 
     UNPACKED_DIR_SUFFIX = ".unpacked"
 
@@ -52,19 +68,44 @@ class RemoteLOCAL(RemoteBASE):
     def cache_dir(self, value):
         self.path_info = PathInfo(value) if value else None
 
+    @cached_property
+    def _work_tree(self):
+        if self.repo:
+            return WorkingTree(self.repo.root_dir)
+        return None
+
+    @property
+    def work_tree(self):
+        # When using repo.brancher, repo.tree may change to/from WorkingTree to
+        # GitTree arbitarily. When repo.tree is GitTree, local cache needs to
+        # use its own WorkingTree instance.
+        if self.repo and not is_working_tree(self.repo.tree):
+            return self._work_tree
+        return None
+
     @classmethod
     def supported(cls, config):
         return True
 
+    @cached_property
+    def cache_path(self):
+        return os.path.abspath(self.cache_dir)
+
+    def checksum_to_path(self, checksum):
+        return os.path.join(self.cache_path, checksum[0:2], checksum[2:])
+
     def list_cache_paths(self, prefix=None, progress_callback=None):
-        assert prefix is None
         assert self.path_info is not None
+        if prefix:
+            path_info = self.path_info / prefix[:2]
+        else:
+            path_info = self.path_info
         if progress_callback:
-            for path in walk_files(self.path_info):
+            for path in walk_files(path_info):
                 progress_callback()
                 yield path
         else:
-            yield from walk_files(self.path_info)
+            yield from walk_files(path_info)
 
     def get(self, md5):
         if not md5:
@@ -72,10 +113,13 @@ class RemoteLOCAL(RemoteBASE):
 
         return self.checksum_to_path_info(md5).url
 
-    @staticmethod
-    def exists(path_info):
-        assert path_info.scheme == "local"
-        return os.path.lexists(fspath_py35(path_info))
+    def exists(self, path_info):
+        assert isinstance(path_info, str) or path_info.scheme == "local"
+        if not self.repo:
+            return os.path.exists(path_info)
+        if self.work_tree and self.work_tree.exists(path_info):
+            return True
+        return self.repo.tree.exists(path_info)
 
     def makedirs(self, path_info):
         makedirs(path_info, exist_ok=True, mode=self._dir_mode)
@@ -107,13 +151,19 @@ class RemoteLOCAL(RemoteBASE):
 
         return False
 
-    @staticmethod
-    def isfile(path_info):
-        return os.path.isfile(fspath_py35(path_info))
+    def isfile(self, path_info):
+        if not self.repo:
+            return os.path.isfile(path_info)
+        if self.work_tree and self.work_tree.isfile(path_info):
+            return True
+        return self.repo.tree.isfile(path_info)
 
-    @staticmethod
-    def isdir(path_info):
-        return os.path.isdir(fspath_py35(path_info))
+    def isdir(self, path_info):
+        if not self.repo:
+            return os.path.isdir(path_info)
+        if self.work_tree and self.work_tree.isdir(path_info):
+            return True
+        return self.repo.tree.isdir(path_info)
 
     def iscopy(self, path_info):
         return not (
@@ -122,23 +172,29 @@ class RemoteLOCAL(RemoteBASE):
 
     @staticmethod
     def getsize(path_info):
-        return os.path.getsize(fspath_py35(path_info))
+        return os.path.getsize(path_info)
 
     def walk_files(self, path_info):
-        assert is_working_tree(self.repo.tree)
-
-        for fname in self.repo.tree.walk_files(path_info):
+        if self.work_tree:
+            tree = self.work_tree
+        else:
+            tree = self.repo.tree
+        for fname in tree.walk_files(path_info):
             yield PathInfo(fname)
 
     def get_file_checksum(self, path_info):
         return file_md5(path_info)[0]
 
     def remove(self, path_info):
-        if path_info.scheme != "local":
-            raise NotImplementedError
+        if isinstance(path_info, PathInfo):
+            if path_info.scheme != "local":
+                raise NotImplementedError
+            path = path_info.fspath
+        else:
+            path = path_info
 
-        if self.exists(path_info):
-            remove(path_info.fspath)
+        if self.exists(path):
+            remove(path)
 
     def move(self, from_info, to_info, mode=None):
         if from_info.scheme != "local" or to_info.scheme != "local":
@@ -158,8 +214,19 @@ class RemoteLOCAL(RemoteBASE):
         tmp_info = to_info.parent / tmp_fname(to_info.name)
         try:
             System.copy(from_info, tmp_info)
-            os.chmod(fspath_py35(tmp_info), self._file_mode)
-            os.rename(fspath_py35(tmp_info), fspath_py35(to_info))
+            os.chmod(tmp_info, self._file_mode)
+            os.rename(tmp_info, to_info)
+        except Exception:
+            self.remove(tmp_info)
+            raise
+
+    def copy_fobj(self, fobj, to_info):
+        self.makedirs(to_info.parent)
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        try:
+            copy_fobj_to_file(fobj, tmp_info)
+            os.chmod(tmp_info, self._file_mode)
+            os.rename(tmp_info, to_info)
         except Exception:
             self.remove(tmp_info)
             raise
@@ -207,8 +274,8 @@ class RemoteLOCAL(RemoteBASE):
         System.reflink(from_info, tmp_info)
         # NOTE: reflink has its own separate inode, so you can set permissions
         # that are different from the source.
-        os.chmod(fspath_py35(tmp_info), self._file_mode)
-        os.rename(fspath_py35(tmp_info), fspath_py35(to_info))
+        os.chmod(tmp_info, self._file_mode)
+        os.rename(tmp_info, to_info)
 
     def cache_exists(self, checksums, jobs=None, name=None):
         return [
@@ -231,7 +298,9 @@ class RemoteLOCAL(RemoteBASE):
         copyfile(
             from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
         )
-        os.rename(tmp_file, fspath_py35(to_info))
+
+        self.protect(tmp_file)
+        os.rename(tmp_file, to_info)
 
     def _download(
         self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
@@ -242,8 +311,9 @@ class RemoteLOCAL(RemoteBASE):
 
     @staticmethod
     def open(path_info, mode="r", encoding=None):
-        return open(fspath_py35(path_info), mode=mode, encoding=encoding)
+        return open(path_info, mode=mode, encoding=encoding)
 
+    @index_locked
     def status(
         self,
         named_cache,
@@ -252,37 +322,128 @@ class RemoteLOCAL(RemoteBASE):
         show_checksums=False,
         download=False,
     ):
-        logger.debug(
-            "Preparing to collect status from {}".format(remote.path_info)
+        # Return flattened dict containing all status info
+        dir_status, file_status, _ = self._status(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
         )
-        md5s = list(named_cache[self.scheme])
+        return dict(dir_status, **file_status)
+
+    def _status(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        """Return a tuple of (dir_status_info, file_status_info, dir_contents).
+
+        dir_status_info contains status for .dir files, file_status_info
+        contains status for all other files, and dir_contents is a dict of
+        {dir_checksum: set(file_checksum, ...)} which can be used to map
+        a .dir file to its file contents.
+        """
+        logger.debug(f"Preparing to collect status from {remote.path_info}")
+        md5s = set(named_cache.scheme_keys(self.scheme))
 
         logger.debug("Collecting information from local cache...")
-        local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
+        local_exists = frozenset(
+            self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
+        )
 
         # This is a performance optimization. We can safely assume that,
         # if the resources that we want to fetch are already cached,
         # there's no need to check the remote storage for the existence of
         # those files.
-        if download and sorted(local_exists) == sorted(md5s):
+        if download and local_exists == md5s:
             remote_exists = local_exists
         else:
             logger.debug("Collecting information from remote cache...")
-            remote_exists = list(
-                remote.cache_exists(
-                    md5s, jobs=jobs, name=str(remote.path_info)
+            remote_exists = set()
+            dir_md5s = set(named_cache.dir_keys(self.scheme))
+            if dir_md5s:
+                remote_exists.update(
+                    self._indexed_dir_checksums(named_cache, remote, dir_md5s)
                 )
+                md5s.difference_update(remote_exists)
+            if md5s:
+                remote_exists.update(
+                    remote.cache_exists(
+                        md5s, jobs=jobs, name=str(remote.path_info)
+                    )
+                )
+        return self._make_status(
+            named_cache, show_checksums, local_exists, remote_exists
+        )
+
+    def _make_status(
+        self, named_cache, show_checksums, local_exists, remote_exists
+    ):
+        def make_names(checksum, names):
+            return {"name": checksum if show_checksums else " ".join(names)}
+
+        dir_status = {}
+        file_status = {}
+        dir_contents = {}
+        for checksum, item in named_cache[self.scheme].items():
+            if item.children:
+                dir_status[checksum] = make_names(checksum, item.names)
+                dir_contents[checksum] = set()
+                for child_checksum, child in item.children.items():
+                    file_status[child_checksum] = make_names(
+                        child_checksum, child.names
+                    )
+                    dir_contents[checksum].add(child_checksum)
+            else:
+                file_status[checksum] = make_names(checksum, item.names)
+
+        self._fill_statuses(dir_status, local_exists, remote_exists)
+        self._fill_statuses(file_status, local_exists, remote_exists)
+
+        self._log_missing_caches(dict(dir_status, **file_status))
+
+        return dir_status, file_status, dir_contents
+
+    def _indexed_dir_checksums(self, named_cache, remote, dir_md5s):
+        # Validate our index by verifying all indexed .dir checksums
+        # still exist on the remote
+        indexed_dirs = set(remote.index.dir_checksums())
+        indexed_dir_exists = set()
+        if indexed_dirs:
+            indexed_dir_exists.update(
+                remote._cache_object_exists(indexed_dirs)
             )
+            missing_dirs = indexed_dirs.difference(indexed_dir_exists)
+            if missing_dirs:
+                logger.debug(
+                    "Remote cache missing indexed .dir checksums '{}', "
+                    "clearing remote index".format(", ".join(missing_dirs))
+                )
+                remote.index.clear()
 
-        ret = {
-            checksum: {"name": checksum if show_checksums else " ".join(names)}
-            for checksum, names in named_cache[self.scheme].items()
-        }
-        self._fill_statuses(ret, local_exists, remote_exists)
+        # Check if non-indexed (new) dir checksums exist on remote
+        dir_exists = dir_md5s.intersection(indexed_dir_exists)
+        dir_exists.update(remote._cache_object_exists(dir_md5s - dir_exists))
 
-        self._log_missing_caches(ret)
-
-        return ret
+        # If .dir checksum exists on the remote, assume directory contents
+        # still exists on the remote
+        for dir_checksum in dir_exists:
+            file_checksums = list(
+                named_cache.child_keys(self.scheme, dir_checksum)
+            )
+            if dir_checksum not in remote.index:
+                logger.debug(
+                    "Indexing new .dir '{}' with '{}' nested files".format(
+                        dir_checksum, len(file_checksums)
+                    )
+                )
+                remote.index.update([dir_checksum], file_checksums)
+            yield dir_checksum
+            yield from file_checksums
 
     @staticmethod
     def _fill_statuses(checksum_info_dir, local_exists, remote_exists):
@@ -298,6 +459,7 @@ class RemoteLOCAL(RemoteBASE):
         cache = []
         path_infos = []
         names = []
+        checksums = []
         for md5, info in Tqdm(
             status_info.items(), desc="Analysing status", unit="file"
         ):
@@ -305,6 +467,7 @@ class RemoteLOCAL(RemoteBASE):
                 cache.append(self.checksum_to_path_info(md5))
                 path_infos.append(remote.checksum_to_path_info(md5))
                 names.append(info["name"])
+                checksums.append(md5)
 
         if download:
             to_infos = cache
@@ -313,7 +476,7 @@ class RemoteLOCAL(RemoteBASE):
             to_infos = path_infos
             from_infos = cache
 
-        return from_infos, to_infos, names
+        return from_infos, to_infos, names, checksums
 
     def _process(
         self,
@@ -337,14 +500,16 @@ class RemoteLOCAL(RemoteBASE):
                 file_mode=self._file_mode,
             )
             status = STATUS_DELETED
+            desc = "Downloading"
         else:
             func = remote.upload
             status = STATUS_NEW
+            desc = "Uploading"
 
         if jobs is None:
             jobs = remote.JOBS
 
-        status_info = self.status(
+        dir_status, file_status, dir_contents = self._status(
             named_cache,
             remote,
             jobs=jobs,
@@ -352,24 +517,87 @@ class RemoteLOCAL(RemoteBASE):
             download=download,
         )
 
-        plans = self._get_plans(download, remote, status_info, status)
+        dir_plans = self._get_plans(download, remote, dir_status, status)
+        file_plans = self._get_plans(download, remote, file_status, status)
 
-        if len(plans[0]) == 0:
+        total = len(dir_plans[0]) + len(file_plans[0])
+        if total == 0:
             return 0
 
-        if jobs > 1:
+        with Tqdm(total=total, unit="file", desc=desc) as pbar:
+            func = pbar.wrap_fn(func)
             with ThreadPoolExecutor(max_workers=jobs) as executor:
-                fails = sum(executor.map(func, *plans))
-        else:
-            fails = sum(map(func, *plans))
+                if download:
+                    fails = sum(executor.map(func, *dir_plans))
+                    fails += sum(executor.map(func, *file_plans))
+                else:
+                    # for uploads, push files first, and any .dir files last
+
+                    file_futures = {}
+                    for from_info, to_info, name, checksum in zip(*file_plans):
+                        file_futures[checksum] = executor.submit(
+                            func, from_info, to_info, name
+                        )
+                    dir_futures = {}
+                    for from_info, to_info, name, dir_checksum in zip(
+                        *dir_plans
+                    ):
+                        wait_futures = {
+                            future
+                            for file_checksum, future in file_futures.items()
+                            if file_checksum in dir_contents[dir_checksum]
+                        }
+                        dir_futures[dir_checksum] = executor.submit(
+                            self._dir_upload,
+                            func,
+                            wait_futures,
+                            from_info,
+                            to_info,
+                            name,
+                        )
+                    fails = sum(
+                        future.result()
+                        for future in concat(
+                            file_futures.values(), dir_futures.values()
+                        )
+                    )
 
         if fails:
             if download:
+                remote.index.clear()
                 raise DownloadError(fails)
             raise UploadError(fails)
 
-        return len(plans[0])
+        if not download:
+            # index successfully pushed dirs
+            for dir_checksum, future in dir_futures.items():
+                if future.result() == 0:
+                    file_checksums = dir_contents[dir_checksum]
+                    logger.debug(
+                        "Indexing pushed dir '{}' with "
+                        "'{}' nested files".format(
+                            dir_checksum, len(file_checksums)
+                        )
+                    )
+                    remote.index.update([dir_checksum], file_checksums)
 
+        return len(dir_plans[0]) + len(file_plans[0])
+
+    @staticmethod
+    def _dir_upload(func, futures, from_info, to_info, name):
+        for future in as_completed(futures):
+            if future.result():
+                # do not upload this .dir file if any file in this
+                # directory failed to upload
+                logger.debug(
+                    "failed to upload full contents of '{}', "
+                    "aborting .dir file upload".format(name)
+                )
+                logger.error(f"failed to upload '{from_info}' to '{to_info}'")
+                return 1
+        return func(from_info, to_info, name)
+
+    @index_locked
     def push(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
             named_cache,
@@ -379,6 +607,7 @@ class RemoteLOCAL(RemoteBASE):
             download=False,
         )
 
+    @index_locked
     def pull(self, named_cache, remote, jobs=None, show_checksums=False):
         return self._process(
             named_cache,
@@ -408,7 +637,7 @@ class RemoteLOCAL(RemoteBASE):
 
     def _unprotect_file(self, path):
         if System.is_symlink(path) or System.is_hardlink(path):
-            logger.debug("Unprotecting '{}'".format(path))
+            logger.debug(f"Unprotecting '{path}'")
             tmp = os.path.join(os.path.dirname(path), "." + uuid())
 
             # The operations order is important here - if some application
@@ -437,9 +666,7 @@ class RemoteLOCAL(RemoteBASE):
     def unprotect(self, path_info):
         path = path_info.fspath
         if not os.path.exists(path):
-            raise DvcException(
-                "can't unprotect non-existing data '{}'".format(path)
-            )
+            raise DvcException(f"can't unprotect non-existing data '{path}'")
 
         if os.path.isdir(path):
             self._unprotect_dir(path)
@@ -447,7 +674,7 @@ class RemoteLOCAL(RemoteBASE):
             self._unprotect_file(path)
 
     def protect(self, path_info):
-        path = fspath_py35(path_info)
+        path = os.fspath(path_info)
         mode = self.CACHE_MODE
 
         try:
@@ -491,7 +718,7 @@ class RemoteLOCAL(RemoteBASE):
             dir_info = self.get_dir_cache(checksum)
             self._create_unpacked_dir(checksum, dir_info, unpacked_dir_info)
         except DvcException:
-            logger.warning("Could not create '{}'".format(unpacked_dir_info))
+            logger.warning(f"Could not create '{unpacked_dir_info}'")
 
             self.remove(unpacked_dir_info)
 
@@ -530,6 +757,6 @@ class RemoteLOCAL(RemoteBASE):
         if not self.exists(path_info):
             return False
 
-        mode = os.stat(fspath_py35(path_info)).st_mode
+        mode = os.stat(path_info).st_mode
 
         return stat.S_IMODE(mode) == self.CACHE_MODE

dvc/remote/oss.py

@@ -7,14 +7,13 @@ from funcy import cached_property, wrap_prop
 
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
+from dvc.remote.base import BaseRemote
 from dvc.scheme import Schemes
 
-
 logger = logging.getLogger(__name__)
 
 
-class RemoteOSS(RemoteBASE):
+class OSSRemote(BaseRemote):
     """
     oss2 document:
     https://www.alibabacloud.com/help/doc-detail/32026.htm
@@ -65,9 +64,9 @@ class RemoteOSS(RemoteBASE):
     def oss_service(self):
         import oss2
 
-        logger.debug("URL: {}".format(self.path_info))
-        logger.debug("key id: {}".format(self.key_id))
-        logger.debug("key secret: {}".format(self.key_secret))
+        logger.debug(f"URL: {self.path_info}")
+        logger.debug(f"key id: {self.key_id}")
+        logger.debug(f"key secret: {self.key_secret}")
 
         auth = oss2.Auth(self.key_id, self.key_secret)
         bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)
@@ -88,7 +87,7 @@ class RemoteOSS(RemoteBASE):
         if path_info.scheme != self.scheme:
             raise NotImplementedError
 
-        logger.debug("Removing oss://{}".format(path_info))
+        logger.debug(f"Removing oss://{path_info}")
         self.oss_service.delete_object(path_info.path)
 
     def _list_paths(self, prefix, progress_callback=None):

dvc/remote/pool.py

@@ -1,6 +1,6 @@
+import threading
 from collections import deque
 from contextlib import contextmanager
-import threading
 
 from funcy import memoize, wrap_with
 
@@ -30,7 +30,7 @@ def close_pools():
     get_pool.memory.clear()
 
 
-class Pool(object):
+class Pool:
     def __init__(self, conn_func, *conn_args, **conn_kwargs):
         self._conn_func = conn_func
         self._conn_args = conn_args

dvc/remote/s3.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import logging
 import os
 import posixpath
@@ -8,17 +6,16 @@ import threading
 from funcy import cached_property, wrap_prop
 
 from dvc.config import ConfigError
-from dvc.exceptions import DvcException
-from dvc.exceptions import ETagMismatchError
+from dvc.exceptions import DvcException, ETagMismatchError
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
+from dvc.remote.base import BaseRemote
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
 
 
-class RemoteS3(RemoteBASE):
+class S3Remote(BaseRemote):
     scheme = Schemes.S3
     path_cls = CloudURLInfo
     REQUIRES = {"boto3": "boto3"}
@@ -47,6 +44,10 @@ class RemoteS3(RemoteBASE):
         if self.sse:
             self.extra_args["ServerSideEncryption"] = self.sse
 
+        self.sse_kms_key_id = config.get("sse_kms_key_id")
+        if self.sse_kms_key_id:
+            self.extra_args["SSEKMSKeyId"] = self.sse_kms_key_id
+
         self.acl = config.get("acl")
         if self.acl:
             self.extra_args["ACL"] = self.acl
@@ -85,9 +86,7 @@ class RemoteS3(RemoteBASE):
         try:
             obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
         except Exception as exc:
-            raise DvcException(
-                "s3://{}/{} does not exist".format(bucket, path)
-            ) from exc
+            raise DvcException(f"s3://{bucket}/{path} does not exist") from exc
         return obj
 
     @classmethod
@@ -110,7 +109,7 @@ class RemoteS3(RemoteBASE):
             if lastbyte > size:
                 lastbyte = size - 1
 
-            srange = "bytes={}-{}".format(byte_position, lastbyte)
+            srange = f"bytes={byte_position}-{lastbyte}"
 
             part = s3.upload_part_copy(
                 Bucket=to_info.bucket,
@@ -188,7 +187,7 @@ class RemoteS3(RemoteBASE):
         if path_info.scheme != "s3":
             raise NotImplementedError
 
-        logger.debug("Removing {}".format(path_info))
+        logger.debug(f"Removing {path_info}")
         self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
 
     def _list_objects(

dvc/remote/slow_link_detection.py

@@ -1,10 +1,9 @@
 import logging
 import sys
 import time
-
 from functools import wraps
-import colorama
 
+import colorama
 
 logger = logging.getLogger(__name__)
 this = sys.modules[__name__]

dvc/remote/ssh/__init__.py

@@ -4,16 +4,17 @@ import io
 import itertools
 import logging
 import os
+import posixpath
 import threading
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import closing, contextmanager
 from urllib.parse import urlparse
 
-from funcy import memoize, wrap_with, silent, first
+from funcy import first, memoize, silent, wrap_with
 
 import dvc.prompt as prompt
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteBASE
+from dvc.remote.base import BaseRemote
 from dvc.remote.pool import get_connection
 from dvc.scheme import Schemes
 from dvc.utils import to_chunks
@@ -32,7 +33,7 @@ def ask_password(host, user, port):
     )
 
 
-class RemoteSSH(RemoteBASE):
+class SSHRemote(BaseRemote):
     scheme = Schemes.SSH
     REQUIRES = {"paramiko": "paramiko"}
 
@@ -44,6 +45,7 @@ class RemoteSSH(RemoteBASE):
     # paramiko stuff, so we would ideally have it double of server processors.
     # We use conservative setting of 4 instead to not exhaust max sessions.
     CHECKSUM_JOBS = 4
+    TRAVERSE_PREFIX_LEN = 2
 
     DEFAULT_CACHE_TYPES = ["copy"]
 
@@ -101,7 +103,7 @@ class RemoteSSH(RemoteBASE):
     def _load_user_ssh_config(hostname):
         import paramiko
 
-        user_config_file = RemoteSSH.ssh_config_filename()
+        user_config_file = SSHRemote.ssh_config_filename()
         user_ssh_config = {}
         if hostname and os.path.exists(user_config_file):
             ssh_config = paramiko.SSHConfig()
@@ -258,15 +260,18 @@ class RemoteSSH(RemoteBASE):
                 yield io.TextIOWrapper(fd, encoding=encoding)
 
     def list_cache_paths(self, prefix=None, progress_callback=None):
-        assert prefix is None
+        if prefix:
+            root = posixpath.join(self.path_info.path, prefix[:2])
+        else:
+            root = self.path_info.path
         with self.ssh(self.path_info) as ssh:
             # If we simply return an iterator then with above closes instantly
             if progress_callback:
-                for path in ssh.walk_files(self.path_info.path):
+                for path in ssh.walk_files(root):
                     progress_callback()
                     yield path
             else:
-                yield from ssh.walk_files(self.path_info.path)
+                yield from ssh.walk_files(root)
 
     def walk_files(self, path_info):
         with self.ssh(path_info) as ssh:
@@ -285,7 +290,7 @@ class RemoteSSH(RemoteBASE):
                 try:
                     channel.stat(path)
                     ret.append(True)
-                except IOError as exc:
+                except OSError as exc:
                     if exc.errno != errno.ENOENT:
                         raise
                     ret.append(False)
@@ -325,7 +330,7 @@ class RemoteSSH(RemoteBASE):
         ) as pbar:
 
             def exists_with_progress(chunks):
-                return self.batch_exists(chunks, callback=pbar.update_desc)
+                return self.batch_exists(chunks, callback=pbar.update_msg)
 
             with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
                 path_infos = [self.checksum_to_path_info(x) for x in checksums]

dvc/remote/ssh/connection.py

@@ -7,16 +7,16 @@ from contextlib import suppress
 
 from funcy import cached_property
 
+from dvc.exceptions import DvcException
+from dvc.progress import Tqdm
+from dvc.remote.base import RemoteCmdError
+from dvc.utils import tmp_fname
+
 try:
     import paramiko
 except ImportError:
     paramiko = None
 
-from dvc.utils import tmp_fname
-from dvc.progress import Tqdm
-from dvc.exceptions import DvcException
-from dvc.remote.base import RemoteCmdError
-
 
 logger = logging.getLogger(__name__)
 
@@ -25,9 +25,9 @@ def sizeof_fmt(num, suffix="B"):
     """ Convert number of bytes to human-readable string """
     for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
         if abs(num) < 1024.0:
-            return "%3.1f%s%s" % (num, unit, suffix)
+            return f"{num:3.1f}{unit}{suffix}"
         num /= 1024.0
-    return "%.1f%s%s" % (num, "Y", suffix)
+    return "{:.1f}{}{}".format(num, "Y", suffix)
 
 
 class SSHConnection:
@@ -41,6 +41,14 @@ class SSHConnection:
         self.timeout = kwargs.get("timeout", 1800)
 
         self._ssh = paramiko.SSHClient()
+
+        # Explicitly disable paramiko logger. Due to how paramiko dynamically
+        # loads loggers, it is not disabled by DVC disable_other_loggers().
+        # See https://github.com/iterative/dvc/issues/3482
+        self._ssh.set_log_channel("dvc.paramiko")
+        logging.getLogger("dvc.paramiko").disabled = True
+        logging.getLogger("dvc.paramiko.sftp").disabled = True
+
         self._ssh.load_system_host_keys()
         self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
@@ -92,7 +100,7 @@ class SSHConnection:
 
         if stat.S_ISREG(st_mode) or stat.S_ISLNK(st_mode):
             raise DvcException(
-                "a file with the same name '{}' already exists".format(path)
+                f"a file with the same name '{path}' already exists"
             )
 
         head, tail = posixpath.split(path)
@@ -103,12 +111,12 @@ class SSHConnection:
         if tail:
             try:
                 self.sftp.mkdir(path)
-            except IOError as exc:
+            except OSError as exc:
                 # Since paramiko errors are very vague we need to recheck
                 # whether it's because path already exists or something else
                 if exc.errno == errno.EACCES or not self.exists(path):
                     raise DvcException(
-                        "unable to create remote directory '{}'".format(path)
+                        f"unable to create remote directory '{path}'"
                     ) from exc
 
     def walk(self, directory, topdown=True):
@@ -118,7 +126,7 @@ class SSHConnection:
         # [1] https://github.com/python/cpython/blob/master/Lib/os.py
         try:
             dir_entries = self.sftp.listdir_attr(directory)
-        except IOError as exc:
+        except OSError as exc:
             raise DvcException(
                 "couldn't get the '{}' remote directory files list".format(
                     directory
@@ -284,14 +292,14 @@ class SSHConnection:
             md5 = self.execute("md5 " + path).split()[-1]
         else:
             raise DvcException(
-                "'{}' is not supported as a SSH remote".format(self.uname)
+                f"'{self.uname}' is not supported as a SSH remote"
             )
 
         assert len(md5) == 32
         return md5
 
     def copy(self, src, dest):
-        self.execute("cp {} {}".format(src, dest))
+        self.execute(f"cp {src} {dest}")
 
     def open_max_sftp_channels(self):
         # If there are more than 1 it means we've already opened max amount
@@ -313,14 +321,12 @@ class SSHConnection:
 
     def reflink(self, src, dest):
         if self.uname == "Linux":
-            return self.execute("cp --reflink {} {}".format(src, dest))
+            return self.execute(f"cp --reflink {src} {dest}")
 
         if self.uname == "Darwin":
-            return self.execute("cp -c {} {}".format(src, dest))
+            return self.execute(f"cp -c {src} {dest}")
 
-        raise DvcException(
-            "'{}' is not supported as a SSH remote".format(self.uname)
-        )
+        raise DvcException(f"'{self.uname}' is not supported as a SSH remote")
 
     def hardlink(self, src, dest):
-        self.execute("ln {} {}".format(src, dest))
+        self.execute(f"ln {src} {dest}")

dvc/repo/__init__.py

@@ -2,21 +2,18 @@ import os
 from contextlib import contextmanager
 from functools import wraps
 
-from dvc.ignore import CleanTree
-from dvc.compat import fspath_py35
-
 from funcy import cached_property, cat, first
 
 from dvc.config import Config
-from dvc.exceptions import (
-    FileMissingError,
-    IsADirectoryError,
-    NotDvcRepoError,
-    OutputNotFoundError,
-)
+from dvc.exceptions import FileMissingError
+from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
+from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
+from dvc.ignore import CleanTree
 from dvc.path_info import PathInfo
-from dvc.remote.base import RemoteActionNotImplemented
+from dvc.repo.tree import RepoTree
 from dvc.utils.fs import path_isin
+
+from ..utils import parse_target
 from .graph import check_acyclic, get_pipeline, get_pipelines
 
 
@@ -33,7 +30,7 @@ def locked(f):
     return wrapper
 
 
-class Repo(object):
+class Repo:
     DVC_DIR = ".dvc"
 
     from dvc.repo.destroy import destroy
@@ -60,49 +57,64 @@ class Repo(object):
     from dvc.repo.get_url import get_url
     from dvc.repo.update import update
 
-    def __init__(self, root_dir=None):
-        from dvc.state import State
+    def __init__(self, root_dir=None, scm=None, rev=None):
+        from dvc.state import State, StateNoop
         from dvc.lock import make_lock
         from dvc.scm import SCM
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
         from dvc.repo.metrics import Metrics
+        from dvc.repo.plots import Plots
+        from dvc.repo.params import Params
         from dvc.scm.tree import WorkingTree
-        from dvc.repo.tag import Tag
         from dvc.utils.fs import makedirs
+        from dvc.stage.cache import StageCache
 
-        root_dir = self.find_root(root_dir)
+        if scm:
+            # use GitTree instead of WorkingTree as default repo tree instance
+            tree = scm.get_tree(rev)
+            self.root_dir = self.find_root(root_dir, tree)
+            self.scm = scm
+            self.tree = tree
+            self.state = StateNoop()
+        else:
+            root_dir = self.find_root(root_dir)
+            self.root_dir = os.path.abspath(os.path.realpath(root_dir))
 
-        self.root_dir = os.path.abspath(os.path.realpath(root_dir))
         self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
-
         self.config = Config(self.dvc_dir)
 
-        no_scm = self.config["core"].get("no_scm", False)
-        self.scm = SCM(self.root_dir, no_scm=no_scm)
-
-        self.tree = WorkingTree(self.root_dir)
+        if not scm:
+            no_scm = self.config["core"].get("no_scm", False)
+            self.scm = SCM(self.root_dir, no_scm=no_scm)
+            self.tree = WorkingTree(self.root_dir)
 
         self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
-        makedirs(self.tmp_dir, exist_ok=True)
+        self.index_dir = os.path.join(self.tmp_dir, "index")
+        makedirs(self.index_dir, exist_ok=True)
 
         hardlink_lock = self.config["core"].get("hardlink_lock", False)
         self.lock = make_lock(
-            os.path.join(self.dvc_dir, "lock"),
-            tmp_dir=os.path.join(self.dvc_dir, "tmp"),
+            os.path.join(self.tmp_dir, "lock"),
+            tmp_dir=self.tmp_dir,
             hardlink_lock=hardlink_lock,
             friendly=True,
         )
 
-        # NOTE: storing state and link_state in the repository itself to avoid
-        # any possible state corruption in 'shared cache dir' scenario.
-        self.state = State(self)
+        if not scm:
+            # NOTE: storing state and link_state in the repository itself to
+            # avoid any possible state corruption in 'shared cache dir'
+            # scenario.
+            self.state = State(self)
 
         self.cache = Cache(self)
         self.cloud = DataCloud(self)
 
+        self.stage_cache = StageCache(self)
+
         self.metrics = Metrics(self)
-        self.tag = Tag(self)
+        self.plots = Plots(self)
+        self.params = Params(self)
 
         self._ignore()
 
@@ -118,14 +130,19 @@ class Repo(object):
         self._reset()
 
     def __repr__(self):
-        return "{}: '{}'".format(self.__class__.__name__, self.root_dir)
+        return f"{self.__class__.__name__}: '{self.root_dir}'"
 
     @classmethod
-    def find_root(cls, root=None):
+    def find_root(cls, root=None, tree=None):
         root_dir = os.path.realpath(root or os.curdir)
 
+        if tree:
+            if tree.isdir(os.path.join(root_dir, cls.DVC_DIR)):
+                return root_dir
+            raise NotDvcRepoError(f"'{root}' does not contain DVC directory")
+
         if not os.path.isdir(root_dir):
-            raise NotDvcRepoError("directory '{}' does not exist".format(root))
+            raise NotDvcRepoError(f"directory '{root}' does not exist")
 
         while True:
             dvc_dir = os.path.join(root_dir, cls.DVC_DIR)
@@ -157,15 +174,7 @@ class Repo(object):
         return self.cache.local.unprotect(PathInfo(target))
 
     def _ignore(self):
-        from dvc.updater import Updater
-
-        updater = Updater(self.dvc_dir)
-
-        flist = (
-            [self.config.files["local"], updater.updater_file]
-            + [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir]
-            + self.state.files
-        )
+        flist = [self.config.files["local"], self.tmp_dir]
 
         if path_isin(self.cache.local.cache_dir, self.root_dir):
             flist += [self.cache.local.cache_dir]
@@ -189,41 +198,51 @@ class Repo(object):
         if not getattr(self, "_skip_graph_checks", False):
             self._collect_graph(self.stages + new_stages)
 
+    def _collect_inside(self, path, graph):
+        import networkx as nx
+
+        stages = nx.dfs_postorder_nodes(graph)
+        return [stage for stage in stages if path_isin(stage.path, path)]
+
     def collect(self, target, with_deps=False, recursive=False, graph=None):
         import networkx as nx
-        from dvc.stage import Stage
+        from ..dvcfile import Dvcfile
 
         if not target:
             return list(graph) if graph else self.stages
 
-        target = os.path.abspath(target)
-
         if recursive and os.path.isdir(target):
-            stages = nx.dfs_postorder_nodes(graph or self.graph)
-            return [stage for stage in stages if path_isin(stage.path, target)]
+            return self._collect_inside(
+                os.path.abspath(target), graph or self.graph
+            )
 
-        stage = Stage.load(self, target)
-
-        # Optimization: do not collect the graph for a specific target
+        path, name = parse_target(target)
+        dvcfile = Dvcfile(self, path)
+        stages = list(dvcfile.stages.filter(name).values())
         if not with_deps:
-            return [stage]
+            return stages
 
-        pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
-        return list(nx.dfs_postorder_nodes(pipeline, stage))
+        res = set()
+        for stage in stages:
+            pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
+            res.update(nx.dfs_postorder_nodes(pipeline, stage))
+        return res
 
     def collect_granular(self, target, *args, **kwargs):
-        from dvc.stage import Stage
+        from ..dvcfile import Dvcfile, is_valid_filename
 
         if not target:
             return [(stage, None) for stage in self.stages]
 
-        # Optimization: do not collect the graph for a specific .dvc target
-        if Stage.is_valid_filename(target) and not kwargs.get("with_deps"):
-            return [(Stage.load(self, target), None)]
+        file, name = parse_target(target)
+        if is_valid_filename(file) and not kwargs.get("with_deps"):
+            # Optimization: do not collect the graph for a specific .dvc target
+            stages = Dvcfile(self, file).stages.filter(name)
+            return [(stage, None) for stage in stages.values()]
 
         try:
-            (out,) = self.find_outs_by_path(target, strict=False)
-            filter_info = PathInfo(os.path.abspath(target))
+            (out,) = self.find_outs_by_path(file, strict=False)
+            filter_info = PathInfo(os.path.abspath(file))
             return [(out.stage, filter_info)]
         except OutputNotFoundError:
             stages = self.collect(target, *args, **kwargs)
@@ -240,6 +259,7 @@ class Repo(object):
         force=False,
         jobs=None,
         recursive=False,
+        used_run_cache=None,
     ):
         """Get the stages related to the given target and collect
         the `info` of its outputs.
@@ -251,8 +271,9 @@ class Repo(object):
         `all_branches`/`all_tags`/`all_commits` to expand the scope.
 
         Returns:
-            A dictionary with Schemes (representing output's location) as keys,
-            and a list with the outputs' `dumpd` as values.
+            A dictionary with Schemes (representing output's location) mapped
+            to items containing the output's `dumpd` names and the output's
+            children (if the given output is a directory).
         """
         from dvc.cache import NamedCache
 
@@ -272,7 +293,7 @@ class Repo(object):
                 for target in targets
             )
 
-            suffix = "({})".format(branch) if branch else ""
+            suffix = f"({branch})" if branch else ""
             for stage, filter_info in pairs:
                 used_cache = stage.get_used_cache(
                     remote=remote,
@@ -282,9 +303,15 @@ class Repo(object):
                 )
                 cache.update(used_cache, suffix=suffix)
 
+        if used_run_cache:
+            used_cache = self.stage_cache.get_used_cache(
+                used_run_cache, remote=remote, force=force, jobs=jobs,
+            )
+            cache.update(used_cache)
+
         return cache
 
-    def _collect_graph(self, stages=None):
+    def _collect_graph(self, stages):
         """Generate a graph by using the given stages on the given directory
 
         The nodes of the graph are the stage's path relative to the root.
@@ -333,10 +360,9 @@ class Repo(object):
 
         G = nx.DiGraph()
         stages = stages or self.stages
-        stages = [stage for stage in stages if stage]
         outs = Trie()  # Use trie to efficiently find overlapping outs and deps
 
-        for stage in stages:
+        for stage in filter(bool, stages):
             for out in stage.outs:
                 out_key = out.path_info.parts
 
@@ -359,9 +385,9 @@ class Repo(object):
                         "rerun command with non overlapping outs paths."
                     ).format(
                         str(parent),
-                        parent.stage.relpath,
+                        parent.stage.addressing,
                         str(overlapping),
-                        overlapping.stage.relpath,
+                        overlapping.stage.addressing,
                     )
                     raise OverlappingOutputPathsError(parent, overlapping, msg)
 
@@ -385,14 +411,13 @@ class Repo(object):
                     overlapping.extend(outs.values(prefix=dep_key))
 
                 G.add_edges_from((stage, out.stage) for out in overlapping)
-
         check_acyclic(G)
 
         return G
 
     @cached_property
     def graph(self):
-        return self._collect_graph()
+        return self._collect_graph(self.stages)
 
     @cached_property
     def pipelines(self):
@@ -409,25 +434,33 @@ class Repo(object):
         NOTE: For large repos, this could be an expensive
               operation. Consider using some memoization.
         """
-        from dvc.stage import Stage
+        return self._collect_stages()
+
+    @cached_property
+    def plot_templates(self):
+        from .plots.template import PlotTemplates
+
+        return PlotTemplates(self.dvc_dir)
+
+    def _collect_stages(self):
+        from dvc.dvcfile import Dvcfile, is_valid_filename
 
         stages = []
         outs = set()
 
         for root, dirs, files in self.tree.walk(self.root_dir):
-            for fname in files:
-                path = os.path.join(root, fname)
-                if not Stage.is_valid_filename(path):
-                    continue
-                stage = Stage.load(self, path)
-                stages.append(stage)
-
-                for out in stage.outs:
-                    if out.scheme == "local":
-                        outs.add(out.fspath)
-
+            for file_name in filter(is_valid_filename, files):
+                path = os.path.join(root, file_name)
+                stage_loader = Dvcfile(self, path).stages
+                stages.extend(stage_loader.values())
+                outs.update(
+                    out.fspath
+                    for stage in stages
+                    for out in (
+                        out for out in stage.outs if out.scheme == "local"
+                    )
+                )
             dirs[:] = [d for d in dirs if os.path.join(root, d) not in outs]
-
         return stages
 
     def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
@@ -465,49 +498,22 @@ class Repo(object):
     @contextmanager
     def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
-        cause = None
+
+        tree = RepoTree(self, stream=True)
+        path = os.path.join(self.root_dir, path)
         try:
-            out = self.find_out_by_relpath(path)
-        except OutputNotFoundError as exc:
-            out = None
-            cause = exc
-
-        if out and out.use_cache:
-            try:
-                with self._open_cached(out, remote, mode, encoding) as fd:
-                    yield fd
-                return
-            except FileNotFoundError as exc:
-                raise FileMissingError(path) from exc
-
-        abs_path = os.path.join(self.root_dir, path)
-        if os.path.exists(abs_path):
-            with open(abs_path, mode=mode, encoding=encoding) as fd:
-                yield fd
-            return
-
-        raise FileMissingError(path) from cause
-
-    def _open_cached(self, out, remote=None, mode="r", encoding=None):
-        if out.isdir():
-            raise IsADirectoryError("Can't open a dir")
-
-        cache_file = self.cache.local.checksum_to_path_info(out.checksum)
-        cache_file = fspath_py35(cache_file)
-
-        if os.path.exists(cache_file):
-            return open(cache_file, mode=mode, encoding=encoding)
-
-        try:
-            remote_obj = self.cloud.get_remote(remote)
-            remote_info = remote_obj.checksum_to_path_info(out.checksum)
-            return remote_obj.open(remote_info, mode=mode, encoding=encoding)
-        except RemoteActionNotImplemented:
             with self.state:
-                cache_info = out.get_used_cache(remote=remote)
-                self.cloud.pull(cache_info, remote=remote)
-
-            return open(cache_file, mode=mode, encoding=encoding)
+                with tree.open(
+                    os.path.join(self.root_dir, path),
+                    mode=mode,
+                    encoding=encoding,
+                    remote=remote,
+                ) as fobj:
+                    yield fobj
+        except FileNotFoundError as exc:
+            raise FileMissingError(path) from exc
+        except IsADirectoryError as exc:
+            raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
 
     def close(self):
         self.scm.close()

dvc/repo/add.py

@@ -3,16 +3,18 @@ import os
 
 import colorama
 
-from . import locked
+from dvc.dvcfile import Dvcfile, is_dvc_file
+
 from ..exceptions import (
-    RecursiveAddingWhileUsingFilename,
+    OutputDuplicationError,
     OverlappingOutputPathsError,
+    RecursiveAddingWhileUsingFilename,
 )
 from ..output.base import OutputDoesNotExistError
 from ..progress import Tqdm
 from ..repo.scm_context import scm_context
-from ..stage import Stage
-from ..utils import LARGE_DIR_SIZE
+from ..utils import LARGE_DIR_SIZE, resolve_paths
+from . import locked
 
 logger = logging.getLogger(__name__)
 
@@ -63,11 +65,15 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
                 ).format(
                     out=exc.overlapping_out.path_info,
                     parent=exc.parent.path_info,
-                    parent_stage=exc.parent.stage.relpath,
+                    parent_stage=exc.parent.stage.addressing,
                 )
                 raise OverlappingOutputPathsError(
                     exc.parent, exc.overlapping_out, msg
                 )
+            except OutputDuplicationError as exc:
+                raise OutputDuplicationError(
+                    exc.output, set(exc.stages) - set(stages)
+                )
 
             with Tqdm(
                 total=len(stages),
@@ -85,7 +91,7 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
                     if not no_commit:
                         stage.commit()
 
-                    stage.dump()
+                    Dvcfile(repo, stage.path).dump(stage)
                     pbar_stages.update()
 
             stages_list += stages
@@ -107,7 +113,7 @@ def _find_all_targets(repo, target, recursive):
                 unit="file",
             )
             if not repo.is_dvc_internal(fname)
-            if not Stage.is_stage_file(fname)
+            if not is_dvc_file(fname)
             if not repo.scm.belongs_to_scm(fname)
             if not repo.scm.is_tracked(fname)
         ]
@@ -115,6 +121,8 @@ def _find_all_targets(repo, target, recursive):
 
 
 def _create_stages(repo, targets, fname, pbar=None):
+    from dvc.stage import Stage, create_stage
+
     stages = []
 
     for out in Tqdm(
@@ -123,9 +131,11 @@ def _create_stages(repo, targets, fname, pbar=None):
         disable=len(targets) < LARGE_DIR_SIZE,
         unit="file",
     ):
-        stage = Stage.create(
-            repo, outs=[out], accompany_outs=True, fname=fname
-        )
+        path, wdir, out = resolve_paths(repo, out)
+        stage = create_stage(Stage, repo, fname or path, wdir=wdir, outs=[out])
+        if stage:
+            Dvcfile(repo, stage.path).remove_with_prompt(force=True)
+
         repo._reset()
 
         if not stage:
@@ -135,6 +145,6 @@ def _create_stages(repo, targets, fname, pbar=None):
 
         stages.append(stage)
         if pbar is not None:
-            pbar.update_desc(out)
+            pbar.update_msg(out)
 
     return stages

dvc/repo/brancher.py

@@ -25,13 +25,16 @@ def brancher(  # noqa: E302
         return
 
     saved_tree = self.tree
-    revs = revs or []
+    revs = revs.copy() if revs else []
 
     scm = self.scm
 
     self.tree = WorkingTree(self.root_dir)
     yield "working tree"
 
+    if revs and "working tree" in revs:
+        revs.remove("working tree")
+
     if all_commits:
         revs = scm.list_all_commits()
     else:

dvc/repo/checkout.py

@@ -1,9 +1,7 @@
 import logging
 import os
 
-from dvc.compat import fspath
-from dvc.exceptions import CheckoutError
-from dvc.exceptions import CheckoutErrorSuggestGit
+from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit
 from dvc.progress import Tqdm
 from dvc.utils import relpath
 
@@ -24,7 +22,7 @@ def _fspath_dir(path, root):
     if not os.path.exists(str(path)):
         return str(path)
 
-    path = relpath(fspath(path), root)
+    path = relpath(path, root)
     return os.path.join(path, "") if os.path.isdir(path) else path
 
 
@@ -42,7 +40,10 @@ def _checkout(
     relink=False,
     recursive=False,
 ):
-    from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
+    from dvc.stage.exceptions import (
+        StageFileBadNameError,
+        StageFileDoesNotExistError,
+    )
 
     unused = []
     stats = {
@@ -58,6 +59,9 @@ def _checkout(
     stats["deleted"] = [_fspath_dir(u, self.root_dir) for u in unused]
     self.state.remove_links(unused)
 
+    if isinstance(targets, str):
+        targets = [targets]
+
     pairs = set()
     for target in targets:
         try:
@@ -78,13 +82,13 @@ def _checkout(
         for stage, filter_info in pairs:
             result = stage.checkout(
                 force=force,
-                progress_callback=pbar.update_desc,
+                progress_callback=pbar.update_msg,
                 relink=relink,
                 filter_info=filter_info,
             )
-            for data in ["failed", "added", "modified"]:
-                stats[data].extend(
-                    _fspath_dir(path, self.root_dir) for path in result[data]
+            for key, items in result.items():
+                stats[key].extend(
+                    _fspath_dir(path, self.root_dir) for path in items
                 )
 
     if stats.get("failed"):

dvc/repo/commit.py

@@ -1,10 +1,48 @@
+from dvc import prompt
+from dvc.dvcfile import Dvcfile
+from dvc.stage.exceptions import StageCommitError
+
 from . import locked
 
 
+def _prepare_message(stage, changes):
+    changed_deps, changed_outs, changed_stage = changes
+    if changed_deps and changed_outs:
+        msg = "dependencies {deps} and outputs {outs} of {stage} changed."
+    elif changed_deps:
+        msg = "dependencies {deps} of {stage} changed."
+    elif changed_outs:
+        msg = "outputs {outs} of {stage} changed."
+    else:
+        msg = "{stage_changed}"
+
+    msg += " Are you sure you want to commit it?"
+
+    kw = {
+        "stage": stage,
+        "deps": changed_deps,
+        "outs": changed_outs,
+        "stage_changed": changed_stage,
+    }
+    return msg.format_map(kw)
+
+
+def prompt_to_commit(stage, changes, force=False):
+    if not (force or prompt.confirm(_prepare_message(stage, changes))):
+        raise StageCommitError(
+            "unable to commit changed {}. Use `-f|--force` to "
+            "force.".format(stage)
+        )
+
+
 @locked
 def commit(self, target, with_deps=False, recursive=False, force=False):
     stages = self.collect(target, with_deps=with_deps, recursive=recursive)
     for stage in stages:
-        stage.check_can_commit(force=force)
+        changes = stage.changed_entries()
+        if any(changes):
+            prompt_to_commit(stage, changes, force=force)
+            stage.save()
         stage.commit()
-        stage.dump()
+
+        Dvcfile(self, stage.path).dump(stage)