dvc: bump to 1.6.4

* dvc: use cache state when saving downloaded cache

Related to #4485

* tree: dont compute hash from scratch for output dirs

Fixes #4485

* tree: fix get_hash

.github/release-drafter.yml

@@ -1,5 +1,48 @@
 # Config for https://github.com/apps/release-drafter
-branches:
-  - master
+name-template: "$RESOLVED_VERSION 🦉 "
+tag-template: "$RESOLVED_VERSION"
+categories:
+  - title: "🚀 New Features and Enhancements "
+    labels:
+      - "feature"
+      - "enhancement"
+  - title: "💣 Breaking Changes "
+    labels:
+      - "breaking-change"
+  - title: "🏇 Optimizations "
+    labels:
+      - "optimize"
+      - "performance"
+  - title: "🐛 Bug Fixes "
+    labels:
+      - "bugfix"
+      - "bug"
+  - title: "🔨 Maintenance "
+    labels:
+      - "maintenance"
+      - "refactoring"
+      - "chore"
+      - "build"
+      - "ci"
+change-template: "- $TITLE (#$NUMBER) @$AUTHOR"
+version-resolver:
+  major:
+    labels:
+      - "major"
+  minor:
+    labels:
+      - "minor"
+  patch:
+    labels:
+      - "patch"
+  default: patch
+exclude-labels:
+  - "skip-changelog"
 template: |
+  > Refer to https://dvc.org/doc/install for installation instructions.
+
+  ## Changes
+
   $CHANGES
+
+  Thanks again to $CONTRIBUTORS for the contributions! 🎉

.pre-commit-config.yaml

@@ -4,15 +4,11 @@ repos:
         language_version: python3
     repo: https://github.com/ambv/black
     rev: 19.10b0
-  - repo: https://github.com/asottile/seed-isort-config
-    rev: v2.1.1
-    hooks:
-      - id: seed-isort-config
   - hooks:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 4.3.21
+    rev: 5.4.2
   - hooks:
       - id: flake8
         language_version: python3
@@ -22,7 +18,7 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.9
+    rev: 3.8.3
   - repo: local
     hooks:
     - id: pylint

.restyled.yaml

@@ -10,7 +10,7 @@ restylers:
     interpreters:
     - python
   - name: isort
-    image: restyled/restyler-isort:v4.3.21
+    image: restyled/restyler-isort:v5.4.2
     command:
     - isort
     arguments: []

dvc/cache/base.py

@@ -5,7 +5,12 @@ from copy import copy
 from shortuuid import uuid
 
 import dvc.prompt as prompt
-from dvc.exceptions import CheckoutError, ConfirmRemoveError, DvcException
+from dvc.exceptions import (
+    CheckoutError,
+    ConfirmRemoveError,
+    DvcException,
+    MergeError,
+)
 from dvc.path_info import WindowsPathInfo
 from dvc.progress import Tqdm
 from dvc.remote.slow_link_detection import slow_link_guard
@@ -127,7 +132,9 @@ class CloudCache:
             logger.debug("cache for '%s'('%s') has changed.", path_info, hash_)
             return True
 
-        actual = self.tree.get_hash(path_info)
+        typ, actual = self.tree.get_hash(path_info)
+        assert typ == self.tree.PARAM_CHECKSUM
+
         if hash_ != actual:
             logger.debug(
                 "hash value '%s' for '%s' has changed (actual '%s').",
@@ -276,8 +283,6 @@ class CloudCache:
                 self.tree.scheme,
             )
 
-        if not hash_info:
-            hash_info = self.tree.save_info(path_info, tree=tree, **kwargs)
         hash_ = hash_info[self.tree.PARAM_CHECKSUM]
         return self._save(path_info, tree, hash_, save_link, **kwargs)
 
@@ -314,7 +319,7 @@ class CloudCache:
             )
             return False
 
-        actual = self.tree.get_hash(cache_info)
+        _, actual = self.tree.get_hash(cache_info)
 
         logger.debug(
             "cache '%s' expected '%s' actual '%s'", cache_info, hash_, actual,
@@ -360,7 +365,7 @@ class CloudCache:
         return self.changed_cache_file(hash_)
 
     def already_cached(self, path_info):
-        current = self.tree.get_hash(path_info)
+        _, current = self.tree.get_hash(path_info)
 
         if not current:
             return False
@@ -552,3 +557,82 @@ class CloudCache:
             filter_info.isin_or_eq(path_info / entry[self.tree.PARAM_CHECKSUM])
             for entry in self.get_dir_cache(hash_)
         )
+
+    def _to_dict(self, dir_info):
+        return {
+            entry[self.tree.PARAM_RELPATH]: entry[self.tree.PARAM_CHECKSUM]
+            for entry in dir_info
+        }
+
+    def _from_dict(self, dir_dict):
+        return [
+            {
+                self.tree.PARAM_RELPATH: relpath,
+                self.tree.PARAM_CHECKSUM: checksum,
+            }
+            for relpath, checksum in dir_dict.items()
+        ]
+
+    @staticmethod
+    def _diff(ancestor, other, allow_removed=False):
+        from dictdiffer import diff
+
+        allowed = ["add"]
+        if allow_removed:
+            allowed.append("remove")
+
+        result = list(diff(ancestor, other))
+        for typ, _, _ in result:
+            if typ not in allowed:
+                raise MergeError(
+                    "unable to auto-merge directories with diff that contains "
+                    f"'{typ}'ed files"
+                )
+        return result
+
+    def _merge_dirs(self, ancestor_info, our_info, their_info):
+        from operator import itemgetter
+
+        from dictdiffer import patch
+
+        ancestor = self._to_dict(ancestor_info)
+        our = self._to_dict(our_info)
+        their = self._to_dict(their_info)
+
+        our_diff = self._diff(ancestor, our)
+        if not our_diff:
+            return self._from_dict(their)
+
+        their_diff = self._diff(ancestor, their)
+        if not their_diff:
+            return self._from_dict(our)
+
+        # make sure there are no conflicting files
+        self._diff(our, their, allow_removed=True)
+
+        merged = patch(our_diff + their_diff, ancestor, in_place=True)
+
+        # Sorting the list by path to ensure reproducibility
+        return sorted(
+            self._from_dict(merged), key=itemgetter(self.tree.PARAM_RELPATH)
+        )
+
+    def merge(self, ancestor_info, our_info, their_info):
+        assert our_info
+        assert their_info
+
+        if ancestor_info:
+            ancestor_hash = ancestor_info[self.tree.PARAM_CHECKSUM]
+            ancestor = self.get_dir_cache(ancestor_hash)
+        else:
+            ancestor = []
+
+        our_hash = our_info[self.tree.PARAM_CHECKSUM]
+        our = self.get_dir_cache(our_hash)
+
+        their_hash = their_info[self.tree.PARAM_CHECKSUM]
+        their = self.get_dir_cache(their_hash)
+
+        merged = self._merge_dirs(ancestor, our, their)
+        typ, merged_hash = self.tree.save_dir_info(merged)
+        return {typ: merged_hash}

dvc/cache/local.py

@@ -91,7 +91,9 @@ class LocalCache(CloudCache):
     def already_cached(self, path_info):
         assert path_info.scheme in ["", "local"]
 
-        current_md5 = self.tree.get_hash(path_info)
+        typ, current_md5 = self.tree.get_hash(path_info)
+
+        assert typ == "md5"
 
         if not current_md5:
             return False
@@ -112,6 +114,7 @@ class LocalCache(CloudCache):
         jobs=None,
         show_checksums=False,
         download=False,
+        log_missing=True,
     ):
         # Return flattened dict containing all status info
         dir_status, file_status, _ = self._status(
@@ -120,6 +123,7 @@ class LocalCache(CloudCache):
             jobs=jobs,
             show_checksums=show_checksums,
             download=download,
+            log_missing=log_missing,
         )
         return dict(dir_status, **file_status)
 
@@ -130,6 +134,7 @@ class LocalCache(CloudCache):
         jobs=None,
         show_checksums=False,
         download=False,
+        log_missing=True,
     ):
         """Return a tuple of (dir_status_info, file_status_info, dir_contents).
 
@@ -170,11 +175,20 @@ class LocalCache(CloudCache):
                     )
                 )
         return self._make_status(
-            named_cache, show_checksums, local_exists, remote_exists
+            named_cache,
+            show_checksums,
+            local_exists,
+            remote_exists,
+            log_missing,
         )
 
     def _make_status(
-        self, named_cache, show_checksums, local_exists, remote_exists
+        self,
+        named_cache,
+        show_checksums,
+        local_exists,
+        remote_exists,
+        log_missing,
     ):
         def make_names(hash_, names):
             return {"name": hash_ if show_checksums else " ".join(names)}
@@ -197,7 +211,8 @@ class LocalCache(CloudCache):
         self._fill_statuses(dir_status, local_exists, remote_exists)
         self._fill_statuses(file_status, local_exists, remote_exists)
 
-        self._log_missing_caches(dict(dir_status, **file_status))
+        if log_missing:
+            self._log_missing_caches(dict(dir_status, **file_status))
 
         return dir_status, file_status, dir_contents
 
@@ -255,6 +270,7 @@ class LocalCache(CloudCache):
         path_infos = []
         names = []
         hashes = []
+        missing = []
         for md5, info in Tqdm(
             status_info.items(), desc="Analysing status", unit="file"
         ):
@@ -263,6 +279,8 @@ class LocalCache(CloudCache):
                 path_infos.append(remote.tree.hash_to_path_info(md5))
                 names.append(info["name"])
                 hashes.append(md5)
+            elif info["status"] == STATUS_MISSING:
+                missing.append(md5)
 
         if download:
             to_infos = cache
@@ -271,7 +289,7 @@ class LocalCache(CloudCache):
             to_infos = path_infos
             from_infos = cache
 
-        return from_infos, to_infos, names, hashes
+        return (from_infos, to_infos, names, hashes), missing
 
     def _process(
         self,
@@ -312,8 +330,10 @@ class LocalCache(CloudCache):
             download=download,
         )
 
-        dir_plans = self._get_plans(download, remote, dir_status, status)
-        file_plans = self._get_plans(download, remote, file_status, status)
+        dir_plans, _ = self._get_plans(download, remote, dir_status, status)
+        file_plans, missing_files = self._get_plans(
+            download, remote, file_status, status
+        )
 
         total = len(dir_plans[0]) + len(file_plans[0])
         if total == 0:
@@ -339,19 +359,32 @@ class LocalCache(CloudCache):
                         )
                     dir_futures = {}
                     for from_info, to_info, name, dir_hash in zip(*dir_plans):
-                        wait_futures = {
-                            future
-                            for file_hash, future in file_futures.items()
-                            if file_hash in dir_contents[dir_hash]
-                        }
-                        dir_futures[dir_hash] = executor.submit(
-                            self._dir_upload,
-                            func,
-                            wait_futures,
-                            from_info,
-                            to_info,
-                            name,
-                        )
+                        # if for some reason a file contained in this dir is
+                        # missing both locally and in the remote, we want to
+                        # push whatever file content we have, but should not
+                        # push .dir file
+                        for file_hash in missing_files:
+                            if file_hash in dir_contents[dir_hash]:
+                                logger.debug(
+                                    "directory '%s' contains missing files,"
+                                    "skipping .dir file upload",
+                                    name,
+                                )
+                                break
+                        else:
+                            wait_futures = {
+                                future
+                                for file_hash, future in file_futures.items()
+                                if file_hash in dir_contents[dir_hash]
+                            }
+                            dir_futures[dir_hash] = executor.submit(
+                                self._dir_upload,
+                                func,
+                                wait_futures,
+                                from_info,
+                                to_info,
+                                name,
+                            )
                     fails = sum(
                         future.result()
                         for future in concat(
@@ -404,7 +437,7 @@ class LocalCache(CloudCache):
 
     @index_locked
     def pull(self, named_cache, remote, jobs=None, show_checksums=False):
-        return self._process(
+        ret = self._process(
             named_cache,
             remote,
             jobs=jobs,
@@ -412,6 +445,18 @@ class LocalCache(CloudCache):
             download=True,
         )
 
+        if not remote.tree.verify:
+            for checksum in named_cache.scheme_keys("local"):
+                cache_file = self.tree.hash_to_path_info(checksum)
+                if self.tree.exists(cache_file):
+                    # We can safely save here, as existing corrupted files will
+                    # be removed upon status, while files corrupted during
+                    # download will not be moved from tmp_file
+                    # (see `BaseTree.download()`)
+                    self.tree.state.save(cache_file, checksum)
+
+        return ret
+
     @staticmethod
     def _log_missing_caches(hash_info_dict):
         missing_caches = [

dvc/cli.py

@@ -6,6 +6,7 @@ import sys
 from .command import (
     add,
     cache,
+    check_ignore,
     checkout,
     commit,
     completion,
@@ -79,6 +80,7 @@ COMMANDS = [
     git_hook,
     plots,
     experiments,
+    check_ignore,
 ]
 
 
@@ -135,6 +137,14 @@ def get_parent_parser():
     """
     parent_parser = argparse.ArgumentParser(add_help=False)
 
+    parent_parser.add_argument(
+        "--cprofile",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parent_parser.add_argument("--cprofile-dump", help=argparse.SUPPRESS)
+
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
         "-q", "--quiet", action="count", default=0, help="Be quiet."

dvc/command/check_ignore.py

@@ -0,0 +1,126 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+class CmdCheckIgnore(CmdBase):
+    def __init__(self, args):
+        super().__init__(args)
+        self.ignore_filter = self.repo.tree.dvcignore
+
+    def _show_results(self, result):
+        if not result.match and not self.args.non_matching:
+            return
+
+        if self.args.details:
+            patterns = result.patterns
+            if not self.args.all:
+                patterns = patterns[-1:]
+
+            for pattern in patterns:
+                logger.info("{}\t{}".format(pattern, result.file))
+        else:
+            logger.info(result.file)
+
+    def _check_one_file(self, target):
+        result = self.ignore_filter.check_ignore(target)
+        self._show_results(result)
+        if result.match:
+            return 0
+        return 1
+
+    def _interactive_mode(self):
+        ret = 1
+        while True:
+            try:
+                target = input()
+            except (KeyboardInterrupt, EOFError):
+                break
+            if target == "":
+                break
+            if not self._check_one_file(target):
+                ret = 0
+        return ret
+
+    def _normal_mode(self):
+        ret = 1
+        for target in self.args.targets:
+            if not self._check_one_file(target):
+                ret = 0
+        return ret
+
+    def _check_args(self):
+        if not self.args.stdin and not self.args.targets:
+            raise DvcException("`targets` or `--stdin` needed")
+
+        if self.args.stdin and self.args.targets:
+            raise DvcException("cannot have both `targets` and `--stdin`")
+
+        if self.args.non_matching and not self.args.details:
+            raise DvcException(
+                "`--non-matching` is only valid with `--details`"
+            )
+
+        if self.args.all and not self.args.details:
+            raise DvcException("`--all` is only valid with `--details`")
+
+        if self.args.quiet and self.args.details:
+            raise DvcException("cannot use both `--details` and `--quiet`")
+
+    def run(self):
+        self._check_args()
+        if self.args.stdin:
+            return self._interactive_mode()
+        return self._normal_mode()
+
+
+def add_parser(subparsers, parent_parser):
+    ADD_HELP = (
+        "Check whether files or directories are excluded due to `.dvcignore`."
+    )
+
+    parser = subparsers.add_parser(
+        "check-ignore",
+        parents=[parent_parser],
+        description=append_doc_link(ADD_HELP, "check-ignore"),
+        help=ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "-d",
+        "--details",
+        action="store_true",
+        default=False,
+        help="Show the exclude patterns along with each target path.",
+    )
+    parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        default=False,
+        help="Include the target paths which don’t match any pattern "
+        "in the `--details` list.",
+    )
+    parser.add_argument(
+        "-n",
+        "--non-matching",
+        action="store_true",
+        default=False,
+        help="Include the target paths which don’t match any pattern "
+        "in the `--details` list.",
+    )
+    parser.add_argument(
+        "--stdin",
+        action="store_true",
+        default=False,
+        help="Read paths from standard input instead of providing `targets`.",
+    )
+    parser.add_argument(
+        "targets", nargs="*", help="File or directory paths to check",
+    ).complete = completion.FILE
+    parser.set_defaults(func=CmdCheckIgnore)

dvc/command/daemon.py

@@ -9,8 +9,9 @@ class CmdDaemonBase(CmdBaseNoRepo):
 class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
-        from dvc.repo import Repo
+
         from dvc.config import Config
+        from dvc.repo import Repo
         from dvc.updater import Updater
 
         root_dir = Repo.find_root()

dvc/command/dag.py

@@ -8,8 +8,8 @@ logger = logging.getLogger(__name__)
 
 
 def _show_ascii(G):
-    from dvc.repo.graph import get_pipelines
     from dvc.dagascii import draw
+    from dvc.repo.graph import get_pipelines
 
     pipelines = get_pipelines(G)
 
@@ -22,6 +22,7 @@ def _show_ascii(G):
 
 def _show_dot(G):
     import io
+
     from networkx.drawing.nx_pydot import write_dot
 
     dot_file = io.StringIO()
@@ -31,6 +32,7 @@ def _show_dot(G):
 
 def _build(G, target=None, full=False):
     import networkx as nx
+
     from dvc.repo.graph import get_pipeline, get_pipelines
 
     if target:

dvc/command/data_sync.py

@@ -97,7 +97,11 @@ def shared_parent_parser():
         "-j",
         "--jobs",
         type=int,
-        help="Number of jobs to run simultaneously.",
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
         metavar="<number>",
     )
     parent_parser.add_argument(

dvc/command/diff.py

@@ -11,19 +11,30 @@ from dvc.exceptions import DvcException
 logger = logging.getLogger(__name__)
 
 
-def _show_md(diff):
+def _digest(checksum):
+    if isinstance(checksum, str):
+        return checksum[0:8]
+    return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
+
+
+def _show_md(diff, show_hash=False):
     from dvc.utils.diff import table
 
+    header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
     rows = []
     for status in ["added", "deleted", "modified"]:
         entries = diff.get(status, [])
         if not entries:
             continue
-        paths = sorted(entry["path"] for entry in entries)
-        for path in paths:
-            rows.append([status, path])
+        for entry in entries:
+            path = entry["path"]
+            if show_hash:
+                check_sum = _digest(entry.get("hash", ""))
+                rows.append([status, check_sum, path])
+            else:
+                rows.append([status, path])
 
-    return table(["Status", "Path"], rows, True)
+    return table(header, rows, True)
 
 
 class CmdDiff(CmdBase):
@@ -33,8 +44,7 @@ class CmdDiff(CmdBase):
         Given a diff structure, generate a string of paths separated
         by new lines and grouped together by their state.
 
-        A group's header is colored and its entries are sorted to enhance
-        readability, for example:
+        A group's header is colored to enhance readability, for example:
 
             Added:
                 another_file.txt
@@ -55,11 +65,6 @@ class CmdDiff(CmdBase):
         At the bottom, include a summary with the number of files per state.
         """
 
-        def _digest(checksum):
-            if isinstance(checksum, str):
-                return checksum[0:8]
-            return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
-
         colors = {
             "added": colorama.Fore.GREEN,
             "modified": colorama.Fore.YELLOW,
@@ -110,16 +115,19 @@ class CmdDiff(CmdBase):
     def run(self):
         try:
             diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
+            show_hash = self.args.show_hash
 
-            if not self.args.show_hash:
-                for _, entries in diff.items():
+            for key, entries in diff.items():
+                entries = sorted(entries, key=lambda entry: entry["path"])
+                if not show_hash:
                     for entry in entries:
                         del entry["hash"]
+                diff[key] = entries
 
             if self.args.show_json:
                 logger.info(json.dumps(diff))
             elif self.args.show_md:
-                logger.info(_show_md(diff))
+                logger.info(_show_md(diff, show_hash))
             elif diff:
                 logger.info(self._format(diff))
 

dvc/command/experiments.py

@@ -2,14 +2,57 @@ import argparse
 import io
 import logging
 from collections import OrderedDict
+from itertools import groupby
+from typing import Iterable, Optional
 
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.command.metrics import DEFAULT_PRECISION
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, InvalidArgumentError
 
 logger = logging.getLogger(__name__)
 
 
+def _filter_names(
+    names: Iterable,
+    label: str,
+    include: Optional[Iterable],
+    exclude: Optional[Iterable],
+):
+    if include and exclude:
+        intersection = set(include) & set(exclude)
+        if intersection:
+            values = ", ".join(intersection)
+            raise InvalidArgumentError(
+                f"'{values}' specified in both --include-{label} and"
+                f" --exclude-{label}"
+            )
+
+    names = {tuple(name.split(".")) for name in names}
+
+    def _filter(filters, update_func):
+        filters = {tuple(name.split(".")) for name in filters}
+        for length, groups in groupby(filters, len):
+            for group in groups:
+                matches = [name for name in names if name[:length] == group]
+                if not matches:
+                    name = ".".join(group)
+                    raise InvalidArgumentError(
+                        f"'{name}' does not match any known {label}"
+                    )
+                update_func(matches)
+
+    if include:
+        ret = set()
+        _filter(include, ret.update)
+    else:
+        ret = set(names)
+
+    if exclude:
+        _filter(exclude, ret.difference_update)
+
+    return {".".join(name) for name in ret}
+
+
 def _update_names(names, items):
     from flatten_json import flatten
 
@@ -21,7 +64,7 @@ def _update_names(names, items):
             names.add(name)
 
 
-def _collect_names(all_experiments):
+def _collect_names(all_experiments, **kwargs):
     metric_names = set()
     param_names = set()
 
@@ -30,16 +73,58 @@ def _collect_names(all_experiments):
             _update_names(metric_names, exp.get("metrics", {}).items())
             _update_names(param_names, exp.get("params", {}).items())
 
+    metric_names = _filter_names(
+        metric_names,
+        "metrics",
+        kwargs.get("include_metrics"),
+        kwargs.get("exclude_metrics"),
+    )
+    param_names = _filter_names(
+        param_names,
+        "params",
+        kwargs.get("include_params"),
+        kwargs.get("exclude_params"),
+    )
+
     return sorted(metric_names), sorted(param_names)
 
 
 def _collect_rows(
-    base_rev, experiments, metric_names, param_names, precision=None
+    base_rev,
+    experiments,
+    metric_names,
+    param_names,
+    precision=DEFAULT_PRECISION,
+    no_timestamp=False,
 ):
-    from flatten_json import flatten
+    for i, (rev, exp) in enumerate(experiments.items()):
+        row = []
+        style = None
+        queued = "*" if exp.get("queued", False) else ""
+        if no_timestamp:
+            timestamp = ""
+        else:
+            commit_time = exp.get("timestamp")
+            timestamp = f" ({commit_time})" if commit_time is not None else ""
 
-    if precision is None:
-        precision = DEFAULT_PRECISION
+        if rev == "baseline":
+            row.append(f"{base_rev}{timestamp}")
+            style = "bold"
+        elif i < len(experiments) - 1:
+            row.append(f"├── {queued}{rev[:7]}{timestamp}")
+        else:
+            row.append(f"└── {queued}{rev[:7]}{timestamp}")
+
+        _extend_row(
+            row, metric_names, exp.get("metrics", {}).items(), precision
+        )
+        _extend_row(row, param_names, exp.get("params", {}).items(), precision)
+
+        yield row, style
+
+
+def _extend_row(row, names, items, precision):
+    from flatten_json import flatten
 
     def _round(val):
         if isinstance(val, float):
@@ -47,50 +132,57 @@ def _collect_rows(
 
         return val
 
-    def _extend(row, names, items):
-        if not items:
-            row.extend(["-"] * len(names))
-            return
+    if not items:
+        row.extend(["-"] * len(names))
+        return
 
-        for fname, item in items:
-            if isinstance(item, dict):
-                item = flatten(item, ".")
-            else:
-                item = {fname: item}
-            for name in names:
-                if name in item:
-                    row.append(str(_round(item[name])))
-                else:
-                    row.append("-")
-
-    for i, (rev, exp) in enumerate(experiments.items()):
-        row = []
-        style = None
-        queued = "*" if exp.get("queued", False) else ""
-        if rev == "baseline":
-            row.append(f"{base_rev}")
-            style = "bold"
-        elif i < len(experiments) - 1:
-            row.append(f"├── {queued}{rev[:7]}")
+    for fname, item in items:
+        if isinstance(item, dict):
+            item = flatten(item, ".")
         else:
-            row.append(f"└── {queued}{rev[:7]}")
-
-        _extend(row, metric_names, exp.get("metrics", {}).items())
-        _extend(row, param_names, exp.get("params", {}).items())
-
-        yield row, style
+            item = {fname: item}
+        for name in names:
+            if name in item:
+                value = item[name]
+                text = str(_round(value)) if value is not None else "-"
+                row.append(text)
+            else:
+                row.append("-")
 
 
-def _show_experiments(all_experiments, console, precision=None):
+def _parse_list(param_list):
+    ret = []
+    for param_str in param_list:
+        # we don't care about filename prefixes for show, silently
+        # ignore it if provided to keep usage consistent with other
+        # metric/param list command options
+        _, _, param_str = param_str.rpartition(":")
+        ret.extend(param_str.split(","))
+    return ret
+
+
+def _show_experiments(all_experiments, console, **kwargs):
     from rich.table import Table
+
     from dvc.scm.git import Git
 
-    metric_names, param_names = _collect_names(all_experiments)
+    include_metrics = _parse_list(kwargs.pop("include_metrics", []))
+    exclude_metrics = _parse_list(kwargs.pop("exclude_metrics", []))
+    include_params = _parse_list(kwargs.pop("include_params", []))
+    exclude_params = _parse_list(kwargs.pop("exclude_params", []))
+
+    metric_names, param_names = _collect_names(
+        all_experiments,
+        include_metrics=include_metrics,
+        exclude_metrics=exclude_metrics,
+        include_params=include_params,
+        exclude_params=exclude_params,
+    )
 
     table = Table()
-    table.add_column("Experiment")
+    table.add_column("Experiment", no_wrap=True)
     for name in metric_names:
-        table.add_column(name, justify="right")
+        table.add_column(name, justify="right", no_wrap=True)
     for name in param_names:
         table.add_column(name, justify="left")
 
@@ -99,11 +191,7 @@ def _show_experiments(all_experiments, console, precision=None):
             base_rev = base_rev[:7]
 
         for row, _, in _collect_rows(
-            base_rev,
-            experiments,
-            metric_names,
-            param_names,
-            precision=precision,
+            base_rev, experiments, metric_names, param_names, **kwargs,
         ):
             table.add_row(*row)
 
@@ -113,6 +201,7 @@ def _show_experiments(all_experiments, console, precision=None):
 class CmdExperimentsShow(CmdBase):
     def run(self):
         from rich.console import Console
+
         from dvc.utils.pager import pager
 
         if not self.repo.experiments:
@@ -125,15 +214,27 @@ class CmdExperimentsShow(CmdBase):
                 all_commits=self.args.all_commits,
             )
 
-            # Note: rich does not currently include a native way to force
-            # infinite width for use with a pager
-            console = Console(
-                file=io.StringIO(), force_terminal=True, width=9999
+            if self.args.no_pager:
+                console = Console()
+            else:
+                # Note: rich does not currently include a native way to force
+                # infinite width for use with a pager
+                console = Console(
+                    file=io.StringIO(), force_terminal=True, width=9999
+                )
+
+            _show_experiments(
+                all_experiments,
+                console,
+                include_metrics=self.args.include_metrics,
+                exclude_metrics=self.args.exclude_metrics,
+                include_params=self.args.include_params,
+                exclude_params=self.args.exclude_params,
+                no_timestamp=self.args.no_timestamp,
             )
 
-            _show_experiments(all_experiments, console)
-
-            pager(console.file.getvalue())
+            if not self.args.no_pager:
+                pager(console.file.getvalue())
         except DvcException:
             logger.exception("failed to show experiments")
             return 1
@@ -146,9 +247,7 @@ class CmdExperimentsCheckout(CmdBase):
         if not self.repo.experiments:
             return 0
 
-        self.repo.experiments.checkout(
-            self.args.experiment, force=self.args.force
-        )
+        self.repo.experiments.checkout(self.args.experiment)
 
         return 0
 
@@ -275,6 +374,46 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show metrics for all commits.",
     )
+    experiments_show_parser.add_argument(
+        "--no-pager",
+        action="store_true",
+        default=False,
+        help="Do not pipe output into a pager.",
+    )
+    experiments_show_parser.add_argument(
+        "--include-metrics",
+        action="append",
+        default=[],
+        help="Include the specified metrics in output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-metrics",
+        action="append",
+        default=[],
+        help="Exclude the specified metrics from output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--include-params",
+        action="append",
+        default=[],
+        help="Include the specified params in output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-params",
+        action="append",
+        default=[],
+        help="Exclude the specified params from output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--no-timestamp",
+        action="store_true",
+        default=False,
+        help="Do not show experiment timestamps.",
+    )
     experiments_show_parser.set_defaults(func=CmdExperimentsShow)
 
     EXPERIMENTS_CHECKOUT_HELP = "Checkout experiments."
@@ -287,14 +426,6 @@ def add_parser(subparsers, parent_parser):
         help=EXPERIMENTS_CHECKOUT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    experiments_checkout_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Overwrite your current workspace with changes from the "
-        "experiment.",
-    )
     experiments_checkout_parser.add_argument(
         "experiment", help="Checkout this experiment.",
     )

dvc/command/gc.py

@@ -123,7 +123,11 @@ def add_parser(subparsers, parent_parser):
         "-j",
         "--jobs",
         type=int,
-        help="Number of jobs to run simultaneously.",
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
         metavar="<number>",
     )
     gc_parser.add_argument(

dvc/command/git_hook.py

@@ -61,6 +61,26 @@ class CmdPrePush(CmdHookBase):
         return main(["push"])
 
 
+class CmdMergeDriver(CmdHookBase):
+    def _run(self):
+        from dvc.dvcfile import Dvcfile
+        from dvc.repo import Repo
+
+        dvc = Repo()
+
+        try:
+            with dvc.state:
+                ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
+                our = Dvcfile(dvc, self.args.our, verify=False)
+                their = Dvcfile(dvc, self.args.their, verify=False)
+
+                our.merge(ancestor, their)
+
+                return 0
+        finally:
+            dvc.close()
+
+
 def add_parser(subparsers, parent_parser):
     GIT_HOOK_HELP = "Run GIT hook."
 
@@ -113,3 +133,27 @@ def add_parser(subparsers, parent_parser):
         "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
     )
     pre_push_parser.set_defaults(func=CmdPrePush)
+
+    MERGE_DRIVER_HELP = "Run GIT merge driver."
+    merge_driver_parser = git_hook_subparsers.add_parser(
+        "merge-driver",
+        parents=[parent_parser],
+        description=MERGE_DRIVER_HELP,
+        help=MERGE_DRIVER_HELP,
+    )
+    merge_driver_parser.add_argument(
+        "--ancestor",
+        required=True,
+        help="Ancestor's version of the conflicting file.",
+    )
+    merge_driver_parser.add_argument(
+        "--our",
+        required=True,
+        help="Current version of the conflicting file.",
+    )
+    merge_driver_parser.add_argument(
+        "--their",
+        required=True,
+        help="Other branch's version of the conflicting file.",
+    )
+    merge_driver_parser.set_defaults(func=CmdMergeDriver)

dvc/command/imp_url.py

@@ -42,20 +42,16 @@ def add_parser(subparsers, parent_parser):
     )
     import_parser.add_argument(
         "url",
-        help="Supported urls:\n"
-        "/path/to/file\n"
-        "/path/to/directory\n"
-        "C:\\\\path\\to\\file\n"
-        "C:\\\\path\\to\\directory\n"
+        help="Location of the data to download. Supported URLs:\n"
+        "/absolute/path/to/file/or/dir\n"
+        "relative/path/to/file/or/dir\n"
+        "C:\\\\path\\to\\file\\or\\dir\n"
         "https://example.com/path/to/file\n"
-        "s3://bucket/path/to/file\n"
-        "s3://bucket/path/to/directory\n"
-        "gs://bucket/path/to/file\n"
-        "gs://bucket/path/to/directory\n"
+        "s3://bucket/key/path\n"
+        "gs://bucket/path/to/file/or/dir\n"
         "hdfs://example.com/path/to/file\n"
-        "ssh://example.com:/path/to/file\n"
-        "ssh://example.com:/path/to/directory\n"
-        "remote://myremote/path/to/file (see `dvc remote`)",
+        "ssh://example.com/absolute/path/to/file/or/dir\n"
+        "remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
     )
     import_parser.add_argument(
         "out", nargs="?", help="Destination path to put files to.",

dvc/command/init.py

@@ -8,8 +8,8 @@ logger = logging.getLogger(__name__)
 
 class CmdInit(CmdBaseNoRepo):
     def run(self):
-        from dvc.repo import Repo
         from dvc.exceptions import InitError
+        from dvc.repo import Repo
 
         try:
             self.repo = Repo.init(

dvc/command/metrics.py

@@ -15,6 +15,7 @@ def show_metrics(
     metrics, all_branches=False, all_tags=False, all_commits=False
 ):
     from flatten_json import flatten
+
     from dvc.utils.diff import format_dict
 
     # When `metrics` contains a `None` key, it means that some files

dvc/command/plots.py

@@ -86,7 +86,12 @@ class CmdPlotsShow(CmdPlots):
 
 class CmdPlotsDiff(CmdPlots):
     def _func(self, *args, **kwargs):
-        return self.repo.plots.diff(*args, revs=self.args.revisions, **kwargs)
+        return self.repo.plots.diff(
+            *args,
+            revs=self.args.revisions,
+            experiment=self.args.experiment,
+            **kwargs,
+        )
 
 
 class CmdPlotsModify(CmdPlots):
@@ -151,6 +156,13 @@ def add_parser(subparsers, parent_parser):
         help="Plots file to visualize. Shows all plots by default.",
         metavar="<path>",
     ).complete = completion.FILE
+    plots_diff_parser.add_argument(
+        "-e",
+        "--experiment",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     plots_diff_parser.add_argument(
         "revisions", nargs="*", default=None, help="Git commits to plot from",
     )

dvc/command/remote.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.config import ConfigError
+from dvc.config import ConfigError, merge
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
@@ -18,7 +18,7 @@ class CmdRemote(CmdConfig):
 
     def _check_exists(self, conf):
         if self.args.name not in conf["remote"]:
-            raise ConfigError(f"remote '{self.args.name}' doesn't exists.")
+            raise ConfigError(f"remote '{self.args.name}' doesn't exist.")
 
 
 class CmdRemoteAdd(CmdRemote):
@@ -61,7 +61,12 @@ class CmdRemoteRemove(CmdRemote):
 class CmdRemoteModify(CmdRemote):
     def run(self):
         with self.config.edit(self.args.level) as conf:
-            self._check_exists(conf)
+            merged = self.config.load_config_to_level(self.args.level)
+            merge(merged, conf)
+            self._check_exists(merged)
+
+            if self.args.name not in conf["remote"]:
+                conf["remote"][self.args.name] = {}
             section = conf["remote"][self.args.name]
             if self.args.unset:
                 section.pop(self.args.option, None)
@@ -167,7 +172,7 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser.add_argument("name", help="Name of the remote")
     remote_add_parser.add_argument(
         "url",
-        help="URL. See full list of supported urls at {}".format(
+        help="Remote location. See full list of supported URLs at {}".format(
             format_link("https://man.dvc.org/remote")
         ),
     )
@@ -196,7 +201,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_default_parser.add_argument(
-        "name", nargs="?", help="Name of the remote."
+        "name", nargs="?", help="Name of the remote"
     )
     remote_default_parser.add_argument(
         "-u",
@@ -215,7 +220,7 @@ def add_parser(subparsers, parent_parser):
         help=REMOTE_MODIFY_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remote_modify_parser.add_argument("name", help="Name of the remote.")
+    remote_modify_parser.add_argument("name", help="Name of the remote")
     remote_modify_parser.add_argument(
         "option", help="Name of the option to modify."
     )

dvc/command/repro.py

@@ -44,6 +44,7 @@ class CmdRepro(CmdBase):
                     queue=self.args.queue,
                     run_all=self.args.run_all,
                     jobs=self.args.jobs,
+                    params=self.args.params,
                 )
 
                 if len(stages) == 0:
@@ -177,6 +178,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=argparse.SUPPRESS,
     )
+    repro_parser.add_argument(
+        "--params",
+        action="append",
+        default=[],
+        help="Declare parameter values for an experiment.",
+        metavar="[<filename>:]<params_list>",
+    )
     repro_parser.add_argument(
         "--queue", action="store_true", default=False, help=argparse.SUPPRESS
     )

dvc/config.py

@@ -138,6 +138,7 @@ SCHEMA = {
         Optional("hardlink_lock", default=False): Bool,
         Optional("no_scm", default=False): Bool,
         Optional("experiments", default=False): Bool,
+        Optional("check_update", default=True): Bool,
     },
     "cache": {
         "local": str,
@@ -161,7 +162,7 @@ SCHEMA = {
                     "endpointurl": str,
                     "access_key_id": str,
                     "secret_access_key": str,
-                    Optional("listobjects", default=False): Bool,
+                    Optional("listobjects", default=False): Bool,  # obsoleted
                     Optional("use_ssl", default=True): Bool,
                     "sse": str,
                     "sse_kms_key_id": str,
@@ -270,7 +271,7 @@ class Config(dict):
 
     @classmethod
     def get_dir(cls, level):
-        from appdirs import user_config_dir, site_config_dir
+        from appdirs import site_config_dir, user_config_dir
 
         assert level in ("global", "system")
 
@@ -324,11 +325,17 @@ class Config(dict):
         if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
+    def _get_tree(self, level):
+        # NOTE: this might be a GitTree, which doesn't see things outside of
+        # the repo.
+        return self.tree if level == "repo" else self.wtree
+
     def _load_config(self, level):
         filename = self.files[level]
+        tree = self._get_tree(level)
 
-        if self.tree.exists(filename, use_dvcignore=False):
-            with self.tree.open(filename) as fobj:
+        if tree.exists(filename, use_dvcignore=False):
+            with tree.open(filename) as fobj:
                 conf_obj = configobj.ConfigObj(fobj)
         else:
             conf_obj = configobj.ConfigObj()
@@ -336,13 +343,14 @@ class Config(dict):
 
     def _save_config(self, level, conf_dict):
         filename = self.files[level]
+        tree = self._get_tree(level)
 
         logger.debug(f"Writing '{filename}'.")
 
-        self.tree.makedirs(os.path.dirname(filename))
+        tree.makedirs(os.path.dirname(filename))
 
         config = configobj.ConfigObj(_pack_remotes(conf_dict))
-        with self.tree.open(filename, "wb") as fobj:
+        with tree.open(filename, "wb") as fobj:
             config.write(fobj)
         config.filename = filename
 
@@ -400,7 +408,7 @@ class Config(dict):
             if merge_level == level:
                 break
             if merge_level in self.files:
-                _merge(merged_conf, self.load_one(merge_level))
+                merge(merged_conf, self.load_one(merge_level))
         return merged_conf
 
     @contextmanager
@@ -414,7 +422,7 @@ class Config(dict):
         conf = self._save_paths(conf, self.files[level])
 
         merged_conf = self.load_config_to_level(level)
-        _merge(merged_conf, conf)
+        merge(merged_conf, conf)
         self.validate(merged_conf)
 
         self._save_config(level, conf)
@@ -453,11 +461,11 @@ def _pack_remotes(conf):
     return result
 
 
-def _merge(into, update):
+def merge(into, update):
     """Merges second dict into first recursively"""
     for key, val in update.items():
         if isinstance(into.get(key), dict) and isinstance(val, dict):
-            _merge(into[key], val)
+            merge(into[key], val)
         else:
             into[key] = val
 

dvc/daemon.py

@@ -25,7 +25,7 @@ def _popen(cmd, **kwargs):
 
 
 def _spawn_windows(cmd, env):
-    from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
+    from subprocess import STARTF_USESHOWWINDOW, STARTUPINFO
 
     creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
 

dvc/data_cloud.py

@@ -81,26 +81,18 @@ class DataCloud:
         """
         remote = self.get_remote(remote, "pull")
 
-        downloaded_items_num = self.repo.cache.local.pull(
+        return self.repo.cache.local.pull(
             cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )
 
-        if not remote.tree.verify:
-            self._save_pulled_checksums(cache)
-
-        return downloaded_items_num
-
-    def _save_pulled_checksums(self, cache):
-        for checksum in cache.scheme_keys("local"):
-            cache_file = self.repo.cache.local.tree.hash_to_path_info(checksum)
-            if self.repo.cache.local.tree.exists(cache_file):
-                # We can safely save here, as existing corrupted files will
-                # be removed upon status, while files corrupted during
-                # download will not be moved from tmp_file
-                # (see `RemoteBASE.download()`)
-                self.repo.state.save(cache_file, checksum)
-
-    def status(self, cache, jobs=None, remote=None, show_checksums=False):
+    def status(
+        self,
+        cache,
+        jobs=None,
+        remote=None,
+        show_checksums=False,
+        log_missing=True,
+    ):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
@@ -111,8 +103,14 @@ class DataCloud:
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
+            log_missing (bool): log warning messages if file doesn't exist
+                neither in cache, neither in cloud.
         """
         remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
+            cache,
+            jobs=jobs,
+            remote=remote,
+            show_checksums=show_checksums,
+            log_missing=log_missing,
         )

dvc/dependency/param.py

@@ -2,12 +2,11 @@ import os
 from collections import defaultdict
 
 import dpath.util
-import toml
-import yaml
 from voluptuous import Any
 
 from dvc.dependency.local import LocalDependency
 from dvc.exceptions import DvcException
+from dvc.utils.serialize import LOADERS, ParseError
 
 
 class MissingParamsError(DvcException):
@@ -22,8 +21,6 @@ class ParamsDependency(LocalDependency):
     PARAM_PARAMS = "params"
     PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
     DEFAULT_PARAMS_FILE = "params.yaml"
-    PARAMS_FILE_LOADERS = defaultdict(lambda: yaml.safe_load)
-    PARAMS_FILE_LOADERS.update({".toml": toml.load})
 
     def __init__(self, stage, path, params):
         info = {}
@@ -88,15 +85,14 @@ class ParamsDependency(LocalDependency):
         if not self.exists:
             return {}
 
-        with self.repo.tree.open(self.path_info, "r") as fobj:
-            try:
-                config = self.PARAMS_FILE_LOADERS[
-                    self.path_info.suffix.lower()
-                ](fobj)
-            except (yaml.YAMLError, toml.TomlDecodeError) as exc:
-                raise BadParamFileError(
-                    f"Unable to read parameters from '{self}'"
-                ) from exc
+        suffix = self.path_info.suffix.lower()
+        loader = LOADERS[suffix]
+        try:
+            config = loader(self.path_info, tree=self.repo.tree)
+        except ParseError as exc:
+            raise BadParamFileError(
+                f"Unable to read parameters from '{self}'"
+            ) from exc
 
         ret = {}
         for param in self.params:

dvc/dependency/repo.py

@@ -50,7 +50,7 @@ class RepoDependency(LocalDependency):
         return external_repo(d["url"], rev=rev)
 
     def _get_checksum(self, locked=True):
-        from dvc.repo.tree import RepoTree
+        from dvc.tree.repo import RepoTree
 
         with self._make_repo(locked=locked) as repo:
             try:
@@ -64,7 +64,9 @@ class RepoDependency(LocalDependency):
 
                 # We are polluting our repo cache with some dir listing here
                 if tree.isdir(path):
-                    return self.repo.cache.local.tree.get_hash(path, tree=tree)
+                    return self.repo.cache.local.tree.get_hash(
+                        path, tree=tree
+                    )[1]
                 return tree.get_file_hash(path)
 
     def workspace_status(self):

dvc/dvcfile.py

@@ -16,7 +16,13 @@ from dvc.stage.exceptions import (
 from dvc.stage.loader import SingleStageLoader, StageLoader
 from dvc.utils import relpath
 from dvc.utils.collections import apply_diff
-from dvc.utils.yaml import dump_yaml, parse_yaml, parse_yaml_for_update
+from dvc.utils.serialize import (
+    dump_yaml,
+    load_yaml,
+    modify_yaml,
+    parse_yaml,
+    parse_yaml_for_update,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -56,9 +62,10 @@ def check_dvc_filename(path):
 class FileMixin:
     SCHEMA = None
 
-    def __init__(self, repo, path, **kwargs):
+    def __init__(self, repo, path, verify=True, **kwargs):
         self.repo = repo
         self.path = path
+        self.verify = verify
 
     def __repr__(self):
         return "{}: {}".format(
@@ -90,7 +97,8 @@ class FileMixin:
         # 3. path doesn't represent a regular file
         if not self.exists():
             raise StageFileDoesNotExistError(self.path)
-        check_dvc_filename(self.path)
+        if self.verify:
+            check_dvc_filename(self.path)
         if not self.repo.tree.isfile(self.path):
             raise StageFileIsNotDvcFileError(self.path)
 
@@ -115,6 +123,9 @@ class FileMixin:
     def dump(self, stage, **kwargs):
         raise NotImplementedError
 
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
 
 class SingleStageFile(FileMixin):
     from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
@@ -134,7 +145,8 @@ class SingleStageFile(FileMixin):
         from dvc.stage import PipelineStage
 
         assert not isinstance(stage, PipelineStage)
-        check_dvc_filename(self.path)
+        if self.verify:
+            check_dvc_filename(self.path)
         logger.debug(
             "Saving information to '{file}'.".format(file=relpath(self.path))
         )
@@ -144,6 +156,14 @@ class SingleStageFile(FileMixin):
     def remove_stage(self, stage):  # pylint: disable=unused-argument
         self.remove()
 
+    def merge(self, ancestor, other):
+        assert isinstance(ancestor, SingleStageFile)
+        assert isinstance(other, SingleStageFile)
+
+        stage = self.stage
+        stage.merge(ancestor.stage, other.stage)
+        self.dump(stage)
+
 
 class PipelineFile(FileMixin):
     """Abstraction for pipelines file, .yaml + .lock combined."""
@@ -161,7 +181,8 @@ class PipelineFile(FileMixin):
         from dvc.stage import PipelineStage
 
         assert isinstance(stage, PipelineStage)
-        check_dvc_filename(self.path)
+        if self.verify:
+            check_dvc_filename(self.path)
 
         if update_pipeline and not stage.is_data_source:
             self._dump_pipeline_file(stage)
@@ -173,30 +194,27 @@ class PipelineFile(FileMixin):
         self._lockfile.dump(stage)
 
     def _dump_pipeline_file(self, stage):
-        data = {}
-        if self.exists():
-            with open(self.path) as fd:
-                data = parse_yaml_for_update(fd.read(), self.path)
-        else:
-            logger.info("Creating '%s'", self.relpath)
-            open(self.path, "w+").close()
-
-        data["stages"] = data.get("stages", {})
         stage_data = serialize.to_pipeline_file(stage)
-        existing_entry = stage.name in data["stages"]
 
-        action = "Modifying" if existing_entry else "Adding"
-        logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath)
+        with modify_yaml(self.path, tree=self.repo.tree) as data:
+            if not data:
+                logger.info("Creating '%s'", self.relpath)
 
-        if existing_entry:
-            orig_stage_data = data["stages"][stage.name]
-            if "meta" in orig_stage_data:
-                stage_data[stage.name]["meta"] = orig_stage_data["meta"]
-            apply_diff(stage_data[stage.name], orig_stage_data)
-        else:
-            data["stages"].update(stage_data)
+            data["stages"] = data.get("stages", {})
+            existing_entry = stage.name in data["stages"]
+            action = "Modifying" if existing_entry else "Adding"
+            logger.info(
+                "%s stage '%s' in '%s'", action, stage.name, self.relpath
+            )
+
+            if existing_entry:
+                orig_stage_data = data["stages"][stage.name]
+                if "meta" in orig_stage_data:
+                    stage_data[stage.name]["meta"] = orig_stage_data["meta"]
+                apply_diff(stage_data[stage.name], orig_stage_data)
+            else:
+                data["stages"].update(stage_data)
 
-        dump_yaml(self.path, data)
         self.repo.scm.track_file(self.relpath)
 
     @property
@@ -239,6 +257,9 @@ class PipelineFile(FileMixin):
         else:
             super().remove()
 
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
 
 class Lockfile(FileMixin):
     from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
@@ -246,8 +267,8 @@ class Lockfile(FileMixin):
     def load(self):
         if not self.exists():
             return {}
-        with self.repo.tree.open(self.path) as fd:
-            data = parse_yaml(fd.read(), self.path)
+
+        data = load_yaml(self.path, tree=self.repo.tree)
         try:
             self.validate(data, fname=self.relpath)
         except StageFileFormatError:
@@ -258,21 +279,18 @@ class Lockfile(FileMixin):
 
     def dump(self, stage, **kwargs):
         stage_data = serialize.to_lockfile(stage)
-        if not self.exists():
-            modified = True
-            logger.info("Generating lock file '%s'", self.relpath)
-            data = stage_data
-            open(self.path, "w+").close()
-        else:
-            with self.repo.tree.open(self.path, "r") as fd:
-                data = parse_yaml_for_update(fd.read(), self.path)
+
+        with modify_yaml(self.path, tree=self.repo.tree) as data:
+            if not data:
+                logger.info("Generating lock file '%s'", self.relpath)
+
             modified = data.get(stage.name, {}) != stage_data.get(
                 stage.name, {}
             )
             if modified:
                 logger.info("Updating lock file '%s'", self.relpath)
             data.update(stage_data)
-        dump_yaml(self.path, data)
+
         if modified:
             self.repo.scm.track_file(self.relpath)
 
@@ -295,6 +313,9 @@ class Lockfile(FileMixin):
         else:
             self.remove()
 
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
 
 class Dvcfile:
     def __new__(cls, repo, path, **kwargs):

dvc/exceptions.py

@@ -1,7 +1,7 @@
 """Exceptions raised by the dvc."""
 from funcy import first
 
-from dvc.utils import format_link, relpath
+from dvc.utils import error_link, format_link, relpath
 
 
 class DvcException(Exception):
@@ -171,11 +171,7 @@ class BadMetricError(DvcException):
 
 
 class NoMetricsError(DvcException):
-    def __init__(self):
-        super().__init__(
-            "no metric files in this repository. Use `-m/-M` options for "
-            "`dvc run` to mark stage outputs as  metrics."
-        )
+    pass
 
 
 class NoPlotsError(DvcException):
@@ -186,14 +182,6 @@ class NoPlotsError(DvcException):
         )
 
 
-class YAMLFileCorruptedError(DvcException):
-    def __init__(self, path):
-        path = relpath(path)
-        super().__init__(
-            f"unable to read: '{path}', YAML file structure is corrupted"
-        )
-
-
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
         super().__init__(
@@ -268,8 +256,10 @@ class CheckoutError(DvcException):
         self.stats = stats
         targets = [str(t) for t in target_infos]
         m = (
-            "Checkout failed for following targets:\n{}\nDid you "
-            "forget to fetch?".format("\n".join(targets))
+            "Checkout failed for following targets:\n{}\nIs your "
+            "cache up to date?\n{}".format(
+                "\n".join(targets), error_link("missing-files"),
+            )
         )
         super().__init__(m)
 
@@ -344,3 +334,7 @@ class NoOutputOrStageError(DvcException):
             f"'{target}' "
             f"does not exist as an output or a stage name in '{file}'"
         )
+
+
+class MergeError(DvcException):
+    pass

dvc/external_repo.py

@@ -18,10 +18,11 @@ from dvc.exceptions import (
 )
 from dvc.path_info import PathInfo
 from dvc.repo import Repo
-from dvc.repo.tree import RepoTree
 from dvc.scm.base import CloneError
 from dvc.scm.git import Git
 from dvc.tree.local import LocalTree
+from dvc.tree.repo import RepoTree
+from dvc.utils import relpath
 from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
@@ -90,15 +91,26 @@ class BaseExternalRepo:
         """Use the specified cache in place of default tmpdir cache for
         download operations.
         """
-        if hasattr(self, "cache"):
-            save_cache = self.cache.local
-            self.cache.local = cache
+        has_cache = hasattr(self, "cache")
+
+        if has_cache:
+            save_cache = self.cache.local  # pylint: disable=E0203
+            self.cache.local = cache  # pylint: disable=E0203
+        else:
+            from collections import namedtuple
+
+            mock_cache = namedtuple("MockCache", ["local"])(local=cache)
+            self.cache = mock_cache  # pylint: disable=W0201
+
         self._local_cache = cache
 
         yield
 
-        if hasattr(self, "cache"):
+        if has_cache:
             self.cache.local = save_cache
+        else:
+            del self.cache
+
         self._local_cache = None
 
     @cached_property
@@ -130,14 +142,17 @@ class BaseExternalRepo:
         for path in paths:
             if not self.repo_tree.exists(path):
                 raise PathMissingError(path, self.url)
-            save_info = self.local_cache.save(
+            hash_info = self.repo_tree.save_info(
+                path, download_callback=download_update
+            )
+            self.local_cache.save(
                 path,
                 self.repo_tree,
-                None,
+                hash_info,
                 save_link=False,
                 download_callback=download_update,
             )
-            save_infos.append(save_info)
+            save_infos.append(hash_info)
 
         return sum(download_results), failed, save_infos
 
@@ -351,6 +366,11 @@ def _remove(path):
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
         os_retry = retry(5, errors=OSError, timeout=0.1)
-        os_retry(remove)(path)
+        try:
+            os_retry(remove)(path)
+        except PermissionError:
+            logger.warning(
+                "Failed to remove '%s'", relpath(path), exc_info=True
+            )
     else:
         remove(path)

dvc/ignore.py

@@ -1,16 +1,17 @@
 import logging
 import os
 import re
+from collections import namedtuple
 from itertools import groupby, takewhile
 
 from pathspec.patterns import GitWildMatchPattern
 from pathspec.util import normalize_file
-from pygtrie import StringTrie
 
 from dvc.path_info import PathInfo
-from dvc.pathspec_math import merge_patterns
+from dvc.pathspec_math import PatternInfo, merge_patterns
 from dvc.system import System
 from dvc.utils import relpath
+from dvc.utils.collections import PathStringTrie
 
 logger = logging.getLogger(__name__)
 
@@ -24,18 +25,26 @@ class DvcIgnore:
 
 class DvcIgnorePatterns(DvcIgnore):
     def __init__(self, pattern_list, dirname):
+        if pattern_list:
+            if isinstance(pattern_list[0], str):
+                pattern_list = [
+                    PatternInfo(pattern, "") for pattern in pattern_list
+                ]
 
         self.pattern_list = pattern_list
         self.dirname = dirname
         self.prefix = self.dirname + os.sep
 
-        regex_pattern_list = map(
-            GitWildMatchPattern.pattern_to_regex, pattern_list
-        )
+        self.regex_pattern_list = [
+            GitWildMatchPattern.pattern_to_regex(pattern_info.patterns)
+            for pattern_info in pattern_list
+        ]
 
         self.ignore_spec = [
             (ignore, re.compile("|".join(item[0] for item in group)))
-            for ignore, group in groupby(regex_pattern_list, lambda x: x[1])
+            for ignore, group in groupby(
+                self.regex_pattern_list, lambda x: x[1]
+            )
             if ignore is not None
         ]
 
@@ -43,9 +52,19 @@ class DvcIgnorePatterns(DvcIgnore):
     def from_files(cls, ignore_file_path, tree):
         assert os.path.isabs(ignore_file_path)
         dirname = os.path.normpath(os.path.dirname(ignore_file_path))
+        ignore_file_rel_path = os.path.relpath(
+            ignore_file_path, tree.tree_root
+        )
         with tree.open(ignore_file_path, encoding="utf-8") as fobj:
             path_spec_lines = [
-                line for line in map(str.strip, fobj.readlines()) if line
+                PatternInfo(
+                    line,
+                    "{}:{}:{}".format(ignore_file_rel_path, line_no + 1, line),
+                )
+                for line_no, line in enumerate(
+                    map(str.strip, fobj.readlines())
+                )
+                if line and not (line.strip().startswith("#"))
             ]
 
         return cls(path_spec_lines, dirname)
@@ -56,7 +75,7 @@ class DvcIgnorePatterns(DvcIgnore):
 
         return dirs, files
 
-    def matches(self, dirname, basename, is_dir=False):
+    def _get_normalize_path(self, dirname, basename):
         # NOTE: `relpath` is too slow, so we have to assume that both
         # `dirname` and `self.dirname` are relative or absolute together.
         if dirname == self.dirname:
@@ -70,6 +89,12 @@ class DvcIgnorePatterns(DvcIgnore):
 
         if not System.is_unix():
             path = normalize_file(path)
+        return path
+
+    def matches(self, dirname, basename, is_dir=False):
+        path = self._get_normalize_path(dirname, basename)
+        if not path:
+            return False
         return self.ignore(path, is_dir)
 
     def ignore(self, path, is_dir):
@@ -85,25 +110,57 @@ class DvcIgnorePatterns(DvcIgnore):
                     result = ignore
         return result
 
+    def match_details(self, dirname, basename, is_dir=False):
+        path = self._get_normalize_path(dirname, basename)
+        if not path:
+            return False
+        return self._ignore_details(path, is_dir)
+
+    def _ignore_details(self, path, is_dir):
+        result = []
+        for ignore, pattern in zip(self.regex_pattern_list, self.pattern_list):
+            regex = re.compile(ignore[0])
+            # skip system pattern
+            if not pattern.file_info:
+                continue
+            if is_dir:
+                path_dir = f"{path}/"
+                if regex.match(path) or regex.match(path_dir):
+                    result.append(pattern.file_info)
+            else:
+                if regex.match(path):
+                    result.append(pattern.file_info)
+        return result
+
     def __hash__(self):
-        return hash(self.dirname + ":" + "\n".join(self.pattern_list))
+        return hash(self.dirname + ":" + str(self.pattern_list))
 
     def __eq__(self, other):
         if not isinstance(other, DvcIgnorePatterns):
             return NotImplemented
         return (self.dirname == other.dirname) & (
-            self.pattern_list == other.pattern_list
+            [pattern.patterns for pattern in self.pattern_list]
+            == [pattern.patterns for pattern in other.pattern_list]
         )
 
     def __bool__(self):
         return bool(self.pattern_list)
 
 
+CheckIgnoreResult = namedtuple(
+    "CheckIgnoreResult", ["file", "match", "patterns"]
+)
+
+
+def _no_match(path):
+    return CheckIgnoreResult(path, False, ["::"])
+
+
 class DvcIgnoreFilterNoop:
     def __init__(self, tree, root_dir):
         pass
 
-    def __call__(self, root, dirs, files):
+    def __call__(self, root, dirs, files, **kwargs):
         return dirs, files
 
     def is_ignored_dir(self, _):
@@ -112,6 +169,9 @@ class DvcIgnoreFilterNoop:
     def is_ignored_file(self, _):
         return False
 
+    def check_ignore(self, path):
+        return _no_match(path)
+
 
 class DvcIgnoreFilter:
     @staticmethod
@@ -127,10 +187,11 @@ class DvcIgnoreFilter:
 
         self.tree = tree
         self.root_dir = root_dir
-        self.ignores_trie_tree = StringTrie(separator=os.sep)
+        self.ignores_trie_tree = PathStringTrie()
         self.ignores_trie_tree[root_dir] = DvcIgnorePatterns(
             default_ignore_patterns, root_dir
         )
+        self._ignored_subrepos = PathStringTrie()
         self._update(self.root_dir)
 
     def _update(self, dirname):
@@ -166,27 +227,30 @@ class DvcIgnoreFilter:
     def _update_sub_repo(self, root, dirs):
         for d in dirs:
             if self._is_dvc_repo(root, d):
+                self._ignored_subrepos[root] = self._ignored_subrepos.get(
+                    root, set()
+                ) | {d}
+                new_pattern = DvcIgnorePatterns([f"/{d}/"], root)
                 old_pattern = self.ignores_trie_tree.longest_prefix(root).value
                 if old_pattern:
                     self.ignores_trie_tree[root] = DvcIgnorePatterns(
                         *merge_patterns(
                             old_pattern.pattern_list,
                             old_pattern.dirname,
-                            ["/{}/".format(d)],
-                            root,
+                            new_pattern.pattern_list,
+                            new_pattern.dirname,
                         )
                     )
                 else:
-                    self.ignores_trie_tree[root] = DvcIgnorePatterns(
-                        ["/{}/".format(d)], root
-                    )
+                    self.ignores_trie_tree[root] = new_pattern
 
-    def __call__(self, root, dirs, files):
+    def __call__(self, root, dirs, files, ignore_subrepos=True):
         ignore_pattern = self._get_trie_pattern(root)
         if ignore_pattern:
-            return ignore_pattern(root, dirs, files)
-        else:
-            return dirs, files
+            dirs, files = ignore_pattern(root, dirs, files)
+            if not ignore_subrepos:
+                dirs.extend(self._ignored_subrepos.get(root, []))
+        return dirs, files
 
     def _get_trie_pattern(self, dirname):
         ignore_pattern = self.ignores_trie_tree.get(dirname)
@@ -214,7 +278,7 @@ class DvcIgnoreFilter:
 
     def _is_ignored(self, path, is_dir=False):
         if self._outside_repo(path):
-            return True
+            return False
         dirname, basename = os.path.split(os.path.normpath(path))
         ignore_pattern = self._get_trie_pattern(dirname)
         if ignore_pattern:
@@ -222,14 +286,21 @@ class DvcIgnoreFilter:
         else:
             return False
 
-    def is_ignored_dir(self, path):
+    def _is_subrepo(self, path):
+        dirname, basename = os.path.split(os.path.normpath(path))
+        return basename in self._ignored_subrepos.get(dirname, set())
+
+    def is_ignored_dir(self, path, ignore_subrepos=True):
         path = os.path.abspath(path)
+        if not ignore_subrepos:
+            return not self._is_subrepo(path)
         if path == self.root_dir:
             return False
 
         return self._is_ignored(path, True)
 
     def is_ignored_file(self, path):
+        path = os.path.abspath(path)
         return self._is_ignored(path, False)
 
     def _outside_repo(self, path):
@@ -245,3 +316,32 @@ class DvcIgnoreFilter:
         ):
             return True
         return False
+
+    def check_ignore(self, target):
+        full_target = os.path.abspath(target)
+        if not self._outside_repo(full_target):
+            dirname, basename = os.path.split(os.path.normpath(full_target))
+            pattern = self._get_trie_pattern(dirname)
+            if pattern:
+                matches = pattern.match_details(
+                    dirname, basename, os.path.isdir(full_target)
+                )
+
+                if matches:
+                    return CheckIgnoreResult(target, True, matches)
+        return _no_match(target)
+
+
+def init(path):
+    dvcignore = os.path.join(path, DvcIgnore.DVCIGNORE_FILE)
+    if os.path.exists(dvcignore):
+        return dvcignore
+
+    with open(dvcignore, "w") as fobj:
+        fobj.write(
+            "# Add patterns of files dvc should ignore, which could improve\n"
+            "# the performance. Learn more at\n"
+            "# https://dvc.org/doc/user-guide/dvcignore\n"
+        )
+
+    return dvcignore

dvc/main.py

@@ -2,6 +2,7 @@
 
 import errno
 import logging
+from contextlib import contextmanager
 
 from dvc import analytics
 from dvc.cli import parse_args
@@ -10,7 +11,7 @@ from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
 from dvc.external_repo import clean_repos
 from dvc.logger import FOOTER, disable_other_loggers
 from dvc.tree.pool import close_pools
-from dvc.utils import format_link
+from dvc.utils import error_link
 
 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
@@ -21,6 +22,25 @@ from dvc.utils import format_link
 logger = logging.getLogger("dvc")
 
 
+@contextmanager
+def profile(enable, dump):
+    if not enable:
+        yield
+        return
+
+    import cProfile
+
+    prof = cProfile.Profile()
+    prof.enable()
+
+    yield
+
+    if not dump:
+        prof.print_stats(sort="cumtime")
+        return
+    prof.dump_stats(dump)
+
+
 def main(argv=None):  # noqa: C901
     """Run dvc CLI command.
 
@@ -50,8 +70,9 @@ def main(argv=None):  # noqa: C901
 
         logger.trace(args)
 
-        cmd = args.func(args)
-        ret = cmd.run()
+        with profile(enable=args.cprofile, dump=args.cprofile_dump):
+            cmd = args.func(args)
+            ret = cmd.run()
     except ConfigError:
         logger.exception("configuration error")
         ret = 251
@@ -71,9 +92,7 @@ def main(argv=None):  # noqa: C901
             logger.exception(
                 "too many open files, please visit "
                 "{} to see how to handle this "
-                "problem".format(
-                    format_link("https://error.dvc.org/many-files")
-                ),
+                "problem".format(error_link("many-files")),
                 extra={"tb_only": True},
             )
         else:
@@ -84,7 +103,9 @@ def main(argv=None):  # noqa: C901
         ret = 255
 
     try:
-        if ret != 0:
+        if ret != 0 and (
+            ret != 1 or getattr(args, "cmd", "") != "check-ignore"
+        ):
             logger.info(FOOTER)
 
         if analytics.is_enabled():

dvc/output/base.py

@@ -10,6 +10,7 @@ from dvc.cache import NamedCache
 from dvc.exceptions import (
     CollectCacheError,
     DvcException,
+    MergeError,
     RemoteCacheRequiredError,
 )
 
@@ -45,6 +46,12 @@ class OutputIsStageFileError(DvcException):
         super().__init__(f"Stage file '{path}' cannot be an output.")
 
 
+class OutputIsIgnoredError(DvcException):
+    def __init__(self, match):
+        lines = "\n".join(match.patterns)
+        super().__init__(f"Path '{match.file}' is ignored by\n{lines}")
+
+
 class BaseOutput:
     IS_DEPENDENCY = False
 
@@ -77,6 +84,7 @@ class BaseOutput:
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
     IsStageFileError = OutputIsStageFileError
+    IsIgnoredError = OutputIsIgnoredError
 
     sep = "/"
 
@@ -91,7 +99,7 @@ class BaseOutput:
         plot=False,
         persist=False,
     ):
-        self._validate_output_path(path)
+        self._validate_output_path(path, stage)
         # This output (and dependency) objects have too many paths/urls
         # here is a list and comments:
         #
@@ -164,20 +172,12 @@ class BaseOutput:
     def cache_path(self):
         return self.cache.tree.hash_to_path_info(self.checksum).url
 
-    @property
-    def checksum_type(self):
-        return self.tree.PARAM_CHECKSUM
-
     @property
     def checksum(self):
         return self.info.get(self.tree.PARAM_CHECKSUM)
 
-    @checksum.setter
-    def checksum(self, checksum):
-        self.info[self.tree.PARAM_CHECKSUM] = checksum
-
     def get_checksum(self):
-        return self.tree.get_hash(self.path_info)
+        return self.tree.get_hash(self.path_info)[1]
 
     @property
     def is_dir_checksum(self):
@@ -499,8 +499,47 @@ class BaseOutput:
         return ret
 
     @classmethod
-    def _validate_output_path(cls, path):
+    def _validate_output_path(cls, path, stage=None):
         from dvc.dvcfile import is_valid_filename
 
         if is_valid_filename(path):
             raise cls.IsStageFileError(path)
+
+        if stage:
+            check = stage.repo.tree.dvcignore.check_ignore(path)
+            if check.match:
+                raise cls.IsIgnoredError(check)
+
+    def _check_can_merge(self, out):
+        if self.scheme != out.scheme:
+            raise MergeError("unable to auto-merge outputs of different types")
+
+        my = self.dumpd()
+        other = out.dumpd()
+
+        my.pop(self.tree.PARAM_CHECKSUM)
+        other.pop(self.tree.PARAM_CHECKSUM)
+
+        if my != other:
+            raise MergeError(
+                "unable to auto-merge outputs with different options"
+            )
+
+        if not out.is_dir_checksum:
+            raise MergeError(
+                "unable to auto-merge outputs that are not directories"
+            )
+
+    def merge(self, ancestor, other):
+        assert other
+
+        if ancestor:
+            self._check_can_merge(ancestor)
+            ancestor_info = ancestor.info
+        else:
+            ancestor_info = None
+
+        self._check_can_merge(self)
+        self._check_can_merge(other)
+
+        self.info = self.cache.merge(ancestor_info, self.info, other.info)

dvc/output/local.py

@@ -87,8 +87,9 @@ class LocalOutput(BaseOutput):
 
         name = "metrics" if self.metric else "plot"
         if os.path.isdir(path):
-            msg = "directory '{}' cannot be used as {}."
-            raise IsADirectoryError(msg.format(self.path_info, name))
+            msg = "directory '%s' cannot be used as %s."
+            logger.debug(msg, str(self.path_info), name)
+            return
 
         if not istextfile(path):
             msg = "binary file '{}' cannot be used as {}."

dvc/path_info.py

@@ -317,15 +317,9 @@ class HTTPURLInfo(URLInfo):
         )
 
 
-# See https://github.com/shizacat/dvc/blob/remote-webdav/dvc/path_info.py
-class WebDAVURLInfo(HTTPURLInfo):
+class WebDAVURLInfo(URLInfo):
     @cached_property
     def url(self):
-        return "{}://{}{}{}{}{}".format(
-            self.scheme.replace("webdav", "http"),
-            self.netloc,
-            self._spath,
-            (";" + self.params) if self.params else "",
-            ("?" + self.query) if self.query else "",
-            ("#" + self.fragment) if self.fragment else "",
+        return "{}://{}{}".format(
+            self.scheme.replace("webdav", "http"), self.netloc, self._spath
         )

dvc/pathspec_math.py

@@ -3,11 +3,14 @@
 # of two path specification patterns with different base
 # All the operations follow the documents of `gitignore`
 import os
+from collections import namedtuple
 
 from pathspec.util import normalize_file
 
 from dvc.utils import relpath
 
+PatternInfo = namedtuple("PatternInfo", ["patterns", "file_info"])
+
 
 def _not_ignore(rule):
     return (True, rule[1:]) if rule.startswith("!") else (False, rule)
@@ -59,7 +62,10 @@ def _change_dirname(dirname, pattern_list, new_dirname):
     if rel.startswith(".."):
         raise ValueError("change dirname can only change to parent path")
 
-    return [change_rule(rule, rel) for rule in pattern_list]
+    return [
+        PatternInfo(change_rule(rule.patterns, rel), rule.file_info)
+        for rule in pattern_list
+    ]
 
 
 def merge_patterns(pattern_a, prefix_a, pattern_b, prefix_b):

dvc/prompt.py

@@ -7,7 +7,7 @@ from getpass import getpass
 logger = logging.getLogger(__name__)
 
 
-def _ask(prompt, limited_to=None):
+def ask(prompt, limited_to=None):
     if not sys.stdout.isatty():
         return None
 
@@ -39,7 +39,7 @@ def confirm(statement):
         bool: whether or not specified statement was confirmed.
     """
     prompt = f"{statement} [y/n]"
-    answer = _ask(prompt, limited_to=["yes", "no", "y", "n"])
+    answer = ask(prompt, limited_to=["yes", "no", "y", "n"])
     return answer and answer.startswith("y")
 
 

dvc/repo/__init__.py

@@ -15,7 +15,7 @@ from dvc.exceptions import (
     OutputNotFoundError,
 )
 from dvc.path_info import PathInfo
-from dvc.repo.tree import RepoTree
+from dvc.tree.repo import RepoTree
 from dvc.utils.fs import path_isin
 
 from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
@@ -42,43 +42,43 @@ def locked(f):
 class Repo:
     DVC_DIR = ".dvc"
 
-    from dvc.repo.destroy import destroy
-    from dvc.repo.install import install
     from dvc.repo.add import add
-    from dvc.repo.remove import remove
-    from dvc.repo.ls import ls
-    from dvc.repo.freeze import freeze, unfreeze
-    from dvc.repo.move import move
-    from dvc.repo.run import run
-    from dvc.repo.imp import imp
-    from dvc.repo.imp_url import imp_url
-    from dvc.repo.reproduce import reproduce
-    from dvc.repo.checkout import _checkout
-    from dvc.repo.push import push
-    from dvc.repo.fetch import _fetch
-    from dvc.repo.pull import pull
-    from dvc.repo.status import status
-    from dvc.repo.gc import gc
-    from dvc.repo.commit import commit
-    from dvc.repo.diff import diff
     from dvc.repo.brancher import brancher
+    from dvc.repo.checkout import _checkout
+    from dvc.repo.commit import commit
+    from dvc.repo.destroy import destroy
+    from dvc.repo.diff import diff
+    from dvc.repo.fetch import _fetch
+    from dvc.repo.freeze import freeze, unfreeze
+    from dvc.repo.gc import gc
     from dvc.repo.get import get
     from dvc.repo.get_url import get_url
+    from dvc.repo.imp import imp
+    from dvc.repo.imp_url import imp_url
+    from dvc.repo.install import install
+    from dvc.repo.ls import ls
+    from dvc.repo.move import move
+    from dvc.repo.pull import pull
+    from dvc.repo.push import push
+    from dvc.repo.remove import remove
+    from dvc.repo.reproduce import reproduce
+    from dvc.repo.run import run
+    from dvc.repo.status import status
     from dvc.repo.update import update
 
     def __init__(self, root_dir=None, scm=None, rev=None):
-        from dvc.state import State, StateNoop
-        from dvc.lock import make_lock
-        from dvc.scm import SCM
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
+        from dvc.lock import make_lock
         from dvc.repo.experiments import Experiments
         from dvc.repo.metrics import Metrics
-        from dvc.repo.plots import Plots
         from dvc.repo.params import Params
+        from dvc.repo.plots import Plots
+        from dvc.scm import SCM
+        from dvc.stage.cache import StageCache
+        from dvc.state import State, StateNoop
         from dvc.tree.local import LocalTree
         from dvc.utils.fs import makedirs
-        from dvc.stage.cache import StageCache
 
         if scm:
             tree = scm.get_tree(rev)
@@ -441,10 +441,11 @@ class Repo:
         """
         import networkx as nx
         from pygtrie import Trie
+
         from dvc.exceptions import (
             OutputDuplicationError,
-            StagePathAsOutputError,
             OverlappingOutputPathsError,
+            StagePathAsOutputError,
         )
 
         G = nx.DiGraph()
@@ -615,4 +616,3 @@ class Repo:
         self.__dict__.pop("graph", None)
         self.__dict__.pop("stages", None)
         self.__dict__.pop("pipelines", None)
-        self.tree.__dict__.pop("dvcignore", None)

dvc/repo/diff.py

@@ -37,7 +37,7 @@ def diff(self, a_rev="HEAD", b_rev=None):
 
         def _to_checksum(output):
             if on_working_tree:
-                return self.cache.local.tree.get_hash(output.path_info)
+                return self.cache.local.tree.get_hash(output.path_info)[1]
             return output.checksum
 
         def _exists(output):

dvc/repo/experiments/__init__.py

@@ -2,21 +2,42 @@ import logging
 import os
 import re
 import tempfile
-from concurrent.futures import ProcessPoolExecutor, as_completed
+from collections.abc import Mapping
+from concurrent.futures import (
+    ProcessPoolExecutor,
+    ThreadPoolExecutor,
+    as_completed,
+)
+from contextlib import contextmanager
+from functools import partial, wraps
 from typing import Iterable, Optional
 
-from funcy import cached_property
+from funcy import cached_property, first
 
-from dvc.exceptions import DvcException
+from dvc.exceptions import DownloadError, DvcException, UploadError
+from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
 from dvc.repo.experiments.executor import ExperimentExecutor, LocalExecutor
 from dvc.scm.git import Git
 from dvc.stage.serialize import to_lockfile
+from dvc.tree.repo import RepoTree
 from dvc.utils import dict_sha256, env2bool, relpath
-from dvc.utils.fs import copyfile, remove
+from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
 
+def scm_locked(f):
+    # Lock the experiments workspace so that we don't try to perform two
+    # different sequences of git operations at once
+    @wraps(f)
+    def wrapper(exp, *args, **kwargs):
+        with exp.scm_lock:
+            return f(exp, *args, **kwargs)
+
+    return wrapper
+
+
 def hash_exp(stages):
     exp_data = {}
     for stage in stages:
@@ -26,10 +47,22 @@ def hash_exp(stages):
 
 class UnchangedExperimentError(DvcException):
     def __init__(self, rev):
-        super().__init__("Experiment identical to baseline '{rev[:7]}'.")
+        super().__init__(f"Experiment identical to baseline '{rev[:7]}'.")
         self.rev = rev
 
 
+class BaselineMismatchError(DvcException):
+    def __init__(self, rev, expected):
+        if hasattr(rev, "hexsha"):
+            rev = rev.hexsha
+        rev_str = f"{rev[:7]}" if rev is not None else "dangling commit"
+        super().__init__(
+            f"Experiment derived from '{rev_str}', expected '{expected[:7]}'."
+        )
+        self.rev = rev
+        self.expected_rev = expected
+
+
 class Experiments:
     """Class that manages experiments in a DVC repo.
 
@@ -43,8 +76,13 @@ class Experiments:
     STASH_EXPERIMENT_RE = re.compile(
         r"(?:On \(.*\): )dvc-exp-(?P<baseline_rev>[0-9a-f]+)$"
     )
+    BRANCH_RE = re.compile(
+        r"^(?P<baseline_rev>[a-f0-9]{7})-(?P<exp_sha>[a-f0-9]+)$"
+    )
 
     def __init__(self, repo):
+        from dvc.lock import make_lock
+
         if not (
             env2bool("DVC_TEST")
             or repo.config["core"].get("experiments", False)
@@ -52,6 +90,10 @@ class Experiments:
             raise NotImplementedError
 
         self.repo = repo
+        self.scm_lock = make_lock(
+            os.path.join(self.repo.tmp_dir, "exp_scm_lock"),
+            tmp_dir=self.repo.tmp_dir,
+        )
 
     @cached_property
     def exp_dir(self):
@@ -79,6 +121,13 @@ class Experiments:
 
         return Repo(self.exp_dvc_dir)
 
+    @contextmanager
+    def chdir(self):
+        cwd = os.getcwd()
+        os.chdir(self.exp_dvc.root_dir)
+        yield self.exp_dvc.root_dir
+        os.chdir(cwd)
+
     @cached_property
     def args_file(self):
         return os.path.join(self.exp_dvc.tmp_dir, self.PACKED_ARGS_FILE)
@@ -116,28 +165,60 @@ class Experiments:
     def _scm_checkout(self, rev):
         self.scm.repo.git.reset(hard=True)
         if self.scm.repo.head.is_detached:
-            # switch back to default branch
-            self.scm.repo.heads[0].checkout()
+            self._checkout_default_branch()
         if not Git.is_sha(rev) or not self.scm.has_rev(rev):
             self.scm.pull()
-        logger.debug("Checking out base experiment commit '%s'", rev)
+        logger.debug("Checking out experiment commit '%s'", rev)
         self.scm.checkout(rev)
 
-    def _stash_exp(self, *args, **kwargs):
+    def _checkout_default_branch(self):
+        # switch to default branch
+        git_repo = self.scm.repo
+        origin_refs = git_repo.remotes["origin"].refs
+        ref = origin_refs["HEAD"].reference
+        branch_name = ref.name.split("/")[-1]
+        if branch_name in git_repo.heads:
+            branch = git_repo.heads[branch_name]
+        else:
+            branch = git_repo.create_head(branch_name, ref)
+            branch.set_tracking_branch(ref)
+        branch.checkout()
+
+    def _stash_exp(self, *args, params: Optional[dict] = None, **kwargs):
         """Stash changes from the current (parent) workspace as an experiment.
+
+        Args:
+            params: Optional dictionary of parameter values to be used.
+                Values take priority over any parameters specified in the
+                user's workspace.
         """
         rev = self.scm.get_rev()
+
+        # patch user's workspace into experiments clone
         tmp = tempfile.NamedTemporaryFile(delete=False).name
         try:
-            self.repo.scm.repo.git.diff(patch=True, output=tmp)
+            self.repo.scm.repo.git.diff(
+                patch=True, full_index=True, binary=True, output=tmp
+            )
             if os.path.getsize(tmp):
                 logger.debug("Patching experiment workspace")
                 self.scm.repo.git.apply(tmp)
-            else:
+            elif not params:
+                # experiment matches original baseline
                 raise UnchangedExperimentError(rev)
         finally:
             remove(tmp)
+
+        # update experiment params from command line
+        if params:
+            self._update_params(params)
+
+        # save additional repro command line arguments
         self._pack_args(*args, **kwargs)
+
+        # save experiment as a stash commit w/message containing baseline rev
+        # (stash commits are merge commits and do not contain a parent commit
+        # SHA)
         msg = f"{self.STASH_MSG_PREFIX}{rev}"
         self.scm.repo.git.stash("push", "-m", msg)
         return self.scm.resolve_rev("stash@{0}")
@@ -147,13 +228,35 @@ class Experiments:
         self.scm.add(self.args_file)
 
     def _unpack_args(self, tree=None):
-        args_file = os.path.join(self.exp_dvc.tmp_dir, self.PACKED_ARGS_FILE)
-        return ExperimentExecutor.unpack_repro_args(args_file, tree=tree)
+        return ExperimentExecutor.unpack_repro_args(self.args_file, tree=tree)
+
+    def _update_params(self, params: dict):
+        """Update experiment params files with the specified values."""
+        from dvc.utils.serialize import MODIFIERS
+
+        logger.debug("Using experiment params '%s'", params)
+
+        # recursive dict update
+        def _update(dict_, other):
+            for key, value in other.items():
+                if isinstance(value, Mapping):
+                    dict_[key] = _update(dict_.get(key, {}), value)
+                else:
+                    dict_[key] = value
+            return dict_
+
+        for params_fname in params:
+            path = PathInfo(self.exp_dvc.root_dir) / params_fname
+            suffix = path.suffix.lower()
+            modify_data = MODIFIERS[suffix]
+            with modify_data(path, tree=self.exp_dvc.tree) as data:
+                _update(data, params[params_fname])
 
     def _commit(self, exp_hash, check_exists=True, branch=True):
         """Commit stages as an experiment and return the commit SHA."""
         if not self.scm.is_dirty():
             raise UnchangedExperimentError(self.scm.get_rev())
+
         rev = self.scm.get_rev()
         exp_name = f"{rev[:7]}-{exp_hash}"
         if branch:
@@ -175,8 +278,9 @@ class Experiments:
             )
             return [stash_rev]
         results = self.reproduce([stash_rev], keep_stash=False)
-        for exp_rev in results:
-            self.checkout_exp(exp_rev, force=True)
+        exp_rev = first(results)
+        if exp_rev is not None:
+            self.checkout_exp(exp_rev)
         return results
 
     def reproduce_queued(self, **kwargs):
@@ -191,7 +295,8 @@ class Experiments:
             )
         return results
 
-    def new(self, *args, workspace=True, **kwargs):
+    @scm_locked
+    def new(self, *args, **kwargs):
         """Create a new experiment.
 
         Experiment will be reproduced and checked out into the user's
@@ -199,20 +304,17 @@ class Experiments:
         """
         rev = self.repo.scm.get_rev()
         self._scm_checkout(rev)
-        if workspace:
-            try:
-                stash_rev = self._stash_exp(*args, **kwargs)
-            except UnchangedExperimentError as exc:
-                logger.info("Reproducing existing experiment '%s'.", rev[:7])
-                raise exc
-        else:
-            # configure params via command line here
-            pass
+        try:
+            stash_rev = self._stash_exp(*args, **kwargs)
+        except UnchangedExperimentError as exc:
+            logger.info("Reproducing existing experiment '%s'.", rev[:7])
+            raise exc
         logger.debug(
             "Stashed experiment '%s' for future execution.", stash_rev[:7]
         )
         return stash_rev
 
+    @scm_locked
     def reproduce(
         self,
         revs: Optional[Iterable] = None,
@@ -245,19 +347,26 @@ class Experiments:
                 for rev in revs
             }
 
-        # setup executors
+        logger.debug(
+            "Reproducing experiment revs '%s'",
+            ", ".join((rev[:7] for rev in to_run)),
+        )
+
+        # setup executors - unstash experiment, generate executor, upload
+        # contents of (unstashed) exp workspace to the executor tree
         executors = {}
         for rev, baseline_rev in to_run.items():
-            tree = self.scm.get_tree(rev)
-            repro_args, repro_kwargs = self._unpack_args(tree)
+            self._scm_checkout(baseline_rev)
+            self.scm.repo.git.stash("apply", rev)
+            repro_args, repro_kwargs = self._unpack_args()
             executor = LocalExecutor(
-                tree,
                 baseline_rev,
                 repro_args=repro_args,
                 repro_kwargs=repro_kwargs,
                 dvc_dir=self.dvc_dir,
                 cache_dir=self.repo.cache.local.cache_dir,
             )
+            self._collect_input(executor)
             executors[rev] = executor
 
         exec_results = self._reproduce(executors, **kwargs)
@@ -308,10 +417,19 @@ class Experiments:
                 exc = future.exception()
                 if exc is None:
                     exp_hash = future.result()
-                    logger.debug(f"ran exp based on {executor.baseline_rev}")
                     self._scm_checkout(executor.baseline_rev)
-                    self._collect_output(executor.baseline_rev, executor)
-                    remove(self.args_file)
+                    try:
+                        self._collect_output(executor)
+                    except DownloadError:
+                        logger.error(
+                            "Failed to collect output for experiment '%s'",
+                            rev,
+                        )
+                        continue
+                    finally:
+                        if os.path.exists(self.args_file):
+                            remove(self.args_file)
+
                     try:
                         exp_rev = self._commit(exp_hash)
                     except UnchangedExperimentError:
@@ -331,37 +449,154 @@ class Experiments:
 
         return result
 
-    def _collect_output(self, rev: str, executor: ExperimentExecutor):
-        logger.debug("copying tmp output from '%s'", executor.tmp_dir)
-        tree = self.scm.get_tree(rev)
-        for fname in tree.walk_files(tree.tree_root):
-            src = executor.path_info / relpath(fname, tree.tree_root)
-            copyfile(src, fname)
+    def _collect_input(self, executor: ExperimentExecutor):
+        """Copy (upload) input from the experiments workspace to the executor
+        tree.
+        """
+        logger.debug("Collecting input for '%s'", executor.tmp_dir)
+        repo_tree = RepoTree(self.exp_dvc)
+        self._process(
+            executor.tree,
+            self.exp_dvc.tree,
+            executor.collect_files(self.exp_dvc.tree, repo_tree),
+        )
 
-    def checkout_exp(self, rev, force=False):
+    def _collect_output(self, executor: ExperimentExecutor):
+        """Copy (download) output from the executor tree into experiments
+        workspace.
+        """
+        logger.debug("Collecting output from '%s'", executor.tmp_dir)
+        self._process(
+            self.exp_dvc.tree,
+            executor.tree,
+            executor.collect_output(),
+            download=True,
+        )
+
+    @staticmethod
+    def _process(dest_tree, src_tree, collected_files, download=False):
+        from dvc.cache.local import _log_exceptions
+
+        from_infos = []
+        to_infos = []
+        names = []
+        for from_info in collected_files:
+            from_infos.append(from_info)
+            fname = from_info.relative_to(src_tree.path_info)
+            names.append(str(fname))
+            to_infos.append(dest_tree.path_info / fname)
+        total = len(from_infos)
+
+        if download:
+            func = partial(
+                _log_exceptions(src_tree.download, "download"),
+                dir_mode=dest_tree.dir_mode,
+                file_mode=dest_tree.file_mode,
+            )
+            desc = "Downloading"
+        else:
+            func = partial(_log_exceptions(dest_tree.upload, "upload"))
+            desc = "Uploading"
+
+        with Tqdm(total=total, unit="file", desc=desc) as pbar:
+            func = pbar.wrap_fn(func)
+            # TODO: parallelize this, currently --jobs for repro applies to
+            # number of repro executors not download threads
+            with ThreadPoolExecutor(max_workers=1) as dl_executor:
+                fails = sum(dl_executor.map(func, from_infos, to_infos, names))
+
+        if fails:
+            if download:
+                raise DownloadError(fails)
+            raise UploadError(fails)
+
+    @scm_locked
+    def checkout_exp(self, rev):
         """Checkout an experiment to the user's workspace."""
         from git.exc import GitCommandError
+
         from dvc.repo.checkout import _checkout as dvc_checkout
 
-        if force:
-            self.repo.scm.repo.git.reset(hard=True)
+        self._check_baseline(rev)
         self._scm_checkout(rev)
 
         tmp = tempfile.NamedTemporaryFile(delete=False).name
-        self.scm.repo.head.commit.diff("HEAD~1", patch=True, output=tmp)
+        self.scm.repo.head.commit.diff(
+            "HEAD~1", patch=True, full_index=True, binary=True, output=tmp
+        )
+
+        dirty = self.repo.scm.is_dirty()
+        if dirty:
+            logger.debug("Stashing workspace changes.")
+            self.repo.scm.repo.git.stash("push", "--include-untracked")
+
         try:
             if os.path.getsize(tmp):
                 logger.debug("Patching local workspace")
                 self.repo.scm.repo.git.apply(tmp, reverse=True)
-            dvc_checkout(self.repo)
+                need_checkout = True
+            else:
+                need_checkout = False
         except GitCommandError:
-            raise DvcException(
-                "Checkout failed, experiment contains changes which "
-                "conflict with your current workspace. To overwrite "
-                "your workspace, use `dvc experiments checkout --force`."
-            )
+            raise DvcException("failed to apply experiment changes.")
         finally:
             remove(tmp)
+            if dirty:
+                self._unstash_workspace()
+
+        if need_checkout:
+            dvc_checkout(self.repo)
+
+    def _check_baseline(self, exp_rev):
+        baseline_sha = self.repo.scm.get_rev()
+        exp_commit = self.scm.repo.rev_parse(exp_rev)
+        parent = first(exp_commit.parents)
+        if parent is not None and parent.hexsha == baseline_sha:
+            return
+        raise BaselineMismatchError(parent, baseline_sha)
+
+    def _unstash_workspace(self):
+        # Essentially we want `git stash pop` with `-X ours` merge strategy
+        # to prefer the applied experiment changes over stashed workspace
+        # changes. git stash doesn't support merge strategy parameters, but we
+        # can do it ourselves with checkout/reset.
+        from git.exc import GitCommandError
+
+        logger.debug("Unstashing workspace changes.")
+        git_repo = self.repo.scm.repo.git
+
+        # stage workspace changes, then apply stashed changes on top
+        git_repo.add(A=True)
+        try:
+            git_repo.stash("apply", "stash@{0}")
+        except GitCommandError:
+            # stash apply will return error code on merge conflicts,
+            # prefer workspace changes over stash changes
+            git_repo.checkout("--ours", "--", ".")
+
+        # unstage changes and drop the stash entry
+        git_repo.reset("HEAD")
+        git_repo.stash("drop", "stash@{0}")
+
+    @scm_locked
+    def get_baseline(self, rev):
+        """Return the baseline rev for an experiment rev."""
+        from git.exc import GitCommandError
+
+        rev = self.scm.resolve_rev(rev)
+        try:
+            name = self.scm.repo.git.name_rev(rev, name_only=True)
+        except GitCommandError:
+            return None
+        if not name:
+            return None
+        if name in ("undefined", "stash"):
+            _, baseline = self.stash_revs.get(rev, (None, None))
+            return baseline
+        m = self.BRANCH_RE.match(name)
+        if m:
+            return self.scm.resolve_rev(m.group("baseline_rev"))
+        return None
 
     def checkout(self, *args, **kwargs):
         from dvc.repo.experiments.checkout import checkout

dvc/repo/experiments/executor.py

@@ -2,14 +2,15 @@ import logging
 import os
 import pickle
 from tempfile import TemporaryDirectory
+from typing import Iterable
 
 from funcy import cached_property
 
 from dvc.path_info import PathInfo
 from dvc.stage import PipelineStage
 from dvc.tree.base import BaseTree
-from dvc.utils import relpath
-from dvc.utils.fs import copy_fobj_to_file, makedirs
+from dvc.tree.local import LocalTree
+from dvc.tree.repo import RepoTree
 
 logger = logging.getLogger(__name__)
 
@@ -18,7 +19,6 @@ class ExperimentExecutor:
     """Base class for executing experiments in parallel.
 
     Args:
-        src_tree: source tree for this experiment.
         baseline_rev: baseline revision that this experiment is derived from.
 
     Optional keyword args:
@@ -26,14 +26,28 @@ class ExperimentExecutor:
         repro_kwargs: Keyword args to be passed into reproduce.
     """
 
-    def __init__(self, src_tree: BaseTree, baseline_rev: str, **kwargs):
-        self.src_tree = src_tree
+    def __init__(self, baseline_rev: str, **kwargs):
         self.baseline_rev = baseline_rev
         self.repro_args = kwargs.pop("repro_args", [])
         self.repro_kwargs = kwargs.pop("repro_kwargs", {})
 
-    def run(self):
-        pass
+    @property
+    def tree(self) -> BaseTree:
+        raise NotImplementedError
+
+    @staticmethod
+    def reproduce(dvc_dir, cwd=None, **kwargs):
+        raise NotImplementedError
+
+    def collect_output(self) -> Iterable["PathInfo"]:
+        """Iterate over output pathnames for this executor.
+
+        For DVC outs, only the .dvc file path will be yielded. DVC outs
+        themselves should be fetched from remote executor cache in the normal
+        fetch/pull way. For local executors outs will already be available via
+        shared local cache.
+        """
+        raise NotImplementedError
 
     def cleanup(self):
         pass
@@ -55,26 +69,29 @@ class ExperimentExecutor:
 
 
 class LocalExecutor(ExperimentExecutor):
-    """Local machine exepriment executor."""
+    """Local machine experiment executor."""
+
+    def __init__(self, baseline_rev: str, **kwargs):
+        from dvc.repo import Repo
 
-    def __init__(self, src_tree: BaseTree, baseline_rev: str, **kwargs):
         dvc_dir = kwargs.pop("dvc_dir")
         cache_dir = kwargs.pop("cache_dir")
-        super().__init__(src_tree, baseline_rev, **kwargs)
+        super().__init__(baseline_rev, **kwargs)
         self.tmp_dir = TemporaryDirectory()
-        logger.debug("Init local executor in dir '%s'.", self.tmp_dir)
+
+        # init empty DVC repo (will be overwritten when input is uploaded)
+        Repo.init(root_dir=self.tmp_dir.name, no_scm=True)
+        logger.debug(
+            "Init local executor in dir '%s' with baseline '%s'.",
+            self.tmp_dir,
+            baseline_rev[:7],
+        )
         self.dvc_dir = os.path.join(self.tmp_dir.name, dvc_dir)
-        try:
-            for fname in src_tree.walk_files(src_tree.tree_root):
-                dest = self.path_info / relpath(fname, src_tree.tree_root)
-                if not os.path.exists(dest.parent):
-                    makedirs(dest.parent)
-                with src_tree.open(fname, "rb") as fobj:
-                    copy_fobj_to_file(fobj, dest)
-        except Exception:
-            self.tmp_dir.cleanup()
-            raise
         self._config(cache_dir)
+        self._tree = LocalTree(self.dvc, {"url": self.dvc.root_dir})
+        # override default CACHE_MODE since files must be writable in order
+        # to run repro
+        self._tree.CACHE_MODE = 0o644
 
     def _config(self, cache_dir):
         local_config = os.path.join(self.dvc_dir, "config.local")
@@ -93,6 +110,10 @@ class LocalExecutor(ExperimentExecutor):
     def path_info(self):
         return PathInfo(self.tmp_dir.name)
 
+    @property
+    def tree(self):
+        return self._tree
+
     @staticmethod
     def reproduce(dvc_dir, cwd=None, **kwargs):
         """Run dvc repro and return the result."""
@@ -126,6 +147,16 @@ class LocalExecutor(ExperimentExecutor):
         # multiprocessing calls
         return hash_exp(stages + unchanged)
 
+    def collect_output(self) -> Iterable["PathInfo"]:
+        repo_tree = RepoTree(self.dvc)
+        yield from self.collect_files(self.tree, repo_tree)
+
+    @staticmethod
+    def collect_files(tree: BaseTree, repo_tree: RepoTree):
+        for fname in repo_tree.walk_files(repo_tree.root_dir, dvcfiles=True):
+            if not repo_tree.isdvc(fname):
+                yield tree.path_info / fname.relative_to(repo_tree.root_dir)
+
     def cleanup(self):
         logger.debug("Removing tmpdir '%s'", self.tmp_dir)
         self.tmp_dir.cleanup()

dvc/repo/experiments/show.py

@@ -1,6 +1,6 @@
 import logging
-import re
 from collections import OrderedDict, defaultdict
+from datetime import datetime
 
 from dvc.repo import locked
 from dvc.repo.metrics.show import _collect_metrics, _read_metrics
@@ -9,12 +9,15 @@ from dvc.repo.params.show import _collect_configs, _read_params
 logger = logging.getLogger(__name__)
 
 
-EXP_RE = re.compile(r"(?P<rev_sha>[a-f0-9]{7})-(?P<exp_sha>[a-f0-9]+)")
-
-
 def _collect_experiment(repo, branch, stash=False):
     res = defaultdict(dict)
     for rev in repo.brancher(revs=[branch]):
+        if rev == "workspace":
+            res["timestamp"] = None
+        else:
+            commit = repo.scm.repo.rev_parse(rev)
+            res["timestamp"] = datetime.fromtimestamp(commit.committed_date)
+
         configs = _collect_configs(repo)
         params = _read_params(repo, configs, rev)
         if params:
@@ -53,9 +56,9 @@ def show(
 
     # collect reproduced experiments
     for exp_branch in repo.experiments.scm.list_branches():
-        m = re.match(EXP_RE, exp_branch)
+        m = repo.experiments.BRANCH_RE.match(exp_branch)
         if m:
-            rev = repo.scm.resolve_rev(m.group("rev_sha"))
+            rev = repo.scm.resolve_rev(m.group("baseline_rev"))
             if rev in revs:
                 exp_rev = repo.experiments.scm.resolve_rev(exp_branch)
                 with repo.experiments.chdir():
@@ -66,10 +69,11 @@ def show(
 
     # collect queued (not yet reproduced) experiments
     for stash_rev, (_, baseline_rev) in repo.experiments.stash_revs.items():
-        with repo.experiments.chdir():
-            experiment = _collect_experiment(
-                repo.experiments.exp_dvc, stash_rev, stash=True
-            )
-        res[baseline_rev][stash_rev] = experiment
+        if baseline_rev in revs:
+            with repo.experiments.chdir():
+                experiment = _collect_experiment(
+                    repo.experiments.exp_dvc, stash_rev, stash=True
+                )
+            res[baseline_rev][stash_rev] = experiment
 
     return res

dvc/repo/gc.py

@@ -50,6 +50,7 @@ def gc(
     )
 
     from contextlib import ExitStack
+
     from dvc.repo import Repo
 
     if not repos:

dvc/repo/get.py

@@ -20,8 +20,8 @@ class GetDVCFileError(DvcException):
 
 @staticmethod
 def get(url, path, out=None, rev=None):
-    from dvc.external_repo import external_repo
     from dvc.dvcfile import is_valid_filename
+    from dvc.external_repo import external_repo
 
     out = resolve_output(path, out)
 

dvc/repo/graph.py

@@ -1,5 +1,6 @@
 def check_acyclic(graph):
     import networkx as nx
+
     from dvc.exceptions import CyclicGraphError
 
     try:

dvc/repo/init.py

@@ -6,6 +6,7 @@ import colorama
 from dvc import analytics
 from dvc.config import Config
 from dvc.exceptions import InitError, InvalidArgumentError
+from dvc.ignore import init as init_dvcignore
 from dvc.repo import Repo
 from dvc.scm import SCM
 from dvc.scm.base import SCMError
@@ -99,9 +100,13 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
         with config.edit() as conf:
             conf["core"]["no_scm"] = True
 
+    dvcignore = init_dvcignore(root_dir)
+
     proj = Repo(root_dir)
 
-    scm.add([config.files["repo"], proj.plot_templates.templates_dir])
+    scm.add(
+        [config.files["repo"], dvcignore, proj.plot_templates.templates_dir]
+    )
 
     if scm.ignore_file:
         scm.add([os.path.join(dvc_dir, scm.ignore_file)])

dvc/repo/ls.py

@@ -1,3 +1,5 @@
+from itertools import chain
+
 from dvc.exceptions import PathMissingError
 from dvc.path_info import PathInfo
 
@@ -48,7 +50,7 @@ def ls(
 
 
 def _ls(repo, path_info, recursive=None, dvc_only=False):
-    from dvc.repo.tree import RepoTree
+    from dvc.tree.repo import RepoTree
 
     def onerror(exc):
         raise exc
@@ -57,48 +59,32 @@ def _ls(repo, path_info, recursive=None, dvc_only=False):
     # fetch directory listings, but don't want to fetch file contents.
     tree = RepoTree(repo, stream=True)
 
-    ret = {}
+    infos = []
     try:
         for root, dirs, files in tree.walk(
             path_info.fspath, onerror=onerror, dvcfiles=True
         ):
-            for fname in files:
-                info = PathInfo(root) / fname
-                dvc = tree.isdvc(info)
-                if dvc or not dvc_only:
-                    path = str(info.relative_to(path_info))
-                    ret[path] = {
-                        "isout": dvc,
-                        "isdir": False,
-                        "isexec": False if dvc else tree.isexec(info),
-                    }
-
+            entries = chain(files, dirs) if not recursive else files
+            infos.extend(PathInfo(root) / entry for entry in entries)
             if not recursive:
-                for dname in dirs:
-                    info = PathInfo(root) / dname
-                    if not dvc_only or (
-                        tree.dvctree and tree.dvctree.exists(info)
-                    ):
-                        dvc = tree.isdvc(info)
-                        path = str(info.relative_to(path_info))
-                        ret[path] = {
-                            "isout": dvc,
-                            "isdir": True,
-                            "isexec": False if dvc else tree.isexec(info),
-                        }
                 break
     except NotADirectoryError:
-        dvc = tree.isdvc(path_info)
-        if dvc or not dvc_only:
-            return {
-                path_info.name: {
-                    "isout": dvc,
-                    "isdir": False,
-                    "isexec": False if dvc else tree.isexec(path_info),
-                }
-            }
-        return {}
+        infos.append(path_info)
     except FileNotFoundError:
         return {}
 
+    ret = {}
+    for info in infos:
+        metadata = tree.metadata(info)
+        if metadata.output_exists or not dvc_only:
+            path = (
+                path_info.name
+                if path_info == info
+                else str(info.relative_to(path_info))
+            )
+            ret[path] = {
+                "isout": metadata.is_output,
+                "isdir": metadata.isdir,
+                "isexec": metadata.is_exec,
+            }
     return ret

dvc/repo/metrics/show.py

@@ -1,12 +1,11 @@
 import logging
 import os
 
-import yaml
-
 from dvc.exceptions import NoMetricsError
 from dvc.path_info import PathInfo
 from dvc.repo import locked
-from dvc.repo.tree import RepoTree
+from dvc.tree.repo import RepoTree
+from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml
 
 logger = logging.getLogger(__name__)
 
@@ -36,7 +35,7 @@ def _collect_metrics(repo, targets, recursive):
     return list(filter(_filter, metrics))
 
 
-def _extract_metrics(metrics):
+def _extract_metrics(metrics, path, rev):
     if isinstance(metrics, (int, float)):
         return metrics
 
@@ -45,9 +44,18 @@ def _extract_metrics(metrics):
 
     ret = {}
     for key, val in metrics.items():
-        m = _extract_metrics(val)
+        m = _extract_metrics(val, path, rev)
         if m:
             ret[key] = m
+        else:
+            logger.debug(
+                "Could not parse '%s' metric from '%s' at '%s' "
+                "due to its unsupported type: '%s'",
+                key,
+                path,
+                rev,
+                type(val).__name__,
+            )
 
     return ret
 
@@ -61,16 +69,14 @@ def _read_metrics(repo, metrics, rev):
             continue
 
         try:
-            with tree.open(metric, "r") as fobj:
-                # NOTE this also supports JSON
-                val = yaml.safe_load(fobj)
-        except (FileNotFoundError, yaml.YAMLError):
+            val = load_yaml(metric, tree=tree)
+        except (FileNotFoundError, YAMLFileCorruptedError):
             logger.debug(
                 "failed to read '%s' on '%s'", metric, rev, exc_info=True
             )
             continue
 
-        val = _extract_metrics(val)
+        val = _extract_metrics(val, metric, rev)
         if val:
             res[str(metric)] = val
 
@@ -88,6 +94,7 @@ def show(
     all_commits=False,
 ):
     res = {}
+    metrics_found = False
 
     for rev in repo.brancher(
         revs=revs,
@@ -96,13 +103,27 @@ def show(
         all_commits=all_commits,
     ):
         metrics = _collect_metrics(repo, targets, recursive)
+
+        if not metrics_found and metrics:
+            metrics_found = True
+
         vals = _read_metrics(repo, metrics, rev)
 
         if vals:
             res[rev] = vals
 
     if not res:
-        raise NoMetricsError()
+        if metrics_found:
+            msg = (
+                "Could not parse metric files. Use `-v` option to see more "
+                "details."
+            )
+        else:
+            msg = (
+                "no metric files in this repository. Use `-m/-M` options for "
+                "`dvc run` to mark stage outputs as  metrics."
+            )
+        raise NoMetricsError(msg)
 
     # Hide workspace metrics if they are the same as in the active branch
     try:

dvc/repo/move.py

@@ -34,6 +34,7 @@ def move(self, from_path, to_path):
     """
     import dvc.output as Output
     from dvc.stage import Stage
+
     from ..dvcfile import DVC_FILE_SUFFIX, Dvcfile
 
     from_out = Output.loads_from(Stage(self), [from_path])[0]

dvc/repo/params/show.py

@@ -1,12 +1,10 @@
 import logging
 
-import toml
-import yaml
-
 from dvc.dependency.param import ParamsDependency
 from dvc.exceptions import DvcException
 from dvc.path_info import PathInfo
 from dvc.repo import locked
+from dvc.utils.serialize import LOADERS, ParseError
 
 logger = logging.getLogger(__name__)
 
@@ -33,16 +31,15 @@ def _read_params(repo, configs, rev):
         if not repo.tree.exists(config):
             continue
 
-        with repo.tree.open(config, "r") as fobj:
-            try:
-                res[str(config)] = ParamsDependency.PARAMS_FILE_LOADERS[
-                    config.suffix.lower()
-                ](fobj)
-            except (yaml.YAMLError, toml.TomlDecodeError):
-                logger.debug(
-                    "failed to read '%s' on '%s'", config, rev, exc_info=True
-                )
-                continue
+        suffix = config.suffix.lower()
+        loader = LOADERS[suffix]
+        try:
+            res[str(config)] = loader(config, tree=repo.tree)
+        except ParseError:
+            logger.debug(
+                "failed to read '%s' on '%s'", config, rev, exc_info=True
+            )
+            continue
 
     return res
 

dvc/repo/plots/__init__.py

@@ -3,8 +3,8 @@ import logging
 from funcy import first, project
 
 from dvc.exceptions import DvcException, NoPlotsError, OutputNotFoundError
-from dvc.repo.tree import RepoTree
 from dvc.schema import PLOT_PROPS
+from dvc.tree.repo import RepoTree
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
@@ -75,7 +75,7 @@ class Plots:
             for datafile, desc in plots.items()
         }
 
-    def show(self, targets=None, revs=None, props=None):
+    def show(self, targets=None, revs=None, props=None, templates=None):
         from .data import NoMetricInHistoryError
 
         data = self.collect(targets, revs)
@@ -90,7 +90,9 @@ class Plots:
         if not data:
             raise NoPlotsError()
 
-        return self.render(data, revs, props, self.repo.plot_templates)
+        if templates is None:
+            templates = self.repo.plot_templates
+        return self.render(data, revs, props, templates)
 
     def diff(self, *args, **kwargs):
         from .diff import diff
@@ -210,7 +212,7 @@ def _prepare_plots(data, revs, props):
 
 
 def _render(datafile, datas, props, templates):
-    from .data import plot_data, PlotData
+    from .data import PlotData, plot_data
 
     # Copy it to not modify a passed value
     props = props.copy()

dvc/repo/plots/data.py

@@ -5,11 +5,10 @@ import os
 from collections import OrderedDict
 from copy import copy
 
-import yaml
 from funcy import first
-from yaml import SafeLoader
 
 from dvc.exceptions import DvcException
+from dvc.utils.serialize import loads_yaml
 
 
 class PlotMetricTypeError(DvcException):
@@ -101,7 +100,7 @@ def _apply_path(data, path=None, **kwargs):
 def _lists(dictionary):
     for _, value in dictionary.items():
         if isinstance(value, dict):
-            yield from (_lists(value))
+            yield from _lists(value)
         elif isinstance(value, list):
             yield value
 
@@ -114,10 +113,12 @@ def _find_data(data, fields=None, **kwargs):
         # just look for first list of dicts
         fields = set()
 
-    for l in _lists(data):
-        if all(isinstance(dp, dict) for dp in l):
-            if set(first(l).keys()) & fields == fields:
-                return l
+    for lst in _lists(data):
+        if (
+            all(isinstance(dp, dict) for dp in lst)
+            and set(first(lst).keys()) & fields == fields
+        ):
+            return lst
     raise PlotDataStructureError()
 
 
@@ -206,18 +207,7 @@ class CSVPlotData(PlotData):
 
 class YAMLPlotData(PlotData):
     def raw(self, **kwargs):
-        class OrderedLoader(SafeLoader):
-            pass
-
-        def construct_mapping(loader, node):
-            loader.flatten_mapping(node)
-            return OrderedDict(loader.construct_pairs(node))
-
-        OrderedLoader.add_constructor(
-            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping
-        )
-
-        return yaml.load(self.content, OrderedLoader)
+        return loads_yaml(self.content, typ="rt")
 
     def _processors(self):
         parent_processors = super()._processors()

dvc/repo/plots/diff.py

@@ -1,13 +1,23 @@
-def _revisions(revs, is_dirty):
+def _revisions(repo, revs, experiment):
     revisions = revs or []
+    if experiment and len(revisions) == 1:
+        baseline = repo.experiments.get_baseline(revisions[0])
+        if baseline:
+            revisions.append(baseline[:7])
     if len(revisions) <= 1:
-        if len(revisions) == 0 and is_dirty:
+        if len(revisions) == 0 and repo.scm.is_dirty():
             revisions.append("HEAD")
         revisions.append("workspace")
     return revisions
 
 
-def diff(repo, *args, revs=None, **kwargs):
-    return repo.plots.show(
-        *args, revs=_revisions(revs, repo.scm.is_dirty()), **kwargs
+def diff(repo, *args, revs=None, experiment=False, **kwargs):
+    if experiment:
+        # use experiments repo brancher with templates from the main repo
+        kwargs["templates"] = repo.plot_templates
+        plots_repo = repo.experiments.exp_dvc
+    else:
+        plots_repo = repo
+    return plots_repo.plots.show(
+        *args, revs=_revisions(repo, revs, experiment), **kwargs
     )

dvc/repo/reproduce.py

@@ -60,7 +60,7 @@ def reproduce(
     recursive=False,
     pipeline=False,
     all_pipelines=False,
-    **kwargs
+    **kwargs,
 ):
     from dvc.utils import parse_target
 
@@ -71,6 +71,7 @@ def reproduce(
         )
 
     experiment = kwargs.pop("experiment", False)
+    params = _parse_params(kwargs.pop("params", []))
     queue = kwargs.pop("queue", False)
     run_all = kwargs.pop("run_all", False)
     jobs = kwargs.pop("jobs", 1)
@@ -81,6 +82,7 @@ def reproduce(
                 target=target,
                 recursive=recursive,
                 all_pipelines=all_pipelines,
+                params=params,
                 queue=queue,
                 run_all=run_all,
                 jobs=jobs,
@@ -116,6 +118,33 @@ def reproduce(
     return _reproduce_stages(active_graph, targets, **kwargs)
 
 
+def _parse_params(path_params):
+    from flatten_json import unflatten
+    from ruamel.yaml import YAMLError
+
+    from dvc.dependency.param import ParamsDependency
+    from dvc.utils.serialize import loads_yaml
+
+    ret = {}
+    for path_param in path_params:
+        path, _, params_str = path_param.rpartition(":")
+        # remove empty strings from params, on condition such as `-p "file1:"`
+        params = {}
+        for param_str in filter(bool, params_str.split(",")):
+            try:
+                # interpret value strings using YAML rules
+                key, value = param_str.split("=")
+                params[key] = loads_yaml(value)
+            except (ValueError, YAMLError):
+                raise InvalidArgumentError(
+                    f"Invalid param/value pair '{param_str}'"
+                )
+        if not path:
+            path = ParamsDependency.DEFAULT_PARAMS_FILE
+        ret[path] = unflatten(params, ".")
+    return ret
+
+
 def _reproduce_experiments(repo, run_all=False, jobs=1, **kwargs):
     if run_all:
         return repo.experiments.reproduce_queued(jobs=jobs)

dvc/repo/run.py

@@ -36,7 +36,7 @@ def parse_params(path_params):
 
 
 def _get_file_path(kwargs):
-    from dvc.dvcfile import DVC_FILE_SUFFIX, DVC_FILE
+    from dvc.dvcfile import DVC_FILE, DVC_FILE_SUFFIX
 
     out = first(
         concat(
@@ -76,8 +76,8 @@ def _check_stage_exists(dvcfile, stage):
 @locked
 @scm_context
 def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
+    from dvc.dvcfile import PIPELINE_FILE, Dvcfile
     from dvc.stage import Stage, create_stage
-    from dvc.dvcfile import Dvcfile, PIPELINE_FILE
 
     if not kwargs.get("cmd"):
         raise InvalidArgumentError("command is not specified")

dvc/repo/status.py

@@ -10,31 +10,32 @@ from . import locked
 logger = logging.getLogger(__name__)
 
 
-def _joint_status(stages):
+def _joint_status(pairs):
     status_info = {}
 
-    for stage in stages:
+    for stage, filter_info in pairs:
         if stage.frozen and not stage.is_repo_import:
             logger.warning(
                 "{} is frozen. Its dependencies are"
                 " not going to be shown in the status output.".format(stage)
             )
-
-        status_info.update(stage.status(check_updates=True))
+        if not filter_info:
+            status_info.update(stage.status(check_updates=True))
+        else:
+            for out in stage.filter_outs(filter_info):
+                status_info.update(out.status())
 
     return status_info
 
 
 def _local_status(self, targets=None, with_deps=False, recursive=False):
-    if targets:
-        stages = cat(
-            self.collect(t, with_deps=with_deps, recursive=recursive)
-            for t in targets
-        )
-    else:
-        stages = self.collect(None, with_deps=with_deps, recursive=recursive)
+    targets = targets or [None]
+    pairs = cat(
+        self.collect_granular(t, with_deps=with_deps, recursive=recursive)
+        for t in targets
+    )
 
-    return _joint_status(stages)
+    return _joint_status(pairs)
 
 
 def _cloud_status(
@@ -52,6 +53,7 @@ def _cloud_status(
 
     - new: Remote doesn't have the file
     - deleted: File is no longer in the local cache
+    - missing: File doesn't exist neither in the cache, neither in remote
 
     Example:
             Given the following commands:
@@ -89,14 +91,20 @@ def _cloud_status(
     )
 
     ret = {}
-    status_info = self.cloud.status(used, jobs, remote=remote)
+    status_info = self.cloud.status(
+        used, jobs, remote=remote, log_missing=False
+    )
     for info in status_info.values():
         name = info["name"]
         status_ = info["status"]
-        if status_ in [cloud.STATUS_OK, cloud.STATUS_MISSING]:
+        if status_ == cloud.STATUS_OK:
             continue
 
-        prefix_map = {cloud.STATUS_DELETED: "deleted", cloud.STATUS_NEW: "new"}
+        prefix_map = {
+            cloud.STATUS_DELETED: "deleted",
+            cloud.STATUS_NEW: "new",
+            cloud.STATUS_MISSING: "missing",
+        }
 
         ret[name] = prefix_map[status_]
 

dvc/repo/tree.py

@@ -1,444 +0,0 @@
-import logging
-import os
-
-from dvc.dvcfile import is_valid_filename
-from dvc.exceptions import OutputNotFoundError
-from dvc.path_info import PathInfo
-from dvc.tree.base import BaseTree, RemoteActionNotImplemented
-from dvc.utils import file_md5
-from dvc.utils.fs import copy_fobj_to_file, makedirs
-
-logger = logging.getLogger(__name__)
-
-
-class DvcTree(BaseTree):  # pylint:disable=abstract-method
-    """DVC repo tree.
-
-    Args:
-        repo: DVC repo.
-        fetch: if True, uncached DVC outs will be fetched on `open()`.
-        stream: if True, uncached DVC outs will be streamed directly from
-            remote on `open()`.
-
-    `stream` takes precedence over `fetch`. If `stream` is enabled and
-    a remote does not support streaming, uncached DVC outs will be fetched
-    as a fallback.
-    """
-
-    def __init__(self, repo, fetch=False, stream=False):
-        super().__init__(repo, {"url": repo.root_dir})
-        self.fetch = fetch
-        self.stream = stream
-
-    def _find_outs(self, path, *args, **kwargs):
-        outs = self.repo.find_outs_by_path(path, *args, **kwargs)
-
-        def _is_cached(out):
-            return out.use_cache
-
-        outs = list(filter(_is_cached, outs))
-        if not outs:
-            raise OutputNotFoundError(path, self.repo)
-
-        return outs
-
-    def _get_granular_checksum(self, path, out, remote=None):
-        assert isinstance(path, PathInfo)
-        if not self.fetch and not self.stream:
-            raise FileNotFoundError
-        dir_cache = out.get_dir_cache(remote=remote)
-        for entry in dir_cache:
-            entry_relpath = entry[out.tree.PARAM_RELPATH]
-            if os.name == "nt":
-                entry_relpath = entry_relpath.replace("/", os.sep)
-            if path == out.path_info / entry_relpath:
-                return entry[out.tree.PARAM_CHECKSUM]
-        raise FileNotFoundError
-
-    def open(
-        self, path, mode="r", encoding="utf-8", remote=None
-    ):  # pylint: disable=arguments-differ
-        try:
-            outs = self._find_outs(path, strict=False)
-        except OutputNotFoundError as exc:
-            raise FileNotFoundError from exc
-
-        # NOTE: this handles both dirty and checkout-ed out at the same time
-        if self.repo.tree.exists(path):
-            return self.repo.tree.open(path, mode=mode, encoding=encoding)
-
-        if len(outs) != 1 or (
-            outs[0].is_dir_checksum and path == outs[0].path_info
-        ):
-            raise IsADirectoryError
-
-        out = outs[0]
-        if out.changed_cache(filter_info=path):
-            if not self.fetch and not self.stream:
-                raise FileNotFoundError
-
-            remote_obj = self.repo.cloud.get_remote(remote)
-            if self.stream:
-                if out.is_dir_checksum:
-                    checksum = self._get_granular_checksum(path, out)
-                else:
-                    checksum = out.checksum
-                try:
-                    remote_info = remote_obj.tree.hash_to_path_info(checksum)
-                    return remote_obj.tree.open(
-                        remote_info, mode=mode, encoding=encoding
-                    )
-                except RemoteActionNotImplemented:
-                    pass
-            cache_info = out.get_used_cache(filter_info=path, remote=remote)
-            self.repo.cloud.pull(cache_info, remote=remote)
-
-        if out.is_dir_checksum:
-            checksum = self._get_granular_checksum(path, out)
-            cache_path = out.cache.tree.hash_to_path_info(checksum).url
-        else:
-            cache_path = out.cache_path
-        return open(cache_path, mode=mode, encoding=encoding)
-
-    def exists(self, path):  # pylint: disable=arguments-differ
-        try:
-            self._find_outs(path, strict=False, recursive=True)
-            return True
-        except OutputNotFoundError:
-            return False
-
-    def isdir(self, path):  # pylint: disable=arguments-differ
-        if not self.exists(path):
-            return False
-
-        path_info = PathInfo(os.path.abspath(path))
-        outs = self._find_outs(path, strict=False, recursive=True)
-        if len(outs) != 1:
-            return True
-
-        out = outs[0]
-        if not out.is_dir_checksum:
-            if out.path_info != path_info:
-                return True
-            return False
-
-        if out.path_info == path_info:
-            return True
-
-        # for dir checksum, we need to check if this is a file inside the
-        # directory
-        try:
-            self._get_granular_checksum(path_info, out)
-            return False
-        except FileNotFoundError:
-            return True
-
-    def isfile(self, path):  # pylint: disable=arguments-differ
-        if not self.exists(path):
-            return False
-
-        return not self.isdir(path)
-
-    def _add_dir(self, top, trie, out, download_callback=None, **kwargs):
-        if not self.fetch and not self.stream:
-            return
-
-        # pull dir cache if needed
-        dir_cache = out.get_dir_cache(**kwargs)
-
-        # pull dir contents if needed
-        if self.fetch:
-            if out.changed_cache(filter_info=top):
-                used_cache = out.get_used_cache(filter_info=top)
-                downloaded = self.repo.cloud.pull(used_cache, **kwargs)
-                if download_callback:
-                    download_callback(downloaded)
-
-        for entry in dir_cache:
-            entry_relpath = entry[out.tree.PARAM_RELPATH]
-            if os.name == "nt":
-                entry_relpath = entry_relpath.replace("/", os.sep)
-            path_info = out.path_info / entry_relpath
-            trie[path_info.parts] = None
-
-    def _walk(self, root, trie, topdown=True, **kwargs):
-        dirs = set()
-        files = []
-
-        out = trie.get(root.parts)
-        if out and out.is_dir_checksum:
-            self._add_dir(root, trie, out, **kwargs)
-
-        root_len = len(root.parts)
-        for key, out in trie.iteritems(prefix=root.parts):  # noqa: B301
-            if key == root.parts:
-                continue
-
-            name = key[root_len]
-            if len(key) > root_len + 1 or (out and out.is_dir_checksum):
-                dirs.add(name)
-                continue
-
-            files.append(name)
-
-        assert topdown
-        dirs = list(dirs)
-        yield root.fspath, dirs, files
-
-        for dname in dirs:
-            yield from self._walk(root / dname, trie)
-
-    def walk(self, top, topdown=True, onerror=None, **kwargs):
-        from pygtrie import Trie
-
-        assert topdown
-
-        if not self.exists(top):
-            if onerror is not None:
-                onerror(FileNotFoundError(top))
-            return
-
-        if not self.isdir(top):
-            if onerror is not None:
-                onerror(NotADirectoryError(top))
-            return
-
-        root = PathInfo(os.path.abspath(top))
-        outs = self._find_outs(top, recursive=True, strict=False)
-
-        trie = Trie()
-
-        for out in outs:
-            trie[out.path_info.parts] = out
-
-            if out.is_dir_checksum and root.isin_or_eq(out.path_info):
-                self._add_dir(top, trie, out, **kwargs)
-
-        yield from self._walk(root, trie, topdown=topdown, **kwargs)
-
-    def isdvc(self, path, **kwargs):
-        try:
-            return len(self._find_outs(path, **kwargs)) == 1
-        except OutputNotFoundError:
-            pass
-        return False
-
-    def isexec(self, path):  # pylint: disable=unused-argument
-        return False
-
-    def get_file_hash(self, path_info):
-        outs = self._find_outs(path_info, strict=False)
-        if len(outs) != 1:
-            raise OutputNotFoundError
-        out = outs[0]
-        if out.is_dir_checksum:
-            return self._get_granular_checksum(path_info, out)
-        return out.checksum
-
-
-class RepoTree(BaseTree):  # pylint:disable=abstract-method
-    """DVC + git-tracked files tree.
-
-    Args:
-        repo: DVC or git repo.
-
-    Any kwargs will be passed to `DvcTree()`.
-    """
-
-    def __init__(self, repo, **kwargs):
-        super().__init__(repo, {"url": repo.root_dir})
-        if hasattr(repo, "dvc_dir"):
-            self.dvctree = DvcTree(repo, **kwargs)
-        else:
-            # git-only erepo's do not need dvctree
-            self.dvctree = None
-
-    @property
-    def fetch(self):
-        if self.dvctree:
-            return self.dvctree.fetch
-        return False
-
-    @property
-    def stream(self):
-        if self.dvctree:
-            return self.dvctree.stream
-        return False
-
-    def open(
-        self, path, mode="r", encoding="utf-8", **kwargs
-    ):  # pylint: disable=arguments-differ
-        if "b" in mode:
-            encoding = None
-
-        if self.dvctree and self.dvctree.exists(path):
-            return self.dvctree.open(
-                path, mode=mode, encoding=encoding, **kwargs
-            )
-        return self.repo.tree.open(path, mode=mode, encoding=encoding)
-
-    def exists(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        return self.repo.tree.exists(path) or (
-            self.dvctree and self.dvctree.exists(path)
-        )
-
-    def isdir(self, path):  # pylint: disable=arguments-differ
-        return self.repo.tree.isdir(path) or (
-            self.dvctree and self.dvctree.isdir(path)
-        )
-
-    def isdvc(self, path, **kwargs):
-        return self.dvctree is not None and self.dvctree.isdvc(path, **kwargs)
-
-    def isfile(self, path):  # pylint: disable=arguments-differ
-        return self.repo.tree.isfile(path) or (
-            self.dvctree and self.dvctree.isfile(path)
-        )
-
-    def isexec(self, path):
-        if self.dvctree and self.dvctree.exists(path):
-            return self.dvctree.isexec(path)
-        return self.repo.tree.isexec(path)
-
-    def stat(self, path):
-        return self.repo.tree.stat(path)
-
-    def _walk_one(self, walk):
-        try:
-            root, dirs, files = next(walk)
-        except StopIteration:
-            return
-        yield root, dirs, files
-        for _ in dirs:
-            yield from self._walk_one(walk)
-
-    def _walk(self, dvc_walk, repo_walk, dvcfiles=False):
-        try:
-            _, dvc_dirs, dvc_fnames = next(dvc_walk)
-            repo_root, repo_dirs, repo_fnames = next(repo_walk)
-        except StopIteration:
-            return
-
-        # separate subdirs into shared dirs, dvc-only dirs, repo-only dirs
-        dvc_set = set(dvc_dirs)
-        repo_set = set(repo_dirs)
-        dvc_only = list(dvc_set - repo_set)
-        repo_only = list(repo_set - dvc_set)
-        shared = list(dvc_set & repo_set)
-        dirs = shared + dvc_only + repo_only
-
-        # merge file lists
-        files = {
-            fname
-            for fname in dvc_fnames + repo_fnames
-            if dvcfiles or not is_valid_filename(fname)
-        }
-
-        yield repo_root, dirs, list(files)
-
-        # set dir order for next recursion level - shared dirs first so that
-        # next() for both generators recurses into the same shared directory
-        dvc_dirs[:] = [dirname for dirname in dirs if dirname in dvc_set]
-        repo_dirs[:] = [dirname for dirname in dirs if dirname in repo_set]
-
-        for dirname in dirs:
-            if dirname in shared:
-                yield from self._walk(dvc_walk, repo_walk, dvcfiles=dvcfiles)
-            elif dirname in dvc_set:
-                yield from self._walk_one(dvc_walk)
-            elif dirname in repo_set:
-                yield from self._walk_one(repo_walk)
-
-    def walk(
-        self, top, topdown=True, onerror=None, dvcfiles=False, **kwargs
-    ):  # pylint: disable=arguments-differ
-        """Walk and merge both DVC and repo trees.
-
-        Args:
-            top: path to walk from
-            topdown: if True, tree will be walked from top down.
-            onerror: if set, onerror function will be called if an error
-                occurs (by default errors are ignored).
-            dvcfiles: if True, dvcfiles will be included in the files list
-                for walked directories.
-
-        Any kwargs will be passed into methods used for fetching and/or
-        streaming DVC outs from remotes.
-        """
-        assert topdown
-
-        if not self.exists(top):
-            if onerror is not None:
-                onerror(FileNotFoundError(top))
-            return
-
-        if not self.isdir(top):
-            if onerror is not None:
-                onerror(NotADirectoryError(top))
-            return
-
-        dvc_exists = self.dvctree and self.dvctree.exists(top)
-        repo_exists = self.repo.tree.exists(top)
-        if dvc_exists and not repo_exists:
-            yield from self.dvctree.walk(
-                top, topdown=topdown, onerror=onerror, **kwargs
-            )
-            return
-        if repo_exists and not dvc_exists:
-            yield from self.repo.tree.walk(
-                top, topdown=topdown, onerror=onerror
-            )
-            return
-
-        dvc_walk = self.dvctree.walk(top, topdown=topdown, **kwargs)
-        repo_walk = self.repo.tree.walk(top, topdown=topdown)
-        yield from self._walk(dvc_walk, repo_walk, dvcfiles=dvcfiles)
-
-    def walk_files(self, top, **kwargs):  # pylint: disable=arguments-differ
-        for root, _, files in self.walk(top, **kwargs):
-            for fname in files:
-                yield PathInfo(root) / fname
-
-    def get_file_hash(self, path_info):
-        """Return file checksum for specified path.
-
-        If path_info is a DVC out, the pre-computed checksum for the file
-        will be used. If path_info is a git file, MD5 will be computed for
-        the git object.
-        """
-        if not self.exists(path_info):
-            raise FileNotFoundError
-        if self.dvctree and self.dvctree.exists(path_info):
-            try:
-                return self.dvctree.get_file_hash(path_info)
-            except OutputNotFoundError:
-                pass
-        return file_md5(path_info, self)[0]
-
-    def copytree(self, top, dest):
-        top = PathInfo(top)
-        dest = PathInfo(dest)
-
-        if not self.exists(top):
-            raise FileNotFoundError
-
-        if self.isfile(top):
-            makedirs(dest.parent, exist_ok=True)
-            with self.open(top, mode="rb") as fobj:
-                copy_fobj_to_file(fobj, dest)
-            return
-
-        for root, _, files in self.walk(top):
-            root = PathInfo(root)
-            dest_dir = dest / root.relative_to(top)
-            makedirs(dest_dir, exist_ok=True)
-            for fname in files:
-                src = root / fname
-                with self.open(src, mode="rb") as fobj:
-                    copy_fobj_to_file(fobj, dest_dir / fname)
-
-    @property
-    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
-        return self.repo.tree.hash_jobs

dvc/scm/git.py

@@ -5,7 +5,6 @@ import os
 import shlex
 from functools import partial
 
-import yaml
 from funcy import cached_property
 from pathspec.patterns import GitWildMatchPattern
 
@@ -20,6 +19,7 @@ from dvc.scm.base import (
 )
 from dvc.utils import fix_env, is_binary, relpath
 from dvc.utils.fs import path_isin
+from dvc.utils.serialize import modify_yaml
 
 logger = logging.getLogger(__name__)
 
@@ -307,7 +307,21 @@ class Git(Base):
 
         os.chmod(hook, 0o777)
 
+    def _install_merge_driver(self):
+        self.repo.git.config("merge.dvc.name", "DVC merge driver")
+        self.repo.git.config(
+            "merge.dvc.driver",
+            (
+                "dvc git-hook merge-driver "
+                "--ancestor %O "
+                "--our %A "
+                "--their %B "
+            ),
+        )
+
     def install(self, use_pre_commit_tool=False):
+        self._install_merge_driver()
+
         if not use_pre_commit_tool:
             self._verify_dvc_hooks()
             self._install_hook("post-checkout")
@@ -316,41 +330,32 @@ class Git(Base):
             return
 
         config_path = os.path.join(self.root_dir, ".pre-commit-config.yaml")
+        with modify_yaml(config_path) as config:
+            entry = {
+                "repo": "https://github.com/iterative/dvc",
+                "rev": "master",
+                "hooks": [
+                    {
+                        "id": "dvc-pre-commit",
+                        "language_version": "python3",
+                        "stages": ["commit"],
+                    },
+                    {
+                        "id": "dvc-pre-push",
+                        "language_version": "python3",
+                        "stages": ["push"],
+                    },
+                    {
+                        "id": "dvc-post-checkout",
+                        "language_version": "python3",
+                        "stages": ["post-checkout"],
+                        "always_run": True,
+                    },
+                ],
+            }
 
-        config = {}
-        if os.path.exists(config_path):
-            with open(config_path) as fobj:
-                config = yaml.safe_load(fobj)
-
-        entry = {
-            "repo": "https://github.com/iterative/dvc",
-            "rev": "master",
-            "hooks": [
-                {
-                    "id": "dvc-pre-commit",
-                    "language_version": "python3",
-                    "stages": ["commit"],
-                },
-                {
-                    "id": "dvc-pre-push",
-                    "language_version": "python3",
-                    "stages": ["push"],
-                },
-                {
-                    "id": "dvc-post-checkout",
-                    "language_version": "python3",
-                    "stages": ["post-checkout"],
-                    "always_run": True,
-                },
-            ],
-        }
-
-        if entry in config["repos"]:
-            return
-
-        config["repos"].append(entry)
-        with open(config_path, "w+") as fobj:
-            yaml.dump(config, fobj)
+            if entry not in config["repos"]:
+                config["repos"].append(entry)
 
     def cleanup_ignores(self):
         for path in self.ignored_paths:
@@ -393,9 +398,10 @@ class Git(Base):
         return self.repo.rev_parse("HEAD").hexsha
 
     def resolve_rev(self, rev):
-        from git.exc import BadName, GitCommandError
         from contextlib import suppress
 
+        from git.exc import BadName, GitCommandError
+
         def _resolve_rev(name):
             with suppress(BadName, GitCommandError):
                 try:
@@ -404,6 +410,8 @@ class Git(Base):
                 except NotImplementedError:
                     # Fall back to `git rev-parse` for advanced features
                     return self.repo.git.rev_parse(name)
+                except ValueError:
+                    raise RevError(f"unknown Git revision '{name}'")
 
         # Resolve across local names
         sha = _resolve_rev(rev)

dvc/stage/__init__.py

@@ -7,7 +7,7 @@ from funcy import cached_property, project
 
 import dvc.dependency as dependency
 import dvc.prompt as prompt
-from dvc.exceptions import CheckoutError, DvcException
+from dvc.exceptions import CheckoutError, DvcException, MergeError
 from dvc.utils import relpath
 
 from . import params
@@ -306,7 +306,12 @@ class Stage(params.StageParams):
     @rwlocked(read=["deps"], write=["outs"])
     def reproduce(self, interactive=False, **kwargs):
         if not (kwargs.get("force", False) or self.changed()):
-            logger.info("Stage '%s' didn't change, skipping", self.addressing)
+            if not isinstance(self, PipelineStage) and self.is_data_source:
+                logger.info("'%s' didn't change, skipping", self.addressing)
+            else:
+                logger.info(
+                    "Stage '%s' didn't change, skipping", self.addressing
+                )
             return None
 
         msg = (
@@ -437,7 +442,7 @@ class Stage(params.StageParams):
             if not no_commit:
                 self.commit()
 
-    def _filter_outs(self, path_info):
+    def filter_outs(self, path_info):
         def _func(o):
             return path_info.isin_or_eq(o.path_info)
 
@@ -446,7 +451,7 @@ class Stage(params.StageParams):
     @rwlocked(write=["outs"])
     def checkout(self, **kwargs):
         stats = defaultdict(list)
-        for out in self._filter_outs(kwargs.get("filter_info")):
+        for out in self.filter_outs(kwargs.get("filter_info")):
             key, outs = self._checkout(out, **kwargs)
             if key:
                 stats[key].extend(outs)
@@ -521,18 +526,56 @@ class Stage(params.StageParams):
     def get_all_files_number(self, filter_info=None):
         return sum(
             out.get_files_number(filter_info)
-            for out in self._filter_outs(filter_info)
+            for out in self.filter_outs(filter_info)
         )
 
     def get_used_cache(self, *args, **kwargs):
         from dvc.cache import NamedCache
 
         cache = NamedCache()
-        for out in self._filter_outs(kwargs.get("filter_info")):
+        for out in self.filter_outs(kwargs.get("filter_info")):
             cache.update(out.get_used_cache(*args, **kwargs))
 
         return cache
 
+    @staticmethod
+    def _check_can_merge(stage, ancestor_out=None):
+        if isinstance(stage, PipelineStage):
+            raise MergeError("unable to auto-merge pipeline stages")
+
+        if not stage.is_data_source or stage.deps or len(stage.outs) > 1:
+            raise MergeError(
+                "unable to auto-merge DVC-files that weren't "
+                "created by `dvc add`"
+            )
+
+        if ancestor_out and not stage.outs:
+            raise MergeError(
+                "unable to auto-merge DVC-files with deleted outputs"
+            )
+
+    def merge(self, ancestor, other):
+        assert other
+
+        if not other.outs:
+            return
+
+        if not self.outs:
+            self.outs = other.outs
+            return
+
+        if ancestor:
+            self._check_can_merge(ancestor)
+            outs = ancestor.outs
+            ancestor_out = outs[0] if outs else None
+        else:
+            ancestor_out = None
+
+        self._check_can_merge(self, ancestor_out)
+        self._check_can_merge(other, ancestor_out)
+
+        self.outs[0].merge(ancestor_out, other.outs[0])
+
 
 class PipelineStage(Stage):
     def __init__(self, *args, name=None, **kwargs):
@@ -572,3 +615,6 @@ class PipelineStage(Stage):
 
     def _changed_stage_entry(self):
         return f"'cmd' of {self} has changed."
+
+    def merge(self, ancestor, other):
+        raise NotImplementedError

dvc/stage/cache.py

@@ -2,7 +2,6 @@ import logging
 import os
 from contextlib import contextmanager
 
-import yaml
 from funcy import first
 from voluptuous import Invalid
 
@@ -10,7 +9,7 @@ from dvc.cache.local import _log_exceptions
 from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA
 from dvc.utils import dict_sha256, relpath
 from dvc.utils.fs import makedirs
-from dvc.utils.yaml import dump_yaml
+from dvc.utils.serialize import YAMLFileCorruptedError, dump_yaml, load_yaml
 
 from .loader import StageLoader
 from .serialize import to_single_stage_lockfile
@@ -54,11 +53,10 @@ class StageCache:
         path = self._get_cache_path(key, value)
 
         try:
-            with open(path) as fobj:
-                return COMPILED_LOCK_FILE_STAGE_SCHEMA(yaml.safe_load(fobj))
+            return COMPILED_LOCK_FILE_STAGE_SCHEMA(load_yaml(path))
         except FileNotFoundError:
             return None
-        except (yaml.error.YAMLError, Invalid):
+        except (YAMLFileCorruptedError, Invalid):
             logger.warning("corrupted cache file '%s'.", relpath(path))
             os.unlink(path)
             return None
@@ -80,7 +78,7 @@ class StageCache:
         return None
 
     def _create_stage(self, cache, wdir=None):
-        from . import create_stage, PipelineStage
+        from . import PipelineStage, create_stage
 
         stage = create_stage(
             PipelineStage,

dvc/stage/decorators.py

@@ -6,8 +6,9 @@ from funcy import decorator
 @decorator
 def rwlocked(call, read=None, write=None):
     import sys
-    from dvc.rwlock import rwlock
+
     from dvc.dependency.repo import RepoDependency
+    from dvc.rwlock import rwlock
 
     if read is None:
         read = []

dvc/stage/loader.py

@@ -42,7 +42,9 @@ class StageLoader(Mapping):
             if isinstance(item, dependency.ParamsDependency):
                 item.fill_values(get_in(lock_data, [stage.PARAM_PARAMS, path]))
                 continue
-            item.checksum = get_in(checksums, [key, path, item.checksum_type])
+            item.info = get_in(checksums, [key, path], {})
+            item.info = item.info.copy()
+            item.info.pop("path", None)
 
     @classmethod
     def load_stage(cls, dvcfile, name, stage_data, lock_data=None):

dvc/stage/serialize.py

@@ -8,7 +8,7 @@ from funcy import post_processing
 from dvc.dependency import ParamsDependency
 from dvc.output import BaseOutput
 from dvc.utils.collections import apply_diff
-from dvc.utils.yaml import parse_yaml_for_update
+from dvc.utils.serialize import parse_yaml_for_update
 
 from .params import StageParams
 from .utils import resolve_wdir, split_params_deps
@@ -138,12 +138,7 @@ def to_single_stage_lockfile(stage: "Stage") -> dict:
     params, deps = split_params_deps(stage)
     deps, outs = [
         [
-            OrderedDict(
-                [
-                    (PARAM_PATH, item.def_path),
-                    (item.checksum_type, item.checksum),
-                ]
-            )
+            OrderedDict([(PARAM_PATH, item.def_path), *item.info.items()])
             for item in sort_by_path(items)
         ]
         for items in [deps, stage.outs]

dvc/stage/utils.py

@@ -110,9 +110,10 @@ def check_circular_dependency(stage):
 
 
 def check_duplicated_arguments(stage):
-    from dvc.exceptions import ArgumentDuplicationError
     from collections import Counter
 
+    from dvc.exceptions import ArgumentDuplicationError
+
     path_counts = Counter(edge.path_info for edge in stage.deps + stage.outs)
 
     for path, occurrence in path_counts.items():

dvc/system.py

@@ -119,13 +119,14 @@ class System:
     @staticmethod
     def _getdirinfo(path):
         from collections import namedtuple
+
         from win32file import (  # pylint: disable=import-error
-            CreateFileW,
-            GetFileInformationByHandle,
             FILE_FLAG_BACKUP_SEMANTICS,
             FILE_FLAG_OPEN_REPARSE_POINT,
             FILE_SHARE_READ,
             OPEN_EXISTING,
+            CreateFileW,
+            GetFileInformationByHandle,
         )
 
         # NOTE: use FILE_FLAG_OPEN_REPARSE_POINT to open symlink itself and not

dvc/tree/_metadata.py

@@ -0,0 +1,68 @@
+from dataclasses import dataclass, field
+from typing import List
+
+from dvc.output import BaseOutput
+from dvc.path_info import PathInfo
+
+
+@dataclass
+class Metadata:
+    """
+    Container for storing metadata for a given path, similar in spirit to
+    `os.stat_result`.
+    """
+
+    # required field
+    path_info: PathInfo
+
+    # computed fields
+    is_output: bool = field(init=False, default=False)  # is it an output?
+    # is the path part of an output?
+    part_of_output: bool = field(init=False, default=False)
+    # does the path contain outputs?
+    contains_outputs: bool = field(init=False, default=False)
+    # is the path tracked by dvc? equivalent to (is_output or part_of_output)
+    is_dvc: bool = field(init=False, default=False)
+    # does dvc has any output in that path or inside it? equiv. to bool(outs)
+    output_exists: bool = field(init=False)
+
+    # optional fields
+    isdir: bool = False  # is it a directory?
+    is_exec: bool = False  # is it an executable?
+    outs: List[BaseOutput] = field(default_factory=list)  # list of outputs
+
+    def __post_init__(self):
+        self.output_exists = bool(self.outs)
+
+        if not self.output_exists:
+            return
+
+        # it can contain multiple outputs
+        if len(self.outs) > 1:
+            self.contains_outputs = True
+        else:
+            out = self.outs[0]
+            # or, the path itself could be an output
+            self.is_output = self.path_info == out.path_info
+
+            # or, a directory must have an output somewhere deep inside it
+            if not self.is_output:
+                self.contains_outputs = out.path_info.isin(self.path_info)
+
+        # or, the path could be a part of an output, i.e. inside of an output
+        self.part_of_output = not self.is_output and not self.contains_outputs
+
+        self.is_dvc = self.is_output or self.part_of_output
+        # if it contains outputs, it must be a dir
+        self.isdir = self.contains_outputs
+
+    @property
+    def isfile(self):
+        # self.isdir might require additional computation, especially if it's
+        # a dir that's inside a dvc-tracked dir
+        # since, `Metadata()` does not support being used as null (i.e. when
+        # path does not exist), `isfile` becomes `isdir`'s complement.
+        return not self.isdir
+
+    def __str__(self):
+        return f"Metadata: {self.path_info}"

dvc/tree/azure.py

@@ -35,16 +35,20 @@ class AzureTree(BaseTree):
             container = self._az_config.get("storage", "container_name", None)
             self.path_info = self.PATH_CLS(f"azure://{container}")
 
-        self._conn_kwargs = {
-            opt: config.get(opt) or self._az_config.get("storage", opt, None)
-            for opt in ["connection_string", "sas_token"]
-        }
-        self._conn_kwargs["account_name"] = self._az_config.get(
-            "storage", "account", None
-        )
-        self._conn_kwargs["account_key"] = self._az_config.get(
-            "storage", "key", None
+        self._conn_str = config.get(
+            "connection_string"
+        ) or self._az_config.get("storage", "connection_string", None)
+
+        self._account_url = None
+        if not self._conn_str:
+            name = self._az_config.get("storage", "account", None)
+            self._account_url = f"https://{name}.blob.core.windows.net"
+
+        self._credential = config.get("sas_token") or self._az_config.get(
+            "storage", "sas_token", None
         )
+        if not self._credential:
+            self._credential = self._az_config.get("storage", "key", None)
 
     @cached_property
     def _az_config(self):
@@ -62,64 +66,71 @@ class AzureTree(BaseTree):
     @cached_property
     def blob_service(self):
         # pylint: disable=no-name-in-module
-        from azure.storage.blob import BlockBlobService
-        from azure.common import AzureMissingResourceHttpError
+        from azure.core.exceptions import ResourceNotFoundError
+        from azure.storage.blob import BlobServiceClient
 
         logger.debug(f"URL {self.path_info}")
-        logger.debug(f"Connection options {self._conn_kwargs}")
-        blob_service = BlockBlobService(**self._conn_kwargs)
-        logger.debug(f"Container name {self.path_info.bucket}")
-        try:  # verify that container exists
-            blob_service.list_blobs(
-                self.path_info.bucket, delimiter="/", num_results=1
+
+        if self._conn_str:
+            logger.debug(f"Using connection string '{self._conn_str}'")
+            blob_service = BlobServiceClient.from_connection_string(
+                self._conn_str, credential=self._credential
             )
-        except AzureMissingResourceHttpError:
-            blob_service.create_container(self.path_info.bucket)
+        else:
+            logger.debug(f"Using account url '{self._account_url}'")
+            blob_service = BlobServiceClient(
+                self._account_url, credential=self._credential
+            )
+
+        logger.debug(f"Container name {self.path_info.bucket}")
+        container_client = blob_service.get_container_client(
+            self.path_info.bucket
+        )
+
+        try:  # verify that container exists
+            container_client.get_container_properties()
+        except ResourceNotFoundError:
+            container_client.create_container()
+
         return blob_service
 
     def get_etag(self, path_info):
-        etag = self.blob_service.get_blob_properties(
+        blob_client = self.blob_service.get_blob_client(
             path_info.bucket, path_info.path
-        ).properties.etag
+        )
+        etag = blob_client.get_blob_properties().etag
         return etag.strip('"')
 
     def _generate_download_url(self, path_info, expires=3600):
         from azure.storage.blob import (  # pylint:disable=no-name-in-module
-            BlobPermissions,
+            BlobSasPermissions,
+            generate_blob_sas,
         )
 
         expires_at = datetime.utcnow() + timedelta(seconds=expires)
 
-        sas_token = self.blob_service.generate_blob_shared_access_signature(
-            path_info.bucket,
-            path_info.path,
-            permission=BlobPermissions.READ,
+        blob_client = self.blob_service.get_blob_client(
+            path_info.bucket, path_info.path
+        )
+
+        sas_token = generate_blob_sas(
+            blob_client.account_name,
+            blob_client.container_name,
+            blob_client.blob_name,
+            account_key=blob_client.credential.account_key,
+            permission=BlobSasPermissions(read=True),
             expiry=expires_at,
         )
-        download_url = self.blob_service.make_blob_url(
-            path_info.bucket, path_info.path, sas_token=sas_token
-        )
-        return download_url
+        return blob_client.url + "?" + sas_token
 
     def exists(self, path_info, use_dvcignore=True):
         paths = self._list_paths(path_info.bucket, path_info.path)
         return any(path_info.path == path for path in paths)
 
     def _list_paths(self, bucket, prefix):
-        blob_service = self.blob_service
-        next_marker = None
-        while True:
-            blobs = blob_service.list_blobs(
-                bucket, prefix=prefix, marker=next_marker
-            )
-
-            for blob in blobs:
-                yield blob.name
-
-            if not blobs.next_marker:
-                break
-
-            next_marker = blobs.next_marker
+        container_client = self.blob_service.get_container_client(bucket)
+        for blob in container_client.list_blobs(name_starts_with=prefix):
+            yield blob.name
 
     def walk_files(self, path_info, **kwargs):
         if not kwargs.pop("prefix", False):
@@ -137,29 +148,37 @@ class AzureTree(BaseTree):
             raise NotImplementedError
 
         logger.debug(f"Removing {path_info}")
-        self.blob_service.delete_blob(path_info.bucket, path_info.path)
+        self.blob_service.get_blob_client(
+            path_info.bucket, path_info.path
+        ).delete_blob()
 
     def get_file_hash(self, path_info):
-        return self.get_etag(path_info)
+        return self.PARAM_CHECKSUM, self.get_etag(path_info)
 
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
-            self.blob_service.create_blob_from_path(
-                to_info.bucket,
-                to_info.path,
-                from_file,
-                progress_callback=pbar.update_to,
-            )
+
+        blob_client = self.blob_service.get_blob_client(
+            to_info.bucket, to_info.path
+        )
+        total = os.path.getsize(from_file)
+        with open(from_file, "rb") as fobj:
+            with Tqdm.wrapattr(
+                fobj, "read", desc=name, total=total, disable=no_progress_bar
+            ) as wrapped:
+                blob_client.upload_blob(wrapped, overwrite=True)
 
     def _download(
         self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
     ):
-        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
-            self.blob_service.get_blob_to_path(
-                from_info.bucket,
-                from_info.path,
-                to_file,
-                progress_callback=pbar.update_to,
-            )
+        blob_client = self.blob_service.get_blob_client(
+            from_info.bucket, from_info.path
+        )
+        total = blob_client.get_blob_properties().size
+        stream = blob_client.download_blob()
+        with open(to_file, "wb") as fobj:
+            with Tqdm.wrapattr(
+                fobj, "write", desc=name, total=total, disable=no_progress_bar
+            ) as wrapped:
+                stream.readinto(wrapped)

dvc/tree/base.py

@@ -236,22 +236,17 @@ class BaseTree:
             return False
         return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
 
-    def get_hash(self, path_info, tree=None, **kwargs):
+    def get_hash(self, path_info, **kwargs):
         assert path_info and (
             isinstance(path_info, str) or path_info.scheme == self.scheme
         )
 
-        if not tree:
-            tree = self
+        if not self.exists(path_info):
+            return self.PARAM_CHECKSUM, None
 
-        if not tree.exists(path_info):
-            return None
+        # pylint: disable=assignment-from-none
+        hash_ = self.state.get(path_info)
 
-        if tree == self:
-            # pylint: disable=assignment-from-none
-            hash_ = self.state.get(path_info)
-        else:
-            hash_ = None
         # If we have dir hash in state db, but dir cache file is lost,
         # then we need to recollect the dir via .get_dir_hash() call below,
         # see https://github.com/iterative/dvc/issues/2219 for context
@@ -265,27 +260,27 @@ class BaseTree:
             hash_ = None
 
         if hash_:
-            return hash_
+            return self.PARAM_CHECKSUM, hash_
 
-        if tree.isdir(path_info):
-            hash_ = self.get_dir_hash(path_info, tree, **kwargs)
+        if self.isdir(path_info):
+            typ, hash_ = self.get_dir_hash(path_info, **kwargs)
         else:
-            hash_ = tree.get_file_hash(path_info)
+            typ, hash_ = self.get_file_hash(path_info)
 
         if hash_ and self.exists(path_info):
             self.state.save(path_info, hash_)
 
-        return hash_
+        return typ, hash_
 
     def get_file_hash(self, path_info):
         raise NotImplementedError
 
-    def get_dir_hash(self, path_info, tree, **kwargs):
+    def get_dir_hash(self, path_info, **kwargs):
         if not self.cache:
             raise RemoteCacheRequiredError(path_info)
 
-        dir_info = self._collect_dir(path_info, tree, **kwargs)
-        return self._save_dir_info(dir_info, path_info)
+        dir_info = self._collect_dir(path_info, **kwargs)
+        return self.save_dir_info(dir_info)
 
     def hash_to_path_info(self, hash_):
         return self.path_info / hash_[0:2] / hash_[2:]
@@ -298,29 +293,28 @@ class BaseTree:
 
         return "".join(parts)
 
-    def save_info(self, path_info, tree=None, **kwargs):
-        return {
-            self.PARAM_CHECKSUM: self.get_hash(path_info, tree=tree, **kwargs)
-        }
+    def save_info(self, path_info, **kwargs):
+        typ, hash_ = self.get_hash(path_info, **kwargs)
+        return {typ: hash_}
 
-    @staticmethod
-    def _calculate_hashes(file_infos, tree):
+    def _calculate_hashes(self, file_infos):
         file_infos = list(file_infos)
         with Tqdm(
             total=len(file_infos),
             unit="md5",
             desc="Computing file/dir hashes (only done once)",
         ) as pbar:
-            worker = pbar.wrap_fn(tree.get_file_hash)
-            with ThreadPoolExecutor(max_workers=tree.hash_jobs) as executor:
-                tasks = executor.map(worker, file_infos)
-                hashes = dict(zip(file_infos, tasks))
-        return hashes
+            worker = pbar.wrap_fn(self.get_file_hash)
+            with ThreadPoolExecutor(max_workers=self.hash_jobs) as executor:
+                hashes = (
+                    value for typ, value in executor.map(worker, file_infos)
+                )
+                return dict(zip(file_infos, hashes))
 
-    def _collect_dir(self, path_info, tree, **kwargs):
+    def _collect_dir(self, path_info, **kwargs):
         file_infos = set()
 
-        for fname in tree.walk_files(path_info, **kwargs):
+        for fname in self.walk_files(path_info, **kwargs):
             if DvcIgnore.DVCIGNORE_FILE == fname.name:
                 raise DvcIgnoreInCollectedDirError(fname.parent)
 
@@ -329,7 +323,7 @@ class BaseTree:
         hashes = {fi: self.state.get(fi) for fi in file_infos}
         not_in_state = {fi for fi, hash_ in hashes.items() if hash_ is None}
 
-        new_hashes = self._calculate_hashes(not_in_state, tree)
+        new_hashes = self._calculate_hashes(not_in_state)
         hashes.update(new_hashes)
 
         result = [
@@ -351,8 +345,8 @@ class BaseTree:
         # Sorting the list by path to ensure reproducibility
         return sorted(result, key=itemgetter(self.PARAM_RELPATH))
 
-    def _save_dir_info(self, dir_info, path_info):
-        hash_, tmp_info = self._get_dir_info_hash(dir_info)
+    def save_dir_info(self, dir_info):
+        typ, hash_, tmp_info = self._get_dir_info_hash(dir_info)
         new_info = self.cache.tree.hash_to_path_info(hash_)
         if self.cache.changed_cache_file(hash_):
             self.cache.tree.makedirs(new_info.parent)
@@ -360,13 +354,14 @@ class BaseTree:
                 tmp_info, new_info, mode=self.cache.CACHE_MODE
             )
 
-        if self.exists(path_info):
-            self.state.save(path_info, hash_)
         self.state.save(new_info, hash_)
 
-        return hash_
+        return typ, hash_
 
     def _get_dir_info_hash(self, dir_info):
+        # Sorting the list by path to ensure reproducibility
+        dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
+
         tmp = tempfile.NamedTemporaryFile(delete=False).name
         with open(tmp, "w+") as fobj:
             json.dump(dir_info, fobj, sort_keys=True)
@@ -376,8 +371,8 @@ class BaseTree:
         to_info = tree.path_info / tmp_fname("")
         tree.upload(from_info, to_info, no_progress_bar=True)
 
-        hash_ = tree.get_file_hash(to_info) + self.CHECKSUM_DIR_SUFFIX
-        return hash_, to_info
+        typ, hash_ = tree.get_file_hash(to_info)
+        return typ, hash_ + self.CHECKSUM_DIR_SUFFIX, to_info
 
     def upload(self, from_info, to_info, name=None, no_progress_bar=False):
         if not hasattr(self, "_upload"):

dvc/tree/dvc.py

@@ -0,0 +1,272 @@
+import logging
+import os
+
+from dvc.exceptions import OutputNotFoundError
+from dvc.path_info import PathInfo
+
+from ._metadata import Metadata
+from .base import BaseTree, RemoteActionNotImplemented
+
+logger = logging.getLogger(__name__)
+
+
+class DvcTree(BaseTree):  # pylint:disable=abstract-method
+    """DVC repo tree.
+
+    Args:
+        repo: DVC repo.
+        fetch: if True, uncached DVC outs will be fetched on `open()`.
+        stream: if True, uncached DVC outs will be streamed directly from
+            remote on `open()`.
+
+    `stream` takes precedence over `fetch`. If `stream` is enabled and
+    a remote does not support streaming, uncached DVC outs will be fetched
+    as a fallback.
+    """
+
+    scheme = "local"
+    PARAM_CHECKSUM = "md5"
+
+    def __init__(self, repo, fetch=False, stream=False):
+        super().__init__(repo, {"url": repo.root_dir})
+        self.fetch = fetch
+        self.stream = stream
+
+    def _find_outs(self, path, *args, **kwargs):
+        outs = self.repo.find_outs_by_path(path, *args, **kwargs)
+
+        def _is_cached(out):
+            return out.use_cache
+
+        outs = list(filter(_is_cached, outs))
+        if not outs:
+            raise OutputNotFoundError(path, self.repo)
+
+        return outs
+
+    def _get_granular_checksum(self, path, out, remote=None):
+        assert isinstance(path, PathInfo)
+        if not self.fetch and not self.stream:
+            raise FileNotFoundError
+        dir_cache = out.get_dir_cache(remote=remote)
+        for entry in dir_cache:
+            entry_relpath = entry[out.tree.PARAM_RELPATH]
+            if os.name == "nt":
+                entry_relpath = entry_relpath.replace("/", os.sep)
+            if path == out.path_info / entry_relpath:
+                return entry[out.tree.PARAM_CHECKSUM]
+        raise FileNotFoundError
+
+    def open(
+        self, path, mode="r", encoding="utf-8", remote=None
+    ):  # pylint: disable=arguments-differ
+        try:
+            outs = self._find_outs(path, strict=False)
+        except OutputNotFoundError as exc:
+            raise FileNotFoundError from exc
+
+        # NOTE: this handles both dirty and checkout-ed out at the same time
+        if self.repo.tree.exists(path):
+            return self.repo.tree.open(path, mode=mode, encoding=encoding)
+
+        if len(outs) != 1 or (
+            outs[0].is_dir_checksum and path == outs[0].path_info
+        ):
+            raise IsADirectoryError
+
+        out = outs[0]
+        if out.changed_cache(filter_info=path):
+            if not self.fetch and not self.stream:
+                raise FileNotFoundError
+
+            remote_obj = self.repo.cloud.get_remote(remote)
+            if self.stream:
+                if out.is_dir_checksum:
+                    checksum = self._get_granular_checksum(path, out)
+                else:
+                    checksum = out.checksum
+                try:
+                    remote_info = remote_obj.tree.hash_to_path_info(checksum)
+                    return remote_obj.tree.open(
+                        remote_info, mode=mode, encoding=encoding
+                    )
+                except RemoteActionNotImplemented:
+                    pass
+            cache_info = out.get_used_cache(filter_info=path, remote=remote)
+            self.repo.cloud.pull(cache_info, remote=remote)
+
+        if out.is_dir_checksum:
+            checksum = self._get_granular_checksum(path, out)
+            cache_path = out.cache.tree.hash_to_path_info(checksum).url
+        else:
+            cache_path = out.cache_path
+        return open(cache_path, mode=mode, encoding=encoding)
+
+    def exists(self, path):  # pylint: disable=arguments-differ
+        try:
+            self.metadata(path)
+            return True
+        except OutputNotFoundError:
+            return False
+
+    def isdir(self, path):  # pylint: disable=arguments-differ
+        try:
+            meta = self.metadata(path)
+            return meta.isdir
+        except OutputNotFoundError:
+            return False
+
+    def check_isdir(self, path_info, outs):
+        if len(outs) != 1:
+            return True
+
+        out = outs[0]
+        if not out.is_dir_checksum:
+            return out.path_info != path_info
+        if out.path_info == path_info:
+            return True
+
+        # for dir checksum, we need to check if this is a file inside the
+        # directory
+        try:
+            self._get_granular_checksum(path_info, out)
+            return False
+        except FileNotFoundError:
+            return True
+
+    def isfile(self, path):  # pylint: disable=arguments-differ
+        try:
+            meta = self.metadata(path)
+            return meta.isfile
+        except OutputNotFoundError:
+            return False
+
+    def _fetch_dir(
+        self, out, filter_info=None, download_callback=None, **kwargs
+    ):
+        # pull dir cache if needed
+        out.get_dir_cache(**kwargs)
+
+        # pull dir contents if needed
+        if self.fetch and out.changed_cache(filter_info=filter_info):
+            used_cache = out.get_used_cache(filter_info=filter_info)
+            downloaded = self.repo.cloud.pull(used_cache, **kwargs)
+            if download_callback:
+                download_callback(downloaded)
+
+    def _add_dir(self, top, trie, out, **kwargs):
+        if not self.fetch and not self.stream:
+            return
+
+        self._fetch_dir(out, filter_info=top, **kwargs)
+
+        for entry in out.dir_cache:
+            entry_relpath = entry[out.tree.PARAM_RELPATH]
+            if os.name == "nt":
+                entry_relpath = entry_relpath.replace("/", os.sep)
+            path_info = out.path_info / entry_relpath
+            trie[path_info.parts] = None
+
+    def _walk(self, root, trie, topdown=True, **kwargs):
+        dirs = set()
+        files = []
+
+        out = trie.get(root.parts)
+        if out and out.is_dir_checksum:
+            self._add_dir(root, trie, out, **kwargs)
+
+        root_len = len(root.parts)
+        for key, out in trie.iteritems(prefix=root.parts):  # noqa: B301
+            if key == root.parts:
+                continue
+
+            name = key[root_len]
+            if len(key) > root_len + 1 or (out and out.is_dir_checksum):
+                dirs.add(name)
+                continue
+
+            files.append(name)
+
+        assert topdown
+        dirs = list(dirs)
+        yield root.fspath, dirs, files
+
+        for dname in dirs:
+            yield from self._walk(root / dname, trie)
+
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
+        from pygtrie import Trie
+
+        assert topdown
+        root = PathInfo(os.path.abspath(top))
+        try:
+            meta = self.metadata(root)
+        except OutputNotFoundError:
+            if onerror is not None:
+                onerror(FileNotFoundError(top))
+            return
+
+        if not meta.isdir:
+            if onerror is not None:
+                onerror(NotADirectoryError(top))
+            return
+
+        trie = Trie()
+        for out in meta.outs:
+            trie[out.path_info.parts] = out
+
+            if out.is_dir_checksum and root.isin_or_eq(out.path_info):
+                self._add_dir(top, trie, out, **kwargs)
+
+        yield from self._walk(root, trie, topdown=topdown, **kwargs)
+
+    def walk_files(self, path_info, **kwargs):
+        for root, _, files in self.walk(path_info):
+            for fname in files:
+                # NOTE: os.path.join is ~5.5 times slower
+                yield PathInfo(f"{root}{os.sep}{fname}")
+
+    def isdvc(self, path, recursive=False, strict=True):
+        try:
+            meta = self.metadata(path)
+        except OutputNotFoundError:
+            return False
+
+        recurse = recursive or not strict
+        return meta.output_exists if recurse else meta.is_output
+
+    def isexec(self, path):  # pylint: disable=unused-argument
+        return False
+
+    def get_dir_hash(self, path_info, **kwargs):
+        try:
+            outs = self._find_outs(path_info, strict=True)
+            if len(outs) == 1 and outs[0].is_dir_checksum:
+                out = outs[0]
+                # other code expects us to fetch the dir at this point
+                self._fetch_dir(out, **kwargs)
+                return out.tree.PARAM_CHECKSUM, out.checksum
+        except OutputNotFoundError:
+            pass
+
+        return super().get_dir_hash(path_info, **kwargs)
+
+    def get_file_hash(self, path_info):
+        outs = self._find_outs(path_info, strict=False)
+        if len(outs) != 1:
+            raise OutputNotFoundError
+        out = outs[0]
+        if out.is_dir_checksum:
+            return (
+                out.tree.PARAM_CHECKSUM,
+                self._get_granular_checksum(path_info, out),
+            )
+        return out.tree.PARAM_CHECKSUM, out.checksum
+
+    def metadata(self, path_info):
+        path_info = PathInfo(os.path.abspath(path_info))
+        outs = self._find_outs(path_info, strict=False, recursive=True)
+
+        meta = Metadata(path_info=path_info, outs=outs)
+        meta.isdir = meta.isdir or self.check_isdir(meta.path_info, meta.outs)
+        return meta

dvc/tree/git.py

@@ -1,13 +1,13 @@
 import errno
 import io
 import os
-import stat
+from functools import lru_cache
 
 from funcy import cached_property
 
 from dvc.exceptions import DvcException
 from dvc.tree.base import BaseTree
-from dvc.utils import relpath
+from dvc.utils import is_exec, relpath
 
 # see git-fast-import(1)
 GIT_MODE_DIR = 0o40000
@@ -21,6 +21,30 @@ def _item_basename(item):
     return os.path.basename(item.path)
 
 
+def _is_tree_and_contains(obj, path):
+    if obj.mode != GIT_MODE_DIR:
+        return False
+    # see https://github.com/gitpython-developers/GitPython/issues/851
+    # `return (i in tree)` doesn't work so here is a workaround:
+    for item in obj:
+        if _item_basename(item) == path:
+            return True
+    return False
+
+
+# NOTE: small lru cache covers sequential exists() calls
+@lru_cache(maxsize=1)
+def _get_obj(tree, path):
+    if not path or path == ".":
+        return tree
+    for i in path.split(os.sep):
+        if not _is_tree_and_contains(tree, i):
+            # there is no tree for specified path
+            return None
+        tree = tree[i]
+    return tree
+
+
 class GitTree(BaseTree):  # pylint:disable=abstract-method
     """Proxies the repo file access methods to Git objects"""
 
@@ -84,13 +108,15 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
             path
         ) and not self.dvcignore.is_ignored_dir(path)
 
-    def isdir(self, path):  # pylint: disable=arguments-differ
+    def isdir(
+        self, path, use_dvcignore=True
+    ):  # pylint: disable=arguments-differ
         obj = self._git_object_by_path(path)
         if obj is None:
             return False
         if obj.mode != GIT_MODE_DIR:
             return False
-        return not self.dvcignore.is_ignored_dir(path)
+        return not (use_dvcignore and self.dvcignore.is_ignored_dir(path))
 
     def isfile(self, path):  # pylint: disable=arguments-differ
         obj = self._git_object_by_path(path)
@@ -101,27 +127,12 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
             return False
         return not self.dvcignore.is_ignored_file(path)
 
-    @staticmethod
-    def _is_tree_and_contains(obj, path):
-        if obj.mode != GIT_MODE_DIR:
-            return False
-        # see https://github.com/gitpython-developers/GitPython/issues/851
-        # `return (i in tree)` doesn't work so here is a workaround:
-        for item in obj:
-            if _item_basename(item) == path:
-                return True
-        return False
-
-    def _git_object_by_path(self, path):
+    @cached_property
+    def _tree(self):
         import git
 
-        path = relpath(os.path.realpath(path), self.git.working_dir)
-        if path.split(os.sep, 1)[0] == "..":
-            # path points outside of git repository
-            return None
-
         try:
-            tree = self.git.tree(self.rev)
+            return self.git.tree(self.rev)
         except git.exc.BadName as exc:  # pylint: disable=no-member
             raise DvcException(
                 "revision '{}' not found in Git '{}'".format(
@@ -129,14 +140,13 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
                 )
             ) from exc
 
-        if not path or path == ".":
-            return tree
-        for i in path.split(os.sep):
-            if not self._is_tree_and_contains(tree, i):
-                # there is no tree for specified path
-                return None
-            tree = tree[i]
-        return tree
+    def _git_object_by_path(self, path):
+        path = relpath(os.path.realpath(path), self.git.working_dir)
+        if path.split(os.sep, 1)[0] == "..":
+            # path points outside of git repository
+            return None
+
+        return _get_obj(self._tree, path)
 
     def _walk(self, tree, topdown=True):
         dirs, nondirs = [], []
@@ -156,7 +166,14 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
         if not topdown:
             yield os.path.normpath(tree.abspath), dirs, nondirs
 
-    def walk(self, top, topdown=True, onerror=None, use_dvcignore=True):
+    def walk(
+        self,
+        top,
+        topdown=True,
+        onerror=None,
+        use_dvcignore=True,
+        ignore_subrepos=True,
+    ):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
@@ -176,7 +193,10 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
         for root, dirs, files in self._walk(tree, topdown=topdown):
             if use_dvcignore:
                 dirs[:], files[:] = self.dvcignore(
-                    os.path.abspath(root), dirs, files
+                    os.path.abspath(root),
+                    dirs,
+                    files,
+                    ignore_subrepos=ignore_subrepos,
                 )
             yield root, dirs, files
 
@@ -185,7 +205,7 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
             return False
 
         mode = self.stat(path).st_mode
-        return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+        return is_exec(mode)
 
     def stat(self, path):
         import git
@@ -228,3 +248,6 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
             for file in files:
                 # NOTE: os.path.join is ~5.5 times slower
                 yield f"{root}{os.sep}{file}"
+
+    def _reset(self):
+        return self.__dict__.pop("dvcignore", None)

dvc/tree/gs.py

@@ -28,7 +28,6 @@ def dynamic_chunk_size(func):
         # Note: must be multiple of 256K.
         #
         # [#2572]: https://github.com/iterative/dvc/issues/2572
-
         # skipcq: PYL-W0212
         multiplier = 40
         while True:
@@ -190,11 +189,14 @@ class GSTree(BaseTree):
         path = path_info.path
         blob = self.gs.bucket(bucket).get_blob(path)
         if not blob:
-            return None
+            return self.PARAM_CHECKSUM, None
 
         b64_md5 = blob.md5_hash
         md5 = base64.b64decode(b64_md5)
-        return codecs.getencoder("hex")(md5)[0].decode("utf-8")
+        return (
+            self.PARAM_CHECKSUM,
+            codecs.getencoder("hex")(md5)[0].decode("utf-8"),
+        )
 
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
         bucket = self.gs.bucket(to_info.bucket)

dvc/tree/hdfs.py

@@ -130,6 +130,14 @@ class HDFSTree(BaseTree):
                     self.remove(tmp_info)
                     raise
 
+    def isfile(self, path_info):
+        with self.hdfs(path_info) as hdfs:
+            return hdfs.isfile(path_info.path)
+
+    def isdir(self, path_info):
+        with self.hdfs(path_info) as hdfs:
+            return hdfs.isdir(path_info.path)
+
     def hadoop_fs(self, cmd, user=None):
         cmd = "hadoop fs -" + cmd
         if user:
@@ -167,7 +175,7 @@ class HDFSTree(BaseTree):
         stdout = self.hadoop_fs(
             f"checksum {path_info.url}", user=path_info.user
         )
-        return self._group(regex, stdout, "checksum")
+        return self.PARAM_CHECKSUM, self._group(regex, stdout, "checksum")
 
     def _upload(self, from_file, to_info, **_kwargs):
         with self.hdfs(to_info) as hdfs:

dvc/tree/http.py

@@ -122,12 +122,27 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
         except requests.exceptions.RequestException:
             raise DvcException(f"could not perform a {method} request")
 
+    def _head(self, url):
+        response = self.request("HEAD", url)
+        if response.ok:
+            return response
+
+        # Sometimes servers are configured to forbid HEAD requests
+        # Context: https://github.com/iterative/dvc/issues/4131
+        with self.request("GET", url, stream=True) as r:
+            if r.ok:
+                return r
+
+        return response
+
     def exists(self, path_info, use_dvcignore=True):
-        return bool(self.request("HEAD", path_info.url))
+        return bool(self._head(path_info.url))
 
     def get_file_hash(self, path_info):
         url = path_info.url
-        headers = self.request("HEAD", url).headers
+
+        headers = self._head(url).headers
+
         etag = headers.get("ETag") or headers.get("Content-MD5")
 
         if not etag:
@@ -136,13 +151,7 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
                 "Content-MD5 header for '{url}'".format(url=url)
             )
 
-        if etag.startswith("W/"):
-            raise DvcException(
-                "Weak ETags are not supported."
-                " (Etag: '{etag}', URL: '{url}')".format(etag=etag, url=url)
-            )
-
-        return etag
+        return self.PARAM_CHECKSUM, etag
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
         response = self.request("GET", from_info.url, stream=True)

dvc/tree/local.py

@@ -10,7 +10,7 @@ from dvc.exceptions import DvcException
 from dvc.path_info import PathInfo
 from dvc.scheme import Schemes
 from dvc.system import System
-from dvc.utils import file_md5, relpath, tmp_fname
+from dvc.utils import file_md5, is_exec, relpath, tmp_fname
 from dvc.utils.fs import (
     copy_fobj_to_file,
     copyfile,
@@ -86,18 +86,26 @@ class LocalTree(BaseTree):
 
         return not self.dvcignore.is_ignored_file(path_info)
 
-    def isdir(self, path_info):
+    def isdir(
+        self, path_info, use_dvcignore=True
+    ):  # pylint: disable=arguments-differ
         if not os.path.isdir(path_info):
             return False
-
-        return not self.dvcignore.is_ignored_dir(path_info)
+        return not (use_dvcignore and self.dvcignore.is_ignored_dir(path_info))
 
     def iscopy(self, path_info):
         return not (
             System.is_symlink(path_info) or System.is_hardlink(path_info)
         )
 
-    def walk(self, top, topdown=True, onerror=None, use_dvcignore=True):
+    def walk(
+        self,
+        top,
+        topdown=True,
+        onerror=None,
+        use_dvcignore=True,
+        ignore_subrepos=True,
+    ):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
@@ -108,7 +116,10 @@ class LocalTree(BaseTree):
         ):
             if use_dvcignore:
                 dirs[:], files[:] = self.dvcignore(
-                    os.path.abspath(root), dirs, files
+                    os.path.abspath(root),
+                    dirs,
+                    files,
+                    ignore_subrepos=ignore_subrepos,
                 )
 
             yield os.path.normpath(root), dirs, files
@@ -146,7 +157,7 @@ class LocalTree(BaseTree):
 
     def isexec(self, path):
         mode = os.stat(path).st_mode
-        return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+        return is_exec(mode)
 
     @staticmethod
     def stat(path):
@@ -298,7 +309,7 @@ class LocalTree(BaseTree):
         return stat.S_IMODE(mode) == self.CACHE_MODE
 
     def get_file_hash(self, path_info):
-        return file_md5(path_info)[0]
+        return self.PARAM_CHECKSUM, file_md5(path_info)[0]
 
     @staticmethod
     def getsize(path_info):
@@ -315,7 +326,7 @@ class LocalTree(BaseTree):
         )
 
         self.protect(tmp_file)
-        os.rename(tmp_file, to_info)
+        os.replace(tmp_file, to_info)
 
     @staticmethod
     def _download(
@@ -346,3 +357,6 @@ class LocalTree(BaseTree):
         info = self.hash_to_path_info(hash_)
         path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
         self.remove(path_info)
+
+    def _reset(self):
+        return self.__dict__.pop("dvcignore", None)

dvc/tree/repo.py

@@ -0,0 +1,386 @@
+import logging
+import os
+import stat
+import threading
+from contextlib import suppress
+from itertools import takewhile
+from typing import TYPE_CHECKING, Callable, Optional, Tuple, Union
+
+from funcy import wrap_with
+from pygtrie import StringTrie
+
+from dvc.dvcfile import is_valid_filename
+from dvc.exceptions import OutputNotFoundError
+from dvc.path_info import PathInfo
+from dvc.utils import file_md5, is_exec
+from dvc.utils.fs import copy_fobj_to_file, makedirs
+
+from ._metadata import Metadata
+from .base import BaseTree
+from .dvc import DvcTree
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+    from .git import GitTree
+    from .local import LocalTree
+
+
+logger = logging.getLogger(__name__)
+
+
+class RepoTree(BaseTree):  # pylint:disable=abstract-method
+    """DVC + git-tracked files tree.
+
+    Args:
+        repo: DVC or git repo.
+        subrepos: traverse to subrepos (by default, it ignores subrepos)
+        repo_factory: A function to initialize subrepo with, default is Repo.
+        kwargs: Additional keyword arguments passed to the `DvcTree()`.
+    """
+
+    scheme = "local"
+    PARAM_CHECKSUM = "md5"
+
+    def __init__(
+        self,
+        repo,
+        subrepos=False,
+        repo_factory: Callable[[str], "Repo"] = None,
+        **kwargs
+    ):
+        super().__init__(repo, {"url": repo.root_dir})
+
+        if not repo_factory:
+            from dvc.repo import Repo
+
+            self.repo_factory = Repo
+        else:
+            self.repo_factory = repo_factory
+
+        self._main_repo = repo
+        self.root_dir = repo.root_dir
+        self._traverse_subrepos = subrepos
+
+        self._subrepos_trie = StringTrie(separator=os.sep)
+        """Keeps track of each and every path with the corresponding repo."""
+
+        self._subrepos_trie[self.root_dir] = repo
+
+        self._dvctrees = {}
+        """Keep a dvctree instance of each repo."""
+
+        self._dvctree_configs = kwargs
+
+        if hasattr(repo, "dvc_dir"):
+            self._dvctrees[repo.root_dir] = DvcTree(repo, **kwargs)
+
+    def _get_repo(self, path) -> Optional["Repo"]:
+        """Returns repo that the path falls in, using prefix.
+
+        If the path is already tracked/collected, it just returns the repo.
+
+        Otherwise, it collects the repos that might be in the path's parents
+        and then returns the appropriate one.
+        """
+        repo = self._subrepos_trie.get(path)
+        if repo:
+            return repo
+
+        prefix, repo = self._subrepos_trie.longest_prefix(path)
+        if not prefix:
+            return None
+
+        parents = (parent.fspath for parent in PathInfo(path).parents)
+        dirs = [path] + list(takewhile(lambda p: p != prefix, parents))
+        dirs.reverse()
+        self._update(dirs, starting_repo=repo)
+        return self._subrepos_trie.get(path)
+
+    @wrap_with(threading.Lock())
+    def _update(self, dirs, starting_repo):
+        """Checks for subrepo in directories and updates them."""
+        repo = starting_repo
+        for d in dirs:
+            if self._is_dvc_repo(d):
+                repo = self.repo_factory(d)
+                self._dvctrees[repo.root_dir] = DvcTree(
+                    repo, **self._dvctree_configs
+                )
+            self._subrepos_trie[d] = repo
+
+    def _is_dvc_repo(self, dir_path):
+        """Check if the directory is a dvc repo."""
+        if not self._traverse_subrepos:
+            return False
+
+        from dvc.repo import Repo
+
+        repo_path = os.path.join(dir_path, Repo.DVC_DIR)
+        # dvcignore will ignore subrepos, therefore using `use_dvcignore=False`
+        return self._main_repo.tree.isdir(repo_path, use_dvcignore=False)
+
+    def _get_tree_pair(
+        self, path
+    ) -> Tuple[Union["GitTree", "LocalTree"], DvcTree]:
+        """
+        Returns a pair of trees based on repo the path falls in, using prefix.
+        """
+        path = os.path.abspath(path)
+
+        # fallback to the top-level repo if repo was not found
+        # this can happen if the path is outside of the repo
+        repo = self._get_repo(path) or self._main_repo
+
+        dvc_tree = self._dvctrees.get(repo.root_dir)
+        return repo.tree, dvc_tree
+
+    @property
+    def fetch(self):
+        return "fetch" in self._dvctree_configs
+
+    @property
+    def stream(self):
+        return "stream" in self._dvctree_configs
+
+    def open(
+        self, path, mode="r", encoding="utf-8", **kwargs
+    ):  # pylint: disable=arguments-differ
+        if "b" in mode:
+            encoding = None
+
+        tree, dvc_tree = self._get_tree_pair(path)
+        if dvc_tree and dvc_tree.exists(path):
+            return dvc_tree.open(path, mode=mode, encoding=encoding, **kwargs)
+        return tree.open(path, mode=mode, encoding=encoding)
+
+    def exists(
+        self, path, use_dvcignore=True
+    ):  # pylint: disable=arguments-differ
+        tree, dvc_tree = self._get_tree_pair(path)
+        return tree.exists(path) or (dvc_tree and dvc_tree.exists(path))
+
+    def isdir(self, path):  # pylint: disable=arguments-differ
+        tree, dvc_tree = self._get_tree_pair(path)
+        return tree.isdir(path) or (dvc_tree and dvc_tree.isdir(path))
+
+    def isdvc(self, path, **kwargs):
+        _, dvc_tree = self._get_tree_pair(path)
+        return dvc_tree is not None and dvc_tree.isdvc(path, **kwargs)
+
+    def isfile(self, path):  # pylint: disable=arguments-differ
+        tree, dvc_tree = self._get_tree_pair(path)
+        return tree.isfile(path) or (dvc_tree and dvc_tree.isfile(path))
+
+    def isexec(self, path):
+        tree, dvc_tree = self._get_tree_pair(path)
+        if dvc_tree and dvc_tree.exists(path):
+            return dvc_tree.isexec(path)
+        return tree.isexec(path)
+
+    def stat(self, path):
+        tree, _ = self._get_tree_pair(path)
+        return tree.stat(path)
+
+    def _dvc_walk(self, walk):
+        try:
+            root, dirs, files = next(walk)
+        except StopIteration:
+            return
+        yield root, dirs, files
+        for _ in dirs:
+            yield from self._dvc_walk(walk)
+
+    def _subrepo_walk(self, dir_path, **kwargs):
+        """Walk into a new repo.
+
+         NOTE: subrepo will only be discovered when walking if
+         ignore_subrepos is set to False.
+        """
+        tree, dvc_tree = self._get_tree_pair(dir_path)
+        tree_walk = tree.walk(
+            dir_path, topdown=True, ignore_subrepos=not self._traverse_subrepos
+        )
+        if dvc_tree:
+            dvc_walk = dvc_tree.walk(dir_path, topdown=True, **kwargs)
+        else:
+            dvc_walk = None
+        yield from self._walk(tree_walk, dvc_walk, **kwargs)
+
+    def _walk(self, repo_walk, dvc_walk=None, dvcfiles=False):
+        assert repo_walk
+        try:
+            _, dvc_dirs, dvc_fnames = (
+                next(dvc_walk) if dvc_walk else (None, [], [])
+            )
+            repo_root, repo_dirs, repo_fnames = next(repo_walk)
+        except StopIteration:
+            return
+
+        # separate subdirs into shared dirs, dvc-only dirs, repo-only dirs
+        dvc_set = set(dvc_dirs)
+        repo_set = set(repo_dirs)
+        dvc_only = list(dvc_set - repo_set)
+        repo_only = list(repo_set - dvc_set)
+        shared = list(dvc_set & repo_set)
+        dirs = shared + dvc_only + repo_only
+
+        # merge file lists
+        files = {
+            fname
+            for fname in dvc_fnames + repo_fnames
+            if dvcfiles or not is_valid_filename(fname)
+        }
+
+        yield repo_root, dirs, list(files)
+
+        # set dir order for next recursion level - shared dirs first so that
+        # next() for both generators recurses into the same shared directory
+        dvc_dirs[:] = [dirname for dirname in dirs if dirname in dvc_set]
+        repo_dirs[:] = [dirname for dirname in dirs if dirname in repo_set]
+
+        for dirname in dirs:
+            dir_path = os.path.join(repo_root, dirname)
+            if self._is_dvc_repo(dir_path):
+                yield from self._subrepo_walk(dir_path, dvcfiles=dvcfiles)
+            elif dirname in shared:
+                yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
+            elif dirname in dvc_set:
+                yield from self._dvc_walk(dvc_walk)
+            elif dirname in repo_set:
+                yield from self._walk(repo_walk, None, dvcfiles=dvcfiles)
+
+    def walk(
+        self, top, topdown=True, onerror=None, dvcfiles=False, **kwargs
+    ):  # pylint: disable=arguments-differ
+        """Walk and merge both DVC and repo trees.
+
+        Args:
+            top: path to walk from
+            topdown: if True, tree will be walked from top down.
+            onerror: if set, onerror function will be called if an error
+                occurs (by default errors are ignored).
+            dvcfiles: if True, dvcfiles will be included in the files list
+                for walked directories.
+
+        Any kwargs will be passed into methods used for fetching and/or
+        streaming DVC outs from remotes.
+        """
+        assert topdown
+
+        if not self.exists(top):
+            if onerror is not None:
+                onerror(FileNotFoundError(top))
+            return
+
+        if not self.isdir(top):
+            if onerror is not None:
+                onerror(NotADirectoryError(top))
+            return
+
+        tree, dvc_tree = self._get_tree_pair(top)
+        dvc_exists = dvc_tree and dvc_tree.exists(top)
+        repo_exists = tree.exists(top)
+        if dvc_exists:
+            dvc_walk = dvc_tree.walk(top, topdown=topdown, **kwargs)
+            if repo_exists:
+                repo_walk = tree.walk(
+                    top,
+                    topdown=topdown,
+                    ignore_subrepos=not self._traverse_subrepos,
+                )
+                yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
+            else:
+                yield from dvc_walk
+        else:
+            repo_walk = tree.walk(
+                top,
+                topdown=topdown,
+                onerror=onerror,
+                ignore_subrepos=not self._traverse_subrepos,
+            )
+            yield from self._walk(repo_walk, None, dvcfiles=dvcfiles)
+
+    def walk_files(self, top, **kwargs):  # pylint: disable=arguments-differ
+        for root, _, files in self.walk(top, **kwargs):
+            for fname in files:
+                yield PathInfo(root) / fname
+
+    def get_dir_hash(self, path_info, **kwargs):
+        if not self.exists(path_info):
+            raise FileNotFoundError
+        _, dvc_tree = self._get_tree_pair(path_info)
+        if dvc_tree and dvc_tree.exists(path_info):
+            try:
+                return dvc_tree.get_dir_hash(path_info, **kwargs)
+            except OutputNotFoundError:
+                pass
+        return super().get_dir_hash(path_info, **kwargs)
+
+    def get_file_hash(self, path_info):
+        """Return file checksum for specified path.
+
+        If path_info is a DVC out, the pre-computed checksum for the file
+        will be used. If path_info is a git file, MD5 will be computed for
+        the git object.
+        """
+        if not self.exists(path_info):
+            raise FileNotFoundError
+        _, dvc_tree = self._get_tree_pair(path_info)
+        if dvc_tree and dvc_tree.exists(path_info):
+            try:
+                return dvc_tree.get_file_hash(path_info)
+            except OutputNotFoundError:
+                pass
+        return self.PARAM_CHECKSUM, file_md5(path_info, self)[0]
+
+    def copytree(self, top, dest):
+        top = PathInfo(top)
+        dest = PathInfo(dest)
+
+        if not self.exists(top):
+            raise FileNotFoundError
+
+        if self.isfile(top):
+            makedirs(dest.parent, exist_ok=True)
+            with self.open(top, mode="rb") as fobj:
+                copy_fobj_to_file(fobj, dest)
+            return
+
+        for root, _, files in self.walk(top):
+            root = PathInfo(root)
+            dest_dir = dest / root.relative_to(top)
+            makedirs(dest_dir, exist_ok=True)
+            for fname in files:
+                src = root / fname
+                with self.open(src, mode="rb") as fobj:
+                    copy_fobj_to_file(fobj, dest_dir / fname)
+
+    @property
+    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
+        return self._main_repo.tree.hash_jobs
+
+    def metadata(self, path):
+        path_info = PathInfo(os.path.abspath(path))
+        tree, dvc_tree = self._get_tree_pair(path_info)
+
+        dvc_meta = None
+        if dvc_tree:
+            with suppress(OutputNotFoundError):
+                dvc_meta = dvc_tree.metadata(path_info)
+
+        stat_result = None
+        with suppress(FileNotFoundError):
+            stat_result = tree.stat(path_info)
+
+        if not stat_result and not dvc_meta:
+            raise FileNotFoundError
+
+        meta = dvc_meta or Metadata(path_info=path_info)
+
+        isdir = bool(stat_result) and stat.S_ISDIR(stat_result.st_mode)
+        meta.isdir = meta.isdir or isdir
+
+        if not dvc_meta:
+            meta.is_exec = bool(stat_result) and is_exec(stat_result.st_mode)
+        return meta

dvc/tree/s3.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import threading
+from contextlib import contextmanager
 
 from funcy import cached_property, wrap_prop
 
@@ -9,6 +10,7 @@ from dvc.exceptions import DvcException, ETagMismatchError
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
+from dvc.utils import error_link
 
 from .base import BaseTree
 
@@ -31,11 +33,6 @@ class S3Tree(BaseTree):
         self.profile = config.get("profile")
         self.endpoint_url = config.get("endpointurl")
 
-        if config.get("listobjects"):
-            self.list_objects_api = "list_objects"
-        else:
-            self.list_objects_api = "list_objects_v2"
-
         self.use_ssl = config.get("use_ssl", True)
 
         self.extra_args = {}
@@ -75,24 +72,49 @@ class S3Tree(BaseTree):
 
         session = boto3.session.Session(**session_opts)
 
-        return session.client(
+        return session.resource(
             "s3", endpoint_url=self.endpoint_url, use_ssl=self.use_ssl
         )
 
-    @classmethod
-    def get_etag(cls, s3, bucket, path):
-        obj = cls.get_head_object(s3, bucket, path)
-
-        return obj["ETag"].strip('"')
-
-    @staticmethod
-    def get_head_object(s3, bucket, path, *args, **kwargs):
+    @contextmanager
+    def _get_s3(self):
+        from botocore.exceptions import (
+            EndpointConnectionError,
+            NoCredentialsError,
+        )
 
         try:
-            obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
-        except Exception as exc:
-            raise DvcException(f"s3://{bucket}/{path} does not exist") from exc
-        return obj
+            yield self.s3
+        except NoCredentialsError as exc:
+            link = error_link("no-credentials")
+            raise DvcException(
+                f"Unable to find AWS credentials. {link}"
+            ) from exc
+        except EndpointConnectionError as exc:
+            link = error_link("connection-error")
+            name = self.endpoint_url or "AWS S3"
+            raise DvcException(
+                f"Unable to connect to '{name}'. {link}"
+            ) from exc
+
+    @contextmanager
+    def _get_bucket(self, bucket):
+        with self._get_s3() as s3:
+            try:
+                yield s3.Bucket(bucket)
+            except s3.meta.client.exceptions.NoSuchBucket as exc:
+                link = error_link("no-bucket")
+                raise DvcException(
+                    f"Bucket '{bucket}' does not exist. {link}"
+                ) from exc
+
+    @contextmanager
+    def _get_obj(self, path_info):
+        with self._get_bucket(path_info.bucket) as bucket:
+            try:
+                yield bucket.Object(path_info.path)
+            except bucket.meta.client.exceptions.NoSuchKey as exc:
+                raise DvcException(f"{path_info.url} does not exist") from exc
 
     def _append_aws_grants_to_extra_args(self, config):
         # Keys for extra_args can be one of the following list:
@@ -127,9 +149,12 @@ class S3Tree(BaseTree):
 
     def _generate_download_url(self, path_info, expires=3600):
         params = {"Bucket": path_info.bucket, "Key": path_info.path}
-        return self.s3.generate_presigned_url(
-            ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
-        )
+        with self._get_s3() as s3:
+            return s3.meta.client.generate_presigned_url(
+                ClientMethod="get_object",
+                Params=params,
+                ExpiresIn=int(expires),
+            )
 
     def exists(self, path_info, use_dvcignore=True):
         """Check if the blob exists. If it does not exist,
@@ -164,36 +189,19 @@ class S3Tree(BaseTree):
         return bool(list(self._list_paths(dir_path, max_items=1)))
 
     def isfile(self, path_info):
-        from botocore.exceptions import ClientError
-
         if path_info.path.endswith("/"):
             return False
 
-        try:
-            self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
-        except ClientError as exc:
-            if exc.response["Error"]["Code"] != "404":
-                raise
-            return False
-
-        return True
-
-    def _list_objects(self, path_info, max_items=None):
-        """ Read config for list object api, paginate through list objects."""
-        kwargs = {
-            "Bucket": path_info.bucket,
-            "Prefix": path_info.path,
-            "PaginationConfig": {"MaxItems": max_items},
-        }
-        paginator = self.s3.get_paginator(self.list_objects_api)
-        for page in paginator.paginate(**kwargs):
-            contents = page.get("Contents", ())
-            yield from contents
+        return path_info.path in self._list_paths(path_info)
 
     def _list_paths(self, path_info, max_items=None):
-        return (
-            item["Key"] for item in self._list_objects(path_info, max_items)
-        )
+        kwargs = {"Prefix": path_info.path}
+        if max_items is not None:
+            kwargs["MaxKeys"] = max_items
+
+        with self._get_bucket(path_info.bucket) as bucket:
+            for obj_summary in bucket.objects.filter(**kwargs):
+                yield obj_summary.key
 
     def walk_files(self, path_info, **kwargs):
         if not kwargs.pop("prefix", False):
@@ -209,7 +217,8 @@ class S3Tree(BaseTree):
             raise NotImplementedError
 
         logger.debug(f"Removing {path_info}")
-        self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
+        with self._get_obj(path_info) as obj:
+            obj.delete()
 
     def makedirs(self, path_info):
         # We need to support creating empty directories, which means
@@ -221,10 +230,12 @@ class S3Tree(BaseTree):
             return
 
         dir_path = path_info / ""
-        self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
+        with self._get_obj(dir_path) as obj:
+            obj.put(Body="")
 
     def copy(self, from_info, to_info):
-        self._copy(self.s3, from_info, to_info, self.extra_args)
+        with self._get_s3() as s3:
+            self._copy(s3.meta.client, from_info, to_info, self.extra_args)
 
     @classmethod
     def _copy_multipart(
@@ -238,8 +249,8 @@ class S3Tree(BaseTree):
         parts = []
         byte_position = 0
         for i in range(1, n_parts + 1):
-            obj = S3Tree.get_head_object(
-                s3, from_info.bucket, from_info.path, PartNumber=i
+            obj = s3.head_object(
+                Bucket=from_info.bucket, Key=from_info.path, PartNumber=i
             )
             part_size = obj["ContentLength"]
             lastbyte = byte_position + part_size - 1
@@ -293,7 +304,7 @@ class S3Tree(BaseTree):
         # object is transfered in the same chunks as it was originally.
         from boto3.s3.transfer import TransferConfig
 
-        obj = S3Tree.get_head_object(s3, from_info.bucket, from_info.path)
+        obj = s3.head_object(Bucket=from_info.bucket, Key=from_info.path)
         etag = obj["ETag"].strip('"')
         size = obj["ContentLength"]
 
@@ -313,36 +324,35 @@ class S3Tree(BaseTree):
                 Config=TransferConfig(multipart_threshold=size + 1),
             )
 
-        cached_etag = S3Tree.get_etag(s3, to_info.bucket, to_info.path)
+        cached_etag = s3.head_object(Bucket=to_info.bucket, Key=to_info.path)[
+            "ETag"
+        ].strip('"')
         if etag != cached_etag:
             raise ETagMismatchError(etag, cached_etag)
 
     def get_file_hash(self, path_info):
-        return self.get_etag(self.s3, path_info.bucket, path_info.path)
+        with self._get_obj(path_info) as obj:
+            return (
+                self.PARAM_CHECKSUM,
+                obj.e_tag.strip('"'),
+            )
 
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
-        total = os.path.getsize(from_file)
-        with Tqdm(
-            disable=no_progress_bar, total=total, bytes=True, desc=name
-        ) as pbar:
-            self.s3.upload_file(
-                from_file,
-                to_info.bucket,
-                to_info.path,
-                Callback=pbar.update,
-                ExtraArgs=self.extra_args,
-            )
+        with self._get_obj(to_info) as obj:
+            total = os.path.getsize(from_file)
+            with Tqdm(
+                disable=no_progress_bar, total=total, bytes=True, desc=name
+            ) as pbar:
+                obj.upload_file(
+                    from_file, Callback=pbar.update, ExtraArgs=self.extra_args,
+                )
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        if no_progress_bar:
-            total = None
-        else:
-            total = self.s3.head_object(
-                Bucket=from_info.bucket, Key=from_info.path
-            )["ContentLength"]
-        with Tqdm(
-            disable=no_progress_bar, total=total, bytes=True, desc=name
-        ) as pbar:
-            self.s3.download_file(
-                from_info.bucket, from_info.path, to_file, Callback=pbar.update
-            )
+        with self._get_obj(from_info) as obj:
+            with Tqdm(
+                disable=no_progress_bar,
+                total=obj.content_length,
+                bytes=True,
+                desc=name,
+            ) as pbar:
+                obj.download_file(to_file, Callback=pbar.update)

dvc/tree/ssh/__init__.py

@@ -238,7 +238,7 @@ class SSHTree(BaseTree):
             raise NotImplementedError
 
         with self.ssh(path_info) as ssh:
-            return ssh.md5(path_info.path)
+            return self.PARAM_CHECKSUM, ssh.md5(path_info.path)
 
     def getsize(self, path_info):
         with self.ssh(path_info) as ssh:

dvc/tree/webdav.py

@@ -142,15 +142,7 @@ class WebDAVTree(BaseTree):  # pylint:disable=abstract-method
                 "Content-MD5 header for '{url}'".format(url=path_info.url)
             )
 
-        if etag.startswith("W/"):
-            raise DvcException(
-                "Weak ETags are not supported."
-                " (Etag: '{etag}', URL: '{url}')".format(
-                    etag=etag, url=path_info.url
-                )
-            )
-
-        return etag
+        return self.PARAM_CHECKSUM, etag
 
     # Checks whether path points to directory
     def isdir(self, path_info):

dvc/updater.py

@@ -7,6 +7,7 @@ import colorama
 from packaging import version
 
 from dvc import __version__
+from dvc.config import Config, to_bool
 from dvc.lock import LockError, make_lock
 from dvc.utils import boxify, env2bool
 from dvc.utils.pkg import PKG
@@ -34,7 +35,7 @@ class Updater:  # pragma: no cover
         ctime = os.path.getmtime(self.updater_file)
         outdated = time.time() - ctime >= self.TIMEOUT
         if outdated:
-            logger.debug(f"'{self.updater_file}' is outdated(")
+            logger.debug(f"'{self.updater_file}' is outdated")
         return outdated
 
     def _with_lock(self, func, action):
@@ -46,7 +47,12 @@ class Updater:  # pragma: no cover
             logger.debug(msg.format(self.lock.lockfile, action))
 
     def check(self):
-        if os.getenv("CI") or env2bool("DVC_TEST") or PKG == "snap":
+        if (
+            os.getenv("CI")
+            or env2bool("DVC_TEST")
+            or PKG == "snap"
+            or not self.is_enabled()
+        ):
             return
 
         self._with_lock(self._check, "checking")
@@ -82,6 +88,7 @@ class Updater:  # pragma: no cover
 
     def _get_latest_version(self):
         import json
+
         import requests
 
         try:
@@ -144,3 +151,12 @@ class Updater:  # pragma: no cover
             package_manager = "binary"
 
         return instructions[package_manager]
+
+    def is_enabled(self):
+        enabled = to_bool(
+            Config(validate=False).get("core", {}).get("check_update", "true")
+        )
+        logger.debug(
+            "Check for update is {}abled.".format("en" if enabled else "dis")
+        )
+        return enabled

dvc/utils/__init__.py

@@ -1,18 +1,17 @@
 """Helpers for other modules."""
 
 import hashlib
-import io
 import json
 import logging
 import math
 import os
 import re
+import stat
 import sys
 import time
 
 import colorama
 import nanotime
-from ruamel.yaml import YAML
 from shortuuid import uuid
 
 logger = logging.getLogger(__name__)
@@ -45,8 +44,8 @@ def _fobj_md5(fobj, hash_md5, binary, progress_func=None):
 
 def file_md5(fname, tree=None):
     """ get the (md5 hexdigest, md5 digest) of a file """
-    from dvc.progress import Tqdm
     from dvc.istextfile import istextfile
+    from dvc.progress import Tqdm
 
     if tree:
         exists_func = tree.exists
@@ -237,18 +236,6 @@ def current_timestamp():
     return int(nanotime.timestamp(time.time()))
 
 
-def from_yaml_string(s):
-    return YAML().load(io.StringIO(s))
-
-
-def to_yaml_string(data):
-    stream = io.StringIO()
-    yaml = YAML()
-    yaml.default_flow_style = False
-    yaml.dump(data, stream)
-    return stream.getvalue()
-
-
 def colorize(message, color=None):
     """Returns a message in a specified color."""
     if not color:
@@ -363,19 +350,23 @@ def resolve_output(inp, out):
 
 
 def resolve_paths(repo, out):
+    from urllib.parse import urlparse
+
     from ..dvcfile import DVC_FILE_SUFFIX
     from ..path_info import PathInfo
     from .fs import contains_symlink_up_to
 
-    abspath = os.path.abspath(out)
+    abspath = PathInfo(os.path.abspath(out))
     dirname = os.path.dirname(abspath)
     base = os.path.basename(os.path.normpath(out))
 
-    # NOTE: `out` might not exist yet, so using `dirname`(aka `wdir`) to check
-    # if it is a local path.
+    scheme = urlparse(out).scheme
+    if os.name == "nt" and scheme == abspath.drive[0].lower():
+        # urlparse interprets windows drive letters as URL scheme
+        scheme = ""
     if (
-        os.path.exists(dirname)  # out might not exist yet, so
-        and PathInfo(abspath).isin_or_eq(repo.root_dir)
+        not scheme
+        and abspath.isin_or_eq(repo.root_dir)
         and not contains_symlink_up_to(abspath, repo.root_dir)
     ):
         wdir = dirname
@@ -394,6 +385,10 @@ def format_link(link):
     )
 
 
+def error_link(name):
+    return format_link(f"https://error.dvc.org/{name}")
+
+
 def parse_target(target, default=None):
     from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK, is_valid_filename
     from dvc.exceptions import DvcException
@@ -425,3 +420,7 @@ def parse_target(target, default=None):
         logger.debug("Assuming file to be '%s'", path)
 
     return path, name
+
+
+def is_exec(mode):
+    return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)

dvc/utils/collections.py

@@ -1,5 +1,16 @@
+import os
 from collections.abc import Mapping
 
+from pygtrie import StringTrie as _StringTrie
+
+
+class PathStringTrie(_StringTrie):
+    """Trie based on platform-dependent separator for pathname components."""
+
+    def __init__(self, *args, **kwargs):
+        kwargs["separator"] = os.sep
+        super().__init__(*args, **kwargs)
+
 
 def apply_diff(src, dest):
     """Recursively apply changes from src to dest.

dvc/utils/pager.py

@@ -12,8 +12,8 @@ logger = logging.getLogger(__name__)
 
 
 DEFAULT_PAGER = "less"
-DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
-    DEFAULT_PAGER
+DEFAULT_PAGER_FORMATTED = (
+    f"{DEFAULT_PAGER} --chop-long-lines --clear-screen --RAW-CONTROL-CHARS"
 )
 
 

dvc/utils/serialize/__init__.py

@@ -0,0 +1,12 @@
+from collections import defaultdict
+
+from ._common import *  # noqa, pylint: disable=wildcard-import
+from ._json import *  # noqa, pylint: disable=wildcard-import
+from ._toml import *  # noqa, pylint: disable=wildcard-import
+from ._yaml import *  # noqa, pylint: disable=wildcard-import
+
+LOADERS = defaultdict(lambda: load_yaml)  # noqa: F405
+LOADERS.update({".toml": load_toml, ".json": load_json})  # noqa: F405
+
+MODIFIERS = defaultdict(lambda: modify_yaml)  # noqa: F405
+MODIFIERS.update({".toml": modify_toml, ".json": modify_json})  # noqa: F405

dvc/utils/serialize/_common.py

@@ -0,0 +1,34 @@
+"""Common utilities for serialize."""
+import os
+from contextlib import contextmanager
+
+from dvc.exceptions import DvcException
+from dvc.utils import relpath
+
+
+class ParseError(DvcException):
+    """Errors while parsing files"""
+
+    def __init__(self, path, message):
+        path = relpath(path)
+        super().__init__(f"unable to read: '{path}', {message}")
+
+
+def _load_data(path, parser, tree=None):
+    open_fn = tree.open if tree else open
+    with open_fn(path, encoding="utf-8") as fd:
+        return parser(fd.read(), path)
+
+
+def _dump_data(path, data, dumper, tree=None):
+    open_fn = tree.open if tree else open
+    with open_fn(path, "w+", encoding="utf-8") as fd:
+        dumper(data, fd)
+
+
+@contextmanager
+def _modify_data(path, parser, dumper, tree=None):
+    exists = tree.exists if tree else os.path.exists
+    data = _load_data(path, parser=parser, tree=tree) if exists(path) else {}
+    yield data
+    dumper(path, data, tree=tree)

dvc/utils/serialize/_json.py

@@ -0,0 +1,29 @@
+import json
+
+from funcy import contextmanager, reraise
+
+from ._common import ParseError, _dump_data, _load_data, _modify_data
+
+
+class JSONFileCorruptedError(ParseError):
+    def __init__(self, path):
+        super().__init__(path, "JSON file structure is corrupted")
+
+
+def load_json(path, tree=None):
+    return _load_data(path, parser=parse_json, tree=tree)
+
+
+def parse_json(text, path):
+    with reraise(json.JSONDecodeError, JSONFileCorruptedError(path)):
+        return json.loads(text) or {}
+
+
+def dump_json(path, data, tree=None):
+    return _dump_data(path, data, dumper=json.dump, tree=tree)
+
+
+@contextmanager
+def modify_json(path, tree=None):
+    with _modify_data(path, parse_json, dump_json, tree=tree) as d:
+        yield d

dvc/utils/serialize/_toml.py

@@ -0,0 +1,45 @@
+from contextlib import contextmanager
+
+import toml
+from funcy import reraise
+
+from ._common import ParseError, _dump_data, _load_data, _modify_data
+
+
+class TOMLFileCorruptedError(ParseError):
+    def __init__(self, path):
+        super().__init__(path, "TOML file structure is corrupted")
+
+
+def load_toml(path, tree=None):
+    return _load_data(path, parser=parse_toml, tree=tree)
+
+
+def parse_toml(text, path, decoder=None):
+    with reraise(toml.TomlDecodeError, TOMLFileCorruptedError(path)):
+        return toml.loads(text, decoder=decoder)
+
+
+def parse_toml_for_update(text, path):
+    """Parses text into Python structure.
+
+    NOTE: Python toml package does not currently use ordered dicts, so
+    keys may be re-ordered between load/dump, but this function will at
+    least preserve comments.
+    """
+    decoder = toml.TomlPreserveCommentDecoder()
+    return parse_toml(text, path, decoder=decoder)
+
+
+def _dump(data, stream):
+    return toml.dump(data, stream, encoder=toml.TomlPreserveCommentEncoder())
+
+
+def dump_toml(path, data, tree=None):
+    return _dump_data(path, data, dumper=_dump, tree=tree)
+
+
+@contextmanager
+def modify_toml(path, tree=None):
+    with _modify_data(path, parse_toml_for_update, dump_toml, tree=tree) as d:
+        yield d

dvc/utils/serialize/_yaml.py

@@ -0,0 +1,71 @@
+import io
+from collections import OrderedDict
+from contextlib import contextmanager
+
+from funcy import reraise
+from ruamel.yaml import YAML
+from ruamel.yaml.error import YAMLError
+
+from ._common import ParseError, _dump_data, _load_data, _modify_data
+
+
+class YAMLFileCorruptedError(ParseError):
+    def __init__(self, path):
+        super().__init__(path, "YAML file structure is corrupted")
+
+
+def load_yaml(path, tree=None):
+    return _load_data(path, parser=parse_yaml, tree=tree)
+
+
+def parse_yaml(text, path, typ="safe"):
+    yaml = YAML(typ=typ)
+    with reraise(YAMLError, YAMLFileCorruptedError(path)):
+        return yaml.load(text) or {}
+
+
+def parse_yaml_for_update(text, path):
+    """Parses text into Python structure.
+
+    Unlike `parse_yaml()` this returns ordered dicts, values have special
+    attributes to store comments and line breaks. This allows us to preserve
+    all of those upon dump.
+
+    This one is, however, several times slower than simple `parse_yaml()`.
+    """
+    return parse_yaml(text, path, typ="rt")
+
+
+def _get_yaml():
+    yaml = YAML()
+    yaml.default_flow_style = False
+
+    # tell Dumper to represent OrderedDict as normal dict
+    yaml_repr_cls = yaml.Representer
+    yaml_repr_cls.add_representer(OrderedDict, yaml_repr_cls.represent_dict)
+    return yaml
+
+
+def _dump(data, stream):
+    yaml = _get_yaml()
+    return yaml.dump(data, stream)
+
+
+def dump_yaml(path, data, tree=None):
+    return _dump_data(path, data, dumper=_dump, tree=tree)
+
+
+def loads_yaml(s, typ="safe"):
+    return YAML(typ=typ).load(s)
+
+
+def dumps_yaml(d):
+    stream = io.StringIO()
+    _dump(d, stream)
+    return stream.getvalue()
+
+
+@contextmanager
+def modify_yaml(path, tree=None):
+    with _modify_data(path, parse_yaml_for_update, dump_yaml, tree=tree) as d:
+        yield d

dvc/utils/yaml.py

@@ -1,53 +0,0 @@
-from collections import OrderedDict
-
-from ruamel.yaml import YAML
-from ruamel.yaml.error import YAMLError
-
-from dvc.exceptions import YAMLFileCorruptedError
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
-
-def load_yaml(path):
-    with open(path, encoding="utf-8") as fd:
-        return parse_yaml(fd.read(), path)
-
-
-def parse_yaml(text, path):
-    try:
-        import yaml
-
-        return yaml.load(text, Loader=SafeLoader) or {}
-    except yaml.error.YAMLError as exc:
-        raise YAMLFileCorruptedError(path) from exc
-
-
-def parse_yaml_for_update(text, path):
-    """Parses text into Python structure.
-
-    Unlike `parse_yaml()` this returns ordered dicts, values have special
-    attributes to store comments and line breaks. This allows us to preserve
-    all of those upon dump.
-
-    This one is, however, several times slower than simple `parse_yaml()`.
-    """
-    try:
-        yaml = YAML()
-        return yaml.load(text) or {}
-    except YAMLError as exc:
-        raise YAMLFileCorruptedError(path) from exc
-
-
-def dump_yaml(path, data):
-    with open(path, "w", encoding="utf-8") as fd:
-        yaml = YAML()
-        yaml.default_flow_style = False
-        # tell Dumper to represent OrderedDict as
-        # normal dict
-        yaml.Representer.add_representer(
-            OrderedDict, yaml.Representer.represent_dict
-        )
-        yaml.dump(data, fd)

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "1.3.0"
+_BASE_VERSION = "1.6.4"
 
 
 def _generate_version(base_version):

setup.cfg

@@ -15,10 +15,6 @@ show_source=true
 count=true
 
 [isort]
-include_trailing_comma=true
+profile=black
 known_first_party=dvc,tests
-known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pygtrie,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,toml,tqdm,voluptuous,yaml,zc
 line_length=79
-force_grid_wrap=0
-use_parentheses=True
-multi_line_output=3

setup.py

@@ -60,7 +60,6 @@ install_requires = [
     "grandalf==0.6",
     "distro>=1.3.0",
     "appdirs>=1.4.3",
-    "PyYAML>=5.1.2,<5.4",  # Compatibility with awscli
     "ruamel.yaml>=0.16.1",
     "toml>=0.10.1",
     "funcy>=1.14",
@@ -75,12 +74,14 @@ install_requires = [
     "networkx>=2.1,<2.5",
     "pydot>=1.2.4",
     "speedcopy>=2.0.1; python_version < '3.8' and sys_platform == 'win32'",
+    "dataclasses; python_version < '3.7'",
     "flatten_json>=0.1.6,<0.1.8",
     "tabulate>=0.8.7",
     "pygtrie==2.3.2",
     "dpath>=2.0.1,<3",
-    "shtab>=1.1.0,<2",
+    "shtab>=1.3.0,<2",
     "rich>=3.0.5",
+    "dictdiffer>=0.8.1",
 ]
 
 
@@ -89,7 +90,7 @@ install_requires = [
 gs = ["google-cloud-storage==1.19.0"]
 gdrive = ["pydrive2>=1.4.14"]
 s3 = ["boto3>=1.9.201"]
-azure = ["azure-storage-blob==2.1.0", "knack"]
+azure = ["azure-storage-blob>=12.0", "knack"]
 oss = ["oss2==2.6.1"]
 ssh = ["paramiko>=2.5.0"]
 hdfs = ["pyarrow>=0.17.0"]
@@ -119,7 +120,7 @@ tests_requirements = [
     "google-compute-engine==2.8.13",
     "Pygments",  # required by collective.checkdocs,
     "collective.checkdocs",
-    "flake8==3.7.9",
+    "flake8==3.8.3",
     "psutil",
     "flake8-docstrings",
     "pydocstyle<4.0",
@@ -131,8 +132,8 @@ tests_requirements = [
     "flake8-bugbear",
     "flake8-comprehensions",
     "flake8-string-format",
-    "pylint",
-    "pylint-pytest",
+    "pylint==2.5.3",
+    "pylint-pytest>=0.3.0",
     "pylint-plugin-utils",
     "wget",
     "filelock",

tests/__init__.py

@@ -2,9 +2,10 @@ import os
 
 # Increasing fd ulimit for tests
 if os.name == "nt":
-    import win32file  # pylint: disable=import-error
     import subprocess
 
+    import win32file  # pylint: disable=import-error
+
     win32file._setmaxstdio(2048)
 
     # Workaround for two bugs:

tests/dir_helpers.py

@@ -74,6 +74,8 @@ disable_other_loggers()
 
 
 class TmpDir(pathlib.Path):
+    scheme = "local"
+
     def __new__(cls, *args, **kwargs):
         if cls is TmpDir:
             cls = (  # pylint: disable=self-cls-assignment
@@ -87,7 +89,7 @@ class TmpDir(pathlib.Path):
         self._init()
         return self
 
-    def init(self, *, scm=False, dvc=False):
+    def init(self, *, scm=False, dvc=False, subdir=False):
         from dvc.repo import Repo
         from dvc.scm.git import Git
 
@@ -100,7 +102,9 @@ class TmpDir(pathlib.Path):
             git_init(str_path)
         if dvc:
             self.dvc = Repo.init(
-                str_path, no_scm=not scm and not hasattr(self, "scm")
+                str_path,
+                no_scm=not scm and not hasattr(self, "scm"),
+                subdir=subdir,
             )
         if scm:
             self.scm = self.dvc.scm if hasattr(self, "dvc") else Git(str_path)
@@ -123,10 +127,10 @@ class TmpDir(pathlib.Path):
         if isinstance(struct, (str, bytes, pathlib.PurePath)):
             struct = {struct: text}
 
-        self._gen(struct)
-        return struct.keys()
+        return self._gen(struct)
 
     def _gen(self, struct, prefix=None):
+        paths = []
         for name, contents in struct.items():
             path = (prefix or self) / name
 
@@ -141,6 +145,8 @@ class TmpDir(pathlib.Path):
                     path.write_bytes(contents)
                 else:
                     path.write_text(contents, encoding="utf-8")
+            paths.append(path)
+        return paths
 
     def dvc_gen(self, struct, text="", commit=None):
         paths = self.gen(struct, text)
@@ -249,10 +255,10 @@ class PosixTmpDir(TmpDir, pathlib.PurePosixPath):
 
 @pytest.fixture(scope="session")
 def make_tmp_dir(tmp_path_factory, request):
-    def make(name, *, scm=False, dvc=False):
+    def make(name, *, scm=False, dvc=False, **kwargs):
         path = tmp_path_factory.mktemp(name) if isinstance(name, str) else name
         new_dir = TmpDir(path)
-        new_dir.init(scm=scm, dvc=dvc)
+        new_dir.init(scm=scm, dvc=dvc, **kwargs)
         request.addfinalizer(new_dir.close)
         return new_dir
 
@@ -293,7 +299,12 @@ def git_init(path):
 def run_copy(tmp_dir, dvc):
     tmp_dir.gen(
         "copy.py",
-        "import sys, shutil\nshutil.copyfile(sys.argv[1], sys.argv[2])",
+        (
+            "import sys, shutil, os\n"
+            "shutil.copyfile(sys.argv[1], sys.argv[2]) "
+            "if os.path.isfile(sys.argv[1]) "
+            "else shutil.copytree(sys.argv[1], sys.argv[2])"
+        ),
     )
 
     def run_copy(src, dst, **run_kwargs):

tests/func/experiments/test_experiments.py

@@ -22,3 +22,81 @@ def test_new_simple(tmp_dir, scm, dvc, mocker):
     assert (
         tmp_dir / ".dvc" / "experiments" / "metrics.yaml"
     ).read_text() == "foo: 2"
+
+
+def test_update_with_pull(tmp_dir, scm, dvc, mocker):
+    tmp_dir.gen("copy.py", COPY_SCRIPT)
+    tmp_dir.gen("params.yaml", "foo: 1")
+    stage = dvc.run(
+        cmd="python copy.py params.yaml metrics.yaml",
+        metrics_no_cache=["metrics.yaml"],
+        params=["foo"],
+        name="copy-file",
+    )
+    scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"])
+    scm.commit("init")
+    expected_revs = [scm.get_rev()]
+
+    tmp_dir.gen("params.yaml", "foo: 2")
+    dvc.reproduce(stage.addressing, experiment=True)
+    scm.add(["dvc.yaml", "dvc.lock", "params.yaml", "metrics.yaml"])
+    scm.commit("promote experiment")
+    expected_revs.append(scm.get_rev())
+
+    tmp_dir.gen("params.yaml", "foo: 3")
+    dvc.reproduce(stage.addressing, experiment=True)
+
+    exp_scm = dvc.experiments.scm
+    for rev in expected_revs:
+        assert exp_scm.has_rev(rev)
+
+
+def test_checkout(tmp_dir, scm, dvc):
+    tmp_dir.gen("copy.py", COPY_SCRIPT)
+    tmp_dir.gen("params.yaml", "foo: 1")
+    stage = dvc.run(
+        cmd="python copy.py params.yaml metrics.yaml",
+        metrics_no_cache=["metrics.yaml"],
+        params=["foo"],
+        name="copy-file",
+    )
+    scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"])
+    scm.commit("init")
+
+    dvc.reproduce(stage.addressing, experiment=True, params=["foo=2"])
+    exp_a = dvc.experiments.scm.get_rev()
+
+    dvc.reproduce(stage.addressing, experiment=True, params=["foo=3"])
+    exp_b = dvc.experiments.scm.get_rev()
+
+    dvc.experiments.checkout(exp_a)
+    assert (tmp_dir / "params.yaml").read_text().strip() == "foo: 2"
+    assert (tmp_dir / "metrics.yaml").read_text().strip() == "foo: 2"
+
+    dvc.experiments.checkout(exp_b)
+    assert (tmp_dir / "params.yaml").read_text().strip() == "foo: 3"
+    assert (tmp_dir / "metrics.yaml").read_text().strip() == "foo: 3"
+
+
+def test_get_baseline(tmp_dir, scm, dvc):
+    tmp_dir.gen("copy.py", COPY_SCRIPT)
+    tmp_dir.gen("params.yaml", "foo: 1")
+    stage = dvc.run(
+        cmd="python copy.py params.yaml metrics.yaml",
+        metrics_no_cache=["metrics.yaml"],
+        params=["foo"],
+        name="copy-file",
+    )
+    scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"])
+    scm.commit("init")
+    expected = scm.get_rev()
+    assert dvc.experiments.get_baseline(expected) is None
+
+    dvc.reproduce(stage.addressing, experiment=True, params=["foo=2"])
+    rev = dvc.experiments.scm.get_rev()
+    assert dvc.experiments.get_baseline(rev) == expected
+
+    dvc.reproduce(
+        stage.addressing, experiment=True, params=["foo=3"], queue=True
+    )
+    assert dvc.experiments.get_baseline("stash@{0}") == expected

tests/func/experiments/test_show.py

@@ -1,3 +1,6 @@
+from datetime import datetime
+
+from dvc.dvcfile import PIPELINE_FILE
 from tests.func.test_repro_multistage import COPY_SCRIPT
 
 
@@ -16,5 +19,86 @@ def test_show_simple(tmp_dir, scm, dvc):
             "metrics": {"metrics.yaml": {"foo": 1}},
             "params": {"params.yaml": {"foo": 1}},
             "queued": False,
+            "timestamp": None,
         }
     }
+
+
+def test_show_experiment(tmp_dir, scm, dvc):
+    tmp_dir.gen("copy.py", COPY_SCRIPT)
+    tmp_dir.gen("params.yaml", "foo: 1")
+    dvc.run(
+        cmd="python copy.py params.yaml metrics.yaml",
+        metrics_no_cache=["metrics.yaml"],
+        params=["foo"],
+        name="foo",
+    )
+    scm.add(["copy.py", "params.yaml", "metrics.yaml", "dvc.yaml", "dvc.lock"])
+    scm.commit("baseline")
+    baseline_rev = scm.get_rev()
+    timestamp = datetime.fromtimestamp(
+        scm.repo.rev_parse(baseline_rev).committed_date
+    )
+
+    dvc.reproduce(PIPELINE_FILE, experiment=True, params=["foo=2"])
+    results = dvc.experiments.show()
+
+    expected_baseline = {
+        "metrics": {"metrics.yaml": {"foo": 1}},
+        "params": {"params.yaml": {"foo": 1}},
+        "queued": False,
+        "timestamp": timestamp,
+    }
+    expected_params = {"foo": 2}
+
+    assert set(results.keys()) == {"workspace", baseline_rev}
+    experiments = results[baseline_rev]
+    assert len(experiments) == 2
+    for rev, exp in experiments.items():
+        if rev == "baseline":
+            assert exp == expected_baseline
+        else:
+            assert exp["metrics"]["metrics.yaml"] == expected_params
+            assert exp["params"]["params.yaml"] == expected_params
+
+
+def test_show_queued(tmp_dir, scm, dvc):
+    tmp_dir.gen("copy.py", COPY_SCRIPT)
+    tmp_dir.gen("params.yaml", "foo: 1")
+    stage = dvc.run(
+        cmd="python copy.py params.yaml metrics.yaml",
+        metrics_no_cache=["metrics.yaml"],
+        params=["foo"],
+        name="foo",
+    )
+    scm.add(["copy.py", "params.yaml", "metrics.yaml", "dvc.yaml", "dvc.lock"])
+    scm.commit("baseline")
+    baseline_rev = scm.get_rev()
+
+    dvc.reproduce(
+        stage.addressing, experiment=True, params=["foo=2"], queue=True
+    )
+    exp_rev = dvc.experiments.scm.resolve_rev("stash@{0}")
+
+    results = dvc.experiments.show()[baseline_rev]
+    assert len(results) == 2
+    exp = results[exp_rev]
+    assert exp["queued"]
+    assert exp["params"]["params.yaml"] == {"foo": 2}
+
+    # test that only queued experiments for the current baseline are returned
+    tmp_dir.gen("foo", "foo")
+    scm.add(["foo"])
+    scm.commit("new commit")
+    new_rev = scm.get_rev()
+
+    dvc.reproduce(
+        stage.addressing, experiment=True, params=["foo=3"], queue=True
+    )
+    exp_rev = dvc.experiments.scm.resolve_rev("stash@{0}")
+
+    results = dvc.experiments.show()[new_rev]
+    assert len(results) == 2
+    exp = results[exp_rev]
+    assert exp["queued"]
+    assert exp["params"]["params.yaml"] == {"foo": 3}

tests/func/metrics/test_diff.py

@@ -1,9 +1,8 @@
 import json
 import logging
 
-import yaml
-
 from dvc.main import main
+from dvc.utils.serialize import dump_yaml
 
 
 def test_metrics_diff_simple(tmp_dir, scm, dvc, run_copy_metrics):
@@ -24,7 +23,7 @@ def test_metrics_diff_simple(tmp_dir, scm, dvc, run_copy_metrics):
 def test_metrics_diff_yaml(tmp_dir, scm, dvc, run_copy_metrics):
     def _gen(val):
         metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
-        tmp_dir.gen({"m_temp.yaml": yaml.dump(metrics)})
+        dump_yaml("m_temp.yaml", metrics)
         run_copy_metrics(
             "m_temp.yaml", "m.yaml", metrics=["m.yaml"], commit=str(val)
         )

tests/func/plots/test_plots.py

@@ -6,7 +6,6 @@ import shutil
 from collections import OrderedDict
 
 import pytest
-import yaml
 from funcy import first
 
 from dvc.repo.plots.data import (
@@ -21,6 +20,7 @@ from dvc.repo.plots.template import (
     NoFieldInDataError,
     TemplateNotFoundError,
 )
+from dvc.utils.serialize import dump_yaml, dumps_yaml
 
 
 def _write_csv(metric, filename, header=True):
@@ -493,9 +493,7 @@ def test_plot_default_choose_column(tmp_dir, scm, dvc, run_copy_metrics):
 
 def test_plot_yaml(tmp_dir, scm, dvc, run_copy_metrics):
     metric = [{"val": 2}, {"val": 3}]
-    with open("metric_t.yaml", "w") as fobj:
-        yaml.dump(metric, fobj)
-
+    dump_yaml("metric_t.yaml", metric)
     run_copy_metrics(
         "metric_t.yaml", "metric.yaml", plots_no_cache=["metric.yaml"]
     )
@@ -543,7 +541,7 @@ def test_load_metric_from_dict_yaml(tmp_dir):
     metric = [{"acccuracy": 1, "loss": 2}, {"accuracy": 3, "loss": 4}]
     dmetric = {"train": metric}
 
-    plot_data = YAMLPlotData("-", "revision", yaml.dump(dmetric))
+    plot_data = YAMLPlotData("-", "revision", dumps_yaml(dmetric))
 
     expected = metric
     for d in expected:

tests/func/test_add.py

@@ -17,7 +17,6 @@ from dvc.exceptions import (
     OutputDuplicationError,
     OverlappingOutputPathsError,
     RecursiveAddingWhileUsingFilename,
-    YAMLFileCorruptedError,
 )
 from dvc.main import main
 from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError
@@ -27,7 +26,7 @@ from dvc.system import System
 from dvc.tree.local import LocalTree
 from dvc.utils import LARGE_DIR_SIZE, file_md5, relpath
 from dvc.utils.fs import path_isin
-from dvc.utils.yaml import load_yaml
+from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml
 from tests.basic_env import TestDvc
 from tests.utils import get_gitignore_content
 
@@ -595,8 +594,9 @@ class TestAddUnprotected(TestDvc):
 @pytest.fixture
 def temporary_windows_drive(tmp_path_factory):
     import string
-    import win32api  # pylint: disable=import-error
     from ctypes import windll
+
+    import win32api  # pylint: disable=import-error
     from win32con import DDD_REMOVE_DEFINITION  # pylint: disable=import-error
 
     drives = [

tests/func/test_check_ignore.py

@@ -0,0 +1,135 @@
+import os
+
+import pytest
+
+from dvc.ignore import DvcIgnore
+from dvc.main import main
+
+
+@pytest.mark.parametrize(
+    "file,ret,output", [("ignored", 0, True), ("not_ignored", 1, False)]
+)
+def test_check_ignore(tmp_dir, dvc, file, ret, output, caplog):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored")
+
+    assert main(["check-ignore", file]) == ret
+    assert (file in caplog.text) is output
+    assert "Having any troubles?" not in caplog.text
+
+
+@pytest.mark.parametrize(
+    "file,ret,output",
+    [
+        ("file", 0, "{}:1:f*\tfile\n".format(DvcIgnore.DVCIGNORE_FILE)),
+        ("foo", 0, "{}:2:!foo\tfoo\n".format(DvcIgnore.DVCIGNORE_FILE)),
+        (
+            os.path.join("dir", "foobar"),
+            0,
+            "{}:1:foobar\t{}\n".format(
+                os.path.join("dir", DvcIgnore.DVCIGNORE_FILE),
+                os.path.join("dir", "foobar"),
+            ),
+        ),
+    ],
+)
+def test_check_ignore_details(tmp_dir, dvc, file, ret, output, caplog):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "f*\n!foo")
+    tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "foobar"}})
+
+    assert main(["check-ignore", "-d", file]) == ret
+    assert output in caplog.text
+
+
+@pytest.mark.parametrize("non_matching", [True, False])
+def test_check_ignore_non_matching(tmp_dir, dvc, non_matching, caplog):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "other")
+    if non_matching:
+        assert main(["check-ignore", "-d", "-n", "file"]) == 1
+    else:
+        assert main(["check-ignore", "-d", "file"]) == 1
+
+    assert ("::\tfile\n" in caplog.text) is non_matching
+
+
+@pytest.mark.parametrize(
+    "args",
+    [
+        ["-n", "file"],
+        ["-a", "file"],
+        ["-q", "-d", "file"],
+        ["--stdin", "file"],
+        [],
+    ],
+)
+def test_check_ignore_error_args_cases(tmp_dir, dvc, args, caplog):
+    assert main(["check-ignore"] + args) == 255
+    assert ("Having any troubles?" in caplog.text) == ("-q" not in args)
+
+
+@pytest.mark.parametrize("path,ret", [({"dir": {}}, 0), ({"dir": "files"}, 1)])
+def test_check_ignore_dir(tmp_dir, dvc, path, ret):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/")
+    tmp_dir.gen(path)
+
+    assert main(["check-ignore", "-q", "dir"]) == ret
+
+
+def test_check_ignore_default_dir(tmp_dir, dvc):
+    assert main(["check-ignore", "-q", ".dvc"]) == 1
+
+
+def test_check_ignore_out_side_repo(tmp_dir, dvc):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "file")
+    assert main(["check-ignore", "-q", "../file"]) == 1
+
+
+def test_check_ignore_sub_repo(tmp_dir, dvc):
+    tmp_dir.gen(
+        {DvcIgnore.DVCIGNORE_FILE: "other", "dir": {".dvc": {}, "foo": "bar"}}
+    )
+
+    assert main(["check-ignore", "-q", os.path.join("dir", "foo")]) == 1
+
+
+def test_check_sub_dir_ignore_file(tmp_dir, dvc, caplog):
+    tmp_dir.gen(
+        {
+            DvcIgnore.DVCIGNORE_FILE: "other",
+            "dir": {DvcIgnore.DVCIGNORE_FILE: "bar\nfoo", "foo": "bar"},
+        }
+    )
+
+    assert main(["check-ignore", "-d", os.path.join("dir", "foo")]) == 0
+    assert (
+        "{}:2:foo\t{}".format(
+            os.path.join("dir", DvcIgnore.DVCIGNORE_FILE),
+            os.path.join("dir", "foo"),
+        )
+        in caplog.text
+    )
+
+    sub_dir = tmp_dir / "dir"
+    with sub_dir.chdir():
+        assert main(["check-ignore", "-d", "foo"]) == 0
+        assert ".dvcignore:2:foo\tfoo" in caplog.text
+
+
+def test_check_ignore_details_all(tmp_dir, dvc, caplog):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "f*\n!foo")
+
+    assert main(["check-ignore", "-d", "-a", "foo"]) == 0
+    assert "{}:1:f*\tfoo\n".format(DvcIgnore.DVCIGNORE_FILE) in caplog.text
+    assert "{}:2:!foo\tfoo\n".format(DvcIgnore.DVCIGNORE_FILE) in caplog.text
+
+
+@pytest.mark.parametrize(
+    "file,ret,output", [("ignored", 0, True), ("not_ignored", 1, False)]
+)
+def test_check_ignore_stdin_mode(
+    tmp_dir, dvc, file, ret, output, caplog, mocker
+):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored")
+    mocker.patch("builtins.input", side_effect=[file, ""])
+
+    assert main(["check-ignore", "--stdin"]) == ret
+    assert (file in caplog.text) is output

tests/func/test_checkout.py

@@ -26,7 +26,7 @@ from dvc.tree.local import LocalTree
 from dvc.tree.s3 import S3Tree
 from dvc.utils import relpath
 from dvc.utils.fs import walk_files
-from dvc.utils.yaml import dump_yaml, load_yaml
+from dvc.utils.serialize import dump_yaml, load_yaml
 from tests.basic_env import TestDvc, TestDvcGit
 from tests.func.test_repro import TestRepro
 from tests.remotes import S3
@@ -764,7 +764,7 @@ def test_checkout_for_external_outputs(tmp_dir, dvc):
 
     remote = Remote(S3Tree(dvc, {"url": S3.get_url()}))
     file_path = remote.tree.path_info / "foo"
-    remote.tree.s3.put_object(
+    remote.tree.s3.meta.client.put_object(
         Bucket=remote.tree.path_info.bucket, Key=file_path.path, Body="foo"
     )
 
@@ -775,7 +775,7 @@ def test_checkout_for_external_outputs(tmp_dir, dvc):
     assert stats == {**empty_checkout, "added": [str(file_path)]}
     assert remote.tree.exists(file_path)
 
-    remote.tree.s3.put_object(
+    remote.tree.s3.meta.client.put_object(
         Bucket=remote.tree.path_info.bucket,
         Key=file_path.path,
         Body="foo\nfoo",

tests/func/test_commit.py

@@ -6,7 +6,7 @@ from dvc.dependency.base import DependencyDoesNotExistError
 from dvc.dvcfile import PIPELINE_FILE
 from dvc.output.base import OutputDoesNotExistError
 from dvc.stage.exceptions import StageCommitError
-from dvc.utils.yaml import dump_yaml, load_yaml
+from dvc.utils.serialize import dump_yaml, load_yaml
 
 
 def test_commit_recursive(tmp_dir, dvc):

tests/func/test_data_cloud.py

@@ -6,13 +6,18 @@ import pytest
 from flaky.flaky_decorator import flaky
 
 from dvc.cache import NamedCache
-from dvc.cache.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
+from dvc.cache.base import (
+    STATUS_DELETED,
+    STATUS_MISSING,
+    STATUS_NEW,
+    STATUS_OK,
+)
 from dvc.external_repo import clean_repos
 from dvc.main import main
 from dvc.stage.exceptions import StageNotFound
 from dvc.tree.local import LocalTree
-from dvc.utils.fs import remove
-from dvc.utils.yaml import dump_yaml, load_yaml
+from dvc.utils.fs import move, remove
+from dvc.utils.serialize import dump_yaml, load_yaml
 
 from .test_api import all_clouds
 
@@ -49,6 +54,22 @@ def test_cloud(tmp_dir, dvc, remote):  # pylint:disable=unused-argument
         expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
         assert status_dir == expected
 
+        # Move cache and check status
+        # See issue https://github.com/iterative/dvc/issues/4383 for details
+        backup_dir = dvc.cache.local.cache_dir + ".backup"
+        move(dvc.cache.local.cache_dir, backup_dir)
+        status = dvc.cloud.status(info, show_checksums=True)
+        expected = {md5: {"name": md5, "status": STATUS_MISSING}}
+        assert status == expected
+
+        status_dir = dvc.cloud.status(info_dir, show_checksums=True)
+        expected = {md5_dir: {"name": md5_dir, "status": STATUS_MISSING}}
+        assert status_dir == expected
+
+        # Restore original cache:
+        remove(dvc.cache.local.cache_dir)
+        move(backup_dir, dvc.cache.local.cache_dir)
+
         # Push and check status
         dvc.cloud.push(info)
         assert os.path.exists(cache)
@@ -227,10 +248,10 @@ def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
     assert bar in caplog.text
 
     caplog.clear()
-    dvc.status(cloud=True)
-    assert header in caplog.text
-    assert foo in caplog.text
-    assert bar in caplog.text
+    assert dvc.status(cloud=True) == {"bar": "missing", "foo": "missing"}
+    assert header not in caplog.text
+    assert foo not in caplog.text
+    assert bar not in caplog.text
 
 
 def test_verify_hashes(