dvc: bump to 1.4.0

* Two new mode `-all` and `--stdin` for check-ignore

Fix #4321

Introduce new arguments --all for dvc check-ignore
Add tests for dvc check-ignore --all

* `--stdin` and targets only one

1. `--stdin` and targets can and only can have one
2. more tests

* New tests for stdin mode and solve a problem

1. Add a new tests with mock stdin for interactive mode
2. Solve the problem of `assert "" in caplog.text` in tests always `True`.

* Remove path_input interface

1. remove path_input and make _ask to a public method instead

* Update dvc/command/check_ignore.py

* Update dvc/command/check_ignore.py

* Update dvc/command/check_ignore.py

* Update dvc/command/check_ignore.py

* Update dvc/command/check_ignore.py

* Update dvc/command/check_ignore.py

* Update dvc/command/check_ignore.py

* Update dvc/command/check_ignore.py

* fix styling

Co-authored-by: karajan1001 <[email protected]>
Co-authored-by: Ruslan Kuprieiev <[email protected]>
Co-authored-by: Ruslan Kuprieiev <[email protected]>

dvc/cache/base.py

@@ -276,8 +276,6 @@ class CloudCache:
                 self.tree.scheme,
             )
 
-        if not hash_info:
-            hash_info = self.tree.save_info(path_info, tree=tree, **kwargs)
         hash_ = hash_info[self.tree.PARAM_CHECKSUM]
         return self._save(path_info, tree, hash_, save_link, **kwargs)
 

dvc/cli.py

@@ -6,6 +6,7 @@ import sys
 from .command import (
     add,
     cache,
+    check_ignore,
     checkout,
     commit,
     completion,
@@ -79,6 +80,7 @@ COMMANDS = [
     git_hook,
     plots,
     experiments,
+    check_ignore,
 ]
 
 

dvc/command/check_ignore.py

@@ -0,0 +1,130 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+from dvc.prompt import ask
+
+logger = logging.getLogger(__name__)
+
+
+class CmdCheckIgnore(CmdBase):
+    def __init__(self, args):
+        super().__init__(args)
+        self.ignore_filter = self.repo.tree.dvcignore
+
+    def _show_results(self, result):
+        if not result.match and not self.args.non_matching:
+            return
+
+        if self.args.details:
+            patterns = result.patterns
+            if not self.args.all:
+                patterns = patterns[-1:]
+
+            for pattern in patterns:
+                logger.info("{}\t{}".format(pattern, result.file))
+        else:
+            logger.info(result.file)
+
+    def _check_one_file(self, target):
+        result = self.ignore_filter.check_ignore(target)
+        self._show_results(result)
+        if result.match:
+            return 0
+        return 1
+
+    def _interactive_mode(self):
+        ret = 1
+        while True:
+            target = ask("")
+            if target == "":
+                logger.info(
+                    "Empty string is not a valid pathspec. Please use . "
+                    "instead if you meant to match all paths."
+                )
+                break
+            if not self._check_one_file(target):
+                ret = 0
+        return ret
+
+    def _normal_mode(self):
+        ret = 1
+        for target in self.args.targets:
+            if not self._check_one_file(target):
+                ret = 0
+        return ret
+
+    def _check_args(self):
+        if not self.args.stdin and not self.args.targets:
+            raise DvcException("`targets` or `--stdin` needed")
+
+        if self.args.stdin and self.args.targets:
+            raise DvcException("cannot have both `targets` and `--stdin`")
+
+        if self.args.non_matching and not self.args.details:
+            raise DvcException(
+                "`--non-matching` is only valid with `--details`"
+            )
+
+        if self.args.all and not self.args.details:
+            raise DvcException("`--all` is only valid with `--details`")
+
+        if self.args.quiet and self.args.details:
+            raise DvcException("cannot use both `--details` and `--quiet`")
+
+    def run(self):
+        self._check_args()
+        if self.args.stdin:
+            return self._interactive_mode()
+        return self._normal_mode()
+
+
+def add_parser(subparsers, parent_parser):
+    ADD_HELP = "Debug DVC ignore/exclude files"
+
+    parser = subparsers.add_parser(
+        "check-ignore",
+        parents=[parent_parser],
+        description=append_doc_link(ADD_HELP, "check-ignore"),
+        help=ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "-d",
+        "--details",
+        action="store_true",
+        default=False,
+        help="Show the exclude pattern together with each target path.",
+    )
+    parser.add_argument(
+        "-n",
+        "--non-matching",
+        action="store_true",
+        default=False,
+        help="Show the target paths which don’t match any pattern. "
+        "Only usable when `--details` is also employed",
+    )
+    parser.add_argument(
+        "--stdin",
+        action="store_true",
+        default=False,
+        help="Read pathnames from the standard input, one per line, "
+        "instead of from the command-line.",
+    )
+    parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show all of the patterns match the target paths. "
+        "Only usable when `--details` is also employed",
+    )
+    parser.add_argument(
+        "targets",
+        nargs="*",
+        help="Exact or wildcard paths of files or directories to check "
+        "ignore patterns.",
+    ).complete = completion.FILE
+    parser.set_defaults(func=CmdCheckIgnore)

dvc/command/experiments.py

@@ -48,6 +48,10 @@ def _collect_rows(
         return val
 
     def _extend(row, names, items):
+        if not items:
+            row.extend(["-"] * len(names))
+            return
+
         for fname, item in items:
             if isinstance(item, dict):
                 item = flatten(item, ".")
@@ -55,20 +59,23 @@ def _collect_rows(
                 item = {fname: item}
             for name in names:
                 if name in item:
-                    row.append(str(_round(item[name])))
+                    value = item[name]
+                    text = str(_round(value)) if value is not None else "-"
+                    row.append(text)
                 else:
                     row.append("-")
 
     for i, (rev, exp) in enumerate(experiments.items()):
         row = []
         style = None
+        queued = "*" if exp.get("queued", False) else ""
         if rev == "baseline":
             row.append(f"{base_rev}")
             style = "bold"
         elif i < len(experiments) - 1:
-            row.append(f"├── {rev[:7]}")
+            row.append(f"├── {queued}{rev[:7]}")
         else:
-            row.append(f"└── {rev[:7]}")
+            row.append(f"└── {queued}{rev[:7]}")
 
         _extend(row, metric_names, exp.get("metrics", {}).items())
         _extend(row, param_names, exp.get("params", {}).items())
@@ -83,9 +90,9 @@ def _show_experiments(all_experiments, console, precision=None):
     metric_names, param_names = _collect_names(all_experiments)
 
     table = Table()
-    table.add_column("Experiment")
+    table.add_column("Experiment", no_wrap=True)
     for name in metric_names:
-        table.add_column(name, justify="right")
+        table.add_column(name, justify="right", no_wrap=True)
     for name in param_names:
         table.add_column(name, justify="left")
 
@@ -120,15 +127,19 @@ class CmdExperimentsShow(CmdBase):
                 all_commits=self.args.all_commits,
             )
 
-            # Note: rich does not currently include a native way to force
-            # infinite width for use with a pager
-            console = Console(
-                file=io.StringIO(), force_terminal=True, width=9999
-            )
+            if self.args.no_pager:
+                console = Console()
+            else:
+                # Note: rich does not currently include a native way to force
+                # infinite width for use with a pager
+                console = Console(
+                    file=io.StringIO(), force_terminal=True, width=9999
+                )
 
             _show_experiments(all_experiments, console)
 
-            pager(console.file.getvalue())
+            if not self.args.no_pager:
+                pager(console.file.getvalue())
         except DvcException:
             logger.exception("failed to show experiments")
             return 1
@@ -141,9 +152,7 @@ class CmdExperimentsCheckout(CmdBase):
         if not self.repo.experiments:
             return 0
 
-        self.repo.experiments.checkout(
-            self.args.experiment, force=self.args.force
-        )
+        self.repo.experiments.checkout(self.args.experiment)
 
         return 0
 
@@ -229,6 +238,7 @@ def add_parser(subparsers, parent_parser):
     experiments_parser = subparsers.add_parser(
         "experiments",
         parents=[parent_parser],
+        aliases=["exp"],
         description=append_doc_link(EXPERIMENTS_HELP, "experiments"),
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -269,6 +279,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show metrics for all commits.",
     )
+    experiments_show_parser.add_argument(
+        "--no-pager",
+        action="store_true",
+        default=False,
+        help="Do not pipe output into a pager.",
+    )
     experiments_show_parser.set_defaults(func=CmdExperimentsShow)
 
     EXPERIMENTS_CHECKOUT_HELP = "Checkout experiments."
@@ -281,14 +297,6 @@ def add_parser(subparsers, parent_parser):
         help=EXPERIMENTS_CHECKOUT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    experiments_checkout_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Overwrite your current workspace with changes from the "
-        "experiment.",
-    )
     experiments_checkout_parser.add_argument(
         "experiment", help="Checkout this experiment.",
     )

dvc/command/remote.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.config import ConfigError
+from dvc.config import ConfigError, merge
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
@@ -18,7 +18,7 @@ class CmdRemote(CmdConfig):
 
     def _check_exists(self, conf):
         if self.args.name not in conf["remote"]:
-            raise ConfigError(f"remote '{self.args.name}' doesn't exists.")
+            raise ConfigError(f"remote '{self.args.name}' doesn't exist.")
 
 
 class CmdRemoteAdd(CmdRemote):
@@ -61,7 +61,12 @@ class CmdRemoteRemove(CmdRemote):
 class CmdRemoteModify(CmdRemote):
     def run(self):
         with self.config.edit(self.args.level) as conf:
-            self._check_exists(conf)
+            merged = self.config.load_config_to_level(self.args.level)
+            merge(merged, conf)
+            self._check_exists(merged)
+
+            if self.args.name not in conf["remote"]:
+                conf["remote"][self.args.name] = {}
             section = conf["remote"][self.args.name]
             if self.args.unset:
                 section.pop(self.args.option, None)

dvc/command/repro.py

@@ -41,6 +41,10 @@ class CmdRepro(CmdBase):
                     recursive=self.args.recursive,
                     force_downstream=self.args.force_downstream,
                     experiment=self.args.experiment,
+                    queue=self.args.queue,
+                    run_all=self.args.run_all,
+                    jobs=self.args.jobs,
+                    params=self.args.params,
                 )
 
                 if len(stages) == 0:
@@ -174,4 +178,23 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=argparse.SUPPRESS,
     )
+    repro_parser.add_argument(
+        "--params",
+        action="append",
+        default=[],
+        help="Declare parameter values for an experiment.",
+        metavar="[<filename>:]<params_list>",
+    )
+    repro_parser.add_argument(
+        "--queue", action="store_true", default=False, help=argparse.SUPPRESS
+    )
+    repro_parser.add_argument(
+        "--run-all",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    repro_parser.add_argument(
+        "-j", "--jobs", type=int, help=argparse.SUPPRESS, metavar="<number>"
+    )
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/version.py

@@ -10,7 +10,7 @@ from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.scm.base import SCMError
 from dvc.system import System
-from dvc.utils import is_binary, relpath
+from dvc.utils import relpath
 from dvc.utils.pkg import PKG
 from dvc.version import __version__
 
@@ -27,13 +27,18 @@ class CmdVersion(CmdBaseNoRepo):
     def run(self):
         from dvc.repo import Repo
 
+        package = PKG
+        if PKG is None:
+            package = ""
+        else:
+            package = f"({PKG})"
+
         info = [
-            f"DVC version: {__version__}",
-            f"Python version: {platform.python_version()}",
-            f"Platform: {platform.platform()}",
-            f"Binary: {is_binary()}",
-            f"Package: {PKG}",
-            f"Supported remotes: {self.get_supported_remotes()}",
+            f"DVC version: {__version__} {package}",
+            "---------------------------------",
+            f"Platform: Python {platform.python_version()} on "
+            f"{platform.platform()}",
+            f"Supports: {self.get_supported_remotes()}",
         ]
 
         try:
@@ -46,13 +51,13 @@ class CmdVersion(CmdBaseNoRepo):
             # `dvc config cache.shared group`.
             if os.path.exists(repo.cache.local.cache_dir):
                 info.append(
-                    "Cache: {}".format(self.get_linktype_support_info(repo))
+                    "Cache types: {}".format(
+                        self.get_linktype_support_info(repo)
+                    )
                 )
                 if psutil:
                     fs_type = self.get_fs_type(repo.cache.local.cache_dir)
-                    info.append(
-                        f"Filesystem type (cache directory): {fs_type}"
-                    )
+                    info.append(f"Cache directory: {fs_type}")
             else:
                 logger.warning(
                     "Unable to detect supported link types, as cache "
@@ -68,11 +73,11 @@ class CmdVersion(CmdBaseNoRepo):
             root_directory = os.getcwd()
             info.append("Repo: dvc, git (broken)")
         else:
-            info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
+            if psutil:
+                fs_root = self.get_fs_type(os.path.abspath(root_directory))
+                info.append(f"Workspace directory: {fs_root}")
 
-        if psutil:
-            fs_root = self.get_fs_type(os.path.abspath(root_directory))
-            info.append(f"Filesystem type (workspace): {fs_root}")
+            info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
 
         logger.info("\n".join(info))
         return 0
@@ -80,7 +85,7 @@ class CmdVersion(CmdBaseNoRepo):
     @staticmethod
     def get_fs_type(path):
         partition = {
-            pathlib.Path(part.mountpoint): (part.fstype, part.device)
+            pathlib.Path(part.mountpoint): (part.fstype + " on " + part.device)
             for part in psutil.disk_partitions(all=True)
         }
 
@@ -115,7 +120,9 @@ class CmdVersion(CmdBaseNoRepo):
                 os.unlink(dst)
             except DvcException:
                 status = "not supported"
-            cache.append(f"{name} - {status}")
+
+            if status == "supported":
+                cache.append(name)
         os.remove(src)
 
         return ", ".join(cache)
@@ -129,6 +136,12 @@ class CmdVersion(CmdBaseNoRepo):
             if not tree_cls.get_missing_deps():
                 supported_remotes.append(tree_cls.scheme)
 
+        if len(supported_remotes) == len(TREES):
+            return "All remotes"
+
+        if len(supported_remotes) == 1:
+            return supported_remotes
+
         return ", ".join(supported_remotes)
 
 

dvc/config.py

@@ -119,6 +119,16 @@ HTTP_COMMON = {
     "password": str,
     "ask_password": Bool,
 }
+WEBDAV_COMMON = {
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "token": str,
+    "cert_path": str,
+    "key_path": str,
+    "timeout": Coerce(int),
+}
+
 SCHEMA = {
     "core": {
         "remote": Lower,
@@ -199,6 +209,8 @@ SCHEMA = {
                 },
                 "http": {**HTTP_COMMON, **REMOTE_COMMON},
                 "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
                 "remote": {str: object},  # Any of the above options are valid
             }
         )
@@ -312,11 +324,17 @@ class Config(dict):
         if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
+    def _get_tree(self, level):
+        # NOTE: this might be a GitTree, which doesn't see things outside of
+        # the repo.
+        return self.tree if level == "repo" else self.wtree
+
     def _load_config(self, level):
         filename = self.files[level]
+        tree = self._get_tree(level)
 
-        if self.tree.exists(filename, use_dvcignore=False):
-            with self.tree.open(filename) as fobj:
+        if tree.exists(filename, use_dvcignore=False):
+            with tree.open(filename) as fobj:
                 conf_obj = configobj.ConfigObj(fobj)
         else:
             conf_obj = configobj.ConfigObj()
@@ -324,13 +342,14 @@ class Config(dict):
 
     def _save_config(self, level, conf_dict):
         filename = self.files[level]
+        tree = self._get_tree(level)
 
         logger.debug(f"Writing '{filename}'.")
 
-        self.tree.makedirs(os.path.dirname(filename))
+        tree.makedirs(os.path.dirname(filename))
 
         config = configobj.ConfigObj(_pack_remotes(conf_dict))
-        with self.tree.open(filename, "wb") as fobj:
+        with tree.open(filename, "wb") as fobj:
             config.write(fobj)
         config.filename = filename
 
@@ -388,7 +407,7 @@ class Config(dict):
             if merge_level == level:
                 break
             if merge_level in self.files:
-                _merge(merged_conf, self.load_one(merge_level))
+                merge(merged_conf, self.load_one(merge_level))
         return merged_conf
 
     @contextmanager
@@ -402,7 +421,7 @@ class Config(dict):
         conf = self._save_paths(conf, self.files[level])
 
         merged_conf = self.load_config_to_level(level)
-        _merge(merged_conf, conf)
+        merge(merged_conf, conf)
         self.validate(merged_conf)
 
         self._save_config(level, conf)
@@ -441,11 +460,11 @@ def _pack_remotes(conf):
     return result
 
 
-def _merge(into, update):
+def merge(into, update):
     """Merges second dict into first recursively"""
     for key, val in update.items():
         if isinstance(into.get(key), dict) and isinstance(val, dict):
-            _merge(into[key], val)
+            merge(into[key], val)
         else:
             into[key] = val
 

dvc/dependency/repo.py

@@ -67,7 +67,7 @@ class RepoDependency(LocalDependency):
                     return self.repo.cache.local.tree.get_hash(path, tree=tree)
                 return tree.get_file_hash(path)
 
-    def status(self):
+    def workspace_status(self):
         current_checksum = self._get_checksum(locked=True)
         updated_checksum = self._get_checksum(locked=False)
 
@@ -76,6 +76,9 @@ class RepoDependency(LocalDependency):
 
         return {}
 
+    def status(self):
+        return self.workspace_status()
+
     def save(self):
         pass
 

dvc/exceptions.py

@@ -194,6 +194,14 @@ class YAMLFileCorruptedError(DvcException):
         )
 
 
+class TOMLFileCorruptedError(DvcException):
+    def __init__(self, path):
+        path = relpath(path)
+        super().__init__(
+            f"unable to read: '{path}', TOML file structure is corrupted"
+        )
+
+
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
         super().__init__(

dvc/external_repo.py

@@ -90,15 +90,26 @@ class BaseExternalRepo:
         """Use the specified cache in place of default tmpdir cache for
         download operations.
         """
-        if hasattr(self, "cache"):
-            save_cache = self.cache.local
-            self.cache.local = cache
+        has_cache = hasattr(self, "cache")
+
+        if has_cache:
+            save_cache = self.cache.local  # pylint: disable=E0203
+            self.cache.local = cache  # pylint: disable=E0203
+        else:
+            from collections import namedtuple
+
+            mock_cache = namedtuple("MockCache", ["local"])(local=cache)
+            self.cache = mock_cache  # pylint: disable=W0201
+
         self._local_cache = cache
 
         yield
 
-        if hasattr(self, "cache"):
+        if has_cache:
             self.cache.local = save_cache
+        else:
+            del self.cache
+
         self._local_cache = None
 
     @cached_property
@@ -130,14 +141,17 @@ class BaseExternalRepo:
         for path in paths:
             if not self.repo_tree.exists(path):
                 raise PathMissingError(path, self.url)
-            save_info = self.local_cache.save(
+            hash_info = self.repo_tree.save_info(
+                path, download_callback=download_update
+            )
+            self.local_cache.save(
                 path,
                 self.repo_tree,
-                None,
+                hash_info,
                 save_link=False,
                 download_callback=download_update,
             )
-            save_infos.append(save_info)
+            save_infos.append(hash_info)
 
         return sum(download_results), failed, save_infos
 

dvc/ignore.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import re
+from collections import namedtuple
 from itertools import groupby, takewhile
 
 from pathspec.patterns import GitWildMatchPattern
@@ -8,7 +9,7 @@ from pathspec.util import normalize_file
 from pygtrie import StringTrie
 
 from dvc.path_info import PathInfo
-from dvc.pathspec_math import merge_patterns
+from dvc.pathspec_math import PatternInfo, merge_patterns
 from dvc.system import System
 from dvc.utils import relpath
 
@@ -24,18 +25,26 @@ class DvcIgnore:
 
 class DvcIgnorePatterns(DvcIgnore):
     def __init__(self, pattern_list, dirname):
+        if pattern_list:
+            if isinstance(pattern_list[0], str):
+                pattern_list = [
+                    PatternInfo(pattern, "") for pattern in pattern_list
+                ]
 
         self.pattern_list = pattern_list
         self.dirname = dirname
         self.prefix = self.dirname + os.sep
 
-        regex_pattern_list = map(
-            GitWildMatchPattern.pattern_to_regex, pattern_list
-        )
+        self.regex_pattern_list = [
+            GitWildMatchPattern.pattern_to_regex(pattern_info.patterns)
+            for pattern_info in pattern_list
+        ]
 
         self.ignore_spec = [
             (ignore, re.compile("|".join(item[0] for item in group)))
-            for ignore, group in groupby(regex_pattern_list, lambda x: x[1])
+            for ignore, group in groupby(
+                self.regex_pattern_list, lambda x: x[1]
+            )
             if ignore is not None
         ]
 
@@ -43,9 +52,19 @@ class DvcIgnorePatterns(DvcIgnore):
     def from_files(cls, ignore_file_path, tree):
         assert os.path.isabs(ignore_file_path)
         dirname = os.path.normpath(os.path.dirname(ignore_file_path))
+        ignore_file_rel_path = os.path.relpath(
+            ignore_file_path, tree.tree_root
+        )
         with tree.open(ignore_file_path, encoding="utf-8") as fobj:
             path_spec_lines = [
-                line for line in map(str.strip, fobj.readlines()) if line
+                PatternInfo(
+                    line,
+                    "{}:{}:{}".format(ignore_file_rel_path, line_no + 1, line),
+                )
+                for line_no, line in enumerate(
+                    map(str.strip, fobj.readlines())
+                )
+                if line
             ]
 
         return cls(path_spec_lines, dirname)
@@ -56,7 +75,7 @@ class DvcIgnorePatterns(DvcIgnore):
 
         return dirs, files
 
-    def matches(self, dirname, basename, is_dir=False):
+    def _get_normalize_path(self, dirname, basename):
         # NOTE: `relpath` is too slow, so we have to assume that both
         # `dirname` and `self.dirname` are relative or absolute together.
         if dirname == self.dirname:
@@ -70,6 +89,12 @@ class DvcIgnorePatterns(DvcIgnore):
 
         if not System.is_unix():
             path = normalize_file(path)
+        return path
+
+    def matches(self, dirname, basename, is_dir=False):
+        path = self._get_normalize_path(dirname, basename)
+        if not path:
+            return False
         return self.ignore(path, is_dir)
 
     def ignore(self, path, is_dir):
@@ -85,20 +110,48 @@ class DvcIgnorePatterns(DvcIgnore):
                     result = ignore
         return result
 
+    def match_details(self, dirname, basename, is_dir=False):
+        path = self._get_normalize_path(dirname, basename)
+        if not path:
+            return False
+        return self._ignore_details(path, is_dir)
+
+    def _ignore_details(self, path, is_dir):
+        result = []
+        for ignore, pattern in zip(self.regex_pattern_list, self.pattern_list):
+            regex = re.compile(ignore[0])
+            # skip system pattern
+            if not pattern.file_info:
+                continue
+            if is_dir:
+                path_dir = f"{path}/"
+                if regex.match(path) or regex.match(path_dir):
+                    result.append(pattern.file_info)
+            else:
+                if regex.match(path):
+                    result.append(pattern.file_info)
+        return result
+
     def __hash__(self):
-        return hash(self.dirname + ":" + "\n".join(self.pattern_list))
+        return hash(self.dirname + ":" + str(self.pattern_list))
 
     def __eq__(self, other):
         if not isinstance(other, DvcIgnorePatterns):
             return NotImplemented
         return (self.dirname == other.dirname) & (
-            self.pattern_list == other.pattern_list
+            [pattern.patterns for pattern in self.pattern_list]
+            == [pattern.patterns for pattern in other.pattern_list]
         )
 
     def __bool__(self):
         return bool(self.pattern_list)
 
 
+CheckIgnoreResult = namedtuple(
+    "CheckIgnoreResult", ["file", "match", "patterns"]
+)
+
+
 class DvcIgnoreFilterNoop:
     def __init__(self, tree, root_dir):
         pass
@@ -112,6 +165,9 @@ class DvcIgnoreFilterNoop:
     def is_ignored_file(self, _):
         return False
 
+    def check_ignore(self, _):
+        return []
+
 
 class DvcIgnoreFilter:
     @staticmethod
@@ -166,20 +222,19 @@ class DvcIgnoreFilter:
     def _update_sub_repo(self, root, dirs):
         for d in dirs:
             if self._is_dvc_repo(root, d):
+                new_pattern = DvcIgnorePatterns(["/{}/".format(d)], root)
                 old_pattern = self.ignores_trie_tree.longest_prefix(root).value
                 if old_pattern:
                     self.ignores_trie_tree[root] = DvcIgnorePatterns(
                         *merge_patterns(
                             old_pattern.pattern_list,
                             old_pattern.dirname,
-                            ["/{}/".format(d)],
-                            root,
+                            new_pattern.pattern_list,
+                            new_pattern.dirname,
                         )
                     )
                 else:
-                    self.ignores_trie_tree[root] = DvcIgnorePatterns(
-                        ["/{}/".format(d)], root
-                    )
+                    self.ignores_trie_tree[root] = new_pattern
 
     def __call__(self, root, dirs, files):
         ignore_pattern = self._get_trie_pattern(root)
@@ -245,3 +300,17 @@ class DvcIgnoreFilter:
         ):
             return True
         return False
+
+    def check_ignore(self, target):
+        full_target = os.path.abspath(target)
+        if not self._outside_repo(full_target):
+            dirname, basename = os.path.split(os.path.normpath(full_target))
+            pattern = self._get_trie_pattern(dirname)
+            if pattern:
+                matches = pattern.match_details(
+                    dirname, basename, os.path.isdir(full_target)
+                )
+
+                if matches:
+                    return CheckIgnoreResult(target, True, matches)
+        return CheckIgnoreResult(target, False, ["::"])

dvc/main.py

@@ -21,7 +21,7 @@ from dvc.utils import format_link
 logger = logging.getLogger("dvc")
 
 
-def main(argv=None):
+def main(argv=None):  # noqa: C901
     """Run dvc CLI command.
 
     Args:
@@ -37,16 +37,17 @@ def main(argv=None):
     try:
         args = parse_args(argv)
 
-        verbosity = args.verbose - args.quiet
-        if verbosity:
-            logger.setLevel(
-                {
-                    -2: logging.CRITICAL,
-                    -1: logging.ERROR,
-                    1: logging.DEBUG,
-                    2: logging.TRACE,
-                }[max(-2, min(verbosity, 2))]
-            )
+        level = None
+        if args.quiet:
+            level = logging.CRITICAL
+        elif args.verbose == 1:
+            level = logging.DEBUG
+        elif args.verbose > 1:
+            level = logging.TRACE
+
+        if level is not None:
+            logger.setLevel(level)
+
         logger.trace(args)
 
         cmd = args.func(args)
@@ -81,6 +82,15 @@ def main(argv=None):
     except Exception:  # noqa, pylint: disable=broad-except
         logger.exception("unexpected error")
         ret = 255
+
+    try:
+        if ret != 0:
+            logger.info(FOOTER)
+
+        if analytics.is_enabled():
+            analytics.collect_and_send_report(args, ret)
+
+        return ret
     finally:
         logger.setLevel(outerLogLevel)
 
@@ -92,11 +102,3 @@ def main(argv=None):
         # Remove cached repos in the end of the call, these are anonymous
         # so won't be reused by any other subsequent run anyway.
         clean_repos()
-
-    if ret != 0:
-        logger.info(FOOTER)
-
-    if analytics.is_enabled():
-        analytics.collect_and_send_report(args, ret)
-
-    return ret

dvc/output/base.py

@@ -199,10 +199,7 @@ class BaseOutput:
 
         return self.cache.changed_cache(self.checksum, filter_info=filter_info)
 
-    def status(self):
-        if self.checksum and self.use_cache and self.changed_cache():
-            return {str(self): "not in cache"}
-
+    def workspace_status(self):
         if not self.exists:
             return {str(self): "deleted"}
 
@@ -214,6 +211,12 @@ class BaseOutput:
 
         return {}
 
+    def status(self):
+        if self.checksum and self.use_cache and self.changed_cache():
+            return {str(self): "not in cache"}
+
+        return self.workspace_status()
+
     def changed(self):
         status = self.status()
         logger.debug(str(status))
@@ -280,6 +283,7 @@ class BaseOutput:
         self.info = self.save_info()
 
     def commit(self):
+        assert self.info
         if self.use_cache:
             self.cache.save(self.path_info, self.cache.tree, self.info)
 

dvc/path_info.py

@@ -315,3 +315,17 @@ class HTTPURLInfo(URLInfo):
             and self._path == other._path
             and self._extra_parts == other._extra_parts
         )
+
+
+# See https://github.com/shizacat/dvc/blob/remote-webdav/dvc/path_info.py
+class WebDAVURLInfo(HTTPURLInfo):
+    @cached_property
+    def url(self):
+        return "{}://{}{}{}{}{}".format(
+            self.scheme.replace("webdav", "http"),
+            self.netloc,
+            self._spath,
+            (";" + self.params) if self.params else "",
+            ("?" + self.query) if self.query else "",
+            ("#" + self.fragment) if self.fragment else "",
+        )

dvc/pathspec_math.py

@@ -3,11 +3,14 @@
 # of two path specification patterns with different base
 # All the operations follow the documents of `gitignore`
 import os
+from collections import namedtuple
 
 from pathspec.util import normalize_file
 
 from dvc.utils import relpath
 
+PatternInfo = namedtuple("PatternInfo", ["patterns", "file_info"])
+
 
 def _not_ignore(rule):
     return (True, rule[1:]) if rule.startswith("!") else (False, rule)
@@ -59,7 +62,10 @@ def _change_dirname(dirname, pattern_list, new_dirname):
     if rel.startswith(".."):
         raise ValueError("change dirname can only change to parent path")
 
-    return [change_rule(rule, rel) for rule in pattern_list]
+    return [
+        PatternInfo(change_rule(rule.patterns, rel), rule.file_info)
+        for rule in pattern_list
+    ]
 
 
 def merge_patterns(pattern_a, prefix_a, pattern_b, prefix_b):

dvc/prompt.py

@@ -7,7 +7,7 @@ from getpass import getpass
 logger = logging.getLogger(__name__)
 
 
-def _ask(prompt, limited_to=None):
+def ask(prompt, limited_to=None):
     if not sys.stdout.isatty():
         return None
 
@@ -39,7 +39,7 @@ def confirm(statement):
         bool: whether or not specified statement was confirmed.
     """
     prompt = f"{statement} [y/n]"
-    answer = _ask(prompt, limited_to=["yes", "no", "y", "n"])
+    answer = ask(prompt, limited_to=["yes", "no", "y", "n"])
     return answer and answer.startswith("y")
 
 

dvc/repo/experiments/__init__.py

@@ -1,21 +1,45 @@
 import logging
 import os
+import re
 import tempfile
+from collections import defaultdict
+from collections.abc import Mapping
+from concurrent.futures import ProcessPoolExecutor, as_completed
 from contextlib import contextmanager
+from typing import Iterable, Optional
 
 from funcy import cached_property
 
 from dvc.exceptions import DvcException
+from dvc.path_info import PathInfo
+from dvc.repo.experiments.executor import ExperimentExecutor, LocalExecutor
 from dvc.scm.git import Git
 from dvc.stage.serialize import to_lockfile
 from dvc.utils import dict_sha256, env2bool, relpath
-from dvc.utils.fs import remove
+from dvc.utils.fs import copyfile, remove
 
 logger = logging.getLogger(__name__)
 
 
+def hash_exp(stages):
+    exp_data = {}
+    for stage in stages:
+        exp_data.update(to_lockfile(stage))
+    return dict_sha256(exp_data)
+
+
 class UnchangedExperimentError(DvcException):
-    pass
+    def __init__(self, rev):
+        super().__init__(f"Experiment identical to baseline '{rev[:7]}'.")
+        self.rev = rev
+
+
+class BaselineMismatchError(DvcException):
+    def __init__(self, rev):
+        super().__init__(
+            f"Experiment is not derived from current baseline '{rev[:7]}'."
+        )
+        self.rev = rev
 
 
 class Experiments:
@@ -26,6 +50,11 @@ class Experiments:
     """
 
     EXPERIMENTS_DIR = "experiments"
+    PACKED_ARGS_FILE = "repro.dat"
+    STASH_MSG_PREFIX = "dvc-exp-"
+    STASH_EXPERIMENT_RE = re.compile(
+        r"(?:On \(.*\): )dvc-exp-(?P<baseline_rev>[0-9a-f]+)$"
+    )
 
     def __init__(self, repo):
         if not (
@@ -47,10 +76,13 @@ class Experiments:
             return Git(self.exp_dir)
         return self._init_clone()
 
+    @cached_property
+    def dvc_dir(self):
+        return relpath(self.repo.dvc_dir, self.repo.scm.root_dir)
+
     @cached_property
     def exp_dvc_dir(self):
-        dvc_dir = relpath(self.repo.dvc_dir, self.repo.scm.root_dir)
-        return os.path.join(self.exp_dir, dvc_dir)
+        return os.path.join(self.exp_dir, self.dvc_dir)
 
     @cached_property
     def exp_dvc(self):
@@ -59,20 +91,32 @@ class Experiments:
 
         return Repo(self.exp_dvc_dir)
 
-    @staticmethod
-    def exp_hash(stages):
-        exp_data = {}
-        for stage in stages:
-            exp_data.update(to_lockfile(stage))
-        return dict_sha256(exp_data)
-
     @contextmanager
     def chdir(self):
         cwd = os.getcwd()
         os.chdir(self.exp_dvc.root_dir)
-        yield
+        yield self.exp_dvc.root_dir
         os.chdir(cwd)
 
+    @cached_property
+    def args_file(self):
+        return os.path.join(self.exp_dvc.tmp_dir, self.PACKED_ARGS_FILE)
+
+    @property
+    def stash_reflog(self):
+        if "refs/stash" in self.scm.repo.refs:
+            return self.scm.repo.refs["refs/stash"].log()
+        return []
+
+    @property
+    def stash_revs(self):
+        revs = {}
+        for i, entry in enumerate(self.stash_reflog):
+            m = self.STASH_EXPERIMENT_RE.match(entry.message)
+            if m:
+                revs[entry.newhexsha] = (i, m.group("baseline_rev"))
+        return revs
+
     def _init_clone(self):
         src_dir = self.repo.scm.root_dir
         logger.debug("Initializing experiments clone")
@@ -90,32 +134,95 @@ class Experiments:
 
     def _scm_checkout(self, rev):
         self.scm.repo.git.reset(hard=True)
+        if self.scm.repo.head.is_detached:
+            # switch back to default branch
+            self.scm.repo.heads[0].checkout()
         if not Git.is_sha(rev) or not self.scm.has_rev(rev):
-            self.scm.fetch(all=True)
-        logger.debug("Checking out base experiment commit '%s'", rev)
+            self.scm.pull()
+        logger.debug("Checking out experiment commit '%s'", rev)
         self.scm.checkout(rev)
 
-    def _patch_exp(self):
-        """Create a patch based on the current (parent) workspace and apply it
-        to the experiment workspace.
+    def _stash_exp(self, *args, params: Optional[dict] = None, **kwargs):
+        """Stash changes from the current (parent) workspace as an experiment.
+
+        Args:
+            params: Optional dictionary of parameter values to be used.
+                Values take priority over any parameters specified in the
+                user's workspace.
         """
+        rev = self.scm.get_rev()
+
+        # patch user's workspace into experiments clone
         tmp = tempfile.NamedTemporaryFile(delete=False).name
         try:
             self.repo.scm.repo.git.diff(patch=True, output=tmp)
             if os.path.getsize(tmp):
                 logger.debug("Patching experiment workspace")
                 self.scm.repo.git.apply(tmp)
-            else:
-                raise UnchangedExperimentError(
-                    "Experiment identical to baseline commit."
-                )
+            elif not params:
+                # experiment matches original baseline
+                raise UnchangedExperimentError(rev)
         finally:
             remove(tmp)
 
-    def _commit(self, stages, check_exists=True, branch=True, rev=None):
+        # update experiment params from command line
+        if params:
+            self._update_params(params)
+
+        # save additional repro command line arguments
+        self._pack_args(*args, **kwargs)
+
+        # save experiment as a stash commit w/message containing baseline rev
+        # (stash commits are merge commits and do not contain a parent commit
+        # SHA)
+        msg = f"{self.STASH_MSG_PREFIX}{rev}"
+        self.scm.repo.git.stash("push", "-m", msg)
+        return self.scm.resolve_rev("stash@{0}")
+
+    def _pack_args(self, *args, **kwargs):
+        ExperimentExecutor.pack_repro_args(self.args_file, *args, **kwargs)
+        self.scm.add(self.args_file)
+
+    def _unpack_args(self, tree=None):
+        args_file = os.path.join(self.exp_dvc.tmp_dir, self.PACKED_ARGS_FILE)
+        return ExperimentExecutor.unpack_repro_args(args_file, tree=tree)
+
+    def _update_params(self, params: dict):
+        """Update experiment params files with the specified values."""
+        from dvc.utils.toml import dump_toml, parse_toml_for_update
+        from dvc.utils.yaml import dump_yaml, parse_yaml_for_update
+
+        logger.debug("Using experiment params '%s'", params)
+
+        # recursive dict update
+        def _update(dict_, other):
+            for key, value in other.items():
+                if isinstance(value, Mapping):
+                    dict_[key] = _update(dict_.get(key, {}), value)
+                else:
+                    dict_[key] = value
+            return dict_
+
+        loaders = defaultdict(lambda: parse_yaml_for_update)
+        loaders.update({".toml": parse_toml_for_update})
+        dumpers = defaultdict(lambda: dump_yaml)
+        dumpers.update({".toml": dump_toml})
+
+        for params_fname in params:
+            path = PathInfo(self.exp_dvc.root_dir) / params_fname
+            with self.exp_dvc.tree.open(path, "r") as fobj:
+                text = fobj.read()
+            suffix = path.suffix.lower()
+            data = loaders[suffix](text, path)
+            _update(data, params[params_fname])
+            dumpers[suffix](path, data)
+
+    def _commit(self, exp_hash, check_exists=True, branch=True):
         """Commit stages as an experiment and return the commit SHA."""
-        hash_ = self.exp_hash(stages)
-        exp_name = f"{rev[:7]}-{hash_}"
+        if not self.scm.is_dirty():
+            raise UnchangedExperimentError(self.scm.get_rev())
+        rev = self.scm.get_rev()
+        exp_name = f"{rev[:7]}-{exp_hash}"
         if branch:
             if check_exists and exp_name in self.scm.list_branches():
                 logger.debug("Using existing experiment branch '%s'", exp_name)
@@ -126,12 +233,32 @@ class Experiments:
         self.scm.commit(f"Add experiment {exp_name}")
         return self.scm.get_rev()
 
-    def _reproduce(self, *args, **kwargs):
-        """Run `dvc repro` inside the experiments workspace."""
-        with self.chdir():
-            return self.exp_dvc.reproduce(*args, **kwargs)
+    def reproduce_one(self, queue=False, **kwargs):
+        """Reproduce and checkout a single experiment."""
+        stash_rev = self.new(**kwargs)
+        if queue:
+            logger.info(
+                "Queued experiment '%s' for future execution.", stash_rev[:7]
+            )
+            return [stash_rev]
+        results = self.reproduce([stash_rev], keep_stash=False)
+        for exp_rev in results:
+            self.checkout_exp(exp_rev)
+        return results
 
-    def new(self, *args, workspace=True, **kwargs):
+    def reproduce_queued(self, **kwargs):
+        results = self.reproduce(**kwargs)
+        if results:
+            revs = [f"{rev[:7]}" for rev in results]
+            logger.info(
+                "Successfully reproduced experiment(s) '%s'.\n"
+                "Use `dvc exp checkout <exp_rev>` to apply the results of "
+                "a specific experiment to your workspace.",
+                ", ".join(revs),
+            )
+        return results
+
+    def new(self, *args, **kwargs):
         """Create a new experiment.
 
         Experiment will be reproduced and checked out into the user's
@@ -139,47 +266,191 @@ class Experiments:
         """
         rev = self.repo.scm.get_rev()
         self._scm_checkout(rev)
-        if workspace:
-            try:
-                self._patch_exp()
-            except UnchangedExperimentError as exc:
-                logger.info("Reproducing existing experiment '%s'.", rev[:7])
-                raise exc
-        else:
-            # configure params via command line here
-            pass
-        self.exp_dvc.checkout()
-        stages = self._reproduce(*args, **kwargs)
-        exp_rev = self._commit(stages, rev=rev)
-        self.checkout_exp(exp_rev, force=True)
-        logger.info("Generated experiment '%s'.", exp_rev[:7])
-        return stages
+        try:
+            stash_rev = self._stash_exp(*args, **kwargs)
+        except UnchangedExperimentError as exc:
+            logger.info("Reproducing existing experiment '%s'.", rev[:7])
+            raise exc
+        logger.debug(
+            "Stashed experiment '%s' for future execution.", stash_rev[:7]
+        )
+        return stash_rev
 
-    def checkout_exp(self, rev, force=False):
+    def reproduce(
+        self,
+        revs: Optional[Iterable] = None,
+        keep_stash: Optional[bool] = True,
+        **kwargs,
+    ):
+        """Reproduce the specified experiments.
+
+        Args:
+            revs: If revs is not specified, all stashed experiments will be
+                reproduced.
+            keep_stash: If True, stashed experiments will be preserved if they
+                fail to reproduce successfully.
+        """
+        stash_revs = self.stash_revs
+
+        # to_run contains mapping of:
+        #   input_rev: (stash_index, baseline_rev)
+        # where input_rev contains the changes to execute (usually a stash
+        # commit) and baseline_rev is the baseline to compare output against.
+        # The final experiment commit will be branched from baseline_rev.
+        if revs is None:
+            to_run = {
+                rev: baseline_rev
+                for rev, (_, baseline_rev) in stash_revs.items()
+            }
+        else:
+            to_run = {
+                rev: stash_revs[rev][1] if rev in stash_revs else rev
+                for rev in revs
+            }
+
+        # setup executors
+        executors = {}
+        for rev, baseline_rev in to_run.items():
+            tree = self.scm.get_tree(rev)
+            repro_args, repro_kwargs = self._unpack_args(tree)
+            executor = LocalExecutor(
+                tree,
+                baseline_rev,
+                repro_args=repro_args,
+                repro_kwargs=repro_kwargs,
+                dvc_dir=self.dvc_dir,
+                cache_dir=self.repo.cache.local.cache_dir,
+            )
+            executors[rev] = executor
+
+        exec_results = self._reproduce(executors, **kwargs)
+
+        if keep_stash:
+            # only drop successfully run stashed experiments
+            to_drop = sorted(
+                (
+                    stash_revs[rev][0]
+                    for rev in exec_results
+                    if rev in stash_revs
+                ),
+                reverse=True,
+            )
+        else:
+            # drop all stashed experiments
+            to_drop = sorted(
+                (stash_revs[rev][0] for rev in to_run if rev in stash_revs),
+                reverse=True,
+            )
+        for index in to_drop:
+            self.scm.repo.git.stash("drop", index)
+
+        result = {}
+        for _, exp_result in exec_results.items():
+            result.update(exp_result)
+        return result
+
+    def _reproduce(self, executors: dict, jobs: Optional[int] = 1) -> dict:
+        """Run dvc repro for the specified ExperimentExecutors in parallel.
+
+        Returns dict containing successfully executed experiments.
+        """
+        result = {}
+
+        with ProcessPoolExecutor(max_workers=jobs) as workers:
+            futures = {}
+            for rev, executor in executors.items():
+                future = workers.submit(
+                    executor.reproduce,
+                    executor.dvc_dir,
+                    cwd=executor.dvc.root_dir,
+                    **executor.repro_kwargs,
+                )
+                futures[future] = (rev, executor)
+            for future in as_completed(futures):
+                rev, executor = futures[future]
+                exc = future.exception()
+                if exc is None:
+                    exp_hash = future.result()
+                    logger.debug(f"ran exp based on {executor.baseline_rev}")
+                    self._scm_checkout(executor.baseline_rev)
+                    self._collect_output(executor.baseline_rev, executor)
+                    remove(self.args_file)
+                    try:
+                        exp_rev = self._commit(exp_hash)
+                    except UnchangedExperimentError:
+                        logger.debug(
+                            "Experiment '%s' identical to baseline '%s'",
+                            rev,
+                            executor.baseline_rev,
+                        )
+                        exp_rev = executor.baseline_rev
+                    logger.info("Reproduced experiment '%s'.", exp_rev[:7])
+                    result[rev] = {exp_rev: exp_hash}
+                else:
+                    logger.exception(
+                        "Failed to reproduce experiment '%s'", rev
+                    )
+                executor.cleanup()
+
+        return result
+
+    def _collect_output(self, rev: str, executor: ExperimentExecutor):
+        logger.debug("copying tmp output from '%s'", executor.tmp_dir)
+        tree = self.scm.get_tree(rev)
+        for fname in tree.walk_files(tree.tree_root):
+            src = executor.path_info / relpath(fname, tree.tree_root)
+            copyfile(src, fname)
+
+    def checkout_exp(self, rev):
         """Checkout an experiment to the user's workspace."""
         from git.exc import GitCommandError
         from dvc.repo.checkout import _checkout as dvc_checkout
 
-        if force:
-            self.repo.scm.repo.git.reset(hard=True)
-        logger.debug(f"checkout {rev}")
+        self._check_baseline(rev)
         self._scm_checkout(rev)
 
         tmp = tempfile.NamedTemporaryFile(delete=False).name
         self.scm.repo.head.commit.diff("HEAD~1", patch=True, output=tmp)
+
+        dirty = self.repo.scm.is_dirty()
+        if dirty:
+            logger.debug("Stashing workspace changes.")
+            self.repo.scm.repo.git.stash("push")
+
         try:
             if os.path.getsize(tmp):
                 logger.debug("Patching local workspace")
                 self.repo.scm.repo.git.apply(tmp, reverse=True)
-            dvc_checkout(self.repo)
+                need_checkout = True
+            else:
+                need_checkout = False
         except GitCommandError:
-            raise DvcException(
-                "Checkout failed, experiment contains changes which "
-                "conflict with your current workspace. To overwrite "
-                "your workspace, use `dvc experiments checkout --force`."
-            )
+            raise DvcException("failed to apply experiment changes.")
         finally:
             remove(tmp)
+            if dirty:
+                self._unstash_workspace()
+
+        if need_checkout:
+            dvc_checkout(self.repo)
+
+    def _check_baseline(self, exp_rev):
+        baseline_sha = self.repo.scm.get_rev()
+        exp_commit = self.scm.repo.rev_parse(exp_rev)
+        for parent in exp_commit.parents:
+            if parent.hexsha == baseline_sha:
+                return
+        raise BaselineMismatchError(baseline_sha)
+
+    def _unstash_workspace(self):
+        # Essentially we want `git stash pop` with `-X ours` merge strategy
+        # to prefer the applied experiment changes over stashed workspace
+        # changes. git stash doesn't support merge strategy parameters, but we
+        # can do it ourselves with checkout/reset.
+        logger.debug("Unstashing workspace changes.")
+        self.repo.scm.repo.git.checkout("--ours", "stash@{0}", "--", ".")
+        self.repo.scm.repo.git.reset("HEAD")
+        self.repo.scm.repo.git.stash("drop", "stash@{0}")
 
     def checkout(self, *args, **kwargs):
         from dvc.repo.experiments.checkout import checkout

dvc/repo/experiments/executor.py

@@ -0,0 +1,132 @@
+import logging
+import os
+import pickle
+from tempfile import TemporaryDirectory
+
+from funcy import cached_property
+
+from dvc.path_info import PathInfo
+from dvc.stage import PipelineStage
+from dvc.tree.base import BaseTree
+from dvc.utils import relpath
+from dvc.utils.fs import copy_fobj_to_file, makedirs
+
+logger = logging.getLogger(__name__)
+
+
+class ExperimentExecutor:
+    """Base class for executing experiments in parallel.
+
+    Args:
+        src_tree: source tree for this experiment.
+        baseline_rev: baseline revision that this experiment is derived from.
+
+    Optional keyword args:
+        repro_args: Args to be passed into reproduce.
+        repro_kwargs: Keyword args to be passed into reproduce.
+    """
+
+    def __init__(self, src_tree: BaseTree, baseline_rev: str, **kwargs):
+        self.src_tree = src_tree
+        self.baseline_rev = baseline_rev
+        self.repro_args = kwargs.pop("repro_args", [])
+        self.repro_kwargs = kwargs.pop("repro_kwargs", {})
+
+    def run(self):
+        pass
+
+    def cleanup(self):
+        pass
+
+    # TODO: come up with better way to stash repro arguments
+    @staticmethod
+    def pack_repro_args(path, *args, tree=None, **kwargs):
+        open_func = tree.open if tree else open
+        data = {"args": args, "kwargs": kwargs}
+        with open_func(path, "wb") as fobj:
+            pickle.dump(data, fobj)
+
+    @staticmethod
+    def unpack_repro_args(path, tree=None):
+        open_func = tree.open if tree else open
+        with open_func(path, "rb") as fobj:
+            data = pickle.load(fobj)
+        return data["args"], data["kwargs"]
+
+
+class LocalExecutor(ExperimentExecutor):
+    """Local machine exepriment executor."""
+
+    def __init__(self, src_tree: BaseTree, baseline_rev: str, **kwargs):
+        dvc_dir = kwargs.pop("dvc_dir")
+        cache_dir = kwargs.pop("cache_dir")
+        super().__init__(src_tree, baseline_rev, **kwargs)
+        self.tmp_dir = TemporaryDirectory()
+        logger.debug("Init local executor in dir '%s'.", self.tmp_dir)
+        self.dvc_dir = os.path.join(self.tmp_dir.name, dvc_dir)
+        try:
+            for fname in src_tree.walk_files(src_tree.tree_root):
+                dest = self.path_info / relpath(fname, src_tree.tree_root)
+                if not os.path.exists(dest.parent):
+                    makedirs(dest.parent)
+                with src_tree.open(fname, "rb") as fobj:
+                    copy_fobj_to_file(fobj, dest)
+        except Exception:
+            self.tmp_dir.cleanup()
+            raise
+        self._config(cache_dir)
+
+    def _config(self, cache_dir):
+        local_config = os.path.join(self.dvc_dir, "config.local")
+        logger.debug("Writing experiments local config '%s'", local_config)
+        with open(local_config, "w") as fobj:
+            fobj.write("[core]\n    no_scm = true\n")
+            fobj.write(f"[cache]\n    dir = {cache_dir}")
+
+    @cached_property
+    def dvc(self):
+        from dvc.repo import Repo
+
+        return Repo(self.dvc_dir)
+
+    @cached_property
+    def path_info(self):
+        return PathInfo(self.tmp_dir.name)
+
+    @staticmethod
+    def reproduce(dvc_dir, cwd=None, **kwargs):
+        """Run dvc repro and return the result."""
+        from dvc.repo import Repo
+        from dvc.repo.experiments import hash_exp
+
+        unchanged = []
+
+        def filter_pipeline(stage):
+            if isinstance(stage, PipelineStage):
+                unchanged.append(stage)
+
+        if cwd:
+            old_cwd = os.getcwd()
+            os.chdir(cwd)
+        else:
+            old_cwd = None
+            cwd = os.getcwd()
+
+        try:
+            logger.debug("Running repro in '%s'", cwd)
+            dvc = Repo(dvc_dir)
+            dvc.checkout()
+            stages = dvc.reproduce(on_unchanged=filter_pipeline, **kwargs)
+        finally:
+            if old_cwd is not None:
+                os.chdir(old_cwd)
+
+        # ideally we would return stages here like a normal repro() call, but
+        # stages is not currently picklable and cannot be returned across
+        # multiprocessing calls
+        return hash_exp(stages + unchanged)
+
+    def cleanup(self):
+        logger.debug("Removing tmpdir '%s'", self.tmp_dir)
+        self.tmp_dir.cleanup()
+        super().cleanup()

dvc/repo/experiments/show.py

@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)
 EXP_RE = re.compile(r"(?P<rev_sha>[a-f0-9]{7})-(?P<exp_sha>[a-f0-9]+)")
 
 
-def _collect_experiment(repo, branch):
+def _collect_experiment(repo, branch, stash=False):
     res = defaultdict(dict)
     for rev in repo.brancher(revs=[branch]):
         configs = _collect_configs(repo)
@@ -20,9 +20,10 @@ def _collect_experiment(repo, branch):
         if params:
             res["params"] = params
 
-        metrics = _collect_metrics(repo, None, False)
-        vals = _read_metrics(repo, metrics, rev)
-        if vals:
+        res["queued"] = stash
+        if not stash:
+            metrics = _collect_metrics(repo, None, False)
+            vals = _read_metrics(repo, metrics, rev)
             res["metrics"] = vals
 
     return res
@@ -37,8 +38,9 @@ def show(
     if revs is None:
         revs = [repo.scm.get_rev()]
 
-    revs = set(
-        repo.brancher(
+    revs = OrderedDict(
+        (rev, None)
+        for rev in repo.brancher(
             revs=revs,
             all_branches=all_branches,
             all_tags=all_tags,
@@ -49,6 +51,7 @@ def show(
     for rev in revs:
         res[rev]["baseline"] = _collect_experiment(repo, rev)
 
+    # collect reproduced experiments
     for exp_branch in repo.experiments.scm.list_branches():
         m = re.match(EXP_RE, exp_branch)
         if m:
@@ -61,4 +64,12 @@ def show(
                     )
                 res[rev][exp_rev] = experiment
 
+    # collect queued (not yet reproduced) experiments
+    for stash_rev, (_, baseline_rev) in repo.experiments.stash_revs.items():
+        with repo.experiments.chdir():
+            experiment = _collect_experiment(
+                repo.experiments.exp_dvc, stash_rev, stash=True
+            )
+        res[baseline_rev][stash_rev] = experiment
+
     return res

dvc/repo/reproduce.py

@@ -60,7 +60,7 @@ def reproduce(
     recursive=False,
     pipeline=False,
     all_pipelines=False,
-    **kwargs
+    **kwargs,
 ):
     from dvc.utils import parse_target
 
@@ -71,13 +71,22 @@ def reproduce(
         )
 
     experiment = kwargs.pop("experiment", False)
-    if experiment and self.experiments:
+    params = _parse_params(kwargs.pop("params", []))
+    queue = kwargs.pop("queue", False)
+    run_all = kwargs.pop("run_all", False)
+    jobs = kwargs.pop("jobs", 1)
+    if (experiment or run_all) and self.experiments:
         try:
-            return self.experiments.new(
+            return _reproduce_experiments(
+                self,
                 target=target,
                 recursive=recursive,
                 all_pipelines=all_pipelines,
-                **kwargs
+                params=params,
+                queue=queue,
+                run_all=run_all,
+                jobs=jobs,
+                **kwargs,
             )
         except UnchangedExperimentError:
             # If experiment contains no changes, just run regular repro
@@ -109,8 +118,39 @@ def reproduce(
     return _reproduce_stages(active_graph, targets, **kwargs)
 
 
+def _parse_params(path_params):
+    from flatten_json import unflatten
+    from yaml import safe_load, YAMLError
+    from dvc.dependency.param import ParamsDependency
+
+    ret = {}
+    for path_param in path_params:
+        path, _, params_str = path_param.rpartition(":")
+        # remove empty strings from params, on condition such as `-p "file1:"`
+        params = {}
+        for param_str in filter(bool, params_str.split(",")):
+            try:
+                # interpret value strings using YAML rules
+                key, value = param_str.split("=")
+                params[key] = safe_load(value)
+            except (ValueError, YAMLError):
+                raise InvalidArgumentError(
+                    f"Invalid param/value pair '{param_str}'"
+                )
+        if not path:
+            path = ParamsDependency.DEFAULT_PARAMS_FILE
+        ret[path] = unflatten(params, ".")
+    return ret
+
+
+def _reproduce_experiments(repo, run_all=False, jobs=1, **kwargs):
+    if run_all:
+        return repo.experiments.reproduce_queued(jobs=jobs)
+    return repo.experiments.reproduce_one(**kwargs)
+
+
 def _reproduce_stages(
-    G, stages, downstream=False, single_item=False, **kwargs
+    G, stages, downstream=False, single_item=False, on_unchanged=None, **kwargs
 ):
     r"""Derive the evaluation of the given node for the given graph.
 
@@ -194,7 +234,10 @@ def _reproduce_stages(
                 # dependencies didn't change.
                 kwargs["force"] = True
 
-            result.extend(ret)
+            if ret:
+                result.extend(ret)
+            elif on_unchanged is not None:
+                on_unchanged(stage)
         except Exception as exc:
             raise ReproductionError(stage.relpath) from exc
 

dvc/repo/tree.py

@@ -245,6 +245,9 @@ class RepoTree(BaseTree):  # pylint:disable=abstract-method
     Any kwargs will be passed to `DvcTree()`.
     """
 
+    scheme = "local"
+    PARAM_CHECKSUM = "md5"
+
     def __init__(self, repo, **kwargs):
         super().__init__(repo, {"url": repo.root_dir})
         if hasattr(repo, "dvc_dir"):

dvc/scheme.py

@@ -9,3 +9,5 @@ class Schemes:
     GDRIVE = "gdrive"
     LOCAL = "local"
     OSS = "oss"
+    WEBDAV = "webdav"
+    WEBDAVS = "webdavs"

dvc/stage/__init__.py

@@ -306,7 +306,12 @@ class Stage(params.StageParams):
     @rwlocked(read=["deps"], write=["outs"])
     def reproduce(self, interactive=False, **kwargs):
         if not (kwargs.get("force", False) or self.changed()):
-            logger.info("Stage '%s' didn't change, skipping", self.addressing)
+            if not isinstance(self, PipelineStage) and self.is_data_source:
+                logger.info("'%s' didn't change, skipping", self.addressing)
+            else:
+                logger.info(
+                    "Stage '%s' didn't change, skipping", self.addressing
+                )
             return None
 
         msg = (
@@ -396,7 +401,7 @@ class Stage(params.StageParams):
 
     @staticmethod
     def _changed_entries(entries):
-        return [str(entry) for entry in entries if entry.changed_checksum()]
+        return [str(entry) for entry in entries if entry.workspace_status()]
 
     def _changed_stage_entry(self):
         return f"'md5' of {self} changed."

dvc/tree/__init__.py

@@ -11,6 +11,8 @@ from .local import LocalTree
 from .oss import OSSTree
 from .s3 import S3Tree
 from .ssh import SSHTree
+from .webdav import WebDAVTree
+from .webdavs import WebDAVSTree
 
 TREES = [
     AzureTree,
@@ -22,6 +24,8 @@ TREES = [
     S3Tree,
     SSHTree,
     OSSTree,
+    WebDAVTree,
+    WebDAVSTree,
     # NOTE: LocalTree is the default
 ]
 

dvc/tree/base.py

@@ -236,22 +236,17 @@ class BaseTree:
             return False
         return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
 
-    def get_hash(self, path_info, tree=None, **kwargs):
+    def get_hash(self, path_info, **kwargs):
         assert path_info and (
             isinstance(path_info, str) or path_info.scheme == self.scheme
         )
 
-        if not tree:
-            tree = self
-
-        if not tree.exists(path_info):
+        if not self.exists(path_info):
             return None
 
-        if tree == self:
-            # pylint: disable=assignment-from-none
-            hash_ = self.state.get(path_info)
-        else:
-            hash_ = None
+        # pylint: disable=assignment-from-none
+        hash_ = self.state.get(path_info)
+
         # If we have dir hash in state db, but dir cache file is lost,
         # then we need to recollect the dir via .get_dir_hash() call below,
         # see https://github.com/iterative/dvc/issues/2219 for context
@@ -267,10 +262,10 @@ class BaseTree:
         if hash_:
             return hash_
 
-        if tree.isdir(path_info):
-            hash_ = self.get_dir_hash(path_info, tree, **kwargs)
+        if self.isdir(path_info):
+            hash_ = self.get_dir_hash(path_info, **kwargs)
         else:
-            hash_ = tree.get_file_hash(path_info)
+            hash_ = self.get_file_hash(path_info)
 
         if hash_ and self.exists(path_info):
             self.state.save(path_info, hash_)
@@ -280,12 +275,12 @@ class BaseTree:
     def get_file_hash(self, path_info):
         raise NotImplementedError
 
-    def get_dir_hash(self, path_info, tree, **kwargs):
+    def get_dir_hash(self, path_info, **kwargs):
         if not self.cache:
             raise RemoteCacheRequiredError(path_info)
 
-        dir_info = self._collect_dir(path_info, tree, **kwargs)
-        return self._save_dir_info(dir_info, path_info)
+        dir_info = self._collect_dir(path_info, **kwargs)
+        return self._save_dir_info(dir_info)
 
     def hash_to_path_info(self, hash_):
         return self.path_info / hash_[0:2] / hash_[2:]
@@ -298,29 +293,26 @@ class BaseTree:
 
         return "".join(parts)
 
-    def save_info(self, path_info, tree=None, **kwargs):
-        return {
-            self.PARAM_CHECKSUM: self.get_hash(path_info, tree=tree, **kwargs)
-        }
+    def save_info(self, path_info, **kwargs):
+        return {self.PARAM_CHECKSUM: self.get_hash(path_info, **kwargs)}
 
-    @staticmethod
-    def _calculate_hashes(file_infos, tree):
+    def _calculate_hashes(self, file_infos):
         file_infos = list(file_infos)
         with Tqdm(
             total=len(file_infos),
             unit="md5",
             desc="Computing file/dir hashes (only done once)",
         ) as pbar:
-            worker = pbar.wrap_fn(tree.get_file_hash)
-            with ThreadPoolExecutor(max_workers=tree.hash_jobs) as executor:
+            worker = pbar.wrap_fn(self.get_file_hash)
+            with ThreadPoolExecutor(max_workers=self.hash_jobs) as executor:
                 tasks = executor.map(worker, file_infos)
                 hashes = dict(zip(file_infos, tasks))
         return hashes
 
-    def _collect_dir(self, path_info, tree, **kwargs):
+    def _collect_dir(self, path_info, **kwargs):
         file_infos = set()
 
-        for fname in tree.walk_files(path_info, **kwargs):
+        for fname in self.walk_files(path_info, **kwargs):
             if DvcIgnore.DVCIGNORE_FILE == fname.name:
                 raise DvcIgnoreInCollectedDirError(fname.parent)
 
@@ -329,7 +321,7 @@ class BaseTree:
         hashes = {fi: self.state.get(fi) for fi in file_infos}
         not_in_state = {fi for fi, hash_ in hashes.items() if hash_ is None}
 
-        new_hashes = self._calculate_hashes(not_in_state, tree)
+        new_hashes = self._calculate_hashes(not_in_state)
         hashes.update(new_hashes)
 
         result = [
@@ -351,7 +343,7 @@ class BaseTree:
         # Sorting the list by path to ensure reproducibility
         return sorted(result, key=itemgetter(self.PARAM_RELPATH))
 
-    def _save_dir_info(self, dir_info, path_info):
+    def _save_dir_info(self, dir_info):
         hash_, tmp_info = self._get_dir_info_hash(dir_info)
         new_info = self.cache.tree.hash_to_path_info(hash_)
         if self.cache.changed_cache_file(hash_):
@@ -360,8 +352,6 @@ class BaseTree:
                 tmp_info, new_info, mode=self.cache.CACHE_MODE
             )
 
-        if self.exists(path_info):
-            self.state.save(path_info, hash_)
         self.state.save(new_info, hash_)
 
         return hash_

dvc/tree/gdrive.py

@@ -381,6 +381,9 @@ class GDriveTree(BaseTree):
         # it does not create a file on the remote
         gdrive_file = self._drive.CreateFile(param)
 
+        gdrive_file.FetchMetadata(fields="fileSize")
+        size = gdrive_file["fileSize"]
+
         with Tqdm(
             desc=progress_desc,
             disable=no_progress_bar,
@@ -388,7 +391,13 @@ class GDriveTree(BaseTree):
             # explicit `bar_format` as `total` will be set by `update_to`
             bar_format=Tqdm.BAR_FMT_DEFAULT,
         ) as pbar:
-            gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
+            if size:
+                gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
+            else:
+                # PyDrive doesn't like downloading empty files
+                # https://github.com/iterative/dvc/issues/4286
+                with open(to_file, "w"):
+                    pass
 
     @contextmanager
     @_gdrive_retry

dvc/tree/webdav.py

@@ -0,0 +1,253 @@
+import logging
+import os
+import threading
+from collections import deque
+
+from funcy import cached_property, wrap_prop
+
+from dvc.config import ConfigError
+from dvc.exceptions import DvcException
+from dvc.path_info import HTTPURLInfo, WebDAVURLInfo
+from dvc.progress import Tqdm
+from dvc.scheme import Schemes
+
+from .base import BaseTree
+from .http import ask_password
+
+logger = logging.getLogger(__name__)
+
+
+class WebDAVConnectionError(DvcException):
+    def __init__(self, host):
+        super().__init__(f"Unable to connect to WebDAV {host}.")
+
+
+class WebDAVTree(BaseTree):  # pylint:disable=abstract-method
+    # Use webdav scheme
+    scheme = Schemes.WEBDAV
+
+    # URLInfo for Webdav ~ replaces webdav -> http
+    PATH_CLS = WebDAVURLInfo
+
+    # Traversable as walk_files is implemented
+    CAN_TRAVERSE = True
+
+    # Length of walk_files prefix
+    TRAVERSE_PREFIX_LEN = 2
+
+    # Implementation based on webdav3.client
+    REQUIRES = {"webdavclient3": "webdav3.client"}
+
+    # Chunk size for buffered upload/download with progress bar
+    CHUNK_SIZE = 2 ** 16
+
+    # Constructor
+    def __init__(self, repo, config):
+        # Call BaseTree constructor
+        super().__init__(repo, config)
+
+        # Get username from configuration
+        self.user = config.get("user", None)
+
+        # Get password from configuration (might be None ~ not set)
+        self.password = config.get("password", None)
+
+        # Whether to ask for password if it is not set
+        self.ask_password = config.get("ask_password", False)
+
+        # Use token for webdav auth
+        self.token = config.get("token", None)
+
+        # Path to certificate
+        self.cert_path = config.get("cert_path", None)
+
+        # Path to private key
+        self.key_path = config.get("key_path", None)
+
+        # Connection timeout
+        self.timeout = config.get("timeout", 30)
+
+        # Get URL from configuration
+        self.url = config.get("url", None)
+
+        # If URL in config parse path_info
+        if self.url:
+            self.path_info = self.PATH_CLS(self.url)
+
+            # If username not specified try to use from URL
+            if self.user is None and self.path_info.user is not None:
+                self.user = self.path_info.user
+
+            # If username specified add to path_info
+            if self.user is not None:
+                self.path_info.user = self.user
+        else:
+            self.path_info = None
+
+    # Webdav client
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def _client(self):
+        from webdav3.client import Client
+
+        # Construct hostname from path_info by stripping path
+        http_info = HTTPURLInfo(self.path_info.url)
+        hostname = http_info.replace(path="").url
+
+        # Set password or ask for it
+        if self.ask_password and self.password is None and self.token is None:
+            host, user = self.path_info.host, self.path_info.user
+            self.password = ask_password(host, user)
+
+        # Setup webdav client options dictionary
+        options = {
+            "webdav_hostname": hostname,
+            "webdav_login": self.user,
+            "webdav_password": self.password,
+            "webdav_token": self.token,
+            "webdav_cert_path": self.cert_path,
+            "webdav_key_path": self.key_path,
+            "webdav_timeout": self.timeout,
+            "webdav_chunk_size": self.CHUNK_SIZE,
+        }
+
+        client = Client(options)
+
+        # Check whether client options are valid
+        if not client.valid():
+            raise ConfigError(
+                f"Configuration for WebDAV {hostname} is invalid."
+            )
+
+        # Check whether connection is valid (root should always exist)
+        if not client.check(self.path_info.path):
+            raise WebDAVConnectionError(hostname)
+
+        return client
+
+    # Checks whether file/directory exists at remote
+    def exists(self, path_info, use_dvcignore=True):
+        # Use webdav check to test for file existence
+        return self._client.check(path_info.path)
+
+    # Gets file hash 'etag'
+    def get_file_hash(self, path_info):
+        # Use webdav client info method to get etag
+        etag = self._client.info(path_info.path)["etag"].strip('"')
+
+        # From HTTPTree
+        if not etag:
+            raise DvcException(
+                "could not find an ETag or "
+                "Content-MD5 header for '{url}'".format(url=path_info.url)
+            )
+
+        if etag.startswith("W/"):
+            raise DvcException(
+                "Weak ETags are not supported."
+                " (Etag: '{etag}', URL: '{url}')".format(
+                    etag=etag, url=path_info.url
+                )
+            )
+
+        return etag
+
+    # Checks whether path points to directory
+    def isdir(self, path_info):
+        # Use webdav is_dir to test whether path points to a directory
+        return self._client.is_dir(path_info.path)
+
+    # Yields path info to all files
+    def walk_files(self, path_info, **kwargs):
+        # Check whether directory exists
+        if not self.exists(path_info):
+            return
+
+        # Collect directories
+        dirs = deque([path_info.path])
+
+        # Iterate all directories found so far
+        while dirs:
+            # Iterate directory content
+            for entry in self._client.list(dirs.pop(), get_info=True):
+                # Construct path_info to entry
+                info = path_info.replace(path=entry["path"])
+
+                # Check whether entry is a directory
+                if entry["isdir"]:
+                    # Append new found directory to directory list
+                    dirs.append(info.path)
+                else:
+                    # Yield path info to non directory
+                    yield info
+
+    # Removes file/directory
+    def remove(self, path_info):
+        # Use webdav client clean (DELETE) method to remove file/directory
+        self._client.clean(path_info.path)
+
+    # Creates directories
+    def makedirs(self, path_info):
+        # Terminate recursion
+        if path_info.path == self.path_info.path or self.exists(path_info):
+            return
+
+        # Recursively descent to root
+        self.makedirs(path_info.parent)
+
+        # Construct directory at current recursion depth
+        self._client.mkdir(path_info.path)
+
+    # Moves file/directory at remote
+    def move(self, from_info, to_info, mode=None):
+        # Webdav client move
+        self._client.move(from_info.path, to_info.path)
+
+    # Downloads file from remote to file
+    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
+        # Progress from HTTPTree
+        with open(to_file, "wb") as fd:
+            with Tqdm.wrapattr(
+                fd,
+                "write",
+                total=None if no_progress_bar else self._file_size(from_info),
+                leave=False,
+                desc=from_info.url if name is None else name,
+                disable=no_progress_bar,
+            ) as fd_wrapped:
+                # Download from WebDAV via buffer
+                self._client.download_from(
+                    buff=fd_wrapped, remote_path=from_info.path
+                )
+
+    # Uploads file to remote
+    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
+        # First try to create parent directories
+        self.makedirs(to_info.parent)
+
+        # Progress from HTTPTree
+        def chunks():
+            with open(from_file, "rb") as fd:
+                with Tqdm.wrapattr(
+                    fd,
+                    "read",
+                    total=None
+                    if no_progress_bar
+                    else os.path.getsize(from_file),
+                    leave=False,
+                    desc=to_info.url if name is None else name,
+                    disable=no_progress_bar,
+                ) as fd_wrapped:
+                    while True:
+                        chunk = fd_wrapped.read(self.CHUNK_SIZE)
+                        if not chunk:
+                            break
+                        yield chunk
+
+        # Upload to WebDAV via buffer
+        self._client.upload_to(buff=chunks(), remote_path=to_info.path)
+
+    # Queries size of file at remote
+    def _file_size(self, path_info):
+        # Get file size from info dictionary and convert to int (from str)
+        return int(self._client.info(path_info.path)["size"])

dvc/tree/webdavs.py

@@ -0,0 +1,7 @@
+from dvc.scheme import Schemes
+
+from .webdav import WebDAVTree
+
+
+class WebDAVSTree(WebDAVTree):  # pylint:disable=abstract-method
+    scheme = Schemes.WEBDAVS

dvc/updater.py

@@ -34,7 +34,7 @@ class Updater:  # pragma: no cover
         ctime = os.path.getmtime(self.updater_file)
         outdated = time.time() - ctime >= self.TIMEOUT
         if outdated:
-            logger.debug(f"'{self.updater_file}' is outdated(")
+            logger.debug(f"'{self.updater_file}' is outdated")
         return outdated
 
     def _with_lock(self, func, action):

dvc/utils/__init__.py

@@ -363,19 +363,22 @@ def resolve_output(inp, out):
 
 
 def resolve_paths(repo, out):
+    from urllib.parse import urlparse
     from ..dvcfile import DVC_FILE_SUFFIX
     from ..path_info import PathInfo
     from .fs import contains_symlink_up_to
 
-    abspath = os.path.abspath(out)
+    abspath = PathInfo(os.path.abspath(out))
     dirname = os.path.dirname(abspath)
     base = os.path.basename(os.path.normpath(out))
 
-    # NOTE: `out` might not exist yet, so using `dirname`(aka `wdir`) to check
-    # if it is a local path.
+    scheme = urlparse(out).scheme
+    if os.name == "nt" and scheme == abspath.drive[0].lower():
+        # urlparse interprets windows drive letters as URL scheme
+        scheme = ""
     if (
-        os.path.exists(dirname)  # out might not exist yet, so
-        and PathInfo(abspath).isin_or_eq(repo.root_dir)
+        not scheme
+        and abspath.isin_or_eq(repo.root_dir)
         and not contains_symlink_up_to(abspath, repo.root_dir)
     ):
         wdir = dirname

dvc/utils/toml.py

@@ -0,0 +1,21 @@
+import toml
+
+from dvc.exceptions import TOMLFileCorruptedError
+
+
+def parse_toml_for_update(text, path):
+    """Parses text into Python structure.
+
+    NOTE: Python toml package does not currently use ordered dicts, so
+    keys may be re-ordered between load/dump, but this function will at
+    least preserve comments.
+    """
+    try:
+        return toml.loads(text, decoder=toml.TomlPreserveCommentDecoder())
+    except toml.TomlDecodeError as exc:
+        raise TOMLFileCorruptedError(path) from exc
+
+
+def dump_toml(path, data):
+    with open(path, "w", encoding="utf-8") as fobj:
+        toml.dump(data, fobj, encoder=toml.TomlPreserveCommentEncoder())

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "1.2.2"
+_BASE_VERSION = "1.4.0"
 
 
 def _generate_version(base_version):

setup.py

@@ -79,7 +79,7 @@ install_requires = [
     "tabulate>=0.8.7",
     "pygtrie==2.3.2",
     "dpath>=2.0.1,<3",
-    "shtab>=1.1.0,<2",
+    "shtab>=1.3.0,<2",
     "rich>=3.0.5",
 ]
 
@@ -93,18 +93,19 @@ azure = ["azure-storage-blob==2.1.0", "knack"]
 oss = ["oss2==2.6.1"]
 ssh = ["paramiko>=2.5.0"]
 hdfs = ["pyarrow>=0.17.0"]
+webdav = ["webdavclient3>=3.14.5"]
 # gssapi should not be included in all_remotes, because it doesn't have wheels
 # for linux and mac, so it will fail to compile if user doesn't have all the
 # requirements, including kerberos itself. Once all the wheels are available,
 # we can start shipping it by default.
 ssh_gssapi = ["paramiko[gssapi]>=2.5.0"]
-all_remotes = gs + s3 + azure + ssh + oss + gdrive + hdfs
+all_remotes = gs + s3 + azure + ssh + oss + gdrive + hdfs + webdav
 
 # Extra dependecies to run tests
 tests_requirements = [
     "wheel>=0.31.1",
     # Test requirements:
-    "pytest>=4.6.0",
+    "pytest>=6.0.1",
     "pytest-docker>=0.7.2",
     "pytest-timeout>=1.3.3",
     "pytest-cov>=2.6.1",
@@ -160,6 +161,7 @@ setup(
         "ssh": ssh,
         "ssh_gssapi": ssh_gssapi,
         "hdfs": hdfs,
+        "webdav": webdav,
         "tests": tests_requirements,
     },
     keywords="data-science data-version-control machine-learning git"

tests/func/experiments/test_show.py

@@ -15,5 +15,6 @@ def test_show_simple(tmp_dir, scm, dvc):
         "baseline": {
             "metrics": {"metrics.yaml": {"foo": 1}},
             "params": {"params.yaml": {"foo": 1}},
+            "queued": False,
         }
     }

tests/func/test_check_ignore.py

@@ -0,0 +1,135 @@
+import os
+
+import pytest
+
+from dvc.ignore import DvcIgnore
+from dvc.main import main
+
+
+@pytest.mark.parametrize(
+    "file,ret,output", [("ignored", 0, True), ("not_ignored", 1, False)]
+)
+def test_check_ignore(tmp_dir, dvc, file, ret, output, caplog):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored")
+
+    assert main(["check-ignore", file]) == ret
+    assert (file in caplog.text) is output
+
+
+@pytest.mark.parametrize(
+    "file,ret,output",
+    [
+        ("file", 0, "{}:1:f*\tfile\n".format(DvcIgnore.DVCIGNORE_FILE)),
+        ("foo", 0, "{}:2:!foo\tfoo\n".format(DvcIgnore.DVCIGNORE_FILE)),
+        (
+            os.path.join("dir", "foobar"),
+            0,
+            "{}:1:foobar\t{}\n".format(
+                os.path.join("dir", DvcIgnore.DVCIGNORE_FILE),
+                os.path.join("dir", "foobar"),
+            ),
+        ),
+    ],
+)
+def test_check_ignore_details(tmp_dir, dvc, file, ret, output, caplog):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "f*\n!foo")
+    tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "foobar"}})
+
+    assert main(["check-ignore", "-d", file]) == ret
+    assert output in caplog.text
+
+
+@pytest.mark.parametrize("non_matching", [True, False])
+def test_check_ignore_non_matching(tmp_dir, dvc, non_matching, caplog):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "other")
+    if non_matching:
+        assert main(["check-ignore", "-d", "-n", "file"]) == 1
+    else:
+        assert main(["check-ignore", "-d", "file"]) == 1
+
+    assert ("::\tfile\n" in caplog.text) is non_matching
+
+
+@pytest.mark.parametrize(
+    "args",
+    [
+        ["-n", "file"],
+        ["-a", "file"],
+        ["-q", "-d", "file"],
+        ["--stdin", "file"],
+        [],
+    ],
+)
+def test_check_ignore_error_args_cases(tmp_dir, dvc, args):
+    assert main(["check-ignore"] + args) == 255
+
+
+@pytest.mark.parametrize("path,ret", [({"dir": {}}, 0), ({"dir": "files"}, 1)])
+def test_check_ignore_dir(tmp_dir, dvc, path, ret):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/")
+    tmp_dir.gen(path)
+
+    assert main(["check-ignore", "-q", "dir"]) == ret
+
+
+def test_check_ignore_default_dir(tmp_dir, dvc):
+    assert main(["check-ignore", "-q", ".dvc"]) == 1
+
+
+def test_check_ignore_out_side_repo(tmp_dir, dvc):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "file")
+    assert main(["check-ignore", "-q", "../file"]) == 1
+
+
+def test_check_ignore_sub_repo(tmp_dir, dvc):
+    tmp_dir.gen(
+        {DvcIgnore.DVCIGNORE_FILE: "other", "dir": {".dvc": {}, "foo": "bar"}}
+    )
+
+    assert main(["check-ignore", "-q", os.path.join("dir", "foo")]) == 1
+
+
+def test_check_sub_dir_ignore_file(tmp_dir, dvc, caplog):
+    tmp_dir.gen(
+        {
+            DvcIgnore.DVCIGNORE_FILE: "other",
+            "dir": {DvcIgnore.DVCIGNORE_FILE: "bar\nfoo", "foo": "bar"},
+        }
+    )
+
+    assert main(["check-ignore", "-d", os.path.join("dir", "foo")]) == 0
+    assert (
+        "{}:2:foo\t{}".format(
+            os.path.join("dir", DvcIgnore.DVCIGNORE_FILE),
+            os.path.join("dir", "foo"),
+        )
+        in caplog.text
+    )
+
+    sub_dir = tmp_dir / "dir"
+    with sub_dir.chdir():
+        assert main(["check-ignore", "-d", "foo"]) == 0
+        assert ".dvcignore:2:foo\tfoo" in caplog.text
+
+
+def test_check_ignore_details_all(tmp_dir, dvc, caplog):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "f*\n!foo")
+
+    assert main(["check-ignore", "-d", "-a", "foo"]) == 0
+    assert "{}:1:f*\tfoo\n".format(DvcIgnore.DVCIGNORE_FILE) in caplog.text
+    assert "{}:2:!foo\tfoo\n".format(DvcIgnore.DVCIGNORE_FILE) in caplog.text
+
+
+@pytest.mark.parametrize(
+    "file,ret,output", [("ignored", 0, True), ("not_ignored", 1, False)]
+)
+def test_check_ignore_stdin_mode(
+    tmp_dir, dvc, file, ret, output, caplog, mocker
+):
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored")
+    mocker.patch("builtins.input", side_effect=[file, ""])
+    stdout_mock = mocker.patch("sys.stdout")
+    stdout_mock.isatty.return_value = True
+
+    assert main(["check-ignore", "--stdin"]) == ret
+    assert (file in caplog.text) is output

tests/func/test_commit.py

@@ -2,7 +2,9 @@ from os import fspath
 
 import pytest
 
+from dvc.dependency.base import DependencyDoesNotExistError
 from dvc.dvcfile import PIPELINE_FILE
+from dvc.output.base import OutputDoesNotExistError
 from dvc.stage.exceptions import StageCommitError
 from dvc.utils.yaml import dump_yaml, load_yaml
 
@@ -86,6 +88,24 @@ def test_commit_no_exec(tmp_dir, dvc):
     assert dvc.status(stage.path) == {}
 
 
+def test_commit_no_exec_missing_dep(tmp_dir, dvc):
+    stage = dvc.run(
+        name="my", cmd="mycmd", deps=["dep"], outs=["out"], no_exec=True
+    )
+    assert dvc.status(stage.path)
+
+    with pytest.raises(DependencyDoesNotExistError):
+        dvc.commit(stage.path, force=True)
+
+
+def test_commit_no_exec_missing_out(tmp_dir, dvc):
+    stage = dvc.run(name="my", cmd="mycmd", outs=["out"], no_exec=True)
+    assert dvc.status(stage.path)
+
+    with pytest.raises(OutputDoesNotExistError):
+        dvc.commit(stage.path, force=True)
+
+
 def test_commit_pipeline_stage(tmp_dir, dvc, run_copy):
     tmp_dir.gen("foo", "foo")
     stage = run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")

tests/func/test_ignore.py

@@ -6,13 +6,17 @@ import pytest
 from dvc.exceptions import DvcIgnoreInCollectedDirError
 from dvc.ignore import DvcIgnore, DvcIgnorePatterns
 from dvc.path_info import PathInfo
-from dvc.pathspec_math import merge_patterns
+from dvc.pathspec_math import PatternInfo, merge_patterns
 from dvc.repo import Repo
 from dvc.utils import relpath
 from dvc.utils.fs import get_mtime_and_size
 from tests.dir_helpers import TmpDir
 
 
+def _to_pattern_info_list(str_list):
+    return [PatternInfo(a, "") for a in str_list]
+
+
 def test_ignore(tmp_dir, dvc, monkeypatch):
     tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
     tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
@@ -106,9 +110,9 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
     assert (
         DvcIgnorePatterns(
             *merge_patterns(
-                [".hg/", ".git/", ".dvc/"],
+                _to_pattern_info_list([".hg/", ".git/", ".dvc/"]),
                 os.fspath(tmp_dir),
-                [os.path.basename(dname)],
+                _to_pattern_info_list([os.path.basename(dname)]),
                 top_ignore_path,
             )
         )
@@ -309,17 +313,24 @@ def test_pattern_trie_tree(tmp_dir, dvc):
         os.fspath(tmp_dir / "top" / "first" / "middle" / "second" / "bottom")
     )
 
-    base_pattern = [".hg/", ".git/", ".dvc/"], os.fspath(tmp_dir)
+    base_pattern = (
+        _to_pattern_info_list([".hg/", ".git/", ".dvc/"]),
+        os.fspath(tmp_dir),
+    )
     first_pattern = merge_patterns(
-        *base_pattern, ["a", "b", "c"], os.fspath(tmp_dir / "top" / "first")
+        *base_pattern,
+        _to_pattern_info_list(["a", "b", "c"]),
+        os.fspath(tmp_dir / "top" / "first")
     )
     second_pattern = merge_patterns(
         *first_pattern,
-        ["d", "e", "f"],
+        _to_pattern_info_list(["d", "e", "f"]),
         os.fspath(tmp_dir / "top" / "first" / "middle" / "second")
     )
     other_pattern = merge_patterns(
-        *base_pattern, ["1", "2", "3"], os.fspath(tmp_dir / "other")
+        *base_pattern,
+        _to_pattern_info_list(["1", "2", "3"]),
+        os.fspath(tmp_dir / "other")
     )
 
     assert DvcIgnorePatterns(*base_pattern) == ignore_pattern_top

tests/func/test_import.py

@@ -9,6 +9,7 @@ from dvc.cache import Cache
 from dvc.config import NoRemoteError
 from dvc.dvcfile import Dvcfile
 from dvc.exceptions import DownloadError, PathMissingError
+from dvc.stage.exceptions import StagePathNotFoundError
 from dvc.system import System
 from dvc.utils.fs import makedirs, remove
 
@@ -132,6 +133,28 @@ def test_import_file_from_dir(tmp_dir, scm, dvc, erepo_dir):
     assert (tmp_dir / "X.dvc").exists()
 
 
+def test_import_file_from_dir_to_dir(tmp_dir, scm, dvc, erepo_dir):
+    with erepo_dir.chdir():
+        erepo_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="create dir")
+
+    with pytest.raises(StagePathNotFoundError):
+        dvc.imp(
+            os.fspath(erepo_dir),
+            os.path.join("dir", "foo"),
+            out=os.path.join("dir", "foo"),
+        )
+
+    tmp_dir.gen({"dir": {}})
+    dvc.imp(
+        os.fspath(erepo_dir),
+        os.path.join("dir", "foo"),
+        out=os.path.join("dir", "foo"),
+    )
+    assert not (tmp_dir / "foo.dvc").exists()
+    assert (tmp_dir / "dir" / "foo").read_text() == "foo"
+    assert (tmp_dir / "dir" / "foo.dvc").exists()
+
+
 def test_import_non_cached(erepo_dir, tmp_dir, dvc, scm):
     src = "non_cached_output"
     dst = src + "_imported"

tests/func/test_remote.py

@@ -232,6 +232,24 @@ def test_modify_missing_remote(tmp_dir, dvc):
     assert main(["remote", "modify", "myremote", "user", "xxx"]) == 251
 
 
+def test_remote_modify_local_on_repo_config(tmp_dir, dvc):
+    assert main(["remote", "add", "myremote", "http://example.com/path"]) == 0
+    assert (
+        main(["remote", "modify", "myremote", "user", "xxx", "--local"]) == 0
+    )
+    assert dvc.config.load_one("local")["remote"]["myremote"] == {
+        "user": "xxx"
+    }
+    assert dvc.config.load_one("repo")["remote"]["myremote"] == {
+        "url": "http://example.com/path"
+    }
+    dvc.config.load()
+    assert dvc.config["remote"]["myremote"] == {
+        "url": "http://example.com/path",
+        "user": "xxx",
+    }
+
+
 def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
     # https://github.com/iterative/dvc/issues/2647, is some situations
     # (external dir dependency) cache is required to calculate dir md5

tests/func/test_status.py

@@ -1,28 +1,34 @@
 import os
 
-from mock import patch
-
 from dvc.main import main
-from tests.basic_env import TestDvc
 
 
-class TestStatus(TestDvc):
-    def test_quiet(self):
-        self.dvc.add(self.FOO)
+def test_quiet(tmp_dir, dvc, capsys):
+    tmp_dir.dvc_gen("foo", "foo")
 
-        ret = main(["status", "--quiet"])
-        self.assertEqual(ret, 0)
+    # clear
+    capsys.readouterr()
 
-        os.remove(self.FOO)
-        os.rename(self.BAR, self.FOO)
+    assert main(["status", "--quiet"]) == 0
+    out_err = capsys.readouterr()
+    assert not out_err.out
+    assert not out_err.err
 
-        ret = main(["status", "--quiet"])
-        self.assertEqual(ret, 1)
+    tmp_dir.gen("foo", "barr")
 
-    @patch("dvc.repo.status._cloud_status", return_value=True)
-    def test_implied_cloud(self, mock_status):
-        main(["status", "--remote", "something"])
-        mock_status.assert_called()
+    assert main(["status", "--quiet"]) == 1
+    out_err = capsys.readouterr()
+    assert not out_err.out
+    assert not out_err.err
+
+
+def test_implied_cloud(dvc, mocker):
+    mock_status = mocker.patch(
+        "dvc.repo.status._cloud_status", return_value=True
+    )
+
+    main(["status", "--remote", "something"])
+    assert mock_status.called
 
 
 def test_status_non_dvc_repo_import(tmp_dir, dvc, git_dir):

tests/func/test_tree.py

@@ -217,7 +217,10 @@ def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, local_cloud):
     with erepo_dir.dvc.state:
         cache = dvc.cache.local
         with cache.tree.state:
-            cache.save(PathInfo(erepo_dir / "dir"), tree, None)
+            path_info = PathInfo(erepo_dir / "dir")
+            hash_info = cache.tree.save_info(path_info)
+            cache.save(path_info, tree, hash_info)
+
     for hash_ in expected:
         assert os.path.exists(cache.tree.hash_to_path_info(hash_))
 

tests/func/test_version.py

@@ -16,15 +16,10 @@ def test_info_in_repo(scm_init, tmp_dir, caplog):
 
     assert main(["version"]) == 0
 
-    assert re.search(r"DVC version: \d+\.\d+\.\d+", caplog.text)
-    assert re.search(r"Python version: \d\.\d\.\d", caplog.text)
-    assert re.search(r"Platform: .*", caplog.text)
-    assert re.search(r"Binary: (True|False)", caplog.text)
-    assert re.search(r"Package: .*", caplog.text)
-    assert re.search(r"Supported remotes: .*", caplog.text)
-    assert re.search(
-        r"(Cache: (.*link - (not )?supported(,\s)?){3})", caplog.text
-    )
+    assert re.search(r"DVC version: \d+\.\d+\.\d+.*", caplog.text)
+    assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", caplog.text)
+    assert re.search(r"Supports: .*", caplog.text)
+    assert re.search(r"Cache types: .*", caplog.text)
 
     if scm_init:
         assert "Repo: dvc, git" in caplog.text
@@ -58,26 +53,20 @@ def test_fs_info_in_repo(tmp_dir, dvc, caplog):
     os.mkdir(dvc.cache.local.cache_dir)
     assert main(["version"]) == 0
 
-    assert "Filesystem type (cache directory): " in caplog.text
-    assert "Filesystem type (workspace): " in caplog.text
+    assert re.search(r"Cache directory: .* on .*", caplog.text)
+    assert re.search(r"Workspace directory: .* on .*", caplog.text)
 
 
 def test_info_outside_of_repo(tmp_dir, caplog):
     assert main(["version"]) == 0
 
-    assert re.search(r"DVC version: \d+\.\d+\.\d+", caplog.text)
-    assert re.search(r"Python version: \d\.\d\.\d", caplog.text)
-    assert re.search(r"Platform: .*", caplog.text)
-    assert re.search(r"Binary: (True|False)", caplog.text)
-    assert re.search(r"Package: .*", caplog.text)
-    assert re.search(r"Supported remotes: .*", caplog.text)
-    assert not re.search(r"(Cache: (.*link - (not )?(,\s)?){3})", caplog.text)
+    assert re.search(r"DVC version: \d+\.\d+\.\d+.*", caplog.text)
+    assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", caplog.text)
+    assert re.search(r"Supports: .*", caplog.text)
+    assert not re.search(r"Cache types: .*", caplog.text)
     assert "Repo:" not in caplog.text
 
 
 @pytest.mark.skipif(psutil is None, reason="No psutil.")
 def test_fs_info_outside_of_repo(tmp_dir, caplog):
     assert main(["version"]) == 0
-
-    assert "Filesystem type (cache directory): " not in caplog.text
-    assert "Filesystem type (workspace): " in caplog.text

tests/unit/command/test_repro.py

@@ -15,6 +15,10 @@ default_arguments = {
     "recursive": False,
     "force_downstream": False,
     "experiment": False,
+    "params": [],
+    "queue": False,
+    "run_all": False,
+    "jobs": None,
 }
 
 

tests/unit/remote/test_webdav.py

@@ -0,0 +1,42 @@
+from dvc.tree.webdav import WebDAVTree
+
+# Test configuration
+url = "webdavs://example.com/public.php/webdav"
+user = "username"
+userurl = f"webdavs://{user}@example.com/public.php/webdav"
+password = "password"
+
+
+# Test minimum requiered configuration (url)
+def test_init(dvc):
+    config = {"url": url}
+    tree = WebDAVTree(dvc, config)
+
+    assert tree.path_info == url
+
+
+# Test username from configuration
+def test_user(dvc):
+    config = {"url": url, "user": user}
+    tree = WebDAVTree(dvc, config)
+
+    assert tree.user == user
+    assert tree.path_info.user == user
+
+
+# Test username extraction from url
+def test_userurl(dvc):
+    config = {"url": userurl}
+    tree = WebDAVTree(dvc, config)
+
+    assert tree.path_info == userurl
+    assert tree.user == user
+    assert tree.path_info.user == user
+
+
+# test password from config
+def test_password(dvc):
+    config = {"url": url, "user": user, "password": password}
+    tree = WebDAVTree(dvc, config)
+
+    assert tree.password == password

tests/unit/test_config.py

@@ -3,6 +3,7 @@ import os
 import pytest
 
 from dvc.config import Config
+from dvc.tree.local import LocalTree
 
 
 @pytest.mark.parametrize(
@@ -16,3 +17,19 @@ from dvc.config import Config
 )
 def test_to_relpath(path, expected):
     assert Config._to_relpath(os.path.join(".", "config"), path) == expected
+
+
+def test_get_tree(tmp_dir, scm):
+    tmp_dir.scm_gen("foo", "foo", commit="add foo")
+
+    tree = scm.get_tree("master")
+    config = Config(tree=tree)
+
+    assert config.tree == tree
+    assert config.wtree != tree
+    assert isinstance(config.wtree, LocalTree)
+
+    assert config._get_tree("repo") == tree
+    assert config._get_tree("local") == config.wtree
+    assert config._get_tree("global") == config.wtree
+    assert config._get_tree("system") == config.wtree

tests/unit/test_pathspec_math.py

@@ -1,6 +1,6 @@
 import pytest
 
-from dvc.pathspec_math import _change_dirname
+from dvc.pathspec_math import PatternInfo, _change_dirname
 
 
 @pytest.mark.parametrize(
@@ -69,4 +69,6 @@ from dvc.pathspec_math import _change_dirname
     ],
 )
 def test_dvcignore_pattern_change_dir(tmp_dir, patterns, dirname, changed):
-    assert _change_dirname(dirname, [patterns], "/") == [changed]
+    assert _change_dirname(dirname, [PatternInfo(patterns, "")], "/") == [
+        PatternInfo(changed, "")
+    ]