run: update -h message (#6197)

* update dvc exp run -h message

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/CODEOWNERS

@@ -0,0 +1 @@
+*       @iterative/dvc

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
            - types-tabulate
            - types-toml
     repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.902'
+    rev: 'v0.910'
   - repo: local
     hooks:
     - id: pylint

dvc/command/experiments.py

@@ -397,7 +397,7 @@ def show_experiments(
     styles.update(
         {
             header: {
-                "justify": "left" if typ == "metrics" else "params",
+                "justify": "right" if typ == "metrics" else "left",
                 "header_style": f"black on {header_bg_colors[typ]}",
                 "collapse": idx != 0,
                 "no_wrap": typ == "metrics",
@@ -937,9 +937,7 @@ def add_parser(subparsers, parent_parser):
     )
     experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)
 
-    EXPERIMENTS_RUN_HELP = (
-        "Reproduce complete or partial experiment pipelines."
-    )
+    EXPERIMENTS_RUN_HELP = "Run or resume an experiment."
     experiments_run_parser = experiments_subparsers.add_parser(
         "run",
         parents=[parent_parser],

dvc/dependency/repo.py

@@ -97,7 +97,7 @@ class RepoDependency(Dependency):
         self, **kwargs
     ) -> Dict[Optional["ObjectDB"], Set["HashFile"]]:
         from dvc.config import NoRemoteError
-        from dvc.exceptions import NoOutputOrStageError
+        from dvc.exceptions import NoOutputOrStageError, PathMissingError
         from dvc.objects.db.git import GitObjectDB
         from dvc.objects.stage import stage
 
@@ -126,12 +126,18 @@ class RepoDependency(Dependency):
             except (NoRemoteError, NoOutputOrStageError):
                 pass
 
-            staged_obj = stage(
-                local_odb,
-                path_info,
-                repo.repo_fs,
-                local_odb.fs.PARAM_CHECKSUM,
-            )
+            try:
+                staged_obj = stage(
+                    local_odb,
+                    path_info,
+                    repo.repo_fs,
+                    local_odb.fs.PARAM_CHECKSUM,
+                )
+            except FileNotFoundError as exc:
+                raise PathMissingError(
+                    self.def_path, self.def_repo[self.PARAM_URL]
+                ) from exc
+
             self._staged_objs[rev] = staged_obj
             git_odb = GitObjectDB(repo.repo_fs, repo.root_dir)
             used_objs[git_odb].add(staged_obj)

dvc/fs/base.py

@@ -33,6 +33,8 @@ class RemoteMissingDepsError(DvcException):
 
 
 class BaseFileSystem:
+    sep = "/"
+
     scheme = "base"
     REQUIRES: ClassVar[Dict[str, str]] = {}
     PATH_CLS = URLInfo  # type: Any

dvc/fs/dvc.py

@@ -23,6 +23,8 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         repo: DVC repo.
     """
 
+    sep = os.sep
+
     scheme = "local"
     PARAM_CHECKSUM = "md5"
 

dvc/fs/fsspec_wrapper.py

@@ -173,7 +173,7 @@ class ObjectFSWrapper(FSSpecWrapper):
         )
 
     def find(self, path_info, detail=False, prefix=None):
-        if prefix is not None:
+        if prefix:
             path = self._with_bucket(path_info.parent)
             files = self.fs.find(
                 path, detail=detail, prefix=path_info.parts[-1]

dvc/fs/git.py

@@ -9,6 +9,8 @@ from .base import BaseFileSystem
 class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     """Proxies the repo file access methods to Git objects"""
 
+    sep = os.sep
+
     scheme = "local"
 
     def __init__(self, root_dir, trie):

dvc/fs/gs.py

@@ -16,6 +16,7 @@ class GSFileSystem(ObjectFSWrapper):
     REQUIRES = {"gcsfs": "gcsfs"}
     PARAM_CHECKSUM = "etag"
     DETAIL_FIELDS = frozenset(("etag", "size"))
+    TRAVERSE_PREFIX_LEN = 2
 
     def _prepare_credentials(self, **config):
         login_info = {"consistency": None}

dvc/fs/local.py

@@ -14,6 +14,8 @@ logger = logging.getLogger(__name__)
 
 
 class LocalFileSystem(BaseFileSystem):
+    sep = os.sep
+
     scheme = Schemes.LOCAL
     PATH_CLS = PathInfo
     PARAM_CHECKSUM = "md5"

dvc/fs/repo.py

@@ -31,6 +31,8 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         kwargs: Additional keyword arguments passed to the `DvcFileSystem()`.
     """
 
+    sep = os.sep
+
     scheme = "local"
     PARAM_CHECKSUM = "md5"
 

dvc/fs/ssh/__init__.py

@@ -119,16 +119,16 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         if self.ask_password and self.password is None:
             self.password = ask_password(self.host, self.user, self.port)
 
-    def ssh(self, path_info):
+    def ssh(self):
         self.ensure_credentials()
 
         from .connection import SSHConnection
 
         return get_connection(
             SSHConnection,
-            path_info.host,
-            username=path_info.user,
-            port=path_info.port,
+            self.host,
+            username=self.user,
+            port=self.port,
             key_filename=self.keyfile,
             timeout=self.timeout,
             password=self.password,
@@ -141,7 +141,7 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     def open(self, path_info, mode="r", encoding=None, **kwargs):
         assert mode in {"r", "rt", "rb", "wb"}
 
-        with self.ssh(path_info) as ssh, closing(
+        with self.ssh() as ssh, closing(
             ssh.sftp.open(path_info.path, mode)
         ) as fd:
             if "b" in mode:
@@ -150,19 +150,19 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 yield io.TextIOWrapper(fd, encoding=encoding)
 
     def exists(self, path_info) -> bool:
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return ssh.exists(path_info.path)
 
     def isdir(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return ssh.isdir(path_info.path)
 
     def isfile(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return ssh.isfile(path_info.path)
 
     def walk_files(self, path_info, **kwargs):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             for fname in ssh.walk_files(path_info.path):
                 yield path_info.replace(path=fname)
 
@@ -170,32 +170,32 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         if path_info.scheme != self.scheme:
             raise NotImplementedError
 
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             ssh.remove(path_info.path)
 
     def makedirs(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             ssh.makedirs(path_info.path)
 
     def move(self, from_info, to_info):
         if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
             raise NotImplementedError
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.move(from_info.path, to_info.path)
 
     def copy(self, from_info, to_info):
         if not from_info.scheme == to_info.scheme == self.scheme:
             raise NotImplementedError
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.atomic_copy(from_info.path, to_info.path)
 
     def symlink(self, from_info, to_info):
         if not from_info.scheme == to_info.scheme == self.scheme:
             raise NotImplementedError
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.symlink(from_info.path, to_info.path)
 
     def hardlink(self, from_info, to_info):
@@ -205,7 +205,7 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         # See dvc/remote/local/__init__.py - hardlink()
         if self.getsize(from_info) == 0:
 
-            with self.ssh(to_info) as ssh:
+            with self.ssh() as ssh:
                 ssh.sftp.open(to_info.path, "w").close()
 
             logger.debug(
@@ -215,18 +215,18 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
             return
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.hardlink(from_info.path, to_info.path)
 
     def reflink(self, from_info, to_info):
         if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
             raise NotImplementedError
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.reflink(from_info.path, to_info.path)
 
     def md5(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return HashInfo(
                 "md5",
                 ssh.md5(path_info.path),
@@ -234,7 +234,7 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
 
     def info(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return ssh.info(path_info.path)
 
     def _upload_fobj(self, fobj, to_info, **kwargs):
@@ -243,7 +243,7 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             shutil.copyfileobj(fobj, fdest)
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.download(
                 from_info.path,
                 to_file,
@@ -254,7 +254,7 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        with self.ssh(to_info) as ssh:
+        with self.ssh() as ssh:
             ssh.upload(
                 from_file,
                 to_info.path,

dvc/objects/db/ssh.py

@@ -28,7 +28,7 @@ class SSHObjectDB(ObjectDB):
                 callback(path)
             return ret
 
-        with self.fs.ssh(path_infos[0]) as ssh:
+        with self.fs.ssh() as ssh:
             channels = ssh.open_max_sftp_channels()
             max_workers = len(channels)
 
@@ -78,7 +78,7 @@ class SSHObjectDB(ObjectDB):
             root = posixpath.join(self.path_info.path, prefix[:2])
         else:
             root = self.path_info.path
-        with self.fs.ssh(self.path_info) as ssh:
+        with self.fs.ssh() as ssh:
             if prefix and not ssh.exists(root):
                 return
             # If we simply return an iterator then with above closes instantly

dvc/objects/stage.py

@@ -52,11 +52,6 @@ def get_file_hash(path_info, fs, name, state=None):
         if hash_info:
             return hash_info
 
-    if not fs.exists(path_info):
-        raise FileNotFoundError(
-            errno.ENOENT, os.strerror(errno.ENOENT), path_info
-        )
-
     hash_info = _get_file_hash(path_info, fs, name)
 
     if state:

dvc/output.py

@@ -45,18 +45,24 @@ CHECKSUM_SCHEMA = Any(
     And(Any(str, And(int, Coerce(str))), Length(min=3), Lower),
 )
 
+CASE_SENSITIVE_CHECKSUM_SCHEMA = Any(
+    None,
+    And(str, Length(max=0), SetTo(None)),
+    And(Any(str, And(int, Coerce(str))), Length(min=3)),
+)
+
 # NOTE: currently there are only 3 possible checksum names:
 #
 #    1) md5 (LOCAL, SSH);
-#    2) etag (S3);
+#    2) etag (S3, GS, OSS, AZURE, HTTP);
 #    3) checksum (HDFS);
 #
 # so when a few types of outputs share the same name, we only need
 # specify it once.
 CHECKSUMS_SCHEMA = {
     LocalFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    S3FileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
     HDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    S3FileSystem.PARAM_CHECKSUM: CASE_SENSITIVE_CHECKSUM_SCHEMA,
 }
 
 
@@ -265,8 +271,6 @@ class Output:
     IsStageFileError = OutputIsStageFileError  # type: Type[DvcException]
     IsIgnoredError = OutputIsIgnoredError  # type: Type[DvcException]
 
-    sep = "/"
-
     def __init__(
         self,
         stage,
@@ -881,7 +885,7 @@ class Output:
         obj.name = str(self)
         if isinstance(obj, Tree):
             for key, entry_obj in obj:
-                entry_obj.name = os.path.join(str(self), *key)
+                entry_obj.name = self.fs.sep.join([obj.name, *key])
 
     def get_used_external(
         self, **kwargs

dvc/repo/add.py

@@ -14,6 +14,7 @@ from ..exceptions import (
 from ..progress import Tqdm
 from ..repo.scm_context import scm_context
 from ..utils import LARGE_DIR_SIZE, glob_targets, resolve_output, resolve_paths
+from ..utils.collections import ensure_list, validate
 from . import locked
 
 if TYPE_CHECKING:
@@ -23,34 +24,29 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-@locked
-@scm_context
-def add(  # noqa: C901
-    repo,
-    targets: "TargetType",
-    recursive=False,
-    no_commit=False,
-    fname=None,
-    to_remote=False,
-    **kwargs,
-):
-    from dvc.utils.collections import ensure_list
-
-    if recursive and fname:
+def check_recursive_and_fname(args):
+    if args.recursive and args.fname:
         raise RecursiveAddingWhileUsingFilename()
 
-    targets = ensure_list(targets)
 
-    to_cache = kwargs.get("out") and not to_remote
+def transform_targets(args):
+    args.targets = ensure_list(args.targets)
+
+
+def check_arg_combinations(args):
+    kwargs = args.kwargs
     invalid_opt = None
+    to_remote = args.to_remote
+    to_cache = kwargs.get("out") and not to_remote
+
     if to_remote or to_cache:
         message = "{option} can't be used with "
         message += "--to-remote" if to_remote else "-o"
-        if len(targets) != 1:
+        if len(args.targets) != 1:
             invalid_opt = "multiple targets"
-        elif no_commit:
+        elif args.no_commit:
             invalid_opt = "--no-commit option"
-        elif recursive:
+        elif args.recursive:
             invalid_opt = "--recursive option"
         elif kwargs.get("external"):
             invalid_opt = "--external option"
@@ -64,9 +60,30 @@ def add(  # noqa: C901
     if invalid_opt is not None:
         raise InvalidArgumentError(message.format(option=invalid_opt))
 
+
+VALIDATORS = (
+    check_recursive_and_fname,
+    transform_targets,
+    check_arg_combinations,
+)
+
+
+@locked
+@scm_context
+@validate(*VALIDATORS)
+def add(  # noqa: C901
+    repo,
+    targets: "TargetType",
+    recursive=False,
+    no_commit=False,
+    fname=None,
+    to_remote=False,
+    **kwargs,
+):
     link_failures = []
     stages_list = []
     num_targets = len(targets)
+    to_cache = kwargs.get("out") and not to_remote
     with Tqdm(total=num_targets, desc="Add", unit="file", leave=True) as pbar:
         if num_targets == 1:
             # clear unneeded top-level progress bar for single target
@@ -98,8 +115,11 @@ def add(  # noqa: C901
                 **kwargs,
             )
 
+            # remove existing stages that are to-be replaced with these
+            # new stages for the graph checks.
+            old_stages = set(repo.stages) - set(stages)
             try:
-                repo.check_modified_graph(stages)
+                repo.check_modified_graph(stages, list(old_stages))
             except OverlappingOutputPathsError as exc:
                 msg = (
                     "Cannot add '{out}', because it is overlapping with other "
@@ -233,7 +253,6 @@ def _create_stages(
     transfer=False,
     **kwargs,
 ):
-    from dvc.dvcfile import Dvcfile
     from dvc.stage import Stage, create_stage, restore_meta
 
     expanded_targets = glob_targets(targets, glob=glob)
@@ -259,12 +278,9 @@ def _create_stages(
             external=external,
         )
         restore_meta(stage)
-        Dvcfile(repo, stage.path).remove()
         if desc:
             stage.outs[0].desc = desc
 
-        repo._reset()  # pylint: disable=protected-access
-
         if not stage:
             if pbar is not None:
                 pbar.total -= 1

dvc/repo/stage.py

@@ -462,7 +462,8 @@ class StageLoad:
 
         def is_out_or_ignored(root, directory):
             dir_path = f"{root}{sep}{directory}"
-            return dir_path in outs or is_ignored(dir_path)
+            # trailing slash needed to check if a directory is gitignored
+            return dir_path in outs or is_ignored(f"{dir_path}{sep}")
 
         stages = []
         for root, dirs, files in self.repo.dvcignore.walk(

dvc/scm/git/backend/dulwich.py

@@ -27,6 +27,8 @@ from ..objects import GitObject
 from .base import BaseGitBackend
 
 if TYPE_CHECKING:
+    from dvc.types import StrPath
+
     from ..objects import GitCommit
 
 logger = logging.getLogger(__name__)
@@ -278,12 +280,14 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
 
         return IgnoreFilterManager.from_repo(self.repo)
 
-    def is_ignored(self, path: str) -> bool:
+    def is_ignored(self, path: "StrPath") -> bool:
         # `is_ignored` returns `false` if excluded in `.gitignore` and
         # `None` if it's not mentioned at all. `True` if it is ignored.
-        return bool(
-            self.ignore_manager.is_ignored(relpath(path, self.root_dir))
-        )
+        relative_path = relpath(path, self.root_dir)
+        # if checking a directory, a trailing slash must be included
+        if str(path)[-1] == os.sep:
+            relative_path += os.sep
+        return bool(self.ignore_manager.is_ignored(relative_path))
 
     def set_ref(
         self,

dvc/scm/git/backend/gitpython.py

@@ -3,7 +3,16 @@ import locale
 import logging
 import os
 from functools import partial
-from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from funcy import ignore
 
@@ -14,6 +23,9 @@ from dvc.utils import fix_env, is_binary, relpath
 from ..objects import GitCommit, GitObject
 from .base import BaseGitBackend
 
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
 logger = logging.getLogger(__name__)
 
 
@@ -111,11 +123,11 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def git(self):
         return self.repo.git
 
-    def is_ignored(self, path: str) -> bool:
+    def is_ignored(self, path: "StrPath") -> bool:
         from git.exc import GitCommandError
 
         func = ignore(GitCommandError)(self.repo.git.check_ignore)
-        return bool(func(path))
+        return bool(func(str(path)))
 
     @property
     def root_dir(self) -> str:

dvc/scm/git/backend/pygit2.py

@@ -4,7 +4,16 @@ import os
 import stat
 from contextlib import contextmanager
 from io import BytesIO, StringIO
-from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from funcy import cached_property
 
@@ -17,6 +26,10 @@ from .base import BaseGitBackend
 logger = logging.getLogger(__name__)
 
 
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
+
 class Pygit2Object(GitObject):
     def __init__(self, obj):
         self.obj = obj
@@ -81,6 +94,15 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
 
         return RefdbFsBackend(self.repo)
 
+    def _resolve_refish(self, refish: str):
+        from pygit2 import GIT_OBJ_COMMIT, Tag
+
+        commit, ref = self.repo.resolve_refish(refish)
+        if isinstance(commit, Tag):
+            ref = commit
+            commit = commit.peel(GIT_OBJ_COMMIT)
+        return commit, ref
+
     @property
     def default_signature(self):
         try:
@@ -140,7 +162,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
                 if branch == "-":
                     branch = "@{-1}"
                 try:
-                    commit, ref = self.repo.resolve_refish(branch)
+                    commit, ref = self._resolve_refish(branch)
                 except (KeyError, GitError):
                     raise RevError(f"unknown Git revision '{branch}'")
                 self.repo.checkout_tree(commit, strategy=checkout_strategy)
@@ -200,7 +222,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         from pygit2 import GitError
 
         try:
-            commit, _ref = self.repo.resolve_refish(rev)
+            commit, _ref = self._resolve_refish(rev)
             return str(commit.id)
         except (KeyError, GitError):
             pass
@@ -220,7 +242,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         from pygit2 import GitError
 
         try:
-            commit, _ref = self.repo.resolve_refish(rev)
+            commit, _ref = self._resolve_refish(rev)
         except (KeyError, GitError):
             raise SCMError(f"Invalid commit '{rev}'")
         return GitCommit(
@@ -234,7 +256,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
     def _get_stash(self, ref: str):
         raise NotImplementedError
 
-    def is_ignored(self, path: str) -> bool:
+    def is_ignored(self, path: "StrPath") -> bool:
         rel = relpath(path, self.root_dir)
         if os.name == "nt":
             rel.replace("\\", "/")
@@ -294,12 +316,12 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         from pygit2 import GitError
 
         def _contains(repo, ref, search_commit):
-            commit, _ref = self.repo.resolve_refish(ref)
+            commit, _ref = self._resolve_refish(ref)
             base = repo.merge_base(search_commit.id, commit.id)
             return base == search_commit.id
 
         try:
-            search_commit, _ref = self.repo.resolve_refish(rev)
+            search_commit, _ref = self._resolve_refish(rev)
         except (KeyError, GitError):
             raise SCMError(f"Invalid rev '{rev}'")
 
@@ -384,7 +406,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
 
         # libgit2 stash apply only accepts refs/stash items by index. If rev is
         # not in refs/stash, we will push it onto the stash, and then pop it
-        commit, _ref = self.repo.resolve_refish(rev)
+        commit, _ref = self._resolve_refish(rev)
         stash = self.repo.references.get(Stash.DEFAULT_STASH)
         if stash:
             for i, entry in enumerate(stash.log()):

dvc/utils/collections.py

@@ -1,6 +1,8 @@
+import inspect
 import os
 from collections.abc import Mapping
-from typing import Dict, Iterable, List, TypeVar, Union
+from functools import wraps
+from typing import Callable, Dict, Iterable, List, TypeVar, Union
 
 from pygtrie import StringTrie as _StringTrie
 
@@ -76,3 +78,59 @@ def chunk_dict(d: Dict[_KT, _VT], size: int = 1) -> List[Dict[_KT, _VT]]:
     from funcy import chunks
 
     return [{key: d[key] for key in chunk} for chunk in chunks(size, d)]
+
+
+class _NamespacedDict(dict):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.__dict__ = self
+
+
+def validate(*validators: Callable, post: bool = False):
+    """
+    Validate and transform arguments and results from function calls.
+
+    The validators functions are passed a dictionary of arguments, which
+    supports dot notation access too.
+
+    The key is derived from the function signature, and hence is the name of
+    the argument, whereas the value is the one passed to the function
+    (if it is not passed, default value from keyword arguments are provided).
+
+    >>> def validator(args):
+    ...    assert args["l"] >= 0 and args.b >= 0 and args.h >= 0
+
+    >>> @validate(validator)
+    ... def cuboid_area(l, b, h=1):
+    ...   return 2*(l*b + l*h + b*h)
+
+    >>> cuboid_area(5, 20)
+    250
+    >>> cuboid_area(-1, -2)
+    Traceback (most recent call last):
+      ...
+    AssertionError
+    """
+
+    def wrapped(func: Callable):
+        sig = inspect.signature(func)
+
+        @wraps(func)
+        def inner(*args, **kwargs):
+            ba = sig.bind(*args, **kwargs)
+            ba.apply_defaults()
+            ba.arguments = _NamespacedDict(ba.arguments)
+
+            if not post:
+                for validator in validators:
+                    validator(ba.arguments)
+
+            result = func(*ba.args, **ba.kwargs)
+            if post:
+                for validator in validators:
+                    result = validator(result)
+            return result
+
+        return inner
+
+    return wrapped

dvc/utils/fs.py

@@ -4,11 +4,15 @@ import os
 import shutil
 import stat
 import sys
+from typing import TYPE_CHECKING
 
 from dvc.exceptions import DvcException
 from dvc.system import System
 from dvc.utils import dict_md5
 
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
 logger = logging.getLogger(__name__)
 
 LOCAL_CHUNK_SIZE = 2 ** 20  # 1 MB
@@ -73,9 +77,9 @@ class BasePathNotInCheckedPathException(DvcException):
         super().__init__(msg)
 
 
-def contains_symlink_up_to(path, base_path):
-    base_path = os.fspath(base_path)
-    path = os.fspath(path)
+def contains_symlink_up_to(path: "StrPath", base_path: "StrPath"):
+    base_path = os.path.normcase(os.fspath(base_path))
+    path = os.path.normcase(os.fspath(path))
 
     if base_path not in path:
         raise BasePathNotInCheckedPathException(path, base_path)
@@ -144,11 +148,11 @@ def remove(path):
             raise
 
 
-def path_isin(child, parent):
+def path_isin(child: "StrPath", parent: "StrPath"):
     """Check if given `child` path is inside `parent`."""
 
-    def normalize_path(path):
-        return os.path.normpath(path)
+    def normalize_path(path) -> str:
+        return os.path.normcase(os.path.normpath(path))
 
     parent = os.path.join(normalize_path(parent), "")
     child = normalize_path(child)

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.4.0"
+_BASE_VERSION = "2.5.0"
 
 
 def _generate_version(base_version):

setup.py

@@ -80,7 +80,7 @@ install_requires = [
     "dataclasses==0.7; python_version < '3.7'",
     "flatten_dict>=0.3.0,<1",
     "tabulate>=0.8.7",
-    "pygtrie==2.3.2",
+    "pygtrie>=2.3.2",
     "dpath>=2.0.1,<3",
     "shtab>=1.3.4,<2",
     "rich>=10.0.0",
@@ -88,16 +88,16 @@ install_requires = [
     "python-benedict>=0.21.1",
     "pyparsing==2.4.7",
     "typing_extensions>=3.7.4",
-    "fsspec @ git+https://github.com/intake/filesystem_spec.git",
+    "fsspec>=2021.6.1",
     "diskcache>=5.2.1",
 ]
 
 
 # Extra dependencies for remote integrations
 
-gs = ["gcsfs==2021.6.0"]
+gs = ["gcsfs==2021.6.1"]
 gdrive = ["pydrive2>=1.8.1", "six >= 1.13.0"]
-s3 = ["s3fs==2021.6.0", "aiobotocore[boto3]==1.3.0"]
+s3 = ["s3fs==2021.6.1", "aiobotocore[boto3]==1.3.0"]
 azure = ["adlfs==0.7.1", "azure-identity>=1.4.0", "knack"]
 # https://github.com/Legrandin/pycryptodome/issues/465
 oss = ["oss2==2.6.1", "pycryptodome>=3.10"]

test_requirements.txt

@@ -17,12 +17,12 @@ beautifulsoup4==4.9.3
 wget==3.2
 filelock==3.0.12
 wsgidav==3.1.0
-crc32c==2.2
+crc32c==2.2.post0
 xmltodict==0.12.0
 google-compute-engine==2.8.13
-google-cloud-storage==1.38.0
+google-cloud-storage==1.40.0
 
-urllib3==1.26.5
+urllib3==1.26.6
 
 # required by collective.checkdocs
 Pygments==2.9.0
@@ -39,7 +39,7 @@ pylint-pytest==1.0.3
 pylint-plugin-utils==0.6
 
 # type-checking
-mypy==0.902
+mypy==0.910
 types-requests
 types-paramiko
 types-tabulate

tests/func/test_add.py

@@ -24,7 +24,11 @@ from dvc.fs.local import LocalFileSystem
 from dvc.hash_info import HashInfo
 from dvc.main import main
 from dvc.objects.db import ODBManager
-from dvc.output import OutputAlreadyTrackedError, OutputIsStageFileError
+from dvc.output import (
+    OutputAlreadyTrackedError,
+    OutputDoesNotExistError,
+    OutputIsStageFileError,
+)
 from dvc.stage import Stage
 from dvc.stage.exceptions import (
     StageExternalOutputsError,
@@ -1190,3 +1194,42 @@ def test_add_ignored(tmp_dir, scm, dvc):
     assert str(exc.value) == ("bad DVC file name '{}' is git-ignored.").format(
         os.path.join("dir", "subdir.dvc")
     )
+
+
+def test_add_on_not_existing_file_should_not_remove_stage_file(tmp_dir, dvc):
+    (stage,) = tmp_dir.dvc_gen("foo", "foo")
+    (tmp_dir / "foo").unlink()
+    dvcfile_contents = (tmp_dir / stage.path).read_text()
+
+    with pytest.raises(OutputDoesNotExistError):
+        dvc.add("foo")
+    assert (tmp_dir / "foo.dvc").exists()
+    assert (tmp_dir / stage.path).read_text() == dvcfile_contents
+
+
+@pytest.mark.parametrize(
+    "target",
+    [
+        "dvc.repo.Repo.check_modified_graph",
+        "dvc.stage.Stage.save",
+        "dvc.stage.Stage.commit",
+    ],
+)
+def test_add_does_not_remove_stage_file_on_failure(
+    tmp_dir, dvc, mocker, target
+):
+    (stage,) = tmp_dir.dvc_gen("foo", "foo")
+    tmp_dir.gen("foo", "foobar")  # update file
+    dvcfile_contents = (tmp_dir / stage.path).read_text()
+
+    exc_msg = f"raising error from mocked '{target}'"
+    mocker.patch(
+        target,
+        side_effect=DvcException(exc_msg),
+    )
+
+    with pytest.raises(DvcException) as exc_info:
+        dvc.add("foo")
+    assert str(exc_info.value) == exc_msg
+    assert (tmp_dir / "foo.dvc").exists()
+    assert (tmp_dir / stage.path).read_text() == dvcfile_contents

tests/func/test_data_cloud.py

@@ -248,7 +248,7 @@ def test_warn_on_outdated_stage(tmp_dir, dvc, local_remote, caplog):
 
 def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
     tmp_dir.gen({"foo": "foo"})
-    test_get_file_hash = mocker.spy(dvc_module.objects.stage, "_get_file_hash")
+    test_file_md5 = mocker.spy(dvc_module.objects.stage, "file_md5")
     ret = main(["config", "cache.type", "hardlink"])
     assert ret == 0
     ret = main(["add", "foo"])
@@ -257,7 +257,7 @@ def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
     assert ret == 0
     ret = main(["run", "--single-stage", "-d", "foo", "echo foo"])
     assert ret == 0
-    assert test_get_file_hash.mock.call_count == 1
+    assert test_file_md5.mock.call_count == 1
 
 
 def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
@@ -307,7 +307,7 @@ def test_verify_hashes(
     remove("dir")
     remove(dvc.odb.local.cache_dir)
 
-    hash_spy = mocker.spy(dvc_module.objects.stage, "_get_file_hash")
+    hash_spy = mocker.spy(dvc_module.objects.stage, "file_md5")
 
     dvc.pull()
     assert hash_spy.call_count == 0
@@ -588,3 +588,23 @@ def test_pull_00_prefix(tmp_dir, dvc, remote, monkeypatch):
     stats = dvc.pull()
     assert stats["fetched"] == 2
     assert set(stats["added"]) == {"foo", "bar"}
+
+
+@pytest.mark.parametrize("remote", full_clouds, indirect=True)
+def test_pull_no_00_prefix(tmp_dir, dvc, remote, monkeypatch):
+    # Related: https://github.com/iterative/dvc/issues/6244
+
+    fs_type = type(dvc.cloud.get_remote("upstream").fs)
+    monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
+    monkeypatch.setattr(fs_type, "LIST_OBJECT_PAGE_SIZE", 256, raising=False)
+
+    # foo's md5 checksum is 14ffd92a6cbf5f2f657067df0d5881a6
+    # bar's md5 checksum is 64020400f00960c0ef04052547b134b3
+    tmp_dir.dvc_gen({"foo": "dvc", "bar": "cml"})
+
+    dvc.push()
+    clean(["foo", "bar"], dvc)
+
+    stats = dvc.pull()
+    assert stats["fetched"] == 2
+    assert set(stats["added"]) == {"foo", "bar"}

tests/func/test_import.py

@@ -8,7 +8,7 @@ from mock import patch
 import dvc.data_cloud as cloud
 from dvc.config import NoRemoteError
 from dvc.dvcfile import Dvcfile
-from dvc.exceptions import DownloadError
+from dvc.exceptions import DownloadError, PathMissingError
 from dvc.objects.db import ODBManager
 from dvc.stage.exceptions import StagePathNotFoundError
 from dvc.system import System
@@ -289,7 +289,7 @@ def test_push_wildcard_from_bare_git_repo(
     dvc_repo = make_tmp_dir("dvc-repo", scm=True, dvc=True)
     with dvc_repo.chdir():
         dvc_repo.dvc.imp(os.fspath(tmp_dir), "dirextra")
-        with pytest.raises(FileNotFoundError):
+        with pytest.raises(PathMissingError):
             dvc_repo.dvc.imp(os.fspath(tmp_dir), "dir123")
 
 
@@ -347,11 +347,11 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir):
 
 
 def test_import_non_existing(erepo_dir, tmp_dir, dvc):
-    with pytest.raises(FileNotFoundError):
+    with pytest.raises(PathMissingError):
         tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output")
 
     # https://github.com/iterative/dvc/pull/2837#discussion_r352123053
-    with pytest.raises(FileNotFoundError):
+    with pytest.raises(PathMissingError):
         tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root")
 
 

tests/func/test_import_url.py

@@ -123,6 +123,7 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
     [
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
+        pytest.lazy_fixture("azure"),
         pytest.param(
             pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
         ),

tests/func/test_scm.py

@@ -99,6 +99,35 @@ def test_ignored(tmp_dir, scm):
     assert not scm.is_ignored(tmp_dir / "dir1" / "file2.txt")
 
 
+def test_ignored_dir_unignored_subdirs(tmp_dir, scm):
+    tmp_dir.gen({".gitignore": "data/**\n!data/**/\n!data/**/*.csv"})
+    scm.add([".gitignore"])
+    tmp_dir.gen(
+        {
+            os.path.join("data", "raw", "tracked.csv"): "cont",
+            os.path.join("data", "raw", "not_tracked.json"): "cont",
+        }
+    )
+
+    assert not scm.is_ignored(tmp_dir / "data" / "raw" / "tracked.csv")
+    assert scm.is_ignored(tmp_dir / "data" / "raw" / "not_tracked.json")
+    assert not scm.is_ignored(tmp_dir / "data" / "raw" / "non_existent.csv")
+    assert scm.is_ignored(tmp_dir / "data" / "raw" / "non_existent.json")
+    assert not scm.is_ignored(tmp_dir / "data" / "non_existent.csv")
+    assert scm.is_ignored(tmp_dir / "data" / "non_existent.json")
+
+    assert not scm.is_ignored(f"data{os.sep}")
+    # git check-ignore would now mark "data/raw" as ignored
+    # after detecting it's a directory in the file system;
+    # instead, we rely on the trailing separator to determine if handling a
+    # a directory - for consistency between existent and non-existent paths
+    assert scm.is_ignored(os.path.join("data", "raw"))
+    assert not scm.is_ignored(os.path.join("data", f"raw{os.sep}"))
+
+    assert scm.is_ignored(os.path.join("data", "non_existent"))
+    assert not scm.is_ignored(os.path.join("data", f"non_existent{os.sep}"))
+
+
 def test_get_gitignore(tmp_dir, scm):
     tmp_dir.gen({"file1": "contents", "dir": {}})
 

tests/func/test_stage.py

@@ -227,6 +227,13 @@ def test_parent_repo_collect_stages(tmp_dir, scm, dvc):
     assert deep_subrepo_stages != []
 
 
+def test_collect_repo_ignored_dir_unignored_pattern(tmp_dir, dvc, scm):
+    tmp_dir.gen({".gitignore": "data/**\n!data/**/\n!data/**/*.dvc"})
+    scm.add([".gitignore"])
+    (stage,) = tmp_dir.dvc_gen({"data/raw/tracked.csv": "5,6,7,8"})
+    assert dvc.stage.collect_repo() == [stage]
+
+
 def test_stage_strings_representation(tmp_dir, dvc, run_copy):
     tmp_dir.dvc_gen("foo", "foo")
     stage1 = run_copy("foo", "bar", single_stage=True)

tests/unit/scm/test_git.py

@@ -553,3 +553,30 @@ def test_add(tmp_dir, scm, git):
     assert set(staged["modify"]) == {"dir/baz"}
     assert len(unstaged) == 1
     assert len(untracked) == 1
+
+
+@pytest.mark.parametrize("use_sha", [True, False])
+def test_pygit_resolve_refish(tmp_dir, scm, git, use_sha):
+    import pygit2
+
+    if git.test_backend != "pygit2":
+        pytest.skip()
+
+    tmp_dir.scm_gen("foo", "foo", commit="foo")
+    head = scm.get_rev()
+    tag = "my_tag"
+    scm.gitpython.git.tag("-a", tag, "-m", "create annotated tag")
+
+    if use_sha:
+        # refish will be annotated tag SHA (not commit SHA)
+        ref = git.pygit2.repo.references.get(f"refs/tags/{tag}")
+        refish = str(ref.target)
+    else:
+        refish = tag
+
+    assert refish != head
+    commit, ref = git.pygit2._resolve_refish(refish)
+    assert isinstance(commit, pygit2.Commit)
+    assert str(commit.id) == head
+    if not use_sha:
+        assert ref.name == f"refs/tags/{tag}"

tests/unit/utils/test_collections.py

@@ -1,5 +1,7 @@
 # pylint: disable=unidiomatic-typecheck
-from dvc.utils.collections import apply_diff, chunk_dict
+from mock import create_autospec
+
+from dvc.utils.collections import apply_diff, chunk_dict, validate
 
 
 class MyDict(dict):
@@ -58,3 +60,78 @@ def test_chunk_dict():
     assert chunk_dict(d, 2) == [{"a": 1, "b": 2}, {"c": 3}]
     assert chunk_dict(d, 3) == [d]
     assert chunk_dict(d, 4) == [d]
+
+
+# pylint: disable=unused-argument
+
+
+def _test_func(x, y, *args, j=3, k=5, **kwargs):
+    pass
+
+
+def test_pre_validate_decorator_required_args(mocker):
+    mock = mocker.MagicMock()
+
+    func_mock = create_autospec(_test_func)
+    func = validate(mock)(func_mock)
+
+    func("x", "y")
+
+    func_mock.assert_called_once_with("x", "y", j=3, k=5)
+    mock.assert_called_once()
+
+    (args,) = mock.call_args[0]
+    assert args.x == "x"
+    assert args.y == "y"
+    assert args.args == ()
+    assert args.j == 3
+    assert args.k == 5
+    assert args.kwargs == {}
+
+
+def test_pre_validate_decorator_kwargs_args(mocker):
+    mock = mocker.MagicMock()
+    func_mock = create_autospec(_test_func)
+    func = validate(mock)(func_mock)
+
+    func("x", "y", "part", "of", "args", j=1, k=10, m=100, n=1000)
+
+    func_mock.assert_called_once_with(
+        "x", "y", "part", "of", "args", j=1, k=10, m=100, n=1000
+    )
+    mock.assert_called_once()
+    (args,) = mock.call_args[0]
+    assert args.x == "x"
+    assert args.y == "y"
+    assert args.args == ("part", "of", "args")
+    assert args.j == 1
+    assert args.k == 10
+    assert args.kwargs == {"m": 100, "n": 1000}
+
+
+def test_pre_validate_update_args():
+    def test_validator(args):
+        args.w += 50
+        del args.x
+        args.y = 100
+
+    def test_func(w=1, x=5, y=10, z=15):
+        pass
+
+    mock = create_autospec(test_func)
+    func = validate(test_validator)(mock)
+
+    func(100, 100)
+    mock.assert_called_once_with(w=150, y=100, z=15)
+
+
+def test_post_validate_decorator(mocker):
+    def none_filter(result):
+        return list(filter(None, result))
+
+    test_func = mocker.MagicMock(return_value=[1, None, 2])
+    func = validate(none_filter, post=True)(test_func)
+
+    result = func()
+    test_func.assert_called_once()
+    assert result == [1, 2]

tests/unit/utils/test_fs.py

@@ -216,6 +216,28 @@ def test_path_isin_with_absolute_path():
     assert path_isin(child, parent)
 
 
+def test_path_isin_case_sensitive():
+    child = os.path.join("path", "to", "folder")
+    parent = os.path.join("PATH", "TO")
+
+    assert path_isin(child, parent) == (os.name == "nt")
+
+
+@pytest.mark.skipif(os.name != "nt", reason="Windows specific")
+def test_contains_symlink_case_sensitive_win():
+    child = os.path.join("path", "to", "folder")
+    parent = os.path.join("PATH", "TO")
+    assert contains_symlink_up_to(child, parent) is False
+
+
+@pytest.mark.skipif(os.name == "nt", reason="Posix specific")
+def test_contains_symlink_case_sensitive_posix():
+    child = os.path.join("path", "to", "folder")
+    parent = os.path.join("PATH", "TO")
+    with pytest.raises(BasePathNotInCheckedPathException):
+        contains_symlink_up_to(child, parent)
+
+
 def test_makedirs(tmp_dir):
     path = os.path.join(tmp_dir, "directory")
     path_info = PathInfo(os.path.join(tmp_dir, "another", "directory"))