dvc: bump to 2.5.3

Kudos @skshetry for discovering that we've been publishing py2.py3
wheels on pypi and researching the cause.

Co-authored-by: Saugat Pachhai <[email protected]>

Co-authored-by: Saugat Pachhai <[email protected]>

.github/CODEOWNERS

@@ -0,0 +1 @@
+*       @iterative/dvc

.pre-commit-config.yaml

@@ -7,6 +7,13 @@ repos:
         language_version: python3
     repo: https://github.com/ambv/black
     rev: 21.6b0
+  - hooks:
+      - id: codespell
+        args: 
+          - --ignore-words-list
+          - ba,datas,fo,uptodate
+    repo: https://github.com/codespell-project/codespell
+    rev: v2.1.0
   - hooks:
       - id: isort
         language_version: python3
@@ -32,7 +39,7 @@ repos:
            - types-tabulate
            - types-toml
     repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.902'
+    rev: 'v0.910'
   - repo: local
     hooks:
     - id: pylint

README.rst

@@ -184,7 +184,7 @@ Comparison to related technologies
 
 #. *Data Engineering* tools such as `AirFlow <https://airflow.apache.org/>`_, `Luigi <https://github.com/spotify/luigi>`_ and
    others - in DVC data, model and ML pipelines represent a single ML project and focuses on data scientist' experience while
-   data engineering pipeline orchestrates multiple data projects and focuses on efficent execution. DVC project can be used
+   data engineering pipeline orchestrates multiple data projects and focuses on efficient execution. DVC project can be used
    from data pipeline as a single execution step. `AirFlow DVC <https://github.com/covid-genomics/airflow-dvc>`_ is an example
    of such integration.
 

dvc/command/diff.py

@@ -111,14 +111,12 @@ class CmdDiff(CmdBase):
         if not sum(summary.values()):
             return None
 
-        fmt = (
-            "files summary: {added} added, {deleted} deleted,"
-            " {renamed} renamed, {modified} modified"
+        states_summary = ", ".join(
+            f"{summary[state]} {state}"
+            for state in states
+            if summary[state] > 0
         )
-        if not hide_missing:
-            fmt += ", {not in cache} not in cache"
-
-        ui.write(fmt.format_map(summary))
+        ui.write("files summary:", states_summary)
 
     def run(self):
         from dvc.exceptions import DvcException

dvc/command/experiments.py

@@ -397,7 +397,7 @@ def show_experiments(
     styles.update(
         {
             header: {
-                "justify": "left" if typ == "metrics" else "params",
+                "justify": "right" if typ == "metrics" else "left",
                 "header_style": f"black on {header_bg_colors[typ]}",
                 "collapse": idx != 0,
                 "no_wrap": typ == "metrics",
@@ -937,9 +937,7 @@ def add_parser(subparsers, parent_parser):
     )
     experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)
 
-    EXPERIMENTS_RUN_HELP = (
-        "Reproduce complete or partial experiment pipelines."
-    )
+    EXPERIMENTS_RUN_HELP = "Run or resume an experiment."
     experiments_run_parser = experiments_subparsers.add_parser(
         "run",
         parents=[parent_parser],

dvc/dagascii.py

@@ -163,7 +163,7 @@ class AsciiCanvas:
         self.point(x0 + width, y0 + height, "+")
 
 
-def _build_sugiyama_layout(vertexes, edges):
+def _build_sugiyama_layout(vertices, edges):
     #
     # Just a reminder about naming conventions:
     # +------------X
@@ -174,17 +174,17 @@ def _build_sugiyama_layout(vertexes, edges):
     # Y
     #
 
-    vertexes = {v: Vertex(f" {v} ") for v in vertexes}
+    vertices = {v: Vertex(f" {v} ") for v in vertices}
     # NOTE: reverting edges to correctly orientate the graph
-    edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
-    vertexes = vertexes.values()
-    graph = Graph(vertexes, edges)
+    edges = [Edge(vertices[e], vertices[s]) for s, e in edges]
+    vertices = vertices.values()
+    graph = Graph(vertices, edges)
 
-    for vertex in vertexes:
+    for vertex in vertices:
         vertex.view = VertexViewer(vertex.data)
 
     # NOTE: determine min box length to create the best layout
-    minw = min(v.view.w for v in vertexes)
+    minw = min(v.view.w for v in vertices)
 
     for edge in edges:
         edge.view = EdgeViewer()
@@ -204,11 +204,11 @@ def _build_sugiyama_layout(vertexes, edges):
     return sug
 
 
-def draw(vertexes, edges):
+def draw(vertices, edges):
     """Build a DAG and draw it in ASCII.
 
     Args:
-        vertexes (list): list of graph vertexes.
+        vertices (list): list of graph vertices.
         edges (list): list of graph edges.
     """
     # pylint: disable=too-many-locals
@@ -217,7 +217,7 @@ def draw(vertexes, edges):
     Xs = []  # pylint: disable=invalid-name
     Ys = []  # pylint: disable=invalid-name
 
-    sug = _build_sugiyama_layout(vertexes, edges)
+    sug = _build_sugiyama_layout(vertices, edges)
 
     for vertex in sug.g.sV:
         # NOTE: moving boxes w/2 to the left

dvc/dependency/repo.py

@@ -97,7 +97,7 @@ class RepoDependency(Dependency):
         self, **kwargs
     ) -> Dict[Optional["ObjectDB"], Set["HashFile"]]:
         from dvc.config import NoRemoteError
-        from dvc.exceptions import NoOutputOrStageError
+        from dvc.exceptions import NoOutputOrStageError, PathMissingError
         from dvc.objects.db.git import GitObjectDB
         from dvc.objects.stage import stage
 
@@ -126,12 +126,18 @@ class RepoDependency(Dependency):
             except (NoRemoteError, NoOutputOrStageError):
                 pass
 
-            staged_obj = stage(
-                local_odb,
-                path_info,
-                repo.repo_fs,
-                local_odb.fs.PARAM_CHECKSUM,
-            )
+            try:
+                staged_obj = stage(
+                    local_odb,
+                    path_info,
+                    repo.repo_fs,
+                    local_odb.fs.PARAM_CHECKSUM,
+                )
+            except FileNotFoundError as exc:
+                raise PathMissingError(
+                    self.def_path, self.def_repo[self.PARAM_URL]
+                ) from exc
+
             self._staged_objs[rev] = staged_obj
             git_odb = GitObjectDB(repo.repo_fs, repo.root_dir)
             used_objs[git_odb].add(staged_obj)

dvc/fs/azure.py

@@ -25,7 +25,7 @@ _DEFAULT_CREDS_STEPS = (
 
 @contextmanager
 def _temp_event_loop():
-    """When trying to initalize azure filesystem instances
+    """When trying to initialize azure filesystem instances
     with DefaultCredentials, the authentication process requires
     to have an access to a separate event loop. The normal calls
     are run in a separate loop managed by the fsspec, but the

dvc/fs/base.py

@@ -33,6 +33,8 @@ class RemoteMissingDepsError(DvcException):
 
 
 class BaseFileSystem:
+    sep = "/"
+
     scheme = "base"
     REQUIRES: ClassVar[Dict[str, str]] = {}
     PATH_CLS = URLInfo  # type: Any

dvc/fs/dvc.py

@@ -23,6 +23,8 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         repo: DVC repo.
     """
 
+    sep = os.sep
+
     scheme = "local"
     PARAM_CHECKSUM = "md5"
 
@@ -231,7 +233,7 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
     def info(self, path_info):
         meta = self.metadata(path_info)
-        ret = {"type": "dir" if meta.isdir else "file"}
+        ret = {"type": "directory" if meta.isdir else "file"}
         if meta.is_output and len(meta.outs) == 1 and meta.outs[0].hash_info:
             hash_info = meta.outs[0].hash_info
             ret["size"] = hash_info.size

dvc/fs/fsspec_wrapper.py

@@ -48,7 +48,7 @@ class FSSpecWrapper(BaseFileSystem):
             yield entry
 
     def _entry_hook(self, entry):
-        """Simple hook method to be overriden when wanted to process
+        """Simple hook method to be overridden when wanted to process
         entries within info() and ls(detail=True) calls"""
         return entry
 
@@ -173,7 +173,7 @@ class ObjectFSWrapper(FSSpecWrapper):
         )
 
     def find(self, path_info, detail=False, prefix=None):
-        if prefix is not None:
+        if prefix:
             path = self._with_bucket(path_info.parent)
             files = self.fs.find(
                 path, detail=detail, prefix=path_info.parts[-1]

dvc/fs/gdrive.py

@@ -1,3 +1,4 @@
+import errno
 import io
 import logging
 import os
@@ -11,7 +12,7 @@ from urllib.parse import urlparse
 from funcy import cached_property, retry, wrap_prop, wrap_with
 from funcy.py3 import cat
 
-from dvc.exceptions import DvcException, FileMissingError
+from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
@@ -521,12 +522,14 @@ class GDriveFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             return min(item_ids)
 
         assert not create
-        raise FileMissingError(path_info, hint)
+        raise FileNotFoundError(
+            errno.ENOENT, os.strerror(errno.ENOENT), hint or path_info
+        )
 
     def exists(self, path_info) -> bool:
         try:
             self._get_item_id(path_info)
-        except FileMissingError:
+        except FileNotFoundError:
             return False
         else:
             return True
@@ -616,7 +619,7 @@ class GDriveFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         item_id = self._get_item_id(path_info)
         gdrive_file = self._drive.CreateFile({"id": item_id})
         gdrive_file.FetchMetadata(fields="fileSize")
-        return {"size": gdrive_file.get("fileSize")}
+        return {"size": gdrive_file.get("fileSize"), "type": "file"}
 
     def _upload_fobj(self, fobj, to_info, **kwargs):
         dirname = to_info.parent

dvc/fs/git.py

@@ -1,5 +1,6 @@
 import errno
 import os
+import stat
 
 from dvc.utils import is_exec, relpath
 
@@ -9,6 +10,8 @@ from .base import BaseFileSystem
 class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     """Proxies the repo file access methods to Git objects"""
 
+    sep = os.sep
+
     scheme = "local"
 
     def __init__(self, root_dir, trie):
@@ -104,7 +107,10 @@ class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             raise FileNotFoundError(
                 errno.ENOENT, os.strerror(errno.ENOENT), path_info
             )
-        return {"size": st.st_size}
+        return {
+            "size": st.st_size,
+            "type": "directory" if stat.S_ISDIR(st.st_mode) else "file",
+        }
 
     def stat(self, path):
         key = self._get_key(path)

dvc/fs/gs.py

@@ -16,6 +16,7 @@ class GSFileSystem(ObjectFSWrapper):
     REQUIRES = {"gcsfs": "gcsfs"}
     PARAM_CHECKSUM = "etag"
     DETAIL_FIELDS = frozenset(("etag", "size"))
+    TRAVERSE_PREFIX_LEN = 2
 
     def _prepare_credentials(self, **config):
         login_info = {"consistency": None}

dvc/fs/hdfs.py

@@ -1,3 +1,4 @@
+import errno
 import io
 import logging
 import os
@@ -220,8 +221,19 @@ class HDFSFileSystem(BaseFileSystem):
 
     def info(self, path_info):
         with self.hdfs(path_info) as hdfs:
-            finfo = hdfs.get_file_info(path_info.path)
-            return {"size": finfo.size}
+            from pyarrow.fs import FileType
+
+            file_info = hdfs.get_file_info(path_info.path)
+            if file_info.type is FileType.Directory:
+                kind = "directory"
+            elif file_info.type is FileType.File:
+                kind = "file"
+            else:
+                raise FileNotFoundError(
+                    errno.ENOENT, os.strerror(errno.ENOENT), path_info
+                )
+
+            return {"size": file_info.size, "type": kind}
 
     def checksum(self, path_info):
         return HashInfo(

dvc/fs/http.py

@@ -31,6 +31,7 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     PATH_CLS = HTTPURLInfo
     PARAM_CHECKSUM = "etag"
     CAN_TRAVERSE = False
+    REQUIRES = {"requests": "requests"}
 
     SESSION_RETRIES = 5
     SESSION_BACKOFF_FACTOR = 0.1
@@ -145,7 +146,7 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         resp = self._head(path_info.url)
         etag = resp.headers.get("ETag") or resp.headers.get("Content-MD5")
         size = self._content_length(resp)
-        return {"etag": etag, "size": size}
+        return {"etag": etag, "size": size, "type": "file"}
 
     def _upload_fobj(self, fobj, to_info, **kwargs):
         def chunks(fobj):

dvc/fs/local.py

@@ -14,6 +14,8 @@ logger = logging.getLogger(__name__)
 
 
 class LocalFileSystem(BaseFileSystem):
+    sep = os.sep
+
     scheme = Schemes.LOCAL
     PATH_CLS = PathInfo
     PARAM_CHECKSUM = "md5"
@@ -155,7 +157,7 @@ class LocalFileSystem(BaseFileSystem):
         st = os.stat(path_info)
         return {
             "size": st.st_size,
-            "type": "dir" if stat.S_ISDIR(st.st_mode) else "file",
+            "type": "directory" if stat.S_ISDIR(st.st_mode) else "file",
         }
 
     def _upload(

dvc/fs/repo.py

@@ -31,6 +31,8 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         kwargs: Additional keyword arguments passed to the `DvcFileSystem()`.
     """
 
+    sep = os.sep
+
     scheme = "local"
     PARAM_CHECKSUM = "md5"
 

dvc/fs/s3.py

@@ -188,6 +188,7 @@ class S3FileSystem(BaseS3FileSystem):  # pylint:disable=abstract-method
             "s3",
             endpoint_url=client_kwargs.get("endpoint_url"),
             use_ssl=login_info["use_ssl"],
+            verify=client_kwargs.get("verify"),
         )
 
     def _get_obj(self, path_info):

dvc/fs/ssh/__init__.py

@@ -119,16 +119,16 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         if self.ask_password and self.password is None:
             self.password = ask_password(self.host, self.user, self.port)
 
-    def ssh(self, path_info):
+    def ssh(self):
         self.ensure_credentials()
 
         from .connection import SSHConnection
 
         return get_connection(
             SSHConnection,
-            path_info.host,
-            username=path_info.user,
-            port=path_info.port,
+            self.host,
+            username=self.user,
+            port=self.port,
             key_filename=self.keyfile,
             timeout=self.timeout,
             password=self.password,
@@ -141,7 +141,7 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     def open(self, path_info, mode="r", encoding=None, **kwargs):
         assert mode in {"r", "rt", "rb", "wb"}
 
-        with self.ssh(path_info) as ssh, closing(
+        with self.ssh() as ssh, closing(
             ssh.sftp.open(path_info.path, mode)
         ) as fd:
             if "b" in mode:
@@ -150,19 +150,19 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 yield io.TextIOWrapper(fd, encoding=encoding)
 
     def exists(self, path_info) -> bool:
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return ssh.exists(path_info.path)
 
     def isdir(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return ssh.isdir(path_info.path)
 
     def isfile(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return ssh.isfile(path_info.path)
 
     def walk_files(self, path_info, **kwargs):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             for fname in ssh.walk_files(path_info.path):
                 yield path_info.replace(path=fname)
 
@@ -170,32 +170,32 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         if path_info.scheme != self.scheme:
             raise NotImplementedError
 
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             ssh.remove(path_info.path)
 
     def makedirs(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             ssh.makedirs(path_info.path)
 
     def move(self, from_info, to_info):
         if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
             raise NotImplementedError
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.move(from_info.path, to_info.path)
 
     def copy(self, from_info, to_info):
         if not from_info.scheme == to_info.scheme == self.scheme:
             raise NotImplementedError
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.atomic_copy(from_info.path, to_info.path)
 
     def symlink(self, from_info, to_info):
         if not from_info.scheme == to_info.scheme == self.scheme:
             raise NotImplementedError
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.symlink(from_info.path, to_info.path)
 
     def hardlink(self, from_info, to_info):
@@ -205,7 +205,7 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         # See dvc/remote/local/__init__.py - hardlink()
         if self.getsize(from_info) == 0:
 
-            with self.ssh(to_info) as ssh:
+            with self.ssh() as ssh:
                 ssh.sftp.open(to_info.path, "w").close()
 
             logger.debug(
@@ -215,18 +215,18 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
             return
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.hardlink(from_info.path, to_info.path)
 
     def reflink(self, from_info, to_info):
         if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
             raise NotImplementedError
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.reflink(from_info.path, to_info.path)
 
     def md5(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return HashInfo(
                 "md5",
                 ssh.md5(path_info.path),
@@ -234,7 +234,7 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
 
     def info(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return ssh.info(path_info.path)
 
     def _upload_fobj(self, fobj, to_info, **kwargs):
@@ -243,7 +243,7 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             shutil.copyfileobj(fobj, fdest)
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.download(
                 from_info.path,
                 to_file,
@@ -254,7 +254,7 @@ class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        with self.ssh(to_info) as ssh:
+        with self.ssh() as ssh:
             ssh.upload(
                 from_file,
                 to_info.path,

dvc/fs/ssh/connection.py

@@ -82,7 +82,7 @@ class SSHConnection:
         st = self.sftp.stat(path)
         return {
             "size": st.st_size,
-            "type": "dir" if stat.S_ISDIR(st.st_mode) else "file",
+            "type": "directory" if stat.S_ISDIR(st.st_mode) else "file",
         }
 
     def getsize(self, path):

dvc/fs/webhdfs.py

@@ -123,7 +123,7 @@ class WebHDFSFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
     def info(self, path_info):
         st = self.hdfs_client.status(path_info.path)
-        return {"size": st["length"]}
+        return {"size": st["length"], "type": "file"}
 
     def checksum(self, path_info):
         return HashInfo(

dvc/info.py

@@ -20,6 +20,11 @@ if PKG is None:
 else:
     package = f"({PKG})"
 
+try:
+    import importlib.metadata as importlib_metadata
+except ImportError:  # < 3.8
+    import importlib_metadata  # type: ignore[no-redef]
+
 
 def get_dvc_info():
     info = [
@@ -27,7 +32,7 @@ def get_dvc_info():
         "---------------------------------",
         f"Platform: Python {platform.python_version()} on "
         f"{platform.platform()}",
-        f"Supports: {_get_supported_remotes()}",
+        f"Supports:{_get_supported_remotes()}",
     ]
 
     try:
@@ -113,19 +118,23 @@ def _get_linktype_support_info(repo):
 
 
 def _get_supported_remotes():
-
     supported_remotes = []
     for scheme, fs_cls in FS_MAP.items():
         if not fs_cls.get_missing_deps():
-            supported_remotes.append(scheme)
+            dependencies = []
+            for requirement in fs_cls.REQUIRES:
+                dependencies.append(
+                    f"{requirement} = "
+                    f"{importlib_metadata.version(requirement)}"
+                )
 
-    if len(supported_remotes) == len(FS_MAP):
-        return "All remotes"
+            remote_info = scheme
+            if dependencies:
+                remote_info += " (" + ", ".join(dependencies) + ")"
+            supported_remotes.append(remote_info)
 
-    if len(supported_remotes) == 1:
-        return supported_remotes
-
-    return ", ".join(supported_remotes)
+    assert len(supported_remotes) >= 1
+    return "\n\t" + ",\n\t".join(supported_remotes)
 
 
 def get_fs_type(path):

dvc/objects/db/ssh.py

@@ -28,7 +28,7 @@ class SSHObjectDB(ObjectDB):
                 callback(path)
             return ret
 
-        with self.fs.ssh(path_infos[0]) as ssh:
+        with self.fs.ssh() as ssh:
             channels = ssh.open_max_sftp_channels()
             max_workers = len(channels)
 
@@ -78,7 +78,7 @@ class SSHObjectDB(ObjectDB):
             root = posixpath.join(self.path_info.path, prefix[:2])
         else:
             root = self.path_info.path
-        with self.fs.ssh(self.path_info) as ssh:
+        with self.fs.ssh() as ssh:
             if prefix and not ssh.exists(root):
                 return
             # If we simply return an iterator then with above closes instantly

dvc/objects/file.py

@@ -28,9 +28,7 @@ class HashFile:
 
     @property
     def size(self):
-        if not (self.path_info and self.fs):
-            return None
-        return self.fs.getsize(self.path_info)
+        return self.hash_info.size
 
     def __len__(self):
         return 1

dvc/objects/stage.py

@@ -1,5 +1,3 @@
-import errno
-import os
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 
@@ -52,11 +50,6 @@ def get_file_hash(path_info, fs, name, state=None):
         if hash_info:
             return hash_info
 
-    if not fs.exists(path_info):
-        raise FileNotFoundError(
-            errno.ENOENT, os.strerror(errno.ENOENT), path_info
-        )
-
     hash_info = _get_file_hash(path_info, fs, name)
 
     if state:
@@ -179,11 +172,7 @@ def stage(odb, path_info, fs, name, upload=False, **kwargs):
         isinstance(path_info, str) or path_info.scheme == fs.scheme
     )
 
-    if not fs.exists(path_info):
-        raise FileNotFoundError(
-            errno.ENOENT, os.strerror(errno.ENOENT), path_info
-        )
-
+    details = fs.info(path_info)
     state = odb.state
     # pylint: disable=assignment-from-none
     hash_info = state.get(path_info, fs)
@@ -215,12 +204,12 @@ def stage(odb, path_info, fs, name, upload=False, **kwargs):
         obj.hash_info.size = hash_info.size
         return obj
 
-    if fs.isdir(path_info):
+    if details["type"] == "directory":
         obj = _get_tree_obj(path_info, fs, name, odb, state, upload, **kwargs)
     else:
         _, obj = _get_file_obj(path_info, fs, name, odb, state, upload)
 
-    if obj.hash_info and fs.exists(path_info):
+    if obj.hash_info:
         state.save(path_info, fs, obj.hash_info)
 
     return obj

dvc/objects/tree.py

@@ -28,13 +28,6 @@ class Tree(HashFile):
 
         return Trie(self._dict)
 
-    @property
-    def size(self):
-        try:
-            return sum(obj.size for _, obj in self)
-        except TypeError:
-            return None
-
     def add(self, key, obj):
         self.__dict__.pop("trie", None)
         self._dict[key] = obj
@@ -52,7 +45,10 @@ class Tree(HashFile):
         self.path_info = path_info
         self.hash_info = get_file_hash(path_info, memfs, "md5")
         self.hash_info.value += ".dir"
-        self.hash_info.size = self.size
+        try:
+            self.hash_info.size = sum(obj.size for _, obj in self)
+        except TypeError:
+            self.hash_info.size = None
         self.hash_info.nfiles = len(self)
 
     def __len__(self):

dvc/output.py

@@ -45,18 +45,24 @@ CHECKSUM_SCHEMA = Any(
     And(Any(str, And(int, Coerce(str))), Length(min=3), Lower),
 )
 
+CASE_SENSITIVE_CHECKSUM_SCHEMA = Any(
+    None,
+    And(str, Length(max=0), SetTo(None)),
+    And(Any(str, And(int, Coerce(str))), Length(min=3)),
+)
+
 # NOTE: currently there are only 3 possible checksum names:
 #
 #    1) md5 (LOCAL, SSH);
-#    2) etag (S3);
+#    2) etag (S3, GS, OSS, AZURE, HTTP);
 #    3) checksum (HDFS);
 #
 # so when a few types of outputs share the same name, we only need
 # specify it once.
 CHECKSUMS_SCHEMA = {
     LocalFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    S3FileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
     HDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    S3FileSystem.PARAM_CHECKSUM: CASE_SENSITIVE_CHECKSUM_SCHEMA,
 }
 
 
@@ -265,8 +271,6 @@ class Output:
     IsStageFileError = OutputIsStageFileError  # type: Type[DvcException]
     IsIgnoredError = OutputIsIgnoredError  # type: Type[DvcException]
 
-    sep = "/"
-
     def __init__(
         self,
         stage,
@@ -881,7 +885,7 @@ class Output:
         obj.name = str(self)
         if isinstance(obj, Tree):
             for key, entry_obj in obj:
-                entry_obj.name = os.path.join(str(self), *key)
+                entry_obj.name = self.fs.sep.join([obj.name, *key])
 
     def get_used_external(
         self, **kwargs

dvc/progress.py

@@ -51,7 +51,7 @@ class Tqdm(tqdm):
         file=None,
         total=None,
         postfix=None,
-        **kwargs
+        **kwargs,
     ):
         """
         bytes   : shortcut for
@@ -90,7 +90,7 @@ class Tqdm(tqdm):
             bar_format="!",
             lock_args=(False,),
             total=total,
-            **kwargs
+            **kwargs,
         )
         self.postfix = postfix or {"info": ""}
         if bar_format is None:
@@ -106,13 +106,16 @@ class Tqdm(tqdm):
             self.bar_format = bar_format
         self.refresh()
 
-    def update_msg(self, msg, n=1):
+    def update_msg(self, msg: str, n: int = 1) -> None:
         """
         Sets `msg` as a postfix and calls `update(n)`.
         """
-        self.postfix["info"] = " %s |" % msg
+        self.set_msg(msg)
         self.update(n)
 
+    def set_msg(self, msg: str) -> None:
+        self.postfix["info"] = f" {msg} |"
+
     def update_to(self, current, total=None):
         if total:
             self.total = total

dvc/repo/add.py

@@ -1,9 +1,13 @@
 import logging
 import os
-from typing import TYPE_CHECKING
+from contextlib import contextmanager
+from itertools import tee
+from typing import TYPE_CHECKING, Any, Iterator, List
 
 import colorama
 
+from dvc.ui import ui
+
 from ..exceptions import (
     CacheLinkError,
     InvalidArgumentError,
@@ -11,46 +15,51 @@ from ..exceptions import (
     OverlappingOutputPathsError,
     RecursiveAddingWhileUsingFilename,
 )
-from ..progress import Tqdm
 from ..repo.scm_context import scm_context
 from ..utils import LARGE_DIR_SIZE, glob_targets, resolve_output, resolve_paths
+from ..utils.collections import ensure_list, validate
 from . import locked
 
 if TYPE_CHECKING:
+    from dvc.repo import Repo
+    from dvc.stage import Stage
     from dvc.types import TargetType
 
-
+Stages = List["Stage"]
 logger = logging.getLogger(__name__)
 
 
-@locked
-@scm_context
-def add(  # noqa: C901
-    repo,
-    targets: "TargetType",
-    recursive=False,
-    no_commit=False,
-    fname=None,
-    to_remote=False,
-    **kwargs,
-):
-    from dvc.utils.collections import ensure_list
+OVERLAPPING_OUTPUT_FMT = (
+    "Cannot add '{out}', because it is overlapping with other "
+    "DVC tracked output: '{parent}'.\n"
+    "To include '{out}' in '{parent}', run "
+    "'dvc commit {parent_stage}'"
+)
 
-    if recursive and fname:
+
+def check_recursive_and_fname(args):
+    if args.recursive and args.fname:
         raise RecursiveAddingWhileUsingFilename()
 
-    targets = ensure_list(targets)
 
-    to_cache = kwargs.get("out") and not to_remote
+def transform_targets(args):
+    args.targets = ensure_list(args.targets)
+
+
+def check_arg_combinations(args):
+    kwargs = args.kwargs
     invalid_opt = None
+    to_remote = args.to_remote
+    to_cache = kwargs.get("out") and not to_remote
+
     if to_remote or to_cache:
         message = "{option} can't be used with "
         message += "--to-remote" if to_remote else "-o"
-        if len(targets) != 1:
+        if len(args.targets) != 1:
             invalid_opt = "multiple targets"
-        elif no_commit:
+        elif args.no_commit:
             invalid_opt = "--no-commit option"
-        elif recursive:
+        elif args.recursive:
             invalid_opt = "--recursive option"
         elif kwargs.get("external"):
             invalid_opt = "--external option"
@@ -64,214 +73,192 @@ def add(  # noqa: C901
     if invalid_opt is not None:
         raise InvalidArgumentError(message.format(option=invalid_opt))
 
-    link_failures = []
-    stages_list = []
-    num_targets = len(targets)
-    with Tqdm(total=num_targets, desc="Add", unit="file", leave=True) as pbar:
-        if num_targets == 1:
-            # clear unneeded top-level progress bar for single target
-            pbar.bar_format = "Adding..."
-            pbar.refresh()
-        for target in targets:
-            sub_targets = _find_all_targets(repo, target, recursive)
-            pbar.total += len(sub_targets) - 1
 
-            if os.path.isdir(target) and len(sub_targets) > LARGE_DIR_SIZE:
-                logger.warning(
-                    "You are adding a large directory '{target}' recursively,"
-                    " consider tracking it as a whole instead.\n"
-                    "{purple}HINT:{nc} Remove the generated DVC file and then"
-                    " run `{cyan}dvc add {target}{nc}`".format(
-                        purple=colorama.Fore.MAGENTA,
-                        cyan=colorama.Fore.CYAN,
-                        nc=colorama.Style.RESET_ALL,
-                        target=target,
-                    )
-                )
-
-            stages = _create_stages(
-                repo,
-                sub_targets,
-                fname,
-                pbar=pbar,
-                transfer=to_remote or to_cache,
-                **kwargs,
-            )
-
-            try:
-                repo.check_modified_graph(stages)
-            except OverlappingOutputPathsError as exc:
-                msg = (
-                    "Cannot add '{out}', because it is overlapping with other "
-                    "DVC tracked output: '{parent}'.\n"
-                    "To include '{out}' in '{parent}', run "
-                    "'dvc commit {parent_stage}'"
-                ).format(
-                    out=exc.overlapping_out.path_info,
-                    parent=exc.parent.path_info,
-                    parent_stage=exc.parent.stage.addressing,
-                )
-                raise OverlappingOutputPathsError(
-                    exc.parent, exc.overlapping_out, msg
-                )
-            except OutputDuplicationError as exc:
-                raise OutputDuplicationError(
-                    exc.output, list(set(exc.stages) - set(stages))
-                )
-
-            link_failures.extend(
-                _process_stages(
-                    repo,
-                    sub_targets,
-                    stages,
-                    no_commit,
-                    pbar,
-                    to_remote,
-                    to_cache,
-                    **kwargs,
-                )
-            )
-            stages_list += stages
-
-        if num_targets == 1:  # restore bar format for stats
-            pbar.bar_format = pbar.BAR_FMT_DEFAULT
-
-    if link_failures:
-        msg = (
-            "Some targets could not be linked from cache to workspace.\n{}\n"
-            "To re-link these targets, reconfigure cache types and then run:\n"
-            "\n\tdvc checkout {}"
-        ).format(
-            CacheLinkError.SUPPORT_LINK,
-            " ".join([str(stage.relpath) for stage in link_failures]),
+@contextmanager
+def translate_graph_error(stages: Stages) -> Iterator[None]:
+    try:
+        yield
+    except OverlappingOutputPathsError as exc:
+        msg = OVERLAPPING_OUTPUT_FMT.format(
+            out=exc.overlapping_out.path_info,
+            parent=exc.parent.path_info,
+            parent_stage=exc.parent.stage.addressing,
+        )
+        raise OverlappingOutputPathsError(exc.parent, exc.overlapping_out, msg)
+    except OutputDuplicationError as exc:
+        raise OutputDuplicationError(
+            exc.output, list(set(exc.stages) - set(stages))
         )
-        logger.warning(msg)
-
-    return stages_list
 
 
-def _process_stages(
-    repo, sub_targets, stages, no_commit, pbar, to_remote, to_cache, **kwargs
+def progress_iter(stages: Stages) -> Iterator["Stage"]:
+    total = len(stages)
+    desc = "Adding..."
+    with ui.progress(
+        stages, total=total, desc=desc, unit="file", leave=True
+    ) as pbar:
+        if total == 1:
+            pbar.bar_format = desc
+            pbar.refresh()
+
+        for stage in pbar:
+            if total > 1:
+                pbar.set_msg(f"{stage.outs[0]}")
+            yield stage
+            if total == 1:  # restore bar format for stats
+                # pylint: disable=no-member
+                pbar.bar_format = pbar.BAR_FMT_DEFAULT
+
+
+LINK_FAILURE_MESSAGE = (
+    "\nSome targets could not be linked from cache to workspace.\n{}\n"
+    "To re-link these targets, reconfigure cache types and then run:\n"
+    "\n\tdvc checkout {}"
+)
+
+
+@contextmanager
+def warn_link_failures() -> Iterator[List[str]]:
+    link_failures: List[str] = []
+    try:
+        yield link_failures
+    finally:
+        if link_failures:
+            msg = LINK_FAILURE_MESSAGE.format(
+                CacheLinkError.SUPPORT_LINK,
+                " ".join(link_failures),
+            )
+            ui.error_write(msg)
+
+
+VALIDATORS = (
+    check_recursive_and_fname,
+    transform_targets,
+    check_arg_combinations,
+)
+
+
+@validate(*VALIDATORS)
+@locked
+@scm_context
+def add(  # noqa: C901
+    repo: "Repo",
+    targets: "TargetType",
+    recursive: bool = False,
+    no_commit: bool = False,
+    fname: str = None,
+    to_remote: bool = False,
+    **kwargs: Any,
 ):
-    link_failures = []
-    from dvc.dvcfile import Dvcfile
+    to_cache = bool(kwargs.get("out")) and not to_remote
+    transfer = to_remote or to_cache
 
-    from ..output import OutputDoesNotExistError
+    glob = kwargs.get("glob", False)
+    add_targets = collect_targets(repo, targets, recursive, glob)
+    # pass one for creating stages, other one is used for iterating here
+    add_targets, sources = tee(add_targets)
 
-    if to_remote or to_cache:
-        # Already verified in the add()
-        (stage,) = stages
-        (target,) = sub_targets
-        (out,) = stage.outs
+    # collect targets and build stages as we go
+    desc = "Collecting targets"
+    stages_it = create_stages(repo, add_targets, fname, transfer, **kwargs)
+    stages = list(ui.progress(stages_it, desc=desc, unit="file"))
 
-        odb = None
-        if to_remote:
-            remote = repo.cloud.get_remote(kwargs.get("remote"), "add")
-            odb = remote.odb
+    msg = "Collecting stages from the workspace"
+    with translate_graph_error(stages), ui.status(msg) as status:
+        # remove existing stages that are to-be replaced with these
+        # new stages for the graph checks.
+        old_stages = set(repo.stages) - set(stages)
+        status.update("Checking graph")
+        repo.check_modified_graph(stages, list(old_stages))
 
-        out.transfer(target, odb=odb, jobs=kwargs.get("jobs"))
+    odb = None
+    if to_remote:
+        remote = repo.cloud.get_remote(kwargs.get("remote"), "add")
+        odb = remote.odb
 
-        if to_cache:
-            out.checkout()
-
-        Dvcfile(repo, stage.path).dump(stage)
-        return link_failures
-
-    with Tqdm(
-        total=len(stages),
-        desc="Processing",
-        unit="file",
-        disable=len(stages) == 1,
-    ) as pbar_stages:
-        for stage in stages:
-            try:
-                stage.save()
-            except OutputDoesNotExistError:
-                pbar.n -= 1
-                raise
-
-            try:
-                if not no_commit:
-                    stage.commit()
-            except CacheLinkError:
-                link_failures.append(stage)
-
-            Dvcfile(repo, stage.path).dump(stage)
-            pbar_stages.update()
-
-    return link_failures
+    with warn_link_failures() as link_failures:
+        for stage, source in zip(progress_iter(stages), sources):
+            if to_remote or to_cache:
+                stage.transfer(source, to_remote=to_remote, odb=odb, **kwargs)
+            else:
+                try:
+                    stage.save()
+                    if not no_commit:
+                        stage.commit()
+                except CacheLinkError:
+                    link_failures.append(str(stage.relpath))
+            stage.dump()
+    return stages
 
 
-def _find_all_targets(repo, target, recursive):
+LARGE_DIR_RECURSIVE_ADD_WARNING = (
+    "You are adding a large directory '{target}' recursively.\n"
+    "Consider tracking it as a whole instead with "
+    "`{cyan}dvc add {target}{nc}`."
+)
+
+
+def collect_targets(
+    repo: "Repo",
+    targets: "TargetType",
+    recursive: bool = False,
+    glob: bool = False,
+) -> Iterator[str]:
+    for target in glob_targets(ensure_list(targets), glob=glob):
+        expanded_targets = _find_all_targets(repo, target, recursive=recursive)
+        for index, path in enumerate(expanded_targets):
+            if index == LARGE_DIR_SIZE:
+                msg = LARGE_DIR_RECURSIVE_ADD_WARNING.format(
+                    cyan=colorama.Fore.CYAN,
+                    nc=colorama.Style.RESET_ALL,
+                    target=target,
+                )
+                ui.error_write(msg)
+            yield path
+
+
+def _find_all_targets(
+    repo: "Repo", target: str, recursive: bool = False
+) -> Iterator[str]:
     from dvc.dvcfile import is_dvc_file
 
     if os.path.isdir(target) and recursive:
-        return [
-            os.fspath(path)
-            for path in Tqdm(
-                repo.dvcignore.walk_files(repo.fs, target),
-                desc="Searching " + target,
-                bar_format=Tqdm.BAR_FMT_NOTOTAL,
-                unit="file",
-            )
-            if not repo.is_dvc_internal(os.fspath(path))
-            if not is_dvc_file(os.fspath(path))
-            if not repo.scm.belongs_to_scm(os.fspath(path))
-            if not repo.scm.is_tracked(os.fspath(path))
-        ]
-    return [target]
-
-
-def _create_stages(
-    repo,
-    targets,
-    fname,
-    pbar=None,
-    external=False,
-    glob=False,
-    desc=None,
-    transfer=False,
-    **kwargs,
-):
-    from dvc.dvcfile import Dvcfile
-    from dvc.stage import Stage, create_stage, restore_meta
-
-    expanded_targets = glob_targets(targets, glob=glob)
-
-    stages = []
-    for out in Tqdm(
-        expanded_targets,
-        desc="Creating DVC files",
-        disable=len(expanded_targets) < LARGE_DIR_SIZE,
-        unit="file",
-    ):
-        if kwargs.get("out"):
-            out = resolve_output(out, kwargs["out"])
-        path, wdir, out = resolve_paths(
-            repo, out, always_local=transfer and not kwargs.get("out")
+        files = map(os.fspath, repo.dvcignore.walk_files(repo.fs, target))
+        yield from (
+            path
+            for path in files
+            if not repo.is_dvc_internal(path)
+            if not is_dvc_file(path)
+            if not repo.scm.belongs_to_scm(path)
+            if not repo.scm.is_tracked(path)
         )
-        stage = create_stage(
-            Stage,
-            repo,
-            fname or path,
+    else:
+        yield target
+
+
+def create_stages(
+    repo: "Repo",
+    targets: Iterator[str],
+    fname: str = None,
+    transfer: bool = False,
+    external: bool = False,
+    **kwargs: Any,
+) -> Iterator["Stage"]:
+    for target in targets:
+        if kwargs.get("out"):
+            target = resolve_output(target, kwargs["out"])
+        path, wdir, out = resolve_paths(
+            repo, target, always_local=transfer and not kwargs.get("out")
+        )
+
+        stage = repo.stage.create(
+            single_stage=True,
+            validate=False,
+            fname=fname or path,
             wdir=wdir,
             outs=[out],
             external=external,
         )
-        restore_meta(stage)
-        Dvcfile(repo, stage.path).remove()
-        if desc:
-            stage.outs[0].desc = desc
-
-        repo._reset()  # pylint: disable=protected-access
-
-        if not stage:
-            if pbar is not None:
-                pbar.total -= 1
-            continue
-
-        stages.append(stage)
-        if pbar is not None:
-            pbar.update_msg(out)
-
-    return stages
+        if kwargs.get("desc"):
+            stage.outs[0].desc = kwargs["desc"]
+        yield stage

dvc/repo/experiments/__init__.py

@@ -753,7 +753,7 @@ class Experiments:
         assert entry.index == 0
 
         # NOTE: the stash commit to be popped already contains all the current
-        # workspace changes plus CLI modifed --params changes.
+        # workspace changes plus CLI modified --params changes.
         # `checkout --force` here will not lose any data (popping stash commit
         # will result in conflict between workspace params and stashed CLI
         # params, but we always want the stashed version).

dvc/repo/stage.py

@@ -462,7 +462,8 @@ class StageLoad:
 
         def is_out_or_ignored(root, directory):
             dir_path = f"{root}{sep}{directory}"
-            return dir_path in outs or is_ignored(dir_path)
+            # trailing slash needed to check if a directory is gitignored
+            return dir_path in outs or is_ignored(f"{dir_path}{sep}")
 
         stages = []
         for root, dirs, files in self.repo.dvcignore.walk(

dvc/scm/git/backend/base.py

@@ -187,7 +187,7 @@ class BaseGitBackend(ABC):
 
     @abstractmethod
     def get_refs_containing(self, rev: str, pattern: Optional[str] = None):
-        """Iterate over all git refs containing the specfied revision."""
+        """Iterate over all git refs containing the specified revision."""
 
     @abstractmethod
     def push_refspec(

dvc/scm/git/backend/dulwich.py

@@ -27,6 +27,8 @@ from ..objects import GitObject
 from .base import BaseGitBackend
 
 if TYPE_CHECKING:
+    from dvc.types import StrPath
+
     from ..objects import GitCommit
 
 logger = logging.getLogger(__name__)
@@ -278,12 +280,14 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
 
         return IgnoreFilterManager.from_repo(self.repo)
 
-    def is_ignored(self, path: str) -> bool:
+    def is_ignored(self, path: "StrPath") -> bool:
         # `is_ignored` returns `false` if excluded in `.gitignore` and
         # `None` if it's not mentioned at all. `True` if it is ignored.
-        return bool(
-            self.ignore_manager.is_ignored(relpath(path, self.root_dir))
-        )
+        relative_path = relpath(path, self.root_dir)
+        # if checking a directory, a trailing slash must be included
+        if str(path)[-1] == os.sep:
+            relative_path += os.sep
+        return bool(self.ignore_manager.is_ignored(relative_path))
 
     def set_ref(
         self,

dvc/scm/git/backend/gitpython.py

@@ -3,7 +3,16 @@ import locale
 import logging
 import os
 from functools import partial
-from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from funcy import ignore
 
@@ -14,6 +23,9 @@ from dvc.utils import fix_env, is_binary, relpath
 from ..objects import GitCommit, GitObject
 from .base import BaseGitBackend
 
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
 logger = logging.getLogger(__name__)
 
 
@@ -111,11 +123,11 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def git(self):
         return self.repo.git
 
-    def is_ignored(self, path: str) -> bool:
+    def is_ignored(self, path: "StrPath") -> bool:
         from git.exc import GitCommandError
 
         func = ignore(GitCommandError)(self.repo.git.check_ignore)
-        return bool(func(path))
+        return bool(func(str(path)))
 
     @property
     def root_dir(self) -> str:

dvc/scm/git/backend/pygit2.py

@@ -4,7 +4,16 @@ import os
 import stat
 from contextlib import contextmanager
 from io import BytesIO, StringIO
-from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from funcy import cached_property
 
@@ -17,6 +26,10 @@ from .base import BaseGitBackend
 logger = logging.getLogger(__name__)
 
 
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
+
 class Pygit2Object(GitObject):
     def __init__(self, obj):
         self.obj = obj
@@ -81,6 +94,15 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
 
         return RefdbFsBackend(self.repo)
 
+    def _resolve_refish(self, refish: str):
+        from pygit2 import GIT_OBJ_COMMIT, Tag
+
+        commit, ref = self.repo.resolve_refish(refish)
+        if isinstance(commit, Tag):
+            ref = commit
+            commit = commit.peel(GIT_OBJ_COMMIT)
+        return commit, ref
+
     @property
     def default_signature(self):
         try:
@@ -140,7 +162,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
                 if branch == "-":
                     branch = "@{-1}"
                 try:
-                    commit, ref = self.repo.resolve_refish(branch)
+                    commit, ref = self._resolve_refish(branch)
                 except (KeyError, GitError):
                     raise RevError(f"unknown Git revision '{branch}'")
                 self.repo.checkout_tree(commit, strategy=checkout_strategy)
@@ -200,7 +222,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         from pygit2 import GitError
 
         try:
-            commit, _ref = self.repo.resolve_refish(rev)
+            commit, _ref = self._resolve_refish(rev)
             return str(commit.id)
         except (KeyError, GitError):
             pass
@@ -220,7 +242,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         from pygit2 import GitError
 
         try:
-            commit, _ref = self.repo.resolve_refish(rev)
+            commit, _ref = self._resolve_refish(rev)
         except (KeyError, GitError):
             raise SCMError(f"Invalid commit '{rev}'")
         return GitCommit(
@@ -234,7 +256,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
     def _get_stash(self, ref: str):
         raise NotImplementedError
 
-    def is_ignored(self, path: str) -> bool:
+    def is_ignored(self, path: "StrPath") -> bool:
         rel = relpath(path, self.root_dir)
         if os.name == "nt":
             rel.replace("\\", "/")
@@ -294,12 +316,12 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         from pygit2 import GitError
 
         def _contains(repo, ref, search_commit):
-            commit, _ref = self.repo.resolve_refish(ref)
+            commit, _ref = self._resolve_refish(ref)
             base = repo.merge_base(search_commit.id, commit.id)
             return base == search_commit.id
 
         try:
-            search_commit, _ref = self.repo.resolve_refish(rev)
+            search_commit, _ref = self._resolve_refish(rev)
         except (KeyError, GitError):
             raise SCMError(f"Invalid rev '{rev}'")
 
@@ -384,7 +406,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
 
         # libgit2 stash apply only accepts refs/stash items by index. If rev is
         # not in refs/stash, we will push it onto the stash, and then pop it
-        commit, _ref = self.repo.resolve_refish(rev)
+        commit, _ref = self._resolve_refish(rev)
         stash = self.repo.references.get(Stash.DEFAULT_STASH)
         if stash:
             for i, entry in enumerate(stash.log()):

dvc/stage/__init__.py

@@ -4,7 +4,7 @@ import string
 from collections import defaultdict
 from contextlib import suppress
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Dict, Optional, Set
+from typing import TYPE_CHECKING, Any, Dict, Optional, Set
 
 from funcy import cached_property, project
 
@@ -390,6 +390,19 @@ class Stage(params.StageParams):
         if purge:
             self.dvcfile.remove_stage(self)
 
+    def transfer(
+        self,
+        source: str,
+        odb: "ObjectDB" = None,
+        to_remote: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        assert len(self.outs) == 1
+        (out,) = self.outs
+        out.transfer(source, odb=odb, jobs=kwargs.get("jobs"))
+        if not to_remote:
+            out.checkout()
+
     @rwlocked(read=["deps"], write=["outs"])
     def reproduce(self, interactive=False, **kwargs):
         if not (kwargs.get("force", False) or self.changed()):

dvc/ui/__init__.py

@@ -13,6 +13,8 @@ from typing import (
 from funcy import cached_property
 
 if TYPE_CHECKING:
+    from rich.status import Status
+
     from dvc.progress import Tqdm
     from dvc.ui.table import Headers, Styles, TableData
 
@@ -79,18 +81,23 @@ class Console:
         styled: bool = False,
         file: TextIO = None,
     ) -> None:
+        from dvc.progress import Tqdm
+
         sep = " " if sep is None else sep
         end = "\n" if end is None else end
         if not self._enabled and not force:
             return
 
-        if styled:
-            console = self.error_console if stderr else self.rich_console
-            return console.print(*objects, sep=sep, end=end)
-
         file = file or (sys.stderr if stderr else sys.stdout)
-        values = (self.formatter.format(obj, style=style) for obj in objects)
-        return print(*values, sep=sep, end=end, file=file)
+        with Tqdm.external_write_mode(file=file):
+            if styled:
+                console = self.error_console if stderr else self.rich_console
+                return console.print(*objects, sep=sep, end=end)
+
+            values = (
+                self.formatter.format(obj, style=style) for obj in objects
+            )
+            return print(*values, sep=sep, end=end, file=file)
 
     @staticmethod
     def progress(*args, **kwargs) -> "Tqdm":
@@ -181,6 +188,9 @@ class Console:
             self, data, headers, markdown=markdown, pager=pager, force=force
         )
 
+    def status(self, status: str, **kwargs: Any) -> "Status":
+        return self.error_console.status(status, **kwargs)
+
 
 ui = Console()
 

dvc/utils/collections.py

@@ -1,6 +1,8 @@
+import inspect
 import os
 from collections.abc import Mapping
-from typing import Dict, Iterable, List, TypeVar, Union
+from functools import wraps
+from typing import Callable, Dict, Iterable, List, TypeVar, Union
 
 from pygtrie import StringTrie as _StringTrie
 
@@ -76,3 +78,59 @@ def chunk_dict(d: Dict[_KT, _VT], size: int = 1) -> List[Dict[_KT, _VT]]:
     from funcy import chunks
 
     return [{key: d[key] for key in chunk} for chunk in chunks(size, d)]
+
+
+class _NamespacedDict(dict):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.__dict__ = self
+
+
+def validate(*validators: Callable, post: bool = False):
+    """
+    Validate and transform arguments and results from function calls.
+
+    The validators functions are passed a dictionary of arguments, which
+    supports dot notation access too.
+
+    The key is derived from the function signature, and hence is the name of
+    the argument, whereas the value is the one passed to the function
+    (if it is not passed, default value from keyword arguments are provided).
+
+    >>> def validator(args):
+    ...    assert args["l"] >= 0 and args.b >= 0 and args.h >= 0
+
+    >>> @validate(validator)
+    ... def cuboid_area(l, b, h=1):
+    ...   return 2*(l*b + l*h + b*h)
+
+    >>> cuboid_area(5, 20)
+    250
+    >>> cuboid_area(-1, -2)
+    Traceback (most recent call last):
+      ...
+    AssertionError
+    """
+
+    def wrapped(func: Callable):
+        sig = inspect.signature(func)
+
+        @wraps(func)
+        def inner(*args, **kwargs):
+            ba = sig.bind(*args, **kwargs)
+            ba.apply_defaults()
+            ba.arguments = _NamespacedDict(ba.arguments)
+
+            if not post:
+                for validator in validators:
+                    validator(ba.arguments)
+
+            result = func(*ba.args, **ba.kwargs)
+            if post:
+                for validator in validators:
+                    result = validator(result)
+            return result
+
+        return inner
+
+    return wrapped

dvc/utils/fs.py

@@ -4,11 +4,15 @@ import os
 import shutil
 import stat
 import sys
+from typing import TYPE_CHECKING
 
 from dvc.exceptions import DvcException
 from dvc.system import System
 from dvc.utils import dict_md5
 
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
 logger = logging.getLogger(__name__)
 
 LOCAL_CHUNK_SIZE = 2 ** 20  # 1 MB
@@ -73,9 +77,9 @@ class BasePathNotInCheckedPathException(DvcException):
         super().__init__(msg)
 
 
-def contains_symlink_up_to(path, base_path):
-    base_path = os.fspath(base_path)
-    path = os.fspath(path)
+def contains_symlink_up_to(path: "StrPath", base_path: "StrPath"):
+    base_path = os.path.normcase(os.fspath(base_path))
+    path = os.path.normcase(os.fspath(path))
 
     if base_path not in path:
         raise BasePathNotInCheckedPathException(path, base_path)
@@ -144,11 +148,11 @@ def remove(path):
             raise
 
 
-def path_isin(child, parent):
+def path_isin(child: "StrPath", parent: "StrPath"):
     """Check if given `child` path is inside `parent`."""
 
-    def normalize_path(path):
-        return os.path.normpath(path)
+    def normalize_path(path) -> str:
+        return os.path.normcase(os.path.normpath(path))
 
     parent = os.path.join(normalize_path(parent), "")
     child = normalize_path(child)

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.4.0"
+_BASE_VERSION = "2.5.3"
 
 
 def _generate_version(base_version):

scripts/build_package.sh

@@ -11,4 +11,4 @@ fi
 echo 'PKG = "pip"' >dvc/utils/build.py
 
 python setup.py sdist
-python setup.py bdist_wheel --universal
+python setup.py bdist_wheel

scripts/innosetup/setup.iss

@@ -34,7 +34,7 @@ Source: "{#MyAppDir}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs c
 
 [Tasks]
 Name: modifypath; Description: Adds dvc's application directory to environmental path; Flags: checkablealone;
-Name: modifypath\system; Description: Adds dvc's application directory to enviromental path for all users;
+Name: modifypath\system; Description: Adds dvc's application directory to environmental path for all users;
 Name: addsymlinkpermissions; Description: Add permission for creating symbolic links; Flags: checkablealone;
 Name: addsymlinkpermissions\system; Description: Add permissions for creating symbolic links for all users;
 

scripts/pypi_upload.sh

@@ -9,5 +9,5 @@ fi
 
 rm -rf dist/
 pip install twine
-python setup.py sdist bdist_wheel --universal
+python setup.py sdist bdist_wheel
 twine upload dist/*

setup.cfg

@@ -1,6 +1,3 @@
-[bdist_wheel]
-universal=1
-
 [flake8]
 ignore=
     E203, # Whitespace before ':'

setup.py

@@ -78,9 +78,10 @@ install_requires = [
     "pydot>=1.2.4",
     "speedcopy>=2.0.1; python_version < '3.8' and sys_platform == 'win32'",
     "dataclasses==0.7; python_version < '3.7'",
+    "importlib-metadata==1.4; python_version < '3.8'",
     "flatten_dict>=0.3.0,<1",
     "tabulate>=0.8.7",
-    "pygtrie==2.3.2",
+    "pygtrie>=2.3.2",
     "dpath>=2.0.1,<3",
     "shtab>=1.3.4,<2",
     "rich>=10.0.0",
@@ -88,16 +89,16 @@ install_requires = [
     "python-benedict>=0.21.1",
     "pyparsing==2.4.7",
     "typing_extensions>=3.7.4",
-    "fsspec @ git+https://github.com/intake/filesystem_spec.git",
+    "fsspec>=2021.6.1",
     "diskcache>=5.2.1",
 ]
 
 
 # Extra dependencies for remote integrations
 
-gs = ["gcsfs==2021.6.0"]
+gs = ["gcsfs==2021.6.1"]
 gdrive = ["pydrive2>=1.8.1", "six >= 1.13.0"]
-s3 = ["s3fs==2021.6.0", "aiobotocore[boto3]==1.3.0"]
+s3 = ["s3fs==2021.6.1", "aiobotocore[boto3]==1.3.0"]
 azure = ["adlfs==0.7.1", "azure-identity>=1.4.0", "knack"]
 # https://github.com/Legrandin/pycryptodome/issues/465
 oss = ["oss2==2.6.1", "pycryptodome>=3.10"]

test_requirements.txt

@@ -17,12 +17,12 @@ beautifulsoup4==4.9.3
 wget==3.2
 filelock==3.0.12
 wsgidav==3.1.0
-crc32c==2.2
+crc32c==2.2.post0
 xmltodict==0.12.0
 google-compute-engine==2.8.13
-google-cloud-storage==1.38.0
+google-cloud-storage==1.40.0
 
-urllib3==1.26.5
+urllib3==1.26.6
 
 # required by collective.checkdocs
 Pygments==2.9.0
@@ -39,7 +39,7 @@ pylint-pytest==1.0.3
 pylint-plugin-utils==0.6
 
 # type-checking
-mypy==0.902
+mypy==0.910
 types-requests
 types-paramiko
 types-tabulate

tests/func/plots/test_show.py

@@ -571,7 +571,7 @@ def test_raise_on_wrong_field(tmp_dir, scm, dvc, run_copy_metrics):
 
 
 def test_load_metric_from_dict_json(tmp_dir):
-    metric = [{"acccuracy": 1, "loss": 2}, {"accuracy": 3, "loss": 4}]
+    metric = [{"accuracy": 1, "loss": 2}, {"accuracy": 3, "loss": 4}]
     dmetric = {"train": metric}
 
     plot_data = JSONPlotData("-", "revision", json.dumps(dmetric))
@@ -584,7 +584,7 @@ def test_load_metric_from_dict_json(tmp_dir):
 
 
 def test_load_metric_from_dict_yaml(tmp_dir):
-    metric = [{"acccuracy": 1, "loss": 2}, {"accuracy": 3, "loss": 4}]
+    metric = [{"accuracy": 1, "loss": 2}, {"accuracy": 3, "loss": 4}]
     dmetric = {"train": metric}
 
     plot_data = YAMLPlotData("-", "revision", dumps_yaml(dmetric))

tests/func/test_add.py

@@ -1,6 +1,5 @@
 import errno
 import filecmp
-import logging
 import os
 import shutil
 import stat
@@ -24,7 +23,11 @@ from dvc.fs.local import LocalFileSystem
 from dvc.hash_info import HashInfo
 from dvc.main import main
 from dvc.objects.db import ODBManager
-from dvc.output import OutputAlreadyTrackedError, OutputIsStageFileError
+from dvc.output import (
+    OutputAlreadyTrackedError,
+    OutputDoesNotExistError,
+    OutputIsStageFileError,
+)
 from dvc.stage import Stage
 from dvc.stage.exceptions import (
     StageExternalOutputsError,
@@ -127,14 +130,12 @@ class TestAddCmdDirectoryRecursive(TestDvc):
 
     def test_warn_about_large_directories(self):
         warning = (
-            "You are adding a large directory 'large-dir' recursively,"
-            " consider tracking it as a whole instead.\n"
-            "{purple}HINT:{nc} Remove the generated DVC file and then"
-            " run `{cyan}dvc add large-dir{nc}`".format(
-                purple=colorama.Fore.MAGENTA,
-                cyan=colorama.Fore.CYAN,
-                nc=colorama.Style.RESET_ALL,
-            )
+            "You are adding a large directory 'large-dir' recursively."
+            "\nConsider tracking it as a whole instead with "
+            "`{cyan}dvc add large-dir{nc}`"
+        ).format(
+            cyan=colorama.Fore.CYAN,
+            nc=colorama.Style.RESET_ALL,
         )
 
         os.mkdir("large-dir")
@@ -145,9 +146,8 @@ class TestAddCmdDirectoryRecursive(TestDvc):
             with open(path, "w") as fobj:
                 fobj.write(path)
 
-        with self._caplog.at_level(logging.WARNING, logger="dvc"):
-            assert main(["add", "--recursive", "large-dir"]) == 0
-            assert warning in self._caplog.messages
+        assert main(["add", "--recursive", "large-dir"]) == 0
+        assert warning in self._capsys.readouterr()[1]
 
 
 class TestAddDirectoryWithForwardSlash(TestDvc):
@@ -639,7 +639,7 @@ def test_failed_add_cleanup(tmp_dir, scm, dvc):
 
 
 def test_should_not_track_git_internal_files(mocker, dvc, tmp_dir):
-    stage_creator_spy = mocker.spy(dvc_module.repo.add, "_create_stages")
+    stage_creator_spy = mocker.spy(dvc_module.repo.add, "create_stages")
 
     ret = main(["add", "-R", dvc.root_dir])
     assert ret == 0
@@ -947,15 +947,15 @@ def test_add_file_in_symlink_dir(make_tmp_dir, tmp_dir, dvc, external):
         dvc.add(os.path.join("dir", "foo"))
 
 
-def test_add_with_cache_link_error(tmp_dir, dvc, mocker, caplog):
+def test_add_with_cache_link_error(tmp_dir, dvc, mocker, capsys):
     tmp_dir.gen("foo", "foo")
 
     mocker.patch(
         "dvc.checkout._do_link", side_effect=DvcException("link failed")
     )
-    with caplog.at_level(logging.WARNING, logger="dvc"):
-        dvc.add("foo")
-        assert "reconfigure cache types" in caplog.text
+    dvc.add("foo")
+    err = capsys.readouterr()[1]
+    assert "reconfigure cache types" in err
 
     assert not (tmp_dir / "foo").exists()
     assert (tmp_dir / "foo.dvc").exists()
@@ -1190,3 +1190,42 @@ def test_add_ignored(tmp_dir, scm, dvc):
     assert str(exc.value) == ("bad DVC file name '{}' is git-ignored.").format(
         os.path.join("dir", "subdir.dvc")
     )
+
+
+def test_add_on_not_existing_file_should_not_remove_stage_file(tmp_dir, dvc):
+    (stage,) = tmp_dir.dvc_gen("foo", "foo")
+    (tmp_dir / "foo").unlink()
+    dvcfile_contents = (tmp_dir / stage.path).read_text()
+
+    with pytest.raises(OutputDoesNotExistError):
+        dvc.add("foo")
+    assert (tmp_dir / "foo.dvc").exists()
+    assert (tmp_dir / stage.path).read_text() == dvcfile_contents
+
+
+@pytest.mark.parametrize(
+    "target",
+    [
+        "dvc.repo.Repo.check_modified_graph",
+        "dvc.stage.Stage.save",
+        "dvc.stage.Stage.commit",
+    ],
+)
+def test_add_does_not_remove_stage_file_on_failure(
+    tmp_dir, dvc, mocker, target
+):
+    (stage,) = tmp_dir.dvc_gen("foo", "foo")
+    tmp_dir.gen("foo", "foobar")  # update file
+    dvcfile_contents = (tmp_dir / stage.path).read_text()
+
+    exc_msg = f"raising error from mocked '{target}'"
+    mocker.patch(
+        target,
+        side_effect=DvcException(exc_msg),
+    )
+
+    with pytest.raises(DvcException) as exc_info:
+        dvc.add("foo")
+    assert str(exc_info.value) == exc_msg
+    assert (tmp_dir / "foo.dvc").exists()
+    assert (tmp_dir / stage.path).read_text() == dvcfile_contents

tests/func/test_data_cloud.py

@@ -248,7 +248,7 @@ def test_warn_on_outdated_stage(tmp_dir, dvc, local_remote, caplog):
 
 def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
     tmp_dir.gen({"foo": "foo"})
-    test_get_file_hash = mocker.spy(dvc_module.objects.stage, "_get_file_hash")
+    test_file_md5 = mocker.spy(dvc_module.objects.stage, "file_md5")
     ret = main(["config", "cache.type", "hardlink"])
     assert ret == 0
     ret = main(["add", "foo"])
@@ -257,7 +257,7 @@ def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
     assert ret == 0
     ret = main(["run", "--single-stage", "-d", "foo", "echo foo"])
     assert ret == 0
-    assert test_get_file_hash.mock.call_count == 1
+    assert test_file_md5.mock.call_count == 1
 
 
 def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
@@ -307,7 +307,7 @@ def test_verify_hashes(
     remove("dir")
     remove(dvc.odb.local.cache_dir)
 
-    hash_spy = mocker.spy(dvc_module.objects.stage, "_get_file_hash")
+    hash_spy = mocker.spy(dvc_module.objects.stage, "file_md5")
 
     dvc.pull()
     assert hash_spy.call_count == 0
@@ -588,3 +588,23 @@ def test_pull_00_prefix(tmp_dir, dvc, remote, monkeypatch):
     stats = dvc.pull()
     assert stats["fetched"] == 2
     assert set(stats["added"]) == {"foo", "bar"}
+
+
+@pytest.mark.parametrize("remote", full_clouds, indirect=True)
+def test_pull_no_00_prefix(tmp_dir, dvc, remote, monkeypatch):
+    # Related: https://github.com/iterative/dvc/issues/6244
+
+    fs_type = type(dvc.cloud.get_remote("upstream").fs)
+    monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
+    monkeypatch.setattr(fs_type, "LIST_OBJECT_PAGE_SIZE", 256, raising=False)
+
+    # foo's md5 checksum is 14ffd92a6cbf5f2f657067df0d5881a6
+    # bar's md5 checksum is 64020400f00960c0ef04052547b134b3
+    tmp_dir.dvc_gen({"foo": "dvc", "bar": "cml"})
+
+    dvc.push()
+    clean(["foo", "bar"], dvc)
+
+    stats = dvc.pull()
+    assert stats["fetched"] == 2
+    assert set(stats["added"]) == {"foo", "bar"}

tests/func/test_import.py

@@ -8,7 +8,7 @@ from mock import patch
 import dvc.data_cloud as cloud
 from dvc.config import NoRemoteError
 from dvc.dvcfile import Dvcfile
-from dvc.exceptions import DownloadError
+from dvc.exceptions import DownloadError, PathMissingError
 from dvc.objects.db import ODBManager
 from dvc.stage.exceptions import StagePathNotFoundError
 from dvc.system import System
@@ -289,7 +289,7 @@ def test_push_wildcard_from_bare_git_repo(
     dvc_repo = make_tmp_dir("dvc-repo", scm=True, dvc=True)
     with dvc_repo.chdir():
         dvc_repo.dvc.imp(os.fspath(tmp_dir), "dirextra")
-        with pytest.raises(FileNotFoundError):
+        with pytest.raises(PathMissingError):
             dvc_repo.dvc.imp(os.fspath(tmp_dir), "dir123")
 
 
@@ -347,11 +347,11 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir):
 
 
 def test_import_non_existing(erepo_dir, tmp_dir, dvc):
-    with pytest.raises(FileNotFoundError):
+    with pytest.raises(PathMissingError):
         tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output")
 
     # https://github.com/iterative/dvc/pull/2837#discussion_r352123053
-    with pytest.raises(FileNotFoundError):
+    with pytest.raises(PathMissingError):
         tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root")
 
 

tests/func/test_import_url.py

@@ -123,6 +123,7 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
     [
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
+        pytest.lazy_fixture("azure"),
         pytest.param(
             pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
         ),

tests/func/test_scm.py

@@ -99,6 +99,35 @@ def test_ignored(tmp_dir, scm):
     assert not scm.is_ignored(tmp_dir / "dir1" / "file2.txt")
 
 
+def test_ignored_dir_unignored_subdirs(tmp_dir, scm):
+    tmp_dir.gen({".gitignore": "data/**\n!data/**/\n!data/**/*.csv"})
+    scm.add([".gitignore"])
+    tmp_dir.gen(
+        {
+            os.path.join("data", "raw", "tracked.csv"): "cont",
+            os.path.join("data", "raw", "not_tracked.json"): "cont",
+        }
+    )
+
+    assert not scm.is_ignored(tmp_dir / "data" / "raw" / "tracked.csv")
+    assert scm.is_ignored(tmp_dir / "data" / "raw" / "not_tracked.json")
+    assert not scm.is_ignored(tmp_dir / "data" / "raw" / "non_existent.csv")
+    assert scm.is_ignored(tmp_dir / "data" / "raw" / "non_existent.json")
+    assert not scm.is_ignored(tmp_dir / "data" / "non_existent.csv")
+    assert scm.is_ignored(tmp_dir / "data" / "non_existent.json")
+
+    assert not scm.is_ignored(f"data{os.sep}")
+    # git check-ignore would now mark "data/raw" as ignored
+    # after detecting it's a directory in the file system;
+    # instead, we rely on the trailing separator to determine if handling a
+    # a directory - for consistency between existent and non-existent paths
+    assert scm.is_ignored(os.path.join("data", "raw"))
+    assert not scm.is_ignored(os.path.join("data", f"raw{os.sep}"))
+
+    assert scm.is_ignored(os.path.join("data", "non_existent"))
+    assert not scm.is_ignored(os.path.join("data", f"non_existent{os.sep}"))
+
+
 def test_get_gitignore(tmp_dir, scm):
     tmp_dir.gen({"file1": "contents", "dir": {}})
 

tests/func/test_stage.py

@@ -227,6 +227,13 @@ def test_parent_repo_collect_stages(tmp_dir, scm, dvc):
     assert deep_subrepo_stages != []
 
 
+def test_collect_repo_ignored_dir_unignored_pattern(tmp_dir, dvc, scm):
+    tmp_dir.gen({".gitignore": "data/**\n!data/**/\n!data/**/*.dvc"})
+    scm.add([".gitignore"])
+    (stage,) = tmp_dir.dvc_gen({"data/raw/tracked.csv": "5,6,7,8"})
+    assert dvc.stage.collect_repo() == [stage]
+
+
 def test_stage_strings_representation(tmp_dir, dvc, run_copy):
     tmp_dir.dvc_gen("foo", "foo")
     stage1 = run_copy("foo", "bar", single_stage=True)

tests/func/test_status.py

@@ -49,7 +49,7 @@ def test_status_non_dvc_repo_import(tmp_dir, dvc, git_dir):
 
 
 def test_status_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
-    git_dir.scm_gen("file", "first version", commit="first verison")
+    git_dir.scm_gen("file", "first version", commit="first version")
     old_rev = git_dir.scm.get_rev()
 
     dvc.imp(os.fspath(git_dir), "file", "file")

tests/func/test_version.py

@@ -1,7 +1,7 @@
 import re
 
 from dvc.main import main
-from tests.unit.test_info import PYTHON_VERSION_REGEX
+from tests.unit.test_info import PYTHON_VERSION_REGEX, find_supported_remotes
 
 
 def test_(tmp_dir, dvc, scm, capsys):
@@ -10,7 +10,7 @@ def test_(tmp_dir, dvc, scm, capsys):
     out, _ = capsys.readouterr()
     assert re.search(r"DVC version: \d+\.\d+\.\d+.*", out)
     assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", out)
-    assert re.search(r"Supports: .*", out)
+    assert find_supported_remotes(out)
     assert re.search(r"Cache types: .*", out)
     assert re.search(r"Caches: local", out)
     assert re.search(r"Remotes: None", out)

tests/unit/command/test_diff.py

@@ -48,8 +48,7 @@ def test_default(mocker, capsys):
         "Renamed:\n"
         "    data{sep}file_old -> data{sep}file_new\n"
         "\n"
-        "files summary: 1 added, 0 deleted, "
-        "1 renamed, 0 modified, 0 not in cache"
+        "files summary: 1 added, 1 renamed"
     ).format(sep=os.path.sep) in capsys.readouterr()[0]
 
 
@@ -99,8 +98,7 @@ def test_show_hash(mocker, capsys):
         "    CCCCCCCC..DDDDDDDD  file1\n"
         "    AAAAAAAA..BBBBBBBB  file2\n"
         "\n"
-        "files summary: 0 added, 2 deleted, "
-        "1 renamed, 2 modified, 0 not in cache"
+        "files summary: 2 deleted, 1 renamed, 2 modified"
     ) in out
 
 
@@ -350,6 +348,6 @@ def test_hide_missing(mocker, capsys):
         "Renamed:\n"
         "    file_old -> file_new\n"
         "\n"
-        "files summary: 1 added, 0 deleted, 1 renamed, 0 modified"
+        "files summary: 1 added, 1 renamed"
     ) in out
     assert "not in cache" not in out

tests/unit/objects/test_tree.py

@@ -69,19 +69,27 @@ def test_list(lst, trie_dict):
         ({}, 0),
         (
             {
-                ("a",): HashInfo("md5", "abc", size=1),
-                ("b",): HashInfo("md5", "def", size=2),
-                ("c",): HashInfo("md5", "ghi", size=3),
-                ("dir", "foo"): HashInfo("md5", "jkl", size=4),
-                ("dir", "bar"): HashInfo("md5", "mno", size=5),
-                ("dir", "baz"): HashInfo("md5", "pqr", size=6),
+                ("a",): HashFile(None, None, HashInfo("md5", "abc", size=1)),
+                ("b",): HashFile(None, None, HashInfo("md5", "def", size=2)),
+                ("c",): HashFile(None, None, HashInfo("md5", "ghi", size=3)),
+                ("dir", "foo"): HashFile(
+                    None, None, HashInfo("md5", "jkl", size=4)
+                ),
+                ("dir", "bar"): HashFile(
+                    None, None, HashInfo("md5", "mno", size=5)
+                ),
+                ("dir", "baz"): HashFile(
+                    None, None, HashInfo("md5", "pqr", size=6)
+                ),
             },
             21,
         ),
         (
             {
-                ("a",): HashInfo("md5", "abc", size=1),
-                ("b",): HashInfo("md5", "def", size=None),
+                ("a",): HashFile(None, None, HashInfo("md5", "abc", size=1)),
+                ("b",): HashFile(
+                    None, None, HashInfo("md5", "def", size=None)
+                ),
             },
             None,
         ),
@@ -90,6 +98,7 @@ def test_list(lst, trie_dict):
 def test_size(trie_dict, size):
     tree = Tree(None, None, None)
     tree._dict = trie_dict
+    tree.digest()
     assert tree.size == size
 
 

tests/unit/remote/test_http.py

@@ -175,5 +175,9 @@ def test_content_length(mocker, headers, expected_size):
 
     url = URLInfo("https://example.org/file.txt")
 
-    assert fs.info(url) == {"etag": None, "size": expected_size}
+    assert fs.info(url) == {
+        "etag": None,
+        "size": expected_size,
+        "type": "file",
+    }
     assert fs._content_length(res) == expected_size

tests/unit/scm/test_git.py

@@ -553,3 +553,30 @@ def test_add(tmp_dir, scm, git):
     assert set(staged["modify"]) == {"dir/baz"}
     assert len(unstaged) == 1
     assert len(untracked) == 1
+
+
+@pytest.mark.parametrize("use_sha", [True, False])
+def test_pygit_resolve_refish(tmp_dir, scm, git, use_sha):
+    import pygit2
+
+    if git.test_backend != "pygit2":
+        pytest.skip()
+
+    tmp_dir.scm_gen("foo", "foo", commit="foo")
+    head = scm.get_rev()
+    tag = "my_tag"
+    scm.gitpython.git.tag("-a", tag, "-m", "create annotated tag")
+
+    if use_sha:
+        # refish will be annotated tag SHA (not commit SHA)
+        ref = git.pygit2.repo.references.get(f"refs/tags/{tag}")
+        refish = str(ref.target)
+    else:
+        refish = tag
+
+    assert refish != head
+    commit, ref = git.pygit2._resolve_refish(refish)
+    assert isinstance(commit, pygit2.Commit)
+    assert str(commit.id) == head
+    if not use_sha:
+        assert ref.name == f"refs/tags/{tag}"

tests/unit/test_info.py

@@ -13,6 +13,35 @@ from dvc.info import get_dvc_info
 PYTHON_VERSION_REGEX = r"Python \d\.\d+\.\d+\S*"
 
 
+def find_supported_remotes(string):
+    lines = string.splitlines()
+    index = 0
+
+    for index, line in enumerate(lines):
+        if line == "Supports:":
+            index += 1
+            break
+    else:
+        return []
+
+    remotes = {}
+    for line in lines[index:]:
+        if not line.startswith("\t"):
+            break
+
+        remote_name, _, raw_dependencies = (
+            line.strip().strip(",").partition(" ")
+        )
+        remotes[remote_name] = {
+            dependency: version
+            for dependency, _, version in [
+                dependency.partition(" = ")
+                for dependency in raw_dependencies[1:-1].split(", ")
+            ]
+        }
+    return remotes
+
+
 @pytest.mark.parametrize("scm_init", [True, False])
 def test_info_in_repo(scm_init, tmp_dir):
     tmp_dir.init(scm=scm_init, dvc=True)
@@ -23,7 +52,7 @@ def test_info_in_repo(scm_init, tmp_dir):
 
     assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info)
     assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", dvc_info)
-    assert re.search(r"Supports: .*", dvc_info)
+    assert find_supported_remotes(dvc_info)
     assert re.search(r"Cache types: .*", dvc_info)
 
     if scm_init:
@@ -98,7 +127,7 @@ def test_info_outside_of_repo(tmp_dir, caplog):
 
     assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info)
     assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", dvc_info)
-    assert re.search(r"Supports: .*", dvc_info)
+    assert find_supported_remotes(dvc_info)
     assert not re.search(r"Cache types: .*", dvc_info)
     assert "Repo:" not in dvc_info
 
@@ -107,4 +136,14 @@ def test_fs_info_outside_of_repo(tmp_dir, caplog):
     dvc_info = get_dvc_info()
     assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info)
     assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", dvc_info)
-    assert re.search(r"Supports: .*", dvc_info)
+    assert find_supported_remotes(dvc_info)
+
+
+def test_plugin_versions(tmp_dir, dvc):
+    from dvc.fs import FS_MAP
+
+    dvc_info = get_dvc_info()
+    remotes = find_supported_remotes(dvc_info)
+
+    for remote, dependencies in remotes.items():
+        assert dependencies.keys() == FS_MAP[remote].REQUIRES.keys()

tests/unit/utils/test_collections.py

@@ -1,5 +1,7 @@
 # pylint: disable=unidiomatic-typecheck
-from dvc.utils.collections import apply_diff, chunk_dict
+from mock import create_autospec
+
+from dvc.utils.collections import apply_diff, chunk_dict, validate
 
 
 class MyDict(dict):
@@ -58,3 +60,78 @@ def test_chunk_dict():
     assert chunk_dict(d, 2) == [{"a": 1, "b": 2}, {"c": 3}]
     assert chunk_dict(d, 3) == [d]
     assert chunk_dict(d, 4) == [d]
+
+
+# pylint: disable=unused-argument
+
+
+def _test_func(x, y, *args, j=3, k=5, **kwargs):
+    pass
+
+
+def test_pre_validate_decorator_required_args(mocker):
+    mock = mocker.MagicMock()
+
+    func_mock = create_autospec(_test_func)
+    func = validate(mock)(func_mock)
+
+    func("x", "y")
+
+    func_mock.assert_called_once_with("x", "y", j=3, k=5)
+    mock.assert_called_once()
+
+    (args,) = mock.call_args[0]
+    assert args.x == "x"
+    assert args.y == "y"
+    assert args.args == ()
+    assert args.j == 3
+    assert args.k == 5
+    assert args.kwargs == {}
+
+
+def test_pre_validate_decorator_kwargs_args(mocker):
+    mock = mocker.MagicMock()
+    func_mock = create_autospec(_test_func)
+    func = validate(mock)(func_mock)
+
+    func("x", "y", "part", "of", "args", j=1, k=10, m=100, n=1000)
+
+    func_mock.assert_called_once_with(
+        "x", "y", "part", "of", "args", j=1, k=10, m=100, n=1000
+    )
+    mock.assert_called_once()
+    (args,) = mock.call_args[0]
+    assert args.x == "x"
+    assert args.y == "y"
+    assert args.args == ("part", "of", "args")
+    assert args.j == 1
+    assert args.k == 10
+    assert args.kwargs == {"m": 100, "n": 1000}
+
+
+def test_pre_validate_update_args():
+    def test_validator(args):
+        args.w += 50
+        del args.x
+        args.y = 100
+
+    def test_func(w=1, x=5, y=10, z=15):
+        pass
+
+    mock = create_autospec(test_func)
+    func = validate(test_validator)(mock)
+
+    func(100, 100)
+    mock.assert_called_once_with(w=150, y=100, z=15)
+
+
+def test_post_validate_decorator(mocker):
+    def none_filter(result):
+        return list(filter(None, result))
+
+    test_func = mocker.MagicMock(return_value=[1, None, 2])
+    func = validate(none_filter, post=True)(test_func)
+
+    result = func()
+    test_func.assert_called_once()
+    assert result == [1, 2]

tests/unit/utils/test_fs.py

@@ -216,6 +216,28 @@ def test_path_isin_with_absolute_path():
     assert path_isin(child, parent)
 
 
+def test_path_isin_case_sensitive():
+    child = os.path.join("path", "to", "folder")
+    parent = os.path.join("PATH", "TO")
+
+    assert path_isin(child, parent) == (os.name == "nt")
+
+
+@pytest.mark.skipif(os.name != "nt", reason="Windows specific")
+def test_contains_symlink_case_sensitive_win():
+    child = os.path.join("path", "to", "folder")
+    parent = os.path.join("PATH", "TO")
+    assert contains_symlink_up_to(child, parent) is False
+
+
+@pytest.mark.skipif(os.name == "nt", reason="Posix specific")
+def test_contains_symlink_case_sensitive_posix():
+    child = os.path.join("path", "to", "folder")
+    parent = os.path.join("PATH", "TO")
+    with pytest.raises(BasePathNotInCheckedPathException):
+        contains_symlink_up_to(child, parent)
+
+
 def test_makedirs(tmp_dir):
     path = os.path.join(tmp_dir, "directory")
     path_info = PathInfo(os.path.join(tmp_dir, "another", "directory"))