dvc: bump to 2.4.3

* Keep trailing slash when checking if directory is gitignored

* Calling is_ignored with correct type (could be Path-like)

* Fix test to comply to annotated types

* Fix separator issues

* Relaxed typing in is_ignored

* Added test

* Test scm

* Fix test on Windows

* Expanded test_ignored_dir_unignored_subdirs

* Even more assertions

.github/CODEOWNERS

@@ -0,0 +1 @@
+*       @iterative/dvc

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
            - types-tabulate
            - types-toml
     repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.902'
+    rev: 'v0.910'
   - repo: local
     hooks:
     - id: pylint

dvc/objects/stage.py

@@ -52,11 +52,6 @@ def get_file_hash(path_info, fs, name, state=None):
         if hash_info:
             return hash_info
 
-    if not fs.exists(path_info):
-        raise FileNotFoundError(
-            errno.ENOENT, os.strerror(errno.ENOENT), path_info
-        )
-
     hash_info = _get_file_hash(path_info, fs, name)
 
     if state:

dvc/output.py

@@ -45,18 +45,24 @@ CHECKSUM_SCHEMA = Any(
     And(Any(str, And(int, Coerce(str))), Length(min=3), Lower),
 )
 
+CASE_SENSITIVE_CHECKSUM_SCHEMA = Any(
+    None,
+    And(str, Length(max=0), SetTo(None)),
+    And(Any(str, And(int, Coerce(str))), Length(min=3)),
+)
+
 # NOTE: currently there are only 3 possible checksum names:
 #
 #    1) md5 (LOCAL, SSH);
-#    2) etag (S3);
+#    2) etag (S3, GS, OSS, AZURE, HTTP);
 #    3) checksum (HDFS);
 #
 # so when a few types of outputs share the same name, we only need
 # specify it once.
 CHECKSUMS_SCHEMA = {
     LocalFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    S3FileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
     HDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    S3FileSystem.PARAM_CHECKSUM: CASE_SENSITIVE_CHECKSUM_SCHEMA,
 }
 
 

dvc/repo/stage.py

@@ -462,7 +462,8 @@ class StageLoad:
 
         def is_out_or_ignored(root, directory):
             dir_path = f"{root}{sep}{directory}"
-            return dir_path in outs or is_ignored(dir_path)
+            # trailing slash needed to check if a directory is gitignored
+            return dir_path in outs or is_ignored(f"{dir_path}{sep}")
 
         stages = []
         for root, dirs, files in self.repo.dvcignore.walk(

dvc/scm/git/backend/dulwich.py

@@ -27,6 +27,8 @@ from ..objects import GitObject
 from .base import BaseGitBackend
 
 if TYPE_CHECKING:
+    from dvc.types import StrPath
+
     from ..objects import GitCommit
 
 logger = logging.getLogger(__name__)
@@ -278,12 +280,14 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
 
         return IgnoreFilterManager.from_repo(self.repo)
 
-    def is_ignored(self, path: str) -> bool:
+    def is_ignored(self, path: "StrPath") -> bool:
         # `is_ignored` returns `false` if excluded in `.gitignore` and
         # `None` if it's not mentioned at all. `True` if it is ignored.
-        return bool(
-            self.ignore_manager.is_ignored(relpath(path, self.root_dir))
-        )
+        relative_path = relpath(path, self.root_dir)
+        # if checking a directory, a trailing slash must be included
+        if str(path)[-1] == os.sep:
+            relative_path += os.sep
+        return bool(self.ignore_manager.is_ignored(relative_path))
 
     def set_ref(
         self,

dvc/scm/git/backend/gitpython.py

@@ -3,7 +3,16 @@ import locale
 import logging
 import os
 from functools import partial
-from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from funcy import ignore
 
@@ -14,6 +23,9 @@ from dvc.utils import fix_env, is_binary, relpath
 from ..objects import GitCommit, GitObject
 from .base import BaseGitBackend
 
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
 logger = logging.getLogger(__name__)
 
 
@@ -111,11 +123,11 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def git(self):
         return self.repo.git
 
-    def is_ignored(self, path: str) -> bool:
+    def is_ignored(self, path: "StrPath") -> bool:
         from git.exc import GitCommandError
 
         func = ignore(GitCommandError)(self.repo.git.check_ignore)
-        return bool(func(path))
+        return bool(func(str(path)))
 
     @property
     def root_dir(self) -> str:

dvc/scm/git/backend/pygit2.py

@@ -4,7 +4,16 @@ import os
 import stat
 from contextlib import contextmanager
 from io import BytesIO, StringIO
-from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from funcy import cached_property
 
@@ -17,6 +26,10 @@ from .base import BaseGitBackend
 logger = logging.getLogger(__name__)
 
 
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
+
 class Pygit2Object(GitObject):
     def __init__(self, obj):
         self.obj = obj
@@ -234,7 +247,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
     def _get_stash(self, ref: str):
         raise NotImplementedError
 
-    def is_ignored(self, path: str) -> bool:
+    def is_ignored(self, path: "StrPath") -> bool:
         rel = relpath(path, self.root_dir)
         if os.name == "nt":
             rel.replace("\\", "/")

dvc/utils/fs.py

@@ -4,11 +4,15 @@ import os
 import shutil
 import stat
 import sys
+from typing import TYPE_CHECKING
 
 from dvc.exceptions import DvcException
 from dvc.system import System
 from dvc.utils import dict_md5
 
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
 logger = logging.getLogger(__name__)
 
 LOCAL_CHUNK_SIZE = 2 ** 20  # 1 MB
@@ -73,9 +77,9 @@ class BasePathNotInCheckedPathException(DvcException):
         super().__init__(msg)
 
 
-def contains_symlink_up_to(path, base_path):
-    base_path = os.fspath(base_path)
-    path = os.fspath(path)
+def contains_symlink_up_to(path: "StrPath", base_path: "StrPath"):
+    base_path = os.path.normcase(os.fspath(base_path))
+    path = os.path.normcase(os.fspath(path))
 
     if base_path not in path:
         raise BasePathNotInCheckedPathException(path, base_path)
@@ -144,11 +148,11 @@ def remove(path):
             raise
 
 
-def path_isin(child, parent):
+def path_isin(child: "StrPath", parent: "StrPath"):
     """Check if given `child` path is inside `parent`."""
 
-    def normalize_path(path):
-        return os.path.normpath(path)
+    def normalize_path(path) -> str:
+        return os.path.normcase(os.path.normpath(path))
 
     parent = os.path.join(normalize_path(parent), "")
     child = normalize_path(child)

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.4.0"
+_BASE_VERSION = "2.4.3"
 
 
 def _generate_version(base_version):

setup.py

@@ -80,7 +80,7 @@ install_requires = [
     "dataclasses==0.7; python_version < '3.7'",
     "flatten_dict>=0.3.0,<1",
     "tabulate>=0.8.7",
-    "pygtrie==2.3.2",
+    "pygtrie>=2.3.2",
     "dpath>=2.0.1,<3",
     "shtab>=1.3.4,<2",
     "rich>=10.0.0",
@@ -88,16 +88,16 @@ install_requires = [
     "python-benedict>=0.21.1",
     "pyparsing==2.4.7",
     "typing_extensions>=3.7.4",
-    "fsspec @ git+https://github.com/intake/filesystem_spec.git",
+    "fsspec>=2021.6.1",
     "diskcache>=5.2.1",
 ]
 
 
 # Extra dependencies for remote integrations
 
-gs = ["gcsfs==2021.6.0"]
+gs = ["gcsfs==2021.6.1"]
 gdrive = ["pydrive2>=1.8.1", "six >= 1.13.0"]
-s3 = ["s3fs==2021.6.0", "aiobotocore[boto3]==1.3.0"]
+s3 = ["s3fs==2021.6.1", "aiobotocore[boto3]==1.3.0"]
 azure = ["adlfs==0.7.1", "azure-identity>=1.4.0", "knack"]
 # https://github.com/Legrandin/pycryptodome/issues/465
 oss = ["oss2==2.6.1", "pycryptodome>=3.10"]

test_requirements.txt

@@ -17,12 +17,12 @@ beautifulsoup4==4.9.3
 wget==3.2
 filelock==3.0.12
 wsgidav==3.1.0
-crc32c==2.2
+crc32c==2.2.post0
 xmltodict==0.12.0
 google-compute-engine==2.8.13
-google-cloud-storage==1.38.0
+google-cloud-storage==1.39.0
 
-urllib3==1.26.5
+urllib3==1.26.6
 
 # required by collective.checkdocs
 Pygments==2.9.0
@@ -39,7 +39,7 @@ pylint-pytest==1.0.3
 pylint-plugin-utils==0.6
 
 # type-checking
-mypy==0.902
+mypy==0.910
 types-requests
 types-paramiko
 types-tabulate

tests/func/test_data_cloud.py

@@ -248,7 +248,7 @@ def test_warn_on_outdated_stage(tmp_dir, dvc, local_remote, caplog):
 
 def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
     tmp_dir.gen({"foo": "foo"})
-    test_get_file_hash = mocker.spy(dvc_module.objects.stage, "_get_file_hash")
+    test_file_md5 = mocker.spy(dvc_module.objects.stage, "file_md5")
     ret = main(["config", "cache.type", "hardlink"])
     assert ret == 0
     ret = main(["add", "foo"])
@@ -257,7 +257,7 @@ def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
     assert ret == 0
     ret = main(["run", "--single-stage", "-d", "foo", "echo foo"])
     assert ret == 0
-    assert test_get_file_hash.mock.call_count == 1
+    assert test_file_md5.mock.call_count == 1
 
 
 def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
@@ -307,7 +307,7 @@ def test_verify_hashes(
     remove("dir")
     remove(dvc.odb.local.cache_dir)
 
-    hash_spy = mocker.spy(dvc_module.objects.stage, "_get_file_hash")
+    hash_spy = mocker.spy(dvc_module.objects.stage, "file_md5")
 
     dvc.pull()
     assert hash_spy.call_count == 0

tests/func/test_import_url.py

@@ -123,6 +123,7 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
     [
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
+        pytest.lazy_fixture("azure"),
         pytest.param(
             pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
         ),

tests/func/test_scm.py

@@ -99,6 +99,35 @@ def test_ignored(tmp_dir, scm):
     assert not scm.is_ignored(tmp_dir / "dir1" / "file2.txt")
 
 
+def test_ignored_dir_unignored_subdirs(tmp_dir, scm):
+    tmp_dir.gen({".gitignore": "data/**\n!data/**/\n!data/**/*.csv"})
+    scm.add([".gitignore"])
+    tmp_dir.gen(
+        {
+            os.path.join("data", "raw", "tracked.csv"): "cont",
+            os.path.join("data", "raw", "not_tracked.json"): "cont",
+        }
+    )
+
+    assert not scm.is_ignored(tmp_dir / "data" / "raw" / "tracked.csv")
+    assert scm.is_ignored(tmp_dir / "data" / "raw" / "not_tracked.json")
+    assert not scm.is_ignored(tmp_dir / "data" / "raw" / "non_existent.csv")
+    assert scm.is_ignored(tmp_dir / "data" / "raw" / "non_existent.json")
+    assert not scm.is_ignored(tmp_dir / "data" / "non_existent.csv")
+    assert scm.is_ignored(tmp_dir / "data" / "non_existent.json")
+
+    assert not scm.is_ignored(f"data{os.sep}")
+    # git check-ignore would now mark "data/raw" as ignored
+    # after detecting it's a directory in the file system;
+    # instead, we rely on the trailing separator to determine if handling a
+    # a directory - for consistency between existent and non-existent paths
+    assert scm.is_ignored(os.path.join("data", "raw"))
+    assert not scm.is_ignored(os.path.join("data", f"raw{os.sep}"))
+
+    assert scm.is_ignored(os.path.join("data", "non_existent"))
+    assert not scm.is_ignored(os.path.join("data", f"non_existent{os.sep}"))
+
+
 def test_get_gitignore(tmp_dir, scm):
     tmp_dir.gen({"file1": "contents", "dir": {}})
 

tests/func/test_stage.py

@@ -227,6 +227,13 @@ def test_parent_repo_collect_stages(tmp_dir, scm, dvc):
     assert deep_subrepo_stages != []
 
 
+def test_collect_repo_ignored_dir_unignored_pattern(tmp_dir, dvc, scm):
+    tmp_dir.gen({".gitignore": "data/**\n!data/**/\n!data/**/*.dvc"})
+    scm.add([".gitignore"])
+    (stage,) = tmp_dir.dvc_gen({"data/raw/tracked.csv": "5,6,7,8"})
+    assert dvc.stage.collect_repo() == [stage]
+
+
 def test_stage_strings_representation(tmp_dir, dvc, run_copy):
     tmp_dir.dvc_gen("foo", "foo")
     stage1 = run_copy("foo", "bar", single_stage=True)

tests/unit/utils/test_fs.py

@@ -216,6 +216,28 @@ def test_path_isin_with_absolute_path():
     assert path_isin(child, parent)
 
 
+def test_path_isin_case_sensitive():
+    child = os.path.join("path", "to", "folder")
+    parent = os.path.join("PATH", "TO")
+
+    assert path_isin(child, parent) == (os.name == "nt")
+
+
+@pytest.mark.skipif(os.name != "nt", reason="Windows specific")
+def test_contains_symlink_case_sensitive_win():
+    child = os.path.join("path", "to", "folder")
+    parent = os.path.join("PATH", "TO")
+    assert contains_symlink_up_to(child, parent) is False
+
+
+@pytest.mark.skipif(os.name == "nt", reason="Posix specific")
+def test_contains_symlink_case_sensitive_posix():
+    child = os.path.join("path", "to", "folder")
+    parent = os.path.join("PATH", "TO")
+    with pytest.raises(BasePathNotInCheckedPathException):
+        contains_symlink_up_to(child, parent)
+
+
 def test_makedirs(tmp_dir):
     path = os.path.join(tmp_dir, "directory")
     path_info = PathInfo(os.path.join(tmp_dir, "another", "directory"))