dvc: bump to 2.4.2

* utils: normalize case in path_isin

* utils: normalize case in contains_symlink_up_to

* split platform test

dvc/fs/local.py

@@ -32,9 +32,7 @@ class LocalFileSystem(BaseFileSystem):
 
     def exists(self, path_info) -> bool:
         assert isinstance(path_info, str) or path_info.scheme == "local"
-        # NOTE: os.fspath could be removed after fsspec 2021.06.1
-        # https://github.com/intake/filesystem_spec/issues/666
-        return self.fs.exists(os.fspath(path_info))
+        return self.fs.exists(path_info)
 
     def isfile(self, path_info) -> bool:
         return os.path.isfile(path_info)

dvc/objects/stage.py

@@ -52,11 +52,6 @@ def get_file_hash(path_info, fs, name, state=None):
         if hash_info:
             return hash_info
 
-    if not fs.exists(path_info):
-        raise FileNotFoundError(
-            errno.ENOENT, os.strerror(errno.ENOENT), path_info
-        )
-
     hash_info = _get_file_hash(path_info, fs, name)
 
     if state:

dvc/output.py

@@ -45,18 +45,24 @@ CHECKSUM_SCHEMA = Any(
     And(Any(str, And(int, Coerce(str))), Length(min=3), Lower),
 )
 
+CASE_SENSITIVE_CHECKSUM_SCHEMA = Any(
+    None,
+    And(str, Length(max=0), SetTo(None)),
+    And(Any(str, And(int, Coerce(str))), Length(min=3)),
+)
+
 # NOTE: currently there are only 3 possible checksum names:
 #
 #    1) md5 (LOCAL, SSH);
-#    2) etag (S3);
+#    2) etag (S3, GS, OSS, AZURE, HTTP);
 #    3) checksum (HDFS);
 #
 # so when a few types of outputs share the same name, we only need
 # specify it once.
 CHECKSUMS_SCHEMA = {
     LocalFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    S3FileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
     HDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    S3FileSystem.PARAM_CHECKSUM: CASE_SENSITIVE_CHECKSUM_SCHEMA,
 }
 
 

dvc/utils/fs.py

@@ -4,11 +4,15 @@ import os
 import shutil
 import stat
 import sys
+from typing import TYPE_CHECKING
 
 from dvc.exceptions import DvcException
 from dvc.system import System
 from dvc.utils import dict_md5
 
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
 logger = logging.getLogger(__name__)
 
 LOCAL_CHUNK_SIZE = 2 ** 20  # 1 MB
@@ -73,9 +77,9 @@ class BasePathNotInCheckedPathException(DvcException):
         super().__init__(msg)
 
 
-def contains_symlink_up_to(path, base_path):
-    base_path = os.fspath(base_path)
-    path = os.fspath(path)
+def contains_symlink_up_to(path: "StrPath", base_path: "StrPath"):
+    base_path = os.path.normcase(os.fspath(base_path))
+    path = os.path.normcase(os.fspath(path))
 
     if base_path not in path:
         raise BasePathNotInCheckedPathException(path, base_path)
@@ -144,11 +148,11 @@ def remove(path):
             raise
 
 
-def path_isin(child, parent):
+def path_isin(child: "StrPath", parent: "StrPath"):
     """Check if given `child` path is inside `parent`."""
 
-    def normalize_path(path):
-        return os.path.normpath(path)
+    def normalize_path(path) -> str:
+        return os.path.normcase(os.path.normpath(path))
 
     parent = os.path.join(normalize_path(parent), "")
     child = normalize_path(child)

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.4.1"
+_BASE_VERSION = "2.4.2"
 
 
 def _generate_version(base_version):

setup.py

@@ -88,16 +88,16 @@ install_requires = [
     "python-benedict>=0.21.1",
     "pyparsing==2.4.7",
     "typing_extensions>=3.7.4",
-    "fsspec==2021.6.0",
+    "fsspec==2021.6.1",
     "diskcache>=5.2.1",
 ]
 
 
 # Extra dependencies for remote integrations
 
-gs = ["gcsfs==2021.6.0"]
+gs = ["gcsfs==2021.6.1"]
 gdrive = ["pydrive2>=1.8.1", "six >= 1.13.0"]
-s3 = ["s3fs==2021.6.0", "aiobotocore[boto3]==1.3.0"]
+s3 = ["s3fs==2021.6.1", "aiobotocore[boto3]==1.3.0"]
 azure = ["adlfs==0.7.1", "azure-identity>=1.4.0", "knack"]
 # https://github.com/Legrandin/pycryptodome/issues/465
 oss = ["oss2==2.6.1", "pycryptodome>=3.10"]

test_requirements.txt

@@ -17,12 +17,12 @@ beautifulsoup4==4.9.3
 wget==3.2
 filelock==3.0.12
 wsgidav==3.1.0
-crc32c==2.2
+crc32c==2.2.post0
 xmltodict==0.12.0
 google-compute-engine==2.8.13
-google-cloud-storage==1.38.0
+google-cloud-storage==1.39.0
 
-urllib3==1.26.5
+urllib3==1.26.6
 
 # required by collective.checkdocs
 Pygments==2.9.0
@@ -39,7 +39,7 @@ pylint-pytest==1.0.3
 pylint-plugin-utils==0.6
 
 # type-checking
-mypy==0.902
+mypy==0.910
 types-requests
 types-paramiko
 types-tabulate

tests/func/test_data_cloud.py

@@ -248,7 +248,7 @@ def test_warn_on_outdated_stage(tmp_dir, dvc, local_remote, caplog):
 
 def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
     tmp_dir.gen({"foo": "foo"})
-    test_get_file_hash = mocker.spy(dvc_module.objects.stage, "_get_file_hash")
+    test_file_md5 = mocker.spy(dvc_module.objects.stage, "file_md5")
     ret = main(["config", "cache.type", "hardlink"])
     assert ret == 0
     ret = main(["add", "foo"])
@@ -257,7 +257,7 @@ def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
     assert ret == 0
     ret = main(["run", "--single-stage", "-d", "foo", "echo foo"])
     assert ret == 0
-    assert test_get_file_hash.mock.call_count == 1
+    assert test_file_md5.mock.call_count == 1
 
 
 def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
@@ -307,7 +307,7 @@ def test_verify_hashes(
     remove("dir")
     remove(dvc.odb.local.cache_dir)
 
-    hash_spy = mocker.spy(dvc_module.objects.stage, "_get_file_hash")
+    hash_spy = mocker.spy(dvc_module.objects.stage, "file_md5")
 
     dvc.pull()
     assert hash_spy.call_count == 0

tests/func/test_import_url.py

@@ -123,6 +123,7 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
     [
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
+        pytest.lazy_fixture("azure"),
         pytest.param(
             pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
         ),

tests/unit/utils/test_fs.py

@@ -216,6 +216,28 @@ def test_path_isin_with_absolute_path():
     assert path_isin(child, parent)
 
 
+def test_path_isin_case_sensitive():
+    child = os.path.join("path", "to", "folder")
+    parent = os.path.join("PATH", "TO")
+
+    assert path_isin(child, parent) == (os.name == "nt")
+
+
+@pytest.mark.skipif(os.name != "nt", reason="Windows specific")
+def test_contains_symlink_case_sensitive_win():
+    child = os.path.join("path", "to", "folder")
+    parent = os.path.join("PATH", "TO")
+    assert contains_symlink_up_to(child, parent) is False
+
+
+@pytest.mark.skipif(os.name == "nt", reason="Posix specific")
+def test_contains_symlink_case_sensitive_posix():
+    child = os.path.join("path", "to", "folder")
+    parent = os.path.join("PATH", "TO")
+    with pytest.raises(BasePathNotInCheckedPathException):
+        contains_symlink_up_to(child, parent)
+
+
 def test_makedirs(tmp_dir):
     path = os.path.join(tmp_dir, "directory")
     path_info = PathInfo(os.path.join(tmp_dir, "another", "directory"))