dvc: bump to 2.0.7

* Use existing level if defined in addLoggingLevel

Fixes https://github.com/iterative/dvc/issues/4546

* Add addLoggingLevel test

* Format fix.

dvc/fs/azure.py

@@ -48,8 +48,6 @@ class AzureFileSystem(FSSpecWrapper):
         self.path_info = self.PATH_CLS(url)
         self.bucket = self.path_info.bucket
 
-        self.login_method, self.login_info = self._prepare_credentials(config)
-
     def _prepare_credentials(self, config):
         from azure.identity.aio import DefaultAzureCredential
 
@@ -90,7 +88,8 @@ class AzureFileSystem(FSSpecWrapper):
         else:
             login_method = None
 
-        return login_method, login_info
+        self.login_method = login_method
+        return login_info
 
     @cached_property
     def _az_config(self):
@@ -111,7 +110,7 @@ class AzureFileSystem(FSSpecWrapper):
         from azure.core.exceptions import AzureError
 
         try:
-            file_system = AzureBlobFileSystem(**self.login_info)
+            file_system = AzureBlobFileSystem(**self.fs_args)
             if self.bucket not in [
                 container.rstrip("/") for container in file_system.ls("/")
             ]:

dvc/fs/fsspec_wrapper.py

@@ -9,6 +9,11 @@ from .base import BaseFileSystem
 
 # pylint: disable=no-member
 class FSSpecWrapper(BaseFileSystem):
+    def __init__(self, repo, config):
+        super().__init__(repo, config)
+        self.fs_args = {"skip_instance_cache": True}
+        self.fs_args.update(self._prepare_credentials(config))
+
     def fs(self):
         raise NotImplementedError
 
@@ -19,8 +24,14 @@ class FSSpecWrapper(BaseFileSystem):
         return path
 
     def _strip_bucket(self, entry):
-        _, entry = entry.split("/", 1)
-        return entry
+        try:
+            bucket, path = entry.split("/", 1)
+        except ValueError:
+            # If there is no path attached, only returns
+            # the bucket (top-level).
+            bucket, path = entry, None
+
+        return path or bucket
 
     def _strip_buckets(self, entries, detail, prefix=None):
         for entry in entries:
@@ -38,6 +49,11 @@ class FSSpecWrapper(BaseFileSystem):
         entries within info() and ls(detail=True) calls"""
         return entry
 
+    def _prepare_credentials(self, config):  # pylint: disable=unused-argument
+        """Prepare the arguments for authentication to the
+        host filesystem"""
+        return {}
+
     def isdir(self, path_info):
         if not self.exists(path_info):
             return False

dvc/fs/gs.py

@@ -22,10 +22,8 @@ class GSFileSystem(FSSpecWrapper):
         url = config.get("url", "gs://")
         self.path_info = self.PATH_CLS(url)
 
-        self.login_info = self._prepare_credentials(config)
-
     def _prepare_credentials(self, config):
-        login_info = {}
+        login_info = {"consistency": None}
         login_info["project"] = config.get("projectname")
         login_info["token"] = config.get("credentialpath")
         return login_info
@@ -40,4 +38,4 @@ class GSFileSystem(FSSpecWrapper):
     def fs(self):
         from gcsfs import GCSFileSystem
 
-        return GCSFileSystem(**self.login_info, consistency=None)
+        return GCSFileSystem(**self.fs_args)

dvc/logger.py

@@ -23,14 +23,19 @@ def addLoggingLevel(levelName, levelNum, methodName=None):
     Adds a new logging level to the `logging` module and the
     currently configured logging class.
 
+    Uses the existing numeric levelNum if already defined.
+
     Based on https://stackoverflow.com/questions/2183233
     """
     if methodName is None:
         methodName = levelName.lower()
 
-    assert not hasattr(logging, levelName)
-    assert not hasattr(logging, methodName)
-    assert not hasattr(logging.getLoggerClass(), methodName)
+    # If the level name is already defined as a top-level `logging`
+    # constant, then adopt the existing numeric level.
+    if hasattr(logging, levelName):
+        existingLevelNum = getattr(logging, levelName)
+        assert isinstance(existingLevelNum, int)
+        levelNum = existingLevelNum
 
     def logForLevel(self, message, *args, **kwargs):
         if self.isEnabledFor(levelNum):
@@ -40,10 +45,19 @@ def addLoggingLevel(levelName, levelNum, methodName=None):
     def logToRoot(message, *args, **kwargs):
         logging.log(levelNum, message, *args, **kwargs)
 
-    logging.addLevelName(levelNum, levelName)
-    setattr(logging, levelName, levelNum)
-    setattr(logging.getLoggerClass(), methodName, logForLevel)
-    setattr(logging, methodName, logToRoot)
+    # getLevelName resolves the numeric log level if already defined,
+    # otherwise returns a string
+    if not isinstance(logging.getLevelName(levelName), int):
+        logging.addLevelName(levelNum, levelName)
+
+    if not hasattr(logging, levelName):
+        setattr(logging, levelName, levelNum)
+
+    if not hasattr(logging.getLoggerClass(), methodName):
+        setattr(logging.getLoggerClass(), methodName, logForLevel)
+
+    if not hasattr(logging, methodName):
+        setattr(logging, methodName, logToRoot)
 
 
 class LoggingException(Exception):

dvc/repo/diff.py

@@ -1,5 +1,7 @@
 import logging
 import os
+from collections import defaultdict
+from typing import Dict, List
 
 from dvc.exceptions import PathMissingError
 from dvc.objects.stage import get_file_hash
@@ -190,29 +192,23 @@ def _targets_to_path_infos(repo_fs, targets):
 
 
 def _calculate_renamed(new, old, added, deleted):
-    old_inverted = {}
+    old_inverted: Dict[str, List[str]] = defaultdict(list)
     # It is needed to be dict of lists to cover cases
     # when repo has paths with same hash
     for path, path_hash in old.items():
-        bucket = old_inverted.get(path_hash, None)
-        if bucket is None:
-            old_inverted[path_hash] = [path]
-        else:
-            bucket.append(path)
+        old_inverted[path_hash].append(path)
 
     renamed = []
     for path in added:
         path_hash = new[path]
-        old_paths = old_inverted.get(path_hash, None)
-        if not old_paths:
-            continue
-        old_path = None
-        for tmp_old_path in old_paths:
-            if tmp_old_path in deleted:
-                old_path = tmp_old_path
-                break
-        if not old_path:
+        old_paths = old_inverted[path_hash]
+        try:
+            iterator = enumerate(old_paths)
+            index = next(idx for idx, path in iterator if path in deleted)
+        except StopIteration:
             continue
+
+        old_path = old_paths.pop(index)
         renamed.append(
             {"path": {"old": old_path, "new": path}, "hash": path_hash}
         )

dvc/stage/imports.py

@@ -12,7 +12,7 @@ def _update_import_on_remote(stage, remote, jobs):
             "be updated with --to-remote"
         )
 
-    url = stage.deps[0].path_info.url
+    url = stage.deps[0].def_path
     stage.outs[0].hash_info = stage.repo.cloud.transfer(
         url, jobs=jobs, remote=remote, command="update"
     )

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.0.6"
+_BASE_VERSION = "2.0.7"
 
 
 def _generate_version(base_version):

tests/func/params/test_diff.py

@@ -1,7 +1,3 @@
-import os
-
-import pytest
-
 from dvc.utils import relpath
 from dvc.utils.serialize import dump_yaml
 
@@ -18,132 +14,6 @@ def test_diff_no_changes(tmp_dir, scm, dvc):
     assert dvc.params.diff() == {}
 
 
-@pytest.mark.parametrize("commit_last", [True, False])
-def test_diff_add_similar_files(tmp_dir, scm, dvc, commit_last):
-    if commit_last:
-        last_commit_msg = "commit #2"
-        a_rev = "HEAD~1"
-    else:
-        last_commit_msg = None
-        a_rev = "HEAD"
-
-    tmp_dir.dvc_gen(
-        {"dir": {"file": "text1", "subdir": {"file2": "text2"}}},
-        commit="commit #1",
-    )
-    tmp_dir.dvc_gen(
-        {"dir2": {"file": "text1", "subdir": {"file2": "text2"}}},
-        commit=last_commit_msg,
-    )
-    assert dvc.diff(a_rev) == {
-        "added": [
-            {
-                "path": os.path.join("dir2", ""),
-                "hash": "cb58ee07cb01044db229e4d6121a0dfc.dir",
-            },
-            {
-                "path": os.path.join("dir2", "file"),
-                "hash": "cef7ccd89dacf1ced6f5ec91d759953f",
-            },
-            {
-                "path": os.path.join("dir2", "subdir", "file2"),
-                "hash": "fe6123a759017e4a2af4a2d19961ed71",
-            },
-        ],
-        "deleted": [],
-        "modified": [],
-        "renamed": [],
-        "not in cache": [],
-    }
-
-
-@pytest.mark.parametrize("commit_last", [True, False])
-def test_diff_rename_folder(tmp_dir, scm, dvc, commit_last):
-    if commit_last:
-        last_commit_msg = "commit #2"
-        a_rev = "HEAD~1"
-    else:
-        last_commit_msg = None
-        a_rev = "HEAD"
-
-    tmp_dir.dvc_gen(
-        {"dir": {"file": "text1", "subdir": {"file2": "text2"}}},
-        commit="commit #1",
-    )
-    (tmp_dir / "dir").replace(tmp_dir / "dir2")
-    tmp_dir.dvc_add("dir2", commit=last_commit_msg)
-    assert dvc.diff(a_rev) == {
-        "added": [],
-        "deleted": [],
-        "modified": [],
-        "renamed": [
-            {
-                "path": {
-                    "old": os.path.join("dir", ""),
-                    "new": os.path.join("dir2", ""),
-                },
-                "hash": "cb58ee07cb01044db229e4d6121a0dfc.dir",
-            },
-            {
-                "path": {
-                    "old": os.path.join("dir", "file"),
-                    "new": os.path.join("dir2", "file"),
-                },
-                "hash": "cef7ccd89dacf1ced6f5ec91d759953f",
-            },
-            {
-                "path": {
-                    "old": os.path.join("dir", "subdir", "file2"),
-                    "new": os.path.join("dir2", "subdir", "file2"),
-                },
-                "hash": "fe6123a759017e4a2af4a2d19961ed71",
-            },
-        ],
-        "not in cache": [],
-    }
-
-
-@pytest.mark.parametrize("commit_last", [True, False])
-def test_diff_rename_file(tmp_dir, scm, dvc, commit_last):
-    if commit_last:
-        last_commit_msg = "commit #2"
-        a_rev = "HEAD~1"
-    else:
-        last_commit_msg = None
-        a_rev = "HEAD"
-
-    paths = tmp_dir.gen(
-        {"dir": {"file": "text1", "subdir": {"file2": "text2"}}}
-    )
-    tmp_dir.dvc_add(paths, commit="commit #1")
-    (tmp_dir / "dir" / "file").replace(tmp_dir / "dir" / "subdir" / "file3")
-
-    tmp_dir.dvc_add(paths, commit=last_commit_msg)
-    assert dvc.diff(a_rev) == {
-        "added": [],
-        "deleted": [],
-        "modified": [
-            {
-                "path": os.path.join("dir", ""),
-                "hash": {
-                    "old": "cb58ee07cb01044db229e4d6121a0dfc.dir",
-                    "new": "a4ac9c339aacc60b6a3152e362c319c8.dir",
-                },
-            }
-        ],
-        "renamed": [
-            {
-                "path": {
-                    "old": os.path.join("dir", "file"),
-                    "new": os.path.join("dir", "subdir", "file3"),
-                },
-                "hash": "cef7ccd89dacf1ced6f5ec91d759953f",
-            }
-        ],
-        "not in cache": [],
-    }
-
-
 def test_diff(tmp_dir, scm, dvc):
     tmp_dir.gen("params.yaml", "foo: bar")
     dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)

tests/func/test_diff.py

@@ -423,3 +423,175 @@ def test_targets_single_file_in_dir_with_file(tmp_dir, scm, dvc):
         "not in cache": [],
         "renamed": [],
     }
+
+
+@pytest.mark.parametrize("commit_last", [True, False])
+def test_diff_add_similar_files(tmp_dir, scm, dvc, commit_last):
+    if commit_last:
+        last_commit_msg = "commit #2"
+        a_rev = "HEAD~1"
+    else:
+        last_commit_msg = None
+        a_rev = "HEAD"
+
+    tmp_dir.dvc_gen(
+        {"dir": {"file": "text1", "subdir": {"file2": "text2"}}},
+        commit="commit #1",
+    )
+    tmp_dir.dvc_gen(
+        {"dir2": {"file": "text1", "subdir": {"file2": "text2"}}},
+        commit=last_commit_msg,
+    )
+    assert dvc.diff(a_rev) == {
+        "added": [
+            {
+                "path": os.path.join("dir2", ""),
+                "hash": "cb58ee07cb01044db229e4d6121a0dfc.dir",
+            },
+            {
+                "path": os.path.join("dir2", "file"),
+                "hash": "cef7ccd89dacf1ced6f5ec91d759953f",
+            },
+            {
+                "path": os.path.join("dir2", "subdir", "file2"),
+                "hash": "fe6123a759017e4a2af4a2d19961ed71",
+            },
+        ],
+        "deleted": [],
+        "modified": [],
+        "renamed": [],
+        "not in cache": [],
+    }
+
+
+@pytest.mark.parametrize("commit_last", [True, False])
+def test_diff_rename_folder(tmp_dir, scm, dvc, commit_last):
+    if commit_last:
+        last_commit_msg = "commit #2"
+        a_rev = "HEAD~1"
+    else:
+        last_commit_msg = None
+        a_rev = "HEAD"
+
+    tmp_dir.dvc_gen(
+        {"dir": {"file": "text1", "subdir": {"file2": "text2"}}},
+        commit="commit #1",
+    )
+    (tmp_dir / "dir").replace(tmp_dir / "dir2")
+    tmp_dir.dvc_add("dir2", commit=last_commit_msg)
+    assert dvc.diff(a_rev) == {
+        "added": [],
+        "deleted": [],
+        "modified": [],
+        "renamed": [
+            {
+                "path": {
+                    "old": os.path.join("dir", ""),
+                    "new": os.path.join("dir2", ""),
+                },
+                "hash": "cb58ee07cb01044db229e4d6121a0dfc.dir",
+            },
+            {
+                "path": {
+                    "old": os.path.join("dir", "file"),
+                    "new": os.path.join("dir2", "file"),
+                },
+                "hash": "cef7ccd89dacf1ced6f5ec91d759953f",
+            },
+            {
+                "path": {
+                    "old": os.path.join("dir", "subdir", "file2"),
+                    "new": os.path.join("dir2", "subdir", "file2"),
+                },
+                "hash": "fe6123a759017e4a2af4a2d19961ed71",
+            },
+        ],
+        "not in cache": [],
+    }
+
+
+@pytest.mark.parametrize("commit_last", [True, False])
+def test_diff_rename_file(tmp_dir, scm, dvc, commit_last):
+    if commit_last:
+        last_commit_msg = "commit #2"
+        a_rev = "HEAD~1"
+    else:
+        last_commit_msg = None
+        a_rev = "HEAD"
+
+    paths = tmp_dir.gen(
+        {"dir": {"file": "text1", "subdir": {"file2": "text2"}}}
+    )
+    tmp_dir.dvc_add(paths, commit="commit #1")
+    (tmp_dir / "dir" / "file").replace(tmp_dir / "dir" / "subdir" / "file3")
+
+    tmp_dir.dvc_add(paths, commit=last_commit_msg)
+    assert dvc.diff(a_rev) == {
+        "added": [],
+        "deleted": [],
+        "modified": [
+            {
+                "path": os.path.join("dir", ""),
+                "hash": {
+                    "old": "cb58ee07cb01044db229e4d6121a0dfc.dir",
+                    "new": "a4ac9c339aacc60b6a3152e362c319c8.dir",
+                },
+            }
+        ],
+        "renamed": [
+            {
+                "path": {
+                    "old": os.path.join("dir", "file"),
+                    "new": os.path.join("dir", "subdir", "file3"),
+                },
+                "hash": "cef7ccd89dacf1ced6f5ec91d759953f",
+            }
+        ],
+        "not in cache": [],
+    }
+
+
+def test_rename_multiple_files_same_hashes(tmp_dir, scm, dvc):
+    """Test diff by renaming >=2 instances of file with same hashes.
+
+    DVC should be able to detect that they are renames, and should not include
+    them in either of the `added` or the `deleted` section.
+    """
+    tmp_dir.dvc_gen(
+        {"dir": {"foo": "foo", "subdir": {"foo": "foo"}}}, commit="commit #1",
+    )
+    remove(tmp_dir / "dir")
+    # changing foo and subdir/foo to bar and subdir/bar respectively
+    tmp_dir.dvc_gen(
+        {"dir": {"bar": "foo", "subdir": {"bar": "foo"}}}, commit="commit #2"
+    )
+    assert dvc.diff("HEAD~") == {
+        "added": [],
+        "deleted": [],
+        "modified": [
+            {
+                "hash": {
+                    "new": "31b36b3ea5f4485e27f10578c47183b0.dir",
+                    "old": "c7684c8b3b0d28cf80d5305e2d856bfc.dir",
+                },
+                "path": os.path.join("dir", ""),
+            }
+        ],
+        "not in cache": [],
+        "renamed": [
+            {
+                "hash": "acbd18db4cc2f85cedef654fccc4a4d8",
+                "path": {
+                    "new": os.path.join("dir", "bar"),
+                    "old": os.path.join("dir", "foo"),
+                },
+            },
+            {
+                "hash": "acbd18db4cc2f85cedef654fccc4a4d8",
+                "path": {
+                    "new": os.path.join("dir", "subdir", "bar"),
+                    "old": os.path.join("dir", "subdir", "foo"),
+                },
+            },
+        ],
+    }

tests/func/test_fs.py

@@ -349,3 +349,23 @@ def test_fs_ls_with_etag(dvc, cloud):
             fs.info(path_info.replace(path=details["name"]))["etag"]
             == details["etag"]
         )
+
+
+@pytest.mark.parametrize(
+    "cloud", [pytest.lazy_fixture("azure"), pytest.lazy_fixture("gs")]
+)
+def test_fs_fsspec_path_management(dvc, cloud):
+    cloud.gen({"foo": "foo", "data": {"bar": "bar", "baz": {"foo": "foo"}}})
+    fs = get_cloud_fs(dvc, **cloud.config)
+
+    root = cloud.parents[len(cloud.parents) - 1]
+    bucket_details = fs.info(root)
+
+    # special conditions: name always points to the bucket name
+    assert bucket_details["name"] == root.bucket
+    assert bucket_details["type"] == "directory"
+
+    data = cloud / "data"
+    data_details = fs.info(data)
+    assert data_details["name"].rstrip("/") == data.path
+    assert data_details["type"] == "directory"

tests/unit/test_logger.py

@@ -248,3 +248,31 @@ def test_info_with_debug_loglevel_shows_no_datetime(caplog, dt):
         logger.info("message")
 
         assert "message" == formatter.format(caplog.records[0])
+
+
+def test_add_existing_level(caplog, dt):
+    # Common pattern to configure logging level in external libraries
+    # eg:
+    # https://github.com/bokeh/bokeh/blob/04bb30fef2e72e64baaa8b2f330806d5bfdd3b11/
+    # bokeh/util/logconfig.py#L79-L85
+    TRACE2 = 4
+    logging.addLevelName(TRACE2, "TRACE2")
+    logging.TRACE2 = TRACE2
+
+    dvc.logger.addLoggingLevel("TRACE2", 2)
+
+    # DVC sets all expected entrypoints, but doesn't override the level
+    assert logging.TRACE2 == 4
+    assert hasattr(logging, "trace2")
+    assert hasattr(logger, "trace2")
+    assert logging.getLevelName("TRACE2") == 4
+
+    # The TRACE2 logging level uses the original, higher logging level
+    with caplog.at_level(logging.TRACE2, logger="dvc"):
+        logger.trace2("TRACE2")
+    assert len(caplog.records) == 1
+
+    (record,) = caplog.records
+    assert record.levelno == 4
+    assert record.levelname == "TRACE2"
+    assert record.message == "TRACE2"