dvc.objects: move get_mtime_and_size to dvc.objects.utils

dvc/objects/state.py

@@ -8,9 +8,9 @@ from dvc.fs.local import LocalFileSystem
 from dvc.fs.system import inode as get_inode
 from dvc.utils import relpath
 from dvc.utils.decorators import with_diskcache
-from dvc.utils.fs import get_mtime_and_size, remove
 
 from .hash_info import HashInfo
+from .utils import get_mtime_and_size
 
 logger = logging.getLogger(__name__)
 
@@ -183,7 +183,7 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
             return
 
         for path in unused:
-            remove(os.path.join(self.root_dir, path))
+            fs.remove(os.path.join(self.root_dir, path))
 
         with self.links as ref:
             for path in unused:

dvc/objects/utils.py

@@ -0,0 +1,43 @@
+import errno
+import hashlib
+import json
+from typing import TYPE_CHECKING, Tuple
+
+if TYPE_CHECKING:
+    from dvc.fs.base import AnyFSPath, FileSystem
+
+
+def get_mtime_and_size(
+    path: "AnyFSPath", fs: "FileSystem", dvcignore=None
+) -> Tuple[str, int]:
+    import nanotime
+
+    if not fs.isdir(path):
+        base_stat = fs.info(path)
+        size = base_stat["size"]
+        mtime = str(int(nanotime.timestamp(base_stat["mtime"])))
+        return mtime, size
+
+    size = 0
+    files_mtimes = {}
+    if dvcignore:
+        walk_iterator = dvcignore.find(fs, path)
+    else:
+        walk_iterator = fs.find(path)
+    for file_path in walk_iterator:
+        try:
+            stats = fs.info(file_path)
+        except OSError as exc:
+            # NOTE: broken symlink case.
+            if exc.errno != errno.ENOENT:
+                raise
+            continue
+        size += stats["size"]
+        files_mtimes[file_path] = stats["mtime"]
+
+    # We track file changes and moves, which cannot be detected with simply
+    # max(mtime(f) for f in non_ignored_files)
+    hasher = hashlib.md5()
+    hasher.update(json.dumps(files_mtimes, sort_keys=True).encode("utf-8"))
+    mtime = hasher.hexdigest()
+    return mtime, size

dvc/utils/fs.py

@@ -9,7 +9,6 @@ from typing import TYPE_CHECKING
 
 from dvc.exceptions import DvcException
 from dvc.fs import system
-from dvc.utils import dict_md5
 
 if TYPE_CHECKING:
     from dvc.types import StrPath
@@ -19,39 +18,6 @@ logger = logging.getLogger(__name__)
 LOCAL_CHUNK_SIZE = 2**20  # 1 MB
 
 
-def get_mtime_and_size(path, fs, dvcignore=None):
-    import nanotime
-
-    if fs.isdir(path):
-        size = 0
-        files_mtimes = {}
-        if dvcignore:
-            walk_iterator = dvcignore.find(fs, path)
-        else:
-            walk_iterator = fs.find(path)
-        for file_path in walk_iterator:
-            try:
-                stats = fs.info(file_path)
-            except OSError as exc:
-                # NOTE: broken symlink case.
-                if exc.errno != errno.ENOENT:
-                    raise
-                continue
-            size += stats["size"]
-            files_mtimes[file_path] = stats["mtime"]
-
-        # We track file changes and moves, which cannot be detected with simply
-        # max(mtime(f) for f in non_ignored_files)
-        mtime = dict_md5(files_mtimes)
-    else:
-        base_stat = fs.info(path)
-        size = base_stat["size"]
-        mtime = base_stat["mtime"]
-        mtime = int(nanotime.timestamp(mtime))
-
-    return str(mtime), size
-
-
 class BasePathNotInCheckedPathException(DvcException):
     def __init__(self, path, base_path):
         msg = "Path: {} does not overlap with base path: {}".format(

tests/func/test_ignore.py

@@ -6,12 +6,12 @@ import pytest
 
 from dvc.exceptions import DvcIgnoreInCollectedDirError
 from dvc.ignore import DvcIgnore, DvcIgnorePatterns
+from dvc.objects.utils import get_mtime_and_size
 from dvc.output import OutputIsIgnoredError
 from dvc.pathspec_math import PatternInfo, merge_patterns
 from dvc.repo import Repo
 from dvc.testing.tmp_dir import TmpDir
 from dvc.types import List
-from dvc.utils.fs import get_mtime_and_size
 
 
 def _to_pattern_info_list(str_list: List):

tests/unit/utils/test_fs.py

@@ -6,13 +6,13 @@ import pytest
 import dvc
 from dvc.fs import system
 from dvc.fs.local import LocalFileSystem
+from dvc.objects.utils import get_mtime_and_size
 from dvc.utils import relpath
 from dvc.utils.fs import (
     BasePathNotInCheckedPathException,
     contains_symlink_up_to,
     copy_fobj_to_file,
     copyfile,
-    get_mtime_and_size,
     makedirs,
     move,
     path_isin,