utils: don't use unnecessary exists() calls (#5423)

* utils: file_md5: only return hex

* utils: file_md5: don't use exists()

* checkout: don't call exists()

dvc/checkout.py

@@ -51,10 +51,6 @@ def _changed(path_info, tree, hash_info, cache, filter_info=None):
     path = filter_info or path_info
     logger.trace("checking if '%s'('%s') has changed.", path_info, hash_info)
 
-    if not tree.exists(path):
-        logger.debug("'%s' doesn't exist.", path)
-        return True
-
     hi = _filter_hash_info(cache, hash_info, path_info, filter_info)
     if not hi:
         logger.debug("hash value for '%s' is missing.", path)
@@ -64,7 +60,12 @@ def _changed(path_info, tree, hash_info, cache, filter_info=None):
         logger.debug("cache for '%s'('%s') has changed.", path, hi)
         return True
 
-    actual = tree.get_hash(path)
+    try:
+        actual = tree.get_hash(path)
+    except FileNotFoundError:
+        logger.debug("'%s' doesn't exist.", path)
+        return True
+
     if hi != actual:
         logger.debug(
             "hash value '%s' for '%s' has changed (actual '%s').",

dvc/tree/local.py

@@ -208,7 +208,7 @@ class LocalTree(BaseTree):
         System.reflink(from_info, to_info)
 
     def get_file_hash(self, path_info):
-        hash_info = HashInfo(self.PARAM_CHECKSUM, file_md5(path_info)[0],)
+        hash_info = HashInfo(self.PARAM_CHECKSUM, file_md5(path_info))
 
         if hash_info:
             hash_info.size = os.path.getsize(path_info)

dvc/tree/repo.py

@@ -399,7 +399,7 @@ class RepoTree(BaseTree):  # pylint:disable=abstract-method
                 return dvc_tree.get_file_hash(path_info)
             except FileNotFoundError:
                 pass
-        return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info, self)[0])
+        return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info, self))
 
     def _download(
         self, from_info, to_file, name=None, no_progress_bar=False, **kwargs

dvc/utils/__init__.py

@@ -47,40 +47,33 @@ def file_md5(fname, tree=None):
     from dvc.progress import Tqdm
 
     if tree:
-        exists_func = tree.exists
         stat_func = tree.stat
         open_func = tree.open
     else:
-        exists_func = os.path.exists
         stat_func = os.stat
         open_func = open
 
-    if exists_func(fname):
-        hash_md5 = hashlib.md5()
-        binary = not istextfile(fname, tree=tree)
-        size = stat_func(fname).st_size
-        no_progress_bar = True
-        if size >= LARGE_FILE_SIZE:
-            no_progress_bar = False
-            msg = (
-                "Computing md5 for a large file '{}'. This is only done once."
-            )
-            logger.info(msg.format(relpath(fname)))
-        name = relpath(fname)
+    hash_md5 = hashlib.md5()
+    binary = not istextfile(fname, tree=tree)
+    size = stat_func(fname).st_size
+    no_progress_bar = True
+    if size >= LARGE_FILE_SIZE:
+        no_progress_bar = False
+        msg = "Computing md5 for a large file '{}'. This is only done once."
+        logger.info(msg.format(relpath(fname)))
+    name = relpath(fname)
 
-        with Tqdm(
-            desc=name,
-            disable=no_progress_bar,
-            total=size,
-            bytes=True,
-            leave=False,
-        ) as pbar:
-            with open_func(fname, "rb") as fobj:
-                _fobj_md5(fobj, hash_md5, binary, pbar.update)
+    with Tqdm(
+        desc=name,
+        disable=no_progress_bar,
+        total=size,
+        bytes=True,
+        leave=False,
+    ) as pbar:
+        with open_func(fname, "rb") as fobj:
+            _fobj_md5(fobj, hash_md5, binary, pbar.update)
 
-        return (hash_md5.hexdigest(), hash_md5.digest())
-
-    return (None, None)
+    return hash_md5.hexdigest()
 
 
 def bytes_hash(byts, typ):

tests/func/test_add.py

@@ -38,7 +38,7 @@ from tests.utils import get_gitignore_content
 
 def test_add(tmp_dir, dvc):
     (stage,) = tmp_dir.dvc_gen({"foo": "foo"})
-    md5, _ = file_md5("foo")
+    md5 = file_md5("foo")
 
     assert stage is not None
 

tests/func/test_install.py

@@ -64,7 +64,7 @@ class TestInstall:
             conf["core"]["remote"] = "store"
         tmp_dir.dvc_gen("file", "file_content", "commit message")
 
-        file_checksum = file_md5("file")[0]
+        file_checksum = file_md5("file")
         expected_storage_path = (
             storage_path / file_checksum[:2] / file_checksum[2:]
         )

tests/func/test_repro.py

@@ -651,9 +651,7 @@ class TestReproDataSource(TestReproChangedData):
         stages = self.dvc.reproduce(self.foo_stage.path)
 
         self.assertTrue(filecmp.cmp(self.FOO, self.BAR, shallow=False))
-        self.assertEqual(
-            stages[0].outs[0].hash_info.value, file_md5(self.BAR)[0]
-        )
+        self.assertEqual(stages[0].outs[0].hash_info.value, file_md5(self.BAR))
 
 
 class TestReproChangedDir(SingleStageRun, TestDvc):

tests/func/test_run_single_stage.py

@@ -60,7 +60,7 @@ class TestRun(TestDvc):
         self.assertEqual(len(stage.deps), len(deps))
         self.assertEqual(len(stage.outs), len(outs + outs_no_cache))
         self.assertEqual(stage.outs[0].fspath, outs[0])
-        self.assertEqual(stage.outs[0].hash_info.value, file_md5(self.FOO)[0])
+        self.assertEqual(stage.outs[0].hash_info.value, file_md5(self.FOO))
         self.assertTrue(stage.path, fname)
 
         with self.assertRaises(OutputDuplicationError):

tests/func/test_state.py

@@ -12,7 +12,7 @@ def test_state(tmp_dir, dvc):
     tmp_dir.gen("foo", "foo content")
     path = tmp_dir / "foo"
     path_info = PathInfo(path)
-    hash_info = HashInfo("md5", file_md5(path)[0])
+    hash_info = HashInfo("md5", file_md5(path))
 
     state = State(dvc)
 
@@ -25,7 +25,7 @@ def test_state(tmp_dir, dvc):
 
         assert state.get(path_info) is None
 
-        hash_info = HashInfo("md5", file_md5(path)[0])
+        hash_info = HashInfo("md5", file_md5(path))
         state.save(path_info, hash_info)
 
         assert state.get(path_info) == hash_info
@@ -60,7 +60,7 @@ def test_get_state_record_for_inode(get_inode_mock, tmp_dir, dvc):
     assert inode != state._to_sqlite(inode)
 
     foo = tmp_dir / "foo"
-    md5 = file_md5(foo)[0]
+    md5 = file_md5(foo)
     get_inode_mock.side_effect = mock_get_inode(inode)
 
     with state:

tests/func/test_utils.py

@@ -4,7 +4,7 @@ from dvc import utils
 def test_file_md5_crlf(tmp_dir):
     tmp_dir.gen("cr", b"a\nb\nc")
     tmp_dir.gen("crlf", b"a\r\nb\r\nc")
-    assert utils.file_md5("cr")[0] == utils.file_md5("crlf")[0]
+    assert utils.file_md5("cr") == utils.file_md5("crlf")
 
 
 def test_dict_md5():

tests/unit/utils/test_stream.py

@@ -10,7 +10,7 @@ def test_hashed_stream_reader(tmp_dir):
         stream_reader = HashedStreamReader(fobj)
         assert stream_reader.read(3) == b"foo"
 
-    hex_digest, _ = file_md5(foo)
+    hex_digest = file_md5(foo)
     assert stream_reader.is_text_file
     assert hex_digest == stream_reader.hash_info.value
 
@@ -26,6 +26,6 @@ def test_hashed_stream_reader_as_chunks(tmp_dir):
             if not chunk:
                 break
 
-    hex_digest, _ = file_md5(foo)
+    hex_digest = file_md5(foo)
     assert not stream_reader.is_text_file
     assert hex_digest == stream_reader.hash_info.value