diff --git a/dvc/checkout.py b/dvc/checkout.py index a827527df..c04238b4d 100644 --- a/dvc/checkout.py +++ b/dvc/checkout.py @@ -51,10 +51,6 @@ def _changed(path_info, tree, hash_info, cache, filter_info=None): path = filter_info or path_info logger.trace("checking if '%s'('%s') has changed.", path_info, hash_info) - if not tree.exists(path): - logger.debug("'%s' doesn't exist.", path) - return True - hi = _filter_hash_info(cache, hash_info, path_info, filter_info) if not hi: logger.debug("hash value for '%s' is missing.", path) @@ -64,7 +60,12 @@ def _changed(path_info, tree, hash_info, cache, filter_info=None): logger.debug("cache for '%s'('%s') has changed.", path, hi) return True - actual = tree.get_hash(path) + try: + actual = tree.get_hash(path) + except FileNotFoundError: + logger.debug("'%s' doesn't exist.", path) + return True + if hi != actual: logger.debug( "hash value '%s' for '%s' has changed (actual '%s').", diff --git a/dvc/tree/local.py b/dvc/tree/local.py index 33abc76dc..11ed25037 100644 --- a/dvc/tree/local.py +++ b/dvc/tree/local.py @@ -208,7 +208,7 @@ class LocalTree(BaseTree): System.reflink(from_info, to_info) def get_file_hash(self, path_info): - hash_info = HashInfo(self.PARAM_CHECKSUM, file_md5(path_info)[0],) + hash_info = HashInfo(self.PARAM_CHECKSUM, file_md5(path_info)) if hash_info: hash_info.size = os.path.getsize(path_info) diff --git a/dvc/tree/repo.py b/dvc/tree/repo.py index 5e2b4a1e1..0000067e8 100644 --- a/dvc/tree/repo.py +++ b/dvc/tree/repo.py @@ -399,7 +399,7 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method return dvc_tree.get_file_hash(path_info) except FileNotFoundError: pass - return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info, self)[0]) + return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info, self)) def _download( self, from_info, to_file, name=None, no_progress_bar=False, **kwargs diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 49122a53a..7cffdf092 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -47,40 +47,33 @@ def file_md5(fname, tree=None): from dvc.progress import Tqdm if tree: - exists_func = tree.exists stat_func = tree.stat open_func = tree.open else: - exists_func = os.path.exists stat_func = os.stat open_func = open - if exists_func(fname): - hash_md5 = hashlib.md5() - binary = not istextfile(fname, tree=tree) - size = stat_func(fname).st_size - no_progress_bar = True - if size >= LARGE_FILE_SIZE: - no_progress_bar = False - msg = ( - "Computing md5 for a large file '{}'. This is only done once." - ) - logger.info(msg.format(relpath(fname))) - name = relpath(fname) + hash_md5 = hashlib.md5() + binary = not istextfile(fname, tree=tree) + size = stat_func(fname).st_size + no_progress_bar = True + if size >= LARGE_FILE_SIZE: + no_progress_bar = False + msg = "Computing md5 for a large file '{}'. This is only done once." + logger.info(msg.format(relpath(fname))) + name = relpath(fname) - with Tqdm( - desc=name, - disable=no_progress_bar, - total=size, - bytes=True, - leave=False, - ) as pbar: - with open_func(fname, "rb") as fobj: - _fobj_md5(fobj, hash_md5, binary, pbar.update) + with Tqdm( + desc=name, + disable=no_progress_bar, + total=size, + bytes=True, + leave=False, + ) as pbar: + with open_func(fname, "rb") as fobj: + _fobj_md5(fobj, hash_md5, binary, pbar.update) - return (hash_md5.hexdigest(), hash_md5.digest()) - - return (None, None) + return hash_md5.hexdigest() def bytes_hash(byts, typ): diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 49c844577..207ac6276 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -38,7 +38,7 @@ from tests.utils import get_gitignore_content def test_add(tmp_dir, dvc): (stage,) = tmp_dir.dvc_gen({"foo": "foo"}) - md5, _ = file_md5("foo") + md5 = file_md5("foo") assert stage is not None diff --git a/tests/func/test_install.py b/tests/func/test_install.py index 38ee9f04c..c3f480d80 100644 --- a/tests/func/test_install.py +++ b/tests/func/test_install.py @@ -64,7 +64,7 @@ class TestInstall: conf["core"]["remote"] = "store" tmp_dir.dvc_gen("file", "file_content", "commit message") - file_checksum = file_md5("file")[0] + file_checksum = file_md5("file") expected_storage_path = ( storage_path / file_checksum[:2] / file_checksum[2:] ) diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index d82cd7c33..d7fab0f32 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -651,9 +651,7 @@ class TestReproDataSource(TestReproChangedData): stages = self.dvc.reproduce(self.foo_stage.path) self.assertTrue(filecmp.cmp(self.FOO, self.BAR, shallow=False)) - self.assertEqual( - stages[0].outs[0].hash_info.value, file_md5(self.BAR)[0] - ) + self.assertEqual(stages[0].outs[0].hash_info.value, file_md5(self.BAR)) class TestReproChangedDir(SingleStageRun, TestDvc): diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py index 9e8df5cba..2d652cd6f 100644 --- a/tests/func/test_run_single_stage.py +++ b/tests/func/test_run_single_stage.py @@ -60,7 +60,7 @@ class TestRun(TestDvc): self.assertEqual(len(stage.deps), len(deps)) self.assertEqual(len(stage.outs), len(outs + outs_no_cache)) self.assertEqual(stage.outs[0].fspath, outs[0]) - self.assertEqual(stage.outs[0].hash_info.value, file_md5(self.FOO)[0]) + self.assertEqual(stage.outs[0].hash_info.value, file_md5(self.FOO)) self.assertTrue(stage.path, fname) with self.assertRaises(OutputDuplicationError): diff --git a/tests/func/test_state.py b/tests/func/test_state.py index 888d01e4a..2fbf4fe00 100644 --- a/tests/func/test_state.py +++ b/tests/func/test_state.py @@ -12,7 +12,7 @@ def test_state(tmp_dir, dvc): tmp_dir.gen("foo", "foo content") path = tmp_dir / "foo" path_info = PathInfo(path) - hash_info = HashInfo("md5", file_md5(path)[0]) + hash_info = HashInfo("md5", file_md5(path)) state = State(dvc) @@ -25,7 +25,7 @@ def test_state(tmp_dir, dvc): assert state.get(path_info) is None - hash_info = HashInfo("md5", file_md5(path)[0]) + hash_info = HashInfo("md5", file_md5(path)) state.save(path_info, hash_info) assert state.get(path_info) == hash_info @@ -60,7 +60,7 @@ def test_get_state_record_for_inode(get_inode_mock, tmp_dir, dvc): assert inode != state._to_sqlite(inode) foo = tmp_dir / "foo" - md5 = file_md5(foo)[0] + md5 = file_md5(foo) get_inode_mock.side_effect = mock_get_inode(inode) with state: diff --git a/tests/func/test_utils.py b/tests/func/test_utils.py index b3478e650..2c1166f04 100644 --- a/tests/func/test_utils.py +++ b/tests/func/test_utils.py @@ -4,7 +4,7 @@ from dvc import utils def test_file_md5_crlf(tmp_dir): tmp_dir.gen("cr", b"a\nb\nc") tmp_dir.gen("crlf", b"a\r\nb\r\nc") - assert utils.file_md5("cr")[0] == utils.file_md5("crlf")[0] + assert utils.file_md5("cr") == utils.file_md5("crlf") def test_dict_md5(): diff --git a/tests/unit/utils/test_stream.py b/tests/unit/utils/test_stream.py index bde29cdb2..39d6c0f2d 100644 --- a/tests/unit/utils/test_stream.py +++ b/tests/unit/utils/test_stream.py @@ -10,7 +10,7 @@ def test_hashed_stream_reader(tmp_dir): stream_reader = HashedStreamReader(fobj) assert stream_reader.read(3) == b"foo" - hex_digest, _ = file_md5(foo) + hex_digest = file_md5(foo) assert stream_reader.is_text_file assert hex_digest == stream_reader.hash_info.value @@ -26,6 +26,6 @@ def test_hashed_stream_reader_as_chunks(tmp_dir): if not chunk: break - hex_digest, _ = file_md5(foo) + hex_digest = file_md5(foo) assert not stream_reader.is_text_file assert hex_digest == stream_reader.hash_info.value