From 482dfaf9986ca7a2facc0524b3bec83aea5aeef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Redzy=C5=84ski?= Date: Wed, 22 Jan 2020 10:53:26 +0100 Subject: [PATCH] data cloud: refactor verify checksums test --- dvc/data_cloud.py | 3 +-- tests/func/test_data_cloud.py | 37 ++++++++++++++++++++++++++++++++++- tests/func/test_remote.py | 31 ----------------------------- 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index bf4af73f1..e85c8b280 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -89,8 +89,7 @@ class DataCloud(object): if self.repo.cache.local.exists(cache_file): # We can safely save here, as existing corrupted files will be # removed upon status, while files corrupted during download - # will not be copied from tmp_file (see download - # implementation in `remote/base`) + # will not be moved from tmp_file (see `RemoteBASE.download()`) self.repo.state.save(cache_file, checksum) def status(self, cache, jobs=None, remote=None, show_checksums=False): diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 748a7fe63..137fd674c 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -9,10 +9,11 @@ import pytest from mock import patch from dvc.cache import NamedCache +from dvc.compat import fspath from dvc.config import Config from dvc.data_cloud import DataCloud from dvc.main import main -from dvc.remote import RemoteAZURE +from dvc.remote import RemoteAZURE, RemoteConfig from dvc.remote import RemoteGDrive from dvc.remote import RemoteGS from dvc.remote import RemoteHDFS @@ -653,3 +654,37 @@ class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc): assert self.message_header in self._caplog.text assert self.message_foo_part in self._caplog.text assert self.message_bar_part in self._caplog.text + + +def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory): + tmp_dir.dvc_gen({"file": "file1 content"}, commit="add file") + tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir") + + RemoteConfig(dvc.config).add( + "local_remote", + fspath(tmp_path_factory.mktemp("local_remote")), + default=True, + ) + dvc.push() + + # remove artifacts and cache to trigger fetching + os.remove("file") + shutil.rmtree("dir") + shutil.rmtree(dvc.cache.local.cache_dir) + + checksum_spy = mocker.spy(dvc.cache.local, "get_file_checksum") + + dvc.pull() + assert checksum_spy.call_count == 0 + + # Removing cache will invalidate existing state entries + shutil.rmtree(dvc.cache.local.cache_dir) + + dvc.config.set( + Config.SECTION_REMOTE_FMT.format("local_remote"), + Config.SECTION_REMOTE_VERIFY, + "True", + ) + + dvc.pull() + assert checksum_spy.call_count == 3 diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 44580e571..c6d1cffe9 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -13,7 +13,6 @@ from dvc.path_info import PathInfo from dvc.remote import RemoteLOCAL, RemoteConfig from dvc.remote.base import RemoteBASE from dvc.compat import fspath -from dvc.scm import Git from tests.basic_env import TestDvc from tests.remotes import Local @@ -258,33 +257,3 @@ def test_modify_missing_remote(dvc): with pytest.raises(ConfigError, match=r"unable to find remote section"): remote_config.modify("myremote", "gdrive_client_id", "xxx") - - -def test_verify_checksums(tmp_dir, erepo_dir, mocker): - with erepo_dir.chdir(): - erepo_dir.dvc_gen({"file": "file1 content"}, commit="add file") - erepo_dir.dvc_gen( - {"dir": {"subfile": "file2 content"}}, commit="add " "dir" - ) - - Git.clone(fspath(erepo_dir), fspath(tmp_dir)) - - from dvc.repo import Repo - - dvc = Repo(fspath(tmp_dir)) - - file_md5_spy = mocker.spy(dvc.cache.local, "get_file_checksum") - - dvc.pull() - assert file_md5_spy.call_count == 0 - - # Removing cache will invalidate existing state entries - shutil.rmtree(dvc.cache.local.cache_dir) - dvc.config.set( - Config.SECTION_REMOTE_FMT.format("upstream"), - Config.SECTION_REMOTE_VERIFY, - "True", - ) - - dvc.pull() - assert file_md5_spy.call_count == 3