diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 005b1bdc6..a4965cba6 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -13,6 +13,7 @@ from dvc.exceptions import DvcException, ConfirmRemoveError STATUS_OK = 1 +STATUS_MISSING = 2 STATUS_NEW = 3 STATUS_DELETED = 4 @@ -20,7 +21,7 @@ STATUS_DELETED = 4 STATUS_MAP = { # (local_exists, remote_exists) (True, True): STATUS_OK, - (False, False): STATUS_OK, + (False, False): STATUS_MISSING, (True, False): STATUS_NEW, (False, True): STATUS_DELETED, } diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 54bc0a8ad..44cdb06f0 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -13,7 +13,13 @@ from operator import itemgetter import dvc.logger as logger from dvc.system import System -from dvc.remote.base import RemoteBase, STATUS_MAP, STATUS_NEW, STATUS_DELETED +from dvc.remote.base import ( + RemoteBase, + STATUS_MAP, + STATUS_NEW, + STATUS_DELETED, + STATUS_MISSING, +) from dvc.utils import remove, move, copyfile, dict_md5, to_chunks, tmp_fname from dvc.utils import LARGE_DIR_SIZE from dvc.config import Config @@ -590,13 +596,17 @@ class RemoteLOCAL(RemoteBase): progress.finish_target(title) - for md5, info in ret.items(): - info["status"] = STATUS_MAP[ - (md5 in local_exists, md5 in remote_exists) - ] + self._fill_statuses(ret, local_exists, remote_exists) + + self._log_missing_caches(ret) return ret + def _fill_statuses(self, checksum_info_dir, local_exists, remote_exists): + for md5, info in checksum_info_dir.items(): + status = STATUS_MAP[(md5 in local_exists, md5 in remote_exists)] + info["status"] = status + def _get_chunks(self, download, remote, status_info, status, jobs): title = "Analysing status." @@ -710,3 +720,22 @@ class RemoteLOCAL(RemoteBase): return not (mtime == cached_mtime and size == cached_size) return True + + def _log_missing_caches(self, checksum_info_dict): + missing_caches = [ + (md5, info) + for md5, info in checksum_info_dict.items() + if info["status"] == STATUS_MISSING + ] + if missing_caches: + missing_desc = "".join( + [ + "\nname: {}, md5: {}".format(info["name"], md5) + for md5, info in missing_caches + ] + ) + msg = ( + "Some of the cache files do not exist neither locally " + "nor on remote. Missing cache files: {}".format(missing_desc) + ) + logger.warning(msg) diff --git a/tests/test_add.py b/tests/test_add.py index 35ddddd07..a72c419df 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import os -from dvc.utils.compat import StringIO import dvc import time @@ -21,7 +20,7 @@ from dvc.output.base import OutputAlreadyTrackedError from dvc.repo import Repo as DvcRepo from tests.basic_env import TestDvc -from tests.utils import spy, get_gitignore_content +from tests.utils import spy, reset_logger_error_output, get_gitignore_content from tests.utils.logger import MockLoggerHandlers, ConsoleFontColorsRemover @@ -410,7 +409,7 @@ class TestShouldPlaceStageInDataDirIfRepositoryBelowSymlink(TestDvc): class TestShouldThrowProperExceptionOnCorruptedStageFile(TestDvc): def test(self): with MockLoggerHandlers(logger), ConsoleFontColorsRemover(): - logger.handlers[1].stream = StringIO() + reset_logger_error_output() ret = main(["add", self.FOO]) self.assertEqual(0, ret) diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index 6eb826931..1fd864558 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -13,7 +13,7 @@ from dvc.state import State from mock import patch from tests.utils.logger import MockLoggerHandlers, ConsoleFontColorsRemover -import dvc.logger as logger +from dvc.logger import logger from dvc.utils.compat import str from dvc.main import main from dvc.config import Config @@ -31,9 +31,12 @@ from dvc.remote.base import STATUS_OK, STATUS_NEW, STATUS_DELETED from dvc.utils import file_md5, load_stage_file from tests.basic_env import TestDvc -from tests.utils import spy +from tests.utils import ( + spy, + reset_logger_standard_output, + reset_logger_error_output, +) -from dvc.utils.compat import StringIO TEST_REMOTE = "upstream" TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE) @@ -635,15 +638,15 @@ class TestWarnOnOutdatedStage(TestDvc): with open(stage_file_path, "w") as stage_file: yaml.dump(content, stage_file) - with MockLoggerHandlers(logger.logger): - logger.logger.handlers[0].stream = StringIO() + with MockLoggerHandlers(logger): + reset_logger_standard_output() self.main(["status", "-c"]) self.assertIn( "Warning: Output 'bar'(Stage: 'bar.dvc') is " "missing version info. Cache for it will not be " "collected. Use dvc repro to get your pipeline up to " "date.", - logger.logger.handlers[0].stream.getvalue(), + logger.handlers[0].stream.getvalue(), ) def test(self): @@ -749,3 +752,62 @@ class TestCheckSumRecalculation(TestDvc): ret = main(["run", "-d", self.FOO, "echo foo"]) self.assertEqual(ret, 0) self.assertEqual(test_collect.mock.call_count, 1) + + +class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc): + def setUp(self): + super(TestShouldWarnOnNoChecksumInLocalAndRemoteCache, self).setUp() + + cache_dir = self.mkdtemp() + ret = main(["add", self.FOO]) + self.assertEqual(0, ret) + + ret = main(["add", self.BAR]) + self.assertEqual(0, ret) + + # purge cache + shutil.rmtree(self.dvc.cache.local.cache_dir) + + ret = main(["remote", "add", "remote_name", "-d", cache_dir]) + self.assertEqual(0, ret) + + checksum_foo = file_md5(self.FOO)[0] + checksum_bar = file_md5(self.BAR)[0] + self.message_header = ( + "Some of the cache files do not exist neither locally " + "nor on remote. Missing cache files: \n" + ) + self.message_bar_part = "\nname: {}, md5: {}\n".format( + self.BAR, checksum_bar + ) + self.message_foo_part = "\nname: {}, md5: {}\n".format( + self.FOO, checksum_foo + ) + + def stderr_contains_message(self): + self.assertIn( + self.message_header, logger.handlers[1].stream.getvalue() + ) + self.assertIn( + self.message_foo_part, logger.handlers[1].stream.getvalue() + ) + self.assertIn( + self.message_bar_part, logger.handlers[1].stream.getvalue() + ) + + def test(self): + with ConsoleFontColorsRemover(), MockLoggerHandlers(logger): + reset_logger_error_output() + + main(["push"]) + self.stderr_contains_message() + + reset_logger_error_output() + + main(["pull"]) + self.stderr_contains_message() + + reset_logger_error_output() + + main(["status", "-c"]) + self.stderr_contains_message() diff --git a/tests/test_import.py b/tests/test_import.py index 61770c2a2..fa8d67ccf 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -9,11 +9,10 @@ from dvc.exceptions import DvcException from dvc.main import main from mock import patch, mock_open, call from tests.basic_env import TestDvc +from tests.utils import reset_logger_error_output from tests.utils.httpd import StaticFileServer from tests.utils.logger import MockLoggerHandlers, ConsoleFontColorsRemover -from dvc.utils.compat import StringIO - class TestCmdImport(TestDvc): def test(self): @@ -52,7 +51,7 @@ class TestFailedImportMessage(TestDvc): @patch("dvc.command.imp.urlparse") def _test(self, imp_urlparse_patch): with MockLoggerHandlers(logger.logger): - logger.logger.handlers[1].stream = StringIO() + reset_logger_error_output() page_address = "http://somesite.com/file_name" def dvc_exception(*args, **kwargs): diff --git a/tests/test_logger.py b/tests/test_logger.py index 2e8a4e36c..72a3ef21a 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -5,7 +5,7 @@ import logging import dvc.logger as logger from dvc.command.base import CmdBase from dvc.exceptions import DvcException -from dvc.utils.compat import StringIO +from tests.utils import reset_logger_error_output, reset_logger_standard_output from tests.utils.logger import ConsoleFontColorsRemover @@ -17,8 +17,8 @@ class TestLogger(TestCase): logger.logging.StreamHandler(), logger.logging.StreamHandler(), ] - logger.logger.handlers[0].stream = StringIO() - logger.logger.handlers[1].stream = StringIO() + reset_logger_standard_output() + reset_logger_error_output() logger.set_default_level() self.consoleColorRemover = ConsoleFontColorsRemover() diff --git a/tests/test_metrics.py b/tests/test_metrics.py index b17dfece8..78456381b 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -7,7 +7,7 @@ from dvc.exceptions import DvcException, BadMetricError, NoMetricsError from tests.basic_env import TestDvc import dvc.logger as logger -from dvc.utils.compat import StringIO +from tests.utils import reset_logger_error_output, reset_logger_standard_output from tests.utils.logger import MockLoggerHandlers @@ -359,7 +359,7 @@ class TestMetricsCLI(TestMetrics): self.assertEqual(ret, 0) with MockLoggerHandlers(logger.logger): - logger.logger.handlers[1].stream = StringIO() + reset_logger_error_output() ret = main(["metrics", "add", "metric.unknown", "-t", "unknown"]) self.assertEqual(ret, 1) self.assertIn( @@ -372,7 +372,7 @@ class TestMetricsCLI(TestMetrics): ret = main(["metrics", "add", "metric.unknown", "-t", "raw"]) self.assertEqual(ret, 0) - logger.logger.handlers[0].stream = StringIO() + reset_logger_standard_output() ret = main(["metrics", "show", "metric.unknown"]) self.assertEqual(ret, 0) self.assertIn( @@ -389,7 +389,7 @@ class TestMetricsCLI(TestMetrics): self.assertEqual(ret, 0) with MockLoggerHandlers(logger.logger): - logger.logger.handlers[1].stream = StringIO() + reset_logger_error_output() ret = main( ["metrics", "modify", "metric.unknown", "-t", "unknown"] ) @@ -404,7 +404,7 @@ class TestMetricsCLI(TestMetrics): ret = main(["metrics", "modify", "metric.unknown", "-t", "CSV"]) self.assertEqual(ret, 0) - logger.logger.handlers[0].stream = StringIO() + reset_logger_standard_output() ret = main(["metrics", "show", "metric.unknown"]) self.assertEqual(ret, 0) self.assertIn( @@ -421,7 +421,7 @@ class TestMetricsCLI(TestMetrics): self.assertEqual(ret, 0) with MockLoggerHandlers(logger.logger): - logger.logger.handlers[0].stream = StringIO() + reset_logger_standard_output() ret = main( [ "metrics", diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 5188d19a2..39bb8be57 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -2,6 +2,7 @@ import yaml import os from dvc.scm import Git +from dvc.utils.compat import StringIO from mock import MagicMock from contextlib import contextmanager @@ -27,6 +28,18 @@ def load_stage_file(path): return yaml.safe_load(fobj) +def reset_logger_error_output(): + from dvc.logger import logger + + logger.handlers[1].stream = StringIO() + + +def reset_logger_standard_output(): + from dvc.logger import logger + + logger.handlers[0].stream = StringIO() + + @contextmanager def cd(newdir): prevdir = os.getcwd()