push/pull/status -c: should warn on no checksum locally and remotely

dvc/remote/base.py

@@ -13,6 +13,7 @@ from dvc.exceptions import DvcException, ConfirmRemoveError
 
 
 STATUS_OK = 1
+STATUS_MISSING = 2
 STATUS_NEW = 3
 STATUS_DELETED = 4
 
@@ -20,7 +21,7 @@ STATUS_DELETED = 4
 STATUS_MAP = {
     # (local_exists, remote_exists)
     (True, True): STATUS_OK,
-    (False, False): STATUS_OK,
+    (False, False): STATUS_MISSING,
     (True, False): STATUS_NEW,
     (False, True): STATUS_DELETED,
 }

dvc/remote/local.py

@@ -13,7 +13,13 @@ from operator import itemgetter
 
 import dvc.logger as logger
 from dvc.system import System
-from dvc.remote.base import RemoteBase, STATUS_MAP, STATUS_NEW, STATUS_DELETED
+from dvc.remote.base import (
+    RemoteBase,
+    STATUS_MAP,
+    STATUS_NEW,
+    STATUS_DELETED,
+    STATUS_MISSING,
+)
 from dvc.utils import remove, move, copyfile, dict_md5, to_chunks, tmp_fname
 from dvc.utils import LARGE_DIR_SIZE
 from dvc.config import Config
@@ -590,13 +596,17 @@ class RemoteLOCAL(RemoteBase):
 
         progress.finish_target(title)
 
-        for md5, info in ret.items():
-            info["status"] = STATUS_MAP[
-                (md5 in local_exists, md5 in remote_exists)
-            ]
+        self._fill_statuses(ret, local_exists, remote_exists)
+
+        self._log_missing_caches(ret)
 
         return ret
 
+    def _fill_statuses(self, checksum_info_dir, local_exists, remote_exists):
+        for md5, info in checksum_info_dir.items():
+            status = STATUS_MAP[(md5 in local_exists, md5 in remote_exists)]
+            info["status"] = status
+
     def _get_chunks(self, download, remote, status_info, status, jobs):
         title = "Analysing status."
 
@@ -710,3 +720,22 @@ class RemoteLOCAL(RemoteBase):
             return not (mtime == cached_mtime and size == cached_size)
 
         return True
+
+    def _log_missing_caches(self, checksum_info_dict):
+        missing_caches = [
+            (md5, info)
+            for md5, info in checksum_info_dict.items()
+            if info["status"] == STATUS_MISSING
+        ]
+        if missing_caches:
+            missing_desc = "".join(
+                [
+                    "\nname: {}, md5: {}".format(info["name"], md5)
+                    for md5, info in missing_caches
+                ]
+            )
+            msg = (
+                "Some of the cache files do not exist neither locally "
+                "nor on remote. Missing cache files: {}".format(missing_desc)
+            )
+            logger.warning(msg)

tests/test_add.py

@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import os
-from dvc.utils.compat import StringIO
 
 import dvc
 import time
@@ -21,7 +20,7 @@ from dvc.output.base import OutputAlreadyTrackedError
 from dvc.repo import Repo as DvcRepo
 
 from tests.basic_env import TestDvc
-from tests.utils import spy, get_gitignore_content
+from tests.utils import spy, reset_logger_error_output, get_gitignore_content
 from tests.utils.logger import MockLoggerHandlers, ConsoleFontColorsRemover
 
 
@@ -410,7 +409,7 @@ class TestShouldPlaceStageInDataDirIfRepositoryBelowSymlink(TestDvc):
 class TestShouldThrowProperExceptionOnCorruptedStageFile(TestDvc):
     def test(self):
         with MockLoggerHandlers(logger), ConsoleFontColorsRemover():
-            logger.handlers[1].stream = StringIO()
+            reset_logger_error_output()
 
             ret = main(["add", self.FOO])
             self.assertEqual(0, ret)

tests/test_data_cloud.py

@@ -13,7 +13,7 @@ from dvc.state import State
 from mock import patch
 from tests.utils.logger import MockLoggerHandlers, ConsoleFontColorsRemover
 
-import dvc.logger as logger
+from dvc.logger import logger
 from dvc.utils.compat import str
 from dvc.main import main
 from dvc.config import Config
@@ -31,9 +31,12 @@ from dvc.remote.base import STATUS_OK, STATUS_NEW, STATUS_DELETED
 from dvc.utils import file_md5, load_stage_file
 
 from tests.basic_env import TestDvc
-from tests.utils import spy
+from tests.utils import (
+    spy,
+    reset_logger_standard_output,
+    reset_logger_error_output,
+)
 
-from dvc.utils.compat import StringIO
 
 TEST_REMOTE = "upstream"
 TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE)
@@ -635,15 +638,15 @@ class TestWarnOnOutdatedStage(TestDvc):
         with open(stage_file_path, "w") as stage_file:
             yaml.dump(content, stage_file)
 
-        with MockLoggerHandlers(logger.logger):
-            logger.logger.handlers[0].stream = StringIO()
+        with MockLoggerHandlers(logger):
+            reset_logger_standard_output()
             self.main(["status", "-c"])
             self.assertIn(
                 "Warning: Output 'bar'(Stage: 'bar.dvc') is "
                 "missing version info. Cache for it will not be "
                 "collected. Use dvc repro to get your pipeline up to "
                 "date.",
-                logger.logger.handlers[0].stream.getvalue(),
+                logger.handlers[0].stream.getvalue(),
             )
 
     def test(self):
@@ -749,3 +752,62 @@ class TestCheckSumRecalculation(TestDvc):
             ret = main(["run", "-d", self.FOO, "echo foo"])
             self.assertEqual(ret, 0)
         self.assertEqual(test_collect.mock.call_count, 1)
+
+
+class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc):
+    def setUp(self):
+        super(TestShouldWarnOnNoChecksumInLocalAndRemoteCache, self).setUp()
+
+        cache_dir = self.mkdtemp()
+        ret = main(["add", self.FOO])
+        self.assertEqual(0, ret)
+
+        ret = main(["add", self.BAR])
+        self.assertEqual(0, ret)
+
+        # purge cache
+        shutil.rmtree(self.dvc.cache.local.cache_dir)
+
+        ret = main(["remote", "add", "remote_name", "-d", cache_dir])
+        self.assertEqual(0, ret)
+
+        checksum_foo = file_md5(self.FOO)[0]
+        checksum_bar = file_md5(self.BAR)[0]
+        self.message_header = (
+            "Some of the cache files do not exist neither locally "
+            "nor on remote. Missing cache files: \n"
+        )
+        self.message_bar_part = "\nname: {}, md5: {}\n".format(
+            self.BAR, checksum_bar
+        )
+        self.message_foo_part = "\nname: {}, md5: {}\n".format(
+            self.FOO, checksum_foo
+        )
+
+    def stderr_contains_message(self):
+        self.assertIn(
+            self.message_header, logger.handlers[1].stream.getvalue()
+        )
+        self.assertIn(
+            self.message_foo_part, logger.handlers[1].stream.getvalue()
+        )
+        self.assertIn(
+            self.message_bar_part, logger.handlers[1].stream.getvalue()
+        )
+
+    def test(self):
+        with ConsoleFontColorsRemover(), MockLoggerHandlers(logger):
+            reset_logger_error_output()
+
+            main(["push"])
+            self.stderr_contains_message()
+
+            reset_logger_error_output()
+
+            main(["pull"])
+            self.stderr_contains_message()
+
+            reset_logger_error_output()
+
+            main(["status", "-c"])
+            self.stderr_contains_message()

tests/test_import.py

@@ -9,11 +9,10 @@ from dvc.exceptions import DvcException
 from dvc.main import main
 from mock import patch, mock_open, call
 from tests.basic_env import TestDvc
+from tests.utils import reset_logger_error_output
 from tests.utils.httpd import StaticFileServer
 from tests.utils.logger import MockLoggerHandlers, ConsoleFontColorsRemover
 
-from dvc.utils.compat import StringIO
-
 
 class TestCmdImport(TestDvc):
     def test(self):
@@ -52,7 +51,7 @@ class TestFailedImportMessage(TestDvc):
     @patch("dvc.command.imp.urlparse")
     def _test(self, imp_urlparse_patch):
         with MockLoggerHandlers(logger.logger):
-            logger.logger.handlers[1].stream = StringIO()
+            reset_logger_error_output()
             page_address = "http://somesite.com/file_name"
 
             def dvc_exception(*args, **kwargs):

tests/test_logger.py

@@ -5,7 +5,7 @@ import logging
 import dvc.logger as logger
 from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
-from dvc.utils.compat import StringIO
+from tests.utils import reset_logger_error_output, reset_logger_standard_output
 from tests.utils.logger import ConsoleFontColorsRemover
 
 
@@ -17,8 +17,8 @@ class TestLogger(TestCase):
             logger.logging.StreamHandler(),
             logger.logging.StreamHandler(),
         ]
-        logger.logger.handlers[0].stream = StringIO()
-        logger.logger.handlers[1].stream = StringIO()
+        reset_logger_standard_output()
+        reset_logger_error_output()
         logger.set_default_level()
 
         self.consoleColorRemover = ConsoleFontColorsRemover()

tests/test_metrics.py

@@ -7,7 +7,7 @@ from dvc.exceptions import DvcException, BadMetricError, NoMetricsError
 from tests.basic_env import TestDvc
 
 import dvc.logger as logger
-from dvc.utils.compat import StringIO
+from tests.utils import reset_logger_error_output, reset_logger_standard_output
 from tests.utils.logger import MockLoggerHandlers
 
 
@@ -359,7 +359,7 @@ class TestMetricsCLI(TestMetrics):
         self.assertEqual(ret, 0)
 
         with MockLoggerHandlers(logger.logger):
-            logger.logger.handlers[1].stream = StringIO()
+            reset_logger_error_output()
             ret = main(["metrics", "add", "metric.unknown", "-t", "unknown"])
             self.assertEqual(ret, 1)
             self.assertIn(
@@ -372,7 +372,7 @@ class TestMetricsCLI(TestMetrics):
             ret = main(["metrics", "add", "metric.unknown", "-t", "raw"])
             self.assertEqual(ret, 0)
 
-            logger.logger.handlers[0].stream = StringIO()
+            reset_logger_standard_output()
             ret = main(["metrics", "show", "metric.unknown"])
             self.assertEqual(ret, 0)
             self.assertIn(
@@ -389,7 +389,7 @@ class TestMetricsCLI(TestMetrics):
         self.assertEqual(ret, 0)
 
         with MockLoggerHandlers(logger.logger):
-            logger.logger.handlers[1].stream = StringIO()
+            reset_logger_error_output()
             ret = main(
                 ["metrics", "modify", "metric.unknown", "-t", "unknown"]
             )
@@ -404,7 +404,7 @@ class TestMetricsCLI(TestMetrics):
             ret = main(["metrics", "modify", "metric.unknown", "-t", "CSV"])
             self.assertEqual(ret, 0)
 
-            logger.logger.handlers[0].stream = StringIO()
+            reset_logger_standard_output()
             ret = main(["metrics", "show", "metric.unknown"])
             self.assertEqual(ret, 0)
             self.assertIn(
@@ -421,7 +421,7 @@ class TestMetricsCLI(TestMetrics):
         self.assertEqual(ret, 0)
 
         with MockLoggerHandlers(logger.logger):
-            logger.logger.handlers[0].stream = StringIO()
+            reset_logger_standard_output()
             ret = main(
                 [
                     "metrics",

tests/utils/__init__.py

@@ -2,6 +2,7 @@ import yaml
 import os
 
 from dvc.scm import Git
+from dvc.utils.compat import StringIO
 from mock import MagicMock
 from contextlib import contextmanager
 
@@ -27,6 +28,18 @@ def load_stage_file(path):
         return yaml.safe_load(fobj)
 
 
+def reset_logger_error_output():
+    from dvc.logger import logger
+
+    logger.handlers[1].stream = StringIO()
+
+
+def reset_logger_standard_output():
+    from dvc.logger import logger
+
+    logger.handlers[0].stream = StringIO()
+
+
 @contextmanager
 def cd(newdir):
     prevdir = os.getcwd()