stage: always reproduce when --force is given

Fix #1667

dvc/remote/azure.py

@@ -10,7 +10,7 @@ except ImportError:
     BlockBlobService = None
 
 import dvc.logger as logger
-from dvc.utils import tmp_fname
+from dvc.utils import tmp_fname, move
 from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
 from dvc.config import Config
@@ -187,7 +187,7 @@ class RemoteAzure(RemoteBase):
                 msg = "failed to download '{}/{}'".format(bucket, path)
                 logger.warning(msg)
             else:
-                os.rename(tmp_file, to_info["path"])
+                move(tmp_file, to_info["path"])
 
                 if not no_progress_bar:
                     progress.finish_target(name)

dvc/remote/gs.py

@@ -8,7 +8,7 @@ except ImportError:
     storage = None
 
 import dvc.logger as logger
-from dvc.utils import tmp_fname
+from dvc.utils import tmp_fname, move
 from dvc.utils.compat import urlparse, makedirs
 from dvc.remote.base import RemoteBase
 from dvc.config import Config
@@ -207,7 +207,7 @@ class RemoteGS(RemoteBase):
                 )
                 continue
 
-            os.rename(tmp_file, to_info["path"])
+            move(tmp_file, to_info["path"])
 
             if not no_progress_bar:
                 progress.finish_target(name)

dvc/remote/http.py

@@ -10,6 +10,7 @@ from dvc.progress import progress
 from dvc.exceptions import DvcException
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
+from dvc.utils import move
 
 
 class ProgressBarCallback(object):
@@ -147,7 +148,7 @@ class RemoteHTTP(RemoteBase):
             mode, partial_file, request, transferred_bytes, callback
         )
 
-        os.rename(partial_file, target_file)
+        move(partial_file, target_file)
 
     def _write_request_content(
         self, mode, partial_file, request, transferred_bytes, callback=None

dvc/remote/local.py

@@ -534,7 +534,7 @@ class RemoteLOCAL(RemoteBase):
                     name=name,
                 )
 
-                os.rename(tmp_file, to_info["path"])
+                move(tmp_file, to_info["path"])
             except Exception:
                 logger.error(
                     "failed to download '{}' to '{}'".format(

dvc/remote/s3.py

@@ -9,7 +9,7 @@ except ImportError:
     boto3 = None
 
 import dvc.logger as logger
-from dvc.utils import tmp_fname
+from dvc.utils import tmp_fname, move
 from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
 from dvc.config import Config
@@ -251,7 +251,7 @@ class RemoteS3(RemoteBase):
                 logger.error(msg)
                 continue
 
-            os.rename(tmp_file, to_info["path"])
+            move(tmp_file, to_info["path"])
 
             if not no_progress_bar:
                 progress.finish_target(name)

dvc/remote/ssh/connection.py

@@ -160,6 +160,9 @@ class SSHConnection:
             self._sftp.get(src, tmp_file, callback=create_cb(progress_title))
             progress.finish_target(progress_title)
 
+        if os.path.exists(dest):
+            os.remove(dest)
+
         os.rename(tmp_file, dest)
 
     def upload(self, src, dest, no_progress_bar=False, progress_title=None):

dvc/repo/imp.py

@@ -1,7 +1,8 @@
 def imp(self, url, out, resume=False):
     from dvc.stage import Stage
 
-    stage = Stage.create(repo=self, cmd=None, deps=[url], outs=[out])
+    with self.state:
+        stage = Stage.create(repo=self, cmd=None, deps=[url], outs=[out])
 
     if stage is None:
         return None
@@ -9,6 +10,7 @@ def imp(self, url, out, resume=False):
     self.check_dag(self.stages() + [stage])
 
     self.files_to_git_add = []
+
     with self.state:
         stage.run(resume=resume)
 

dvc/stage.py

@@ -289,7 +289,7 @@ class Stage(object):
 
         logger.info("Reproducing '{stage}'".format(stage=self.relpath))
 
-        self.run(dry=dry, no_commit=no_commit)
+        self.run(dry=dry, no_commit=no_commit, force=force)
 
         logger.debug("'{stage}' was reproduced".format(stage=self.relpath))
 
@@ -678,7 +678,7 @@ class Stage(object):
         if p.returncode != 0:
             raise StageCmdFailedError(self)
 
-    def run(self, dry=False, resume=False, no_commit=False):
+    def run(self, dry=False, resume=False, no_commit=False, force=False):
         if self.locked:
             logger.info(
                 "Verifying outputs in locked stage '{stage}'".format(
@@ -695,7 +695,7 @@ class Stage(object):
                 )
             )
             if not dry:
-                if self._already_cached():
+                if self._already_cached() and not force:
                     self.outs[0].checkout()
                 else:
                     self.deps[0].download(
@@ -711,7 +711,11 @@ class Stage(object):
         else:
             logger.info("Running command:\n\t{}".format(self.cmd))
             if not dry:
-                if not self.is_callback and self._already_cached():
+                if (
+                    not force
+                    and not self.is_callback
+                    and self._already_cached()
+                ):
                     self.checkout()
                 else:
                     self._run()

tests/test_repro.py

@@ -21,6 +21,7 @@ from dvc.utils.compat import urljoin
 from dvc.remote.local import RemoteLOCAL
 from dvc.stage import Stage, StageFileDoesNotExistError
 from dvc.system import System
+from dvc.output.base import OutputBase
 from dvc.exceptions import (
     CyclicGraphError,
     StagePathAsOutputError,
@@ -1322,15 +1323,53 @@ class TestReproNoCommit(TestRepro):
 
 class TestReproAlreadyCached(TestRepro):
     def test(self):
-        first_out = self.dvc.run(
-            fname="random.dvc",
+        run_out = self.dvc.run(
+            fname="datetime.dvc",
             deps=[],
-            outs=["random.txt"],
-            cmd='python -c "{}" > random.txt'.format(
-                "from random import random; print(random())"
-            ),
+            outs=["datetime.txt"],
+            cmd='python -c "import time; print(time.time())" > datetime.txt',
         ).outs[0]
 
-        second_out = self.dvc.reproduce(target="random.dvc")[0].outs[0]
+        repro_out = self.dvc.reproduce(target="datetime.dvc")[0].outs[0]
 
-        self.assertNotEqual(first_out.checksum, second_out.checksum)
+        self.assertNotEqual(run_out.checksum, repro_out.checksum)
+
+    def test_force_with_dependencies(self):
+        run_out = self.dvc.run(
+            fname="datetime.dvc",
+            deps=[self.FOO],
+            outs=["datetime.txt"],
+            cmd='python -c "import time; print(time.time())" > datetime.txt',
+        ).outs[0]
+
+        ret = main(["repro", "--force", "datetime.dvc"])
+        self.assertEqual(ret, 0)
+
+        repro_out = Stage.load(self.dvc, "datetime.dvc").outs[0]
+
+        self.assertNotEqual(run_out.checksum, repro_out.checksum)
+
+    def test_force_import(self):
+        ret = main(["import", self.FOO, self.BAR])
+        self.assertEqual(ret, 0)
+
+        patch_download = patch.object(
+            RemoteLOCAL,
+            "download",
+            side_effect=RemoteLOCAL.download,
+            autospec=True,
+        )
+
+        patch_checkout = patch.object(
+            OutputBase,
+            "checkout",
+            side_effect=OutputBase.checkout,
+            autospec=True,
+        )
+
+        with patch_download as mock_download:
+            with patch_checkout as mock_checkout:
+                ret = main(["repro", "--force", "bar.dvc"])
+                self.assertEqual(ret, 0)
+                self.assertEqual(mock_download.call_count, 1)
+                self.assertEqual(mock_checkout.call_count, 0)