From 599ee58a5fca35246ec7a71daed2412bca9f0050 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Thu, 28 Feb 2019 18:01:07 -0600 Subject: [PATCH] stage: always reproduce when --force is given Fix #1667 --- dvc/remote/azure.py | 4 +-- dvc/remote/gs.py | 4 +-- dvc/remote/http.py | 3 +- dvc/remote/local.py | 2 +- dvc/remote/s3.py | 4 +-- dvc/remote/ssh/connection.py | 3 ++ dvc/repo/imp.py | 4 ++- dvc/stage.py | 12 +++++--- tests/test_repro.py | 55 ++++++++++++++++++++++++++++++------ 9 files changed, 70 insertions(+), 21 deletions(-) diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 3853a9d67..740e57b3e 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -10,7 +10,7 @@ except ImportError: BlockBlobService = None import dvc.logger as logger -from dvc.utils import tmp_fname +from dvc.utils import tmp_fname, move from dvc.utils.compat import urlparse, makedirs from dvc.progress import progress from dvc.config import Config @@ -187,7 +187,7 @@ class RemoteAzure(RemoteBase): msg = "failed to download '{}/{}'".format(bucket, path) logger.warning(msg) else: - os.rename(tmp_file, to_info["path"]) + move(tmp_file, to_info["path"]) if not no_progress_bar: progress.finish_target(name) diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 402680e1f..dd57ad9d2 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -8,7 +8,7 @@ except ImportError: storage = None import dvc.logger as logger -from dvc.utils import tmp_fname +from dvc.utils import tmp_fname, move from dvc.utils.compat import urlparse, makedirs from dvc.remote.base import RemoteBase from dvc.config import Config @@ -207,7 +207,7 @@ class RemoteGS(RemoteBase): ) continue - os.rename(tmp_file, to_info["path"]) + move(tmp_file, to_info["path"]) if not no_progress_bar: progress.finish_target(name) diff --git a/dvc/remote/http.py b/dvc/remote/http.py index 35fd8b10e..ed587f35b 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -10,6 +10,7 @@ from dvc.progress import progress from dvc.exceptions import DvcException from dvc.config import Config from dvc.remote.base import RemoteBase +from dvc.utils import move class ProgressBarCallback(object): @@ -147,7 +148,7 @@ class RemoteHTTP(RemoteBase): mode, partial_file, request, transferred_bytes, callback ) - os.rename(partial_file, target_file) + move(partial_file, target_file) def _write_request_content( self, mode, partial_file, request, transferred_bytes, callback=None diff --git a/dvc/remote/local.py b/dvc/remote/local.py index cf5f3b360..54bc0a8ad 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -534,7 +534,7 @@ class RemoteLOCAL(RemoteBase): name=name, ) - os.rename(tmp_file, to_info["path"]) + move(tmp_file, to_info["path"]) except Exception: logger.error( "failed to download '{}' to '{}'".format( diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index a9216ec62..bdd113061 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -9,7 +9,7 @@ except ImportError: boto3 = None import dvc.logger as logger -from dvc.utils import tmp_fname +from dvc.utils import tmp_fname, move from dvc.utils.compat import urlparse, makedirs from dvc.progress import progress from dvc.config import Config @@ -251,7 +251,7 @@ class RemoteS3(RemoteBase): logger.error(msg) continue - os.rename(tmp_file, to_info["path"]) + move(tmp_file, to_info["path"]) if not no_progress_bar: progress.finish_target(name) diff --git a/dvc/remote/ssh/connection.py b/dvc/remote/ssh/connection.py index cc28692cd..7d34aed0c 100644 --- a/dvc/remote/ssh/connection.py +++ b/dvc/remote/ssh/connection.py @@ -160,6 +160,9 @@ class SSHConnection: self._sftp.get(src, tmp_file, callback=create_cb(progress_title)) progress.finish_target(progress_title) + if os.path.exists(dest): + os.remove(dest) + os.rename(tmp_file, dest) def upload(self, src, dest, no_progress_bar=False, progress_title=None): diff --git a/dvc/repo/imp.py b/dvc/repo/imp.py index a51ab7f48..39e1e00f3 100644 --- a/dvc/repo/imp.py +++ b/dvc/repo/imp.py @@ -1,7 +1,8 @@ def imp(self, url, out, resume=False): from dvc.stage import Stage - stage = Stage.create(repo=self, cmd=None, deps=[url], outs=[out]) + with self.state: + stage = Stage.create(repo=self, cmd=None, deps=[url], outs=[out]) if stage is None: return None @@ -9,6 +10,7 @@ def imp(self, url, out, resume=False): self.check_dag(self.stages() + [stage]) self.files_to_git_add = [] + with self.state: stage.run(resume=resume) diff --git a/dvc/stage.py b/dvc/stage.py index 281f3463c..ef2f7c8dc 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -289,7 +289,7 @@ class Stage(object): logger.info("Reproducing '{stage}'".format(stage=self.relpath)) - self.run(dry=dry, no_commit=no_commit) + self.run(dry=dry, no_commit=no_commit, force=force) logger.debug("'{stage}' was reproduced".format(stage=self.relpath)) @@ -678,7 +678,7 @@ class Stage(object): if p.returncode != 0: raise StageCmdFailedError(self) - def run(self, dry=False, resume=False, no_commit=False): + def run(self, dry=False, resume=False, no_commit=False, force=False): if self.locked: logger.info( "Verifying outputs in locked stage '{stage}'".format( @@ -695,7 +695,7 @@ class Stage(object): ) ) if not dry: - if self._already_cached(): + if self._already_cached() and not force: self.outs[0].checkout() else: self.deps[0].download( @@ -711,7 +711,11 @@ class Stage(object): else: logger.info("Running command:\n\t{}".format(self.cmd)) if not dry: - if not self.is_callback and self._already_cached(): + if ( + not force + and not self.is_callback + and self._already_cached() + ): self.checkout() else: self._run() diff --git a/tests/test_repro.py b/tests/test_repro.py index 89a40696d..66154c3a4 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -21,6 +21,7 @@ from dvc.utils.compat import urljoin from dvc.remote.local import RemoteLOCAL from dvc.stage import Stage, StageFileDoesNotExistError from dvc.system import System +from dvc.output.base import OutputBase from dvc.exceptions import ( CyclicGraphError, StagePathAsOutputError, @@ -1322,15 +1323,53 @@ class TestReproNoCommit(TestRepro): class TestReproAlreadyCached(TestRepro): def test(self): - first_out = self.dvc.run( - fname="random.dvc", + run_out = self.dvc.run( + fname="datetime.dvc", deps=[], - outs=["random.txt"], - cmd='python -c "{}" > random.txt'.format( - "from random import random; print(random())" - ), + outs=["datetime.txt"], + cmd='python -c "import time; print(time.time())" > datetime.txt', ).outs[0] - second_out = self.dvc.reproduce(target="random.dvc")[0].outs[0] + repro_out = self.dvc.reproduce(target="datetime.dvc")[0].outs[0] - self.assertNotEqual(first_out.checksum, second_out.checksum) + self.assertNotEqual(run_out.checksum, repro_out.checksum) + + def test_force_with_dependencies(self): + run_out = self.dvc.run( + fname="datetime.dvc", + deps=[self.FOO], + outs=["datetime.txt"], + cmd='python -c "import time; print(time.time())" > datetime.txt', + ).outs[0] + + ret = main(["repro", "--force", "datetime.dvc"]) + self.assertEqual(ret, 0) + + repro_out = Stage.load(self.dvc, "datetime.dvc").outs[0] + + self.assertNotEqual(run_out.checksum, repro_out.checksum) + + def test_force_import(self): + ret = main(["import", self.FOO, self.BAR]) + self.assertEqual(ret, 0) + + patch_download = patch.object( + RemoteLOCAL, + "download", + side_effect=RemoteLOCAL.download, + autospec=True, + ) + + patch_checkout = patch.object( + OutputBase, + "checkout", + side_effect=OutputBase.checkout, + autospec=True, + ) + + with patch_download as mock_download: + with patch_checkout as mock_checkout: + ret = main(["repro", "--force", "bar.dvc"]) + self.assertEqual(ret, 0) + self.assertEqual(mock_download.call_count, 1) + self.assertEqual(mock_checkout.call_count, 0)