diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index 2802b1b15..de2654117 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -12,6 +12,7 @@ class CmdCheckout(CmdBase): targets=self.args.targets, with_deps=self.args.with_deps, force=self.args.force, + relink=self.args.relink, recursive=self.args.recursive, ) return 0 @@ -34,6 +35,12 @@ def add_parser(subparsers, parent_parser): default=False, help="Do not prompt when removing working directory files.", ) + checkout_parser.add_argument( + "--relink", + action="store_true", + default=False, + help="Relink all files to cache, including unchanged.", + ) checkout_parser.add_argument( "-d", "--with-deps", diff --git a/dvc/output/base.py b/dvc/output/base.py index 8344f1f3d..ffb68ac6a 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -282,7 +282,9 @@ class OutputBase(object): def download(self, to): self.remote.download(self.path_info, to.path_info) - def checkout(self, force=False, progress_callback=None, tag=None): + def checkout( + self, force=False, progress_callback=None, tag=None, relink=False + ): if not self.use_cache: if progress_callback: progress_callback(str(self.path_info), self.get_files_number()) @@ -298,6 +300,7 @@ class OutputBase(object): info, force=force, progress_callback=progress_callback, + relink=relink, ) def remove(self, ignore_remove=False): diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 8c94f152f..f72e56127 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -836,7 +836,7 @@ class RemoteBASE(object): pass def _checkout_dir( - self, path_info, checksum, force, progress_callback=None + self, path_info, checksum, force, progress_callback=None, relink=False ): # Create dir separately so that dir is created # even if there are no files in it @@ -854,9 +854,8 @@ class RemoteBASE(object): entry_info = path_info / relative_path entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum} - if self.changed(entry_info, entry_checksum_info): - if self.exists(entry_info): - self.safe_remove(entry_info, force=force) + if relink or self.changed(entry_info, entry_checksum_info): + self.safe_remove(entry_info, force=force) self.link(entry_cache_info, entry_info) self.state.save(entry_info, entry_checksum) if progress_callback: @@ -878,7 +877,12 @@ class RemoteBASE(object): self.safe_remove(path, force) def checkout( - self, path_info, checksum_info, force=False, progress_callback=None + self, + path_info, + checksum_info, + force=False, + progress_callback=None, + relink=False, ): if path_info.scheme not in ["local", self.scheme]: raise NotImplementedError @@ -894,7 +898,7 @@ class RemoteBASE(object): self.safe_remove(path_info, force=force) failed = path_info - elif not self.changed(path_info, checksum_info): + elif not relink and not self.changed(path_info, checksum_info): msg = "Data '{}' didn't change." logger.debug(msg.format(str(path_info))) skip = True @@ -915,18 +919,23 @@ class RemoteBASE(object): msg = "Checking out '{}' with cache '{}'." logger.debug(msg.format(str(path_info), checksum)) - self._checkout(path_info, checksum, force, progress_callback) + self._checkout(path_info, checksum, force, progress_callback, relink) return None def _checkout( - self, path_info, checksum, force=False, progress_callback=None + self, + path_info, + checksum, + force=False, + progress_callback=None, + relink=False, ): if not self.is_dir_checksum(checksum): return self._checkout_file( path_info, checksum, force, progress_callback=progress_callback ) return self._checkout_dir( - path_info, checksum, force, progress_callback=progress_callback + path_info, checksum, force, progress_callback, relink ) def get_files_number(self, checksum): diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index d4e4b805f..040590d86 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -25,7 +25,12 @@ def get_all_files_numbers(stages): def _checkout( - self, targets=None, with_deps=False, force=False, recursive=False + self, + targets=None, + with_deps=False, + force=False, + relink=False, + recursive=False, ): from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError @@ -55,7 +60,11 @@ def _checkout( ) as pbar: for stage in stages: failed.extend( - stage.checkout(force=force, progress_callback=pbar.update_desc) + stage.checkout( + force=force, + progress_callback=pbar.update_desc, + relink=relink, + ) ) if failed: raise CheckoutError(failed) diff --git a/dvc/stage.py b/dvc/stage.py index c3c5daec9..977347a0e 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -904,11 +904,14 @@ class Stage(object): if paths: raise MissingDataSource(paths) - def checkout(self, force=False, progress_callback=None): + def checkout(self, force=False, progress_callback=None, relink=False): failed_checkouts = [] for out in self.outs: failed = out.checkout( - force=force, tag=self.tag, progress_callback=progress_callback + force=force, + tag=self.tag, + progress_callback=progress_callback, + relink=relink, ) if failed: failed_checkouts.append(failed) diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index f2249bee1..5ff3b5f40 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -26,7 +26,6 @@ from dvc.utils.stage import load_stage_file from tests.basic_env import TestDvc from tests.basic_env import TestDvcGit from tests.func.test_repro import TestRepro -from tests.utils import spy logger = logging.getLogger("dvc") @@ -482,58 +481,29 @@ def test_checkout_no_checksum(repo_dir, dvc_repo): assert not os.path.exists(repo_dir.FOO) -@pytest.mark.skip @pytest.mark.parametrize( "link, link_test_func", [("hardlink", System.is_hardlink), ("symlink", System.is_symlink)], ) -def test_should_relink_on_checkout(link, link_test_func, repo_dir, dvc_repo): - dvc_repo.cache.local.cache_types = [link] +def test_checkout_relink(tmp_dir, dvc, link, link_test_func): + dvc.cache.local.cache_types = [link] - dvc_repo.add(repo_dir.DATA_DIR) - dvc_repo.unprotect(repo_dir.DATA_SUB) + tmp_dir.dvc_gen({"dir": {"data": "text"}}) + dvc.unprotect("dir/data") + assert not link_test_func("dir/data") - dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX]) - - assert link_test_func(repo_dir.DATA_SUB) + dvc.checkout(["dir.dvc"], relink=True) + assert link_test_func("dir/data") -@pytest.mark.skip @pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"]) -def test_should_protect_on_checkout(link, dvc_repo, repo_dir): - dvc_repo.cache.local.cache_types = [link] - dvc_repo.cache.local.protected = True +def test_checkout_relink_protected(tmp_dir, dvc, link): + dvc.cache.local.cache_types = [link] + dvc.cache.local.protected = True - dvc_repo.add(repo_dir.FOO) - dvc_repo.unprotect(repo_dir.FOO) + tmp_dir.dvc_gen("foo", "foo") + dvc.unprotect("foo") + assert os.access("foo", os.W_OK) - dvc_repo.checkout([repo_dir.FOO + Stage.STAGE_FILE_SUFFIX]) - - assert not os.access(repo_dir.FOO, os.W_OK) - - -@pytest.mark.skip -def test_should_relink_only_one_file_in_dir(dvc_repo, repo_dir): - dvc_repo.cache.local.cache_types = ["symlink"] - - dvc_repo.add(repo_dir.DATA_DIR) - dvc_repo.unprotect(repo_dir.DATA_SUB) - - link_spy = spy(System.symlink) - with patch.object(dvc_repo.cache.local, "symlink", link_spy): - dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX]) - - assert link_spy.mock.call_count == 1 - - -@pytest.mark.skip -@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"]) -def test_should_not_relink_on_unchanged_dependency(link, dvc_repo, repo_dir): - dvc_repo.cache.local.cache_types = [link] - - dvc_repo.add(repo_dir.DATA_DIR) - - with patch.object(dvc_repo.cache.local, "link") as mock_link: - dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX]) - - assert not mock_link.called + dvc.checkout(["foo.dvc"], relink=True) + assert not os.access("foo", os.W_OK)