Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-01 23:08:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

dvc: add checkout --relink

Browse Source
This commit is contained in:
Alexander Schepanovski
2019-12-09 13:25:32 +03:00
parent 3b2666e46b
commit 2f79e362f4
6 changed files with 60 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/checkout.py
+7
View File
@@ -12,6 +12,7 @@ class CmdCheckout(CmdBase):
targets=self.args.targets,
with_deps=self.args.with_deps,
force=self.args.force,
relink=self.args.relink,
recursive=self.args.recursive,
)
return 0
@@ -34,6 +35,12 @@ def add_parser(subparsers, parent_parser):
default=False,
help="Do not prompt when removing working directory files.",
)
checkout_parser.add_argument(
"--relink",
action="store_true",
default=False,
help="Relink all files to cache, including unchanged.",
)
checkout_parser.add_argument(
"-d",
"--with-deps",
dvc/output/base.py
+4 -1
View File
@@ -282,7 +282,9 @@ class OutputBase(object):
def download(self, to):
self.remote.download(self.path_info, to.path_info)
def checkout(self, force=False, progress_callback=None, tag=None):
def checkout(
self, force=False, progress_callback=None, tag=None, relink=False
):
if not self.use_cache:
if progress_callback:
progress_callback(str(self.path_info), self.get_files_number())
@@ -298,6 +300,7 @@ class OutputBase(object):
info,
force=force,
progress_callback=progress_callback,
relink=relink,
)
def remove(self, ignore_remove=False):
dvc/remote/base.py
+18 -9
View File
@@ -836,7 +836,7 @@ class RemoteBASE(object):
pass
def _checkout_dir(
self, path_info, checksum, force, progress_callback=None
self, path_info, checksum, force, progress_callback=None, relink=False
):
# Create dir separately so that dir is created
# even if there are no files in it
@@ -854,9 +854,8 @@ class RemoteBASE(object):
entry_info = path_info / relative_path
entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
if self.changed(entry_info, entry_checksum_info):
if self.exists(entry_info):
self.safe_remove(entry_info, force=force)
if relink or self.changed(entry_info, entry_checksum_info):
self.safe_remove(entry_info, force=force)
self.link(entry_cache_info, entry_info)
self.state.save(entry_info, entry_checksum)
if progress_callback:
@@ -878,7 +877,12 @@ class RemoteBASE(object):
self.safe_remove(path, force)
def checkout(
self, path_info, checksum_info, force=False, progress_callback=None
self,
path_info,
checksum_info,
force=False,
progress_callback=None,
relink=False,
):
if path_info.scheme not in ["local", self.scheme]:
raise NotImplementedError
@@ -894,7 +898,7 @@ class RemoteBASE(object):
self.safe_remove(path_info, force=force)
failed = path_info
elif not self.changed(path_info, checksum_info):
elif not relink and not self.changed(path_info, checksum_info):
msg = "Data '{}' didn't change."
logger.debug(msg.format(str(path_info)))
skip = True
@@ -915,18 +919,23 @@ class RemoteBASE(object):
msg = "Checking out '{}' with cache '{}'."
logger.debug(msg.format(str(path_info), checksum))
self._checkout(path_info, checksum, force, progress_callback)
self._checkout(path_info, checksum, force, progress_callback, relink)
return None
def _checkout(
self, path_info, checksum, force=False, progress_callback=None
self,
path_info,
checksum,
force=False,
progress_callback=None,
relink=False,
):
if not self.is_dir_checksum(checksum):
return self._checkout_file(
path_info, checksum, force, progress_callback=progress_callback
)
return self._checkout_dir(
path_info, checksum, force, progress_callback=progress_callback
path_info, checksum, force, progress_callback, relink
)
def get_files_number(self, checksum):
dvc/repo/checkout.py
+11 -2
View File
@@ -25,7 +25,12 @@ def get_all_files_numbers(stages):
def _checkout(
self, targets=None, with_deps=False, force=False, recursive=False
self,
targets=None,
with_deps=False,
force=False,
relink=False,
recursive=False,
):
from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
@@ -55,7 +60,11 @@ def _checkout(
) as pbar:
for stage in stages:
failed.extend(
stage.checkout(force=force, progress_callback=pbar.update_desc)
stage.checkout(
force=force,
progress_callback=pbar.update_desc,
relink=relink,
)
)
if failed:
raise CheckoutError(failed)
dvc/stage.py
+5 -2
View File
@@ -904,11 +904,14 @@ class Stage(object):
if paths:
raise MissingDataSource(paths)
def checkout(self, force=False, progress_callback=None):
def checkout(self, force=False, progress_callback=None, relink=False):
failed_checkouts = []
for out in self.outs:
failed = out.checkout(
force=force, tag=self.tag, progress_callback=progress_callback
force=force,
tag=self.tag,
progress_callback=progress_callback,
relink=relink,
)
if failed:
failed_checkouts.append(failed)
tests/func/test_checkout.py
+15 -45
View File
@@ -26,7 +26,6 @@ from dvc.utils.stage import load_stage_file
from tests.basic_env import TestDvc
from tests.basic_env import TestDvcGit
from tests.func.test_repro import TestRepro
from tests.utils import spy
logger = logging.getLogger("dvc")
@@ -482,58 +481,29 @@ def test_checkout_no_checksum(repo_dir, dvc_repo):
assert not os.path.exists(repo_dir.FOO)
@pytest.mark.skip
@pytest.mark.parametrize(
"link, link_test_func",
[("hardlink", System.is_hardlink), ("symlink", System.is_symlink)],
)
def test_should_relink_on_checkout(link, link_test_func, repo_dir, dvc_repo):
dvc_repo.cache.local.cache_types = [link]
def test_checkout_relink(tmp_dir, dvc, link, link_test_func):
dvc.cache.local.cache_types = [link]
dvc_repo.add(repo_dir.DATA_DIR)
dvc_repo.unprotect(repo_dir.DATA_SUB)
tmp_dir.dvc_gen({"dir": {"data": "text"}})
dvc.unprotect("dir/data")
assert not link_test_func("dir/data")
dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX])
assert link_test_func(repo_dir.DATA_SUB)
dvc.checkout(["dir.dvc"], relink=True)
assert link_test_func("dir/data")
@pytest.mark.skip
@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
def test_should_protect_on_checkout(link, dvc_repo, repo_dir):
dvc_repo.cache.local.cache_types = [link]
dvc_repo.cache.local.protected = True
def test_checkout_relink_protected(tmp_dir, dvc, link):
dvc.cache.local.cache_types = [link]
dvc.cache.local.protected = True
dvc_repo.add(repo_dir.FOO)
dvc_repo.unprotect(repo_dir.FOO)
tmp_dir.dvc_gen("foo", "foo")
dvc.unprotect("foo")
assert os.access("foo", os.W_OK)
dvc_repo.checkout([repo_dir.FOO + Stage.STAGE_FILE_SUFFIX])
assert not os.access(repo_dir.FOO, os.W_OK)
@pytest.mark.skip
def test_should_relink_only_one_file_in_dir(dvc_repo, repo_dir):
dvc_repo.cache.local.cache_types = ["symlink"]
dvc_repo.add(repo_dir.DATA_DIR)
dvc_repo.unprotect(repo_dir.DATA_SUB)
link_spy = spy(System.symlink)
with patch.object(dvc_repo.cache.local, "symlink", link_spy):
dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX])
assert link_spy.mock.call_count == 1
@pytest.mark.skip
@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
def test_should_not_relink_on_unchanged_dependency(link, dvc_repo, repo_dir):
dvc_repo.cache.local.cache_types = [link]
dvc_repo.add(repo_dir.DATA_DIR)
with patch.object(dvc_repo.cache.local, "link") as mock_link:
dvc_repo.checkout([repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX])
assert not mock_link.called
dvc.checkout(["foo.dvc"], relink=True)
assert not os.access("foo", os.W_OK)