dvc: use protected mode by default (#3472)

* dvc: use protected mode by default

Fixes #3261
Fixes #2041

* unprotect: adjust help message

https://github.com/iterative/dvc.org/pull/1058/files/21aab371a487acf6f6e6201b29cd832e7c55ed23#r393324668
This commit is contained in:
Ruslan Kuprieiev
2020-03-17 19:16:21 +02:00
committed by GitHub
parent 4b5d4d759a
commit 6559b9d222
21 changed files with 195 additions and 121 deletions
+4 -1
View File
@@ -22,7 +22,10 @@ class CmdUnprotect(CmdBase):
def add_parser(subparsers, parent_parser):
UNPROTECT_HELP = "Unprotect data files or directories."
UNPROTECT_HELP = (
"Unprotect tracked files or directories (when hardlinks or symlinks "
"have been enabled with `dvc config cache.type`)"
)
unprotect_parser = subparsers.add_parser(
"unprotect",
parents=[parent_parser],
+1 -1
View File
@@ -97,7 +97,7 @@ REMOTE_COMMON = {
}
LOCAL_COMMON = {
"type": supported_cache_type,
Optional("protected", default=False): Bool,
Optional("protected", default=False): Bool, # obsoleted
"shared": All(Lower, Choices("group")),
Optional("slow_link_warning", default=True): Bool,
}
+25 -16
View File
@@ -83,6 +83,9 @@ class RemoteBASE(object):
DEFAULT_NO_TRAVERSE = True
DEFAULT_VERIFY = False
CACHE_MODE = None
SHARED_MODE_MAP = {None: (None, None), "group": (None, None)}
state = StateNoop()
def __init__(self, repo, config):
@@ -90,12 +93,14 @@ class RemoteBASE(object):
self._check_requires(config)
shared = config.get("shared")
self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
self.checksum_jobs = (
config.get("checksum_jobs")
or (self.repo and self.repo.config["core"].get("checksum_jobs"))
or self.CHECKSUM_JOBS
)
self.protected = False
self.no_traverse = config.get("no_traverse", self.DEFAULT_NO_TRAVERSE)
self.verify = config.get("verify", self.DEFAULT_VERIFY)
self._dir_info = {}
@@ -221,7 +226,7 @@ class RemoteBASE(object):
new_info = self.cache.checksum_to_path_info(checksum)
if self.cache.changed_cache_file(checksum):
self.cache.makedirs(new_info.parent)
self.cache.move(tmp_info, new_info)
self.cache.move(tmp_info, new_info, mode=self.CACHE_MODE)
self.state.save(path_info, checksum)
self.state.save(new_info, checksum)
@@ -409,15 +414,8 @@ class RemoteBASE(object):
link_method(from_info, to_info)
if self.protected:
self.protect(to_info)
logger.debug(
"Created %s'%s': %s -> %s",
"protected " if self.protected else "",
self.cache_types[0],
from_info,
to_info,
"Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
)
def _save_file(self, path_info, checksum, save_link=True):
@@ -425,14 +423,11 @@ class RemoteBASE(object):
cache_info = self.checksum_to_path_info(checksum)
if self.changed_cache(checksum):
self.move(path_info, cache_info)
self.move(path_info, cache_info, mode=self.CACHE_MODE)
self.link(cache_info, path_info)
elif self.iscopy(path_info) and self._cache_is_copy(path_info):
# Default relink procedure involves unneeded copy
if self.protected:
self.protect(path_info)
else:
self.unprotect(path_info)
self.unprotect(path_info)
else:
self.remove(path_info)
self.link(cache_info, path_info)
@@ -656,7 +651,8 @@ class RemoteBASE(object):
def remove(self, path_info):
raise RemoteActionNotImplemented("remove", self.scheme)
def move(self, from_info, to_info):
def move(self, from_info, to_info, mode=None):
assert mode is None
self.copy(from_info, to_info)
self.remove(from_info)
@@ -718,6 +714,9 @@ class RemoteBASE(object):
removed = True
return removed
def is_protected(self, path_info):
return False
def changed_cache_file(self, checksum):
"""Compare the given checksum with the (corresponding) actual one.
@@ -730,7 +729,14 @@ class RemoteBASE(object):
- Remove the file from cache if it doesn't match the actual checksum
"""
cache_info = self.checksum_to_path_info(checksum)
if self.is_protected(cache_info):
logger.debug(
"Assuming '%s' is unchanged since it is read-only", cache_info
)
return False
actual = self.get_checksum(cache_info)
logger.debug(
@@ -744,6 +750,9 @@ class RemoteBASE(object):
return True
if actual.split(".")[0] == checksum.split(".")[0]:
# making cache file read-only so we don't need to check it
# next time
self.protect(cache_info)
return False
if self.exists(cache_info):
+29 -25
View File
@@ -32,19 +32,11 @@ class RemoteLOCAL(RemoteBASE):
DEFAULT_CACHE_TYPES = ["reflink", "copy"]
CACHE_MODE = 0o444
SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
def __init__(self, repo, config):
super().__init__(repo, config)
self.protected = config.get("protected", False)
shared = config.get("shared")
self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
if self.protected:
# cache files are set to be read-only for everyone
self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
self.cache_dir = config.get("url")
self._dir_info = {}
@@ -142,16 +134,17 @@ class RemoteLOCAL(RemoteBASE):
if self.exists(path_info):
remove(path_info.fspath)
def move(self, from_info, to_info):
def move(self, from_info, to_info, mode=None):
if from_info.scheme != "local" or to_info.scheme != "local":
raise NotImplementedError
self.makedirs(to_info.parent)
if self.isfile(from_info):
mode = self._file_mode
else:
mode = self._dir_mode
if mode is None:
if self.isfile(from_info):
mode = self._file_mode
else:
mode = self._dir_mode
move(from_info, to_info, mode=mode)
@@ -159,6 +152,7 @@ class RemoteLOCAL(RemoteBASE):
tmp_info = to_info.parent / tmp_fname(to_info.name)
try:
System.copy(from_info, tmp_info)
os.chmod(fspath_py35(tmp_info), self._file_mode)
os.rename(fspath_py35(tmp_info), fspath_py35(to_info))
except Exception:
self.remove(tmp_info)
@@ -202,9 +196,13 @@ class RemoteLOCAL(RemoteBASE):
def is_hardlink(path_info):
return System.is_hardlink(path_info)
@staticmethod
def reflink(from_info, to_info):
System.reflink(from_info, to_info)
def reflink(self, from_info, to_info):
tmp_info = to_info.parent / tmp_fname(to_info.name)
System.reflink(from_info, tmp_info)
# NOTE: reflink has its own separate inode, so you can set permissions
# that are different from the source.
os.chmod(fspath_py35(tmp_info), self._file_mode)
os.rename(fspath_py35(tmp_info), fspath_py35(to_info))
def cache_exists(self, checksums, jobs=None, name=None):
return [
@@ -402,8 +400,7 @@ class RemoteLOCAL(RemoteBASE):
)
logger.warning(msg)
@staticmethod
def _unprotect_file(path):
def _unprotect_file(self, path):
if System.is_symlink(path) or System.is_hardlink(path):
logger.debug("Unprotecting '{}'".format(path))
tmp = os.path.join(os.path.dirname(path), "." + uuid())
@@ -423,13 +420,13 @@ class RemoteLOCAL(RemoteBASE):
"a symlink or a hardlink.".format(path)
)
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
os.chmod(path, self._file_mode)
def _unprotect_dir(self, path):
assert is_working_tree(self.repo.tree)
for fname in self.repo.tree.walk_files(path):
RemoteLOCAL._unprotect_file(fname)
self._unprotect_file(fname)
def unprotect(self, path_info):
path = path_info.fspath
@@ -441,12 +438,11 @@ class RemoteLOCAL(RemoteBASE):
if os.path.isdir(path):
self._unprotect_dir(path)
else:
RemoteLOCAL._unprotect_file(path)
self._unprotect_file(path)
@staticmethod
def protect(path_info):
def protect(self, path_info):
path = fspath_py35(path_info)
mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
mode = self.CACHE_MODE
try:
os.chmod(path, mode)
@@ -519,3 +515,11 @@ class RemoteLOCAL(RemoteBASE):
if self.is_dir_checksum(c):
unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
return unpacked
def is_protected(self, path_info):
if not self.exists(path_info):
return False
mode = os.stat(fspath_py35(path_info)).st_mode
return stat.S_IMODE(mode) == self.CACHE_MODE
+2 -1
View File
@@ -209,7 +209,8 @@ class RemoteSSH(RemoteBASE):
with self.ssh(path_info) as ssh:
ssh.remove(path_info.path)
def move(self, from_info, to_info):
def move(self, from_info, to_info, mode=None):
assert mode is None
if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
raise NotImplementedError
+1
View File
@@ -57,6 +57,7 @@ def test_dir_cache_changed_on_single_cache_file_modification(tmp_dir, dvc):
assert os.path.exists(unpacked_dir)
cache_file_path = dvc.cache.local.get(file_md5)
os.chmod(cache_file_path, 0o644)
with open(cache_file_path, "a") as fobj:
fobj.write("modification")
+13 -6
View File
@@ -489,12 +489,12 @@ class TestAddUnprotected(TestDvc):
ret = main(["config", "cache.type", "hardlink"])
self.assertEqual(ret, 0)
ret = main(["config", "cache.protected", "true"])
self.assertEqual(ret, 0)
ret = main(["add", self.FOO])
self.assertEqual(ret, 0)
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(System.is_hardlink(self.FOO))
ret = main(["unprotect", self.FOO])
self.assertEqual(ret, 0)
@@ -561,7 +561,6 @@ def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
tmp_dir.gen("dir/data", "data")
dvc.cache.local.cache_types = ["symlink"]
dvc.cache.local.protected = True
dvc.add("dir")
tmp_dir.gen("dir/new_file", "new_file_content")
@@ -618,7 +617,6 @@ def test_should_relink_on_repeated_add(
@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
def test_should_protect_on_repeated_add(link, tmp_dir, dvc):
dvc.cache.local.cache_types = [link]
dvc.cache.local.protected = True
tmp_dir.dvc_gen({"foo": "foo"})
@@ -626,7 +624,16 @@ def test_should_protect_on_repeated_add(link, tmp_dir, dvc):
dvc.add("foo")
assert not os.access("foo", os.W_OK)
assert not os.access(
os.path.join(".dvc", "cache", "ac", "bd18db4cc2f85cedef654fccc4a4d8"),
os.W_OK,
)
# NOTE: Windows symlink perms don't propagate to the target
if link == "copy" or (link == "symlink" and os.name == "nt"):
assert os.access("foo", os.W_OK)
else:
assert not os.access("foo", os.W_OK)
def test_escape_gitignore_entries(tmp_dir, scm, dvc):
+5 -4
View File
@@ -1,5 +1,4 @@
import os
import shutil
import pytest
@@ -9,6 +8,8 @@ from dvc.compat import fspath
from dvc.exceptions import FileMissingError
from dvc.main import main
from dvc.path_info import URLInfo
from dvc.utils.fs import remove
from tests.remotes import Azure, GCP, HDFS, Local, OSS, S3, SSH
@@ -65,7 +66,7 @@ def test_open(remote_url, tmp_dir, dvc):
run_dvc("push")
# Remove cache to force download
shutil.rmtree(dvc.cache.local.cache_dir)
remove(dvc.cache.local.cache_dir)
with api.open("foo") as fd:
assert fd.read() == "foo-text"
@@ -85,7 +86,7 @@ def test_open_external(remote_url, erepo_dir):
erepo_dir.dvc.push(all_branches=True)
# Remove cache to force download
shutil.rmtree(erepo_dir.dvc.cache.local.cache_dir)
remove(erepo_dir.dvc.cache.local.cache_dir)
# Using file url to force clone to tmp repo
repo_url = "file://{}".format(erepo_dir)
@@ -101,7 +102,7 @@ def test_missing(remote_url, tmp_dir, dvc):
run_dvc("remote", "add", "-d", "upstream", remote_url)
# Remove cache to make foo missing
shutil.rmtree(dvc.cache.local.cache_dir)
remove(dvc.cache.local.cache_dir)
with pytest.raises(FileMissingError):
api.read("foo")
+6 -6
View File
@@ -184,12 +184,12 @@ def test_default_cache_type(dvc):
@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
@pytest.mark.parametrize(
"protected,dir_mode,file_mode",
[(False, 0o775, 0o664), (True, 0o775, 0o444)],
"group, dir_mode", [(False, 0o755), (True, 0o775)],
)
def test_shared_cache(tmp_dir, dvc, protected, dir_mode, file_mode):
with dvc.config.edit() as conf:
conf["cache"].update({"shared": "group", "protected": str(protected)})
def test_shared_cache(tmp_dir, dvc, group, dir_mode):
if group:
with dvc.config.edit() as conf:
conf["cache"].update({"shared": "group"})
dvc.cache = Cache(dvc)
tmp_dir.dvc_gen(
@@ -203,4 +203,4 @@ def test_shared_cache(tmp_dir, dvc, protected, dir_mode, file_mode):
for fname in fnames:
path = os.path.join(root, fname)
assert stat.S_IMODE(os.stat(path).st_mode) == file_mode
assert stat.S_IMODE(os.stat(path).st_mode) == 0o444
+9 -2
View File
@@ -72,6 +72,7 @@ class TestCheckoutCorruptedCacheFile(TestRepro):
def test(self):
cache = self.foo_stage.outs[0].cache_path
os.chmod(cache, 0o644)
with open(cache, "a") as fd:
fd.write("1")
@@ -100,6 +101,7 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
checksum = entry[self.dvc.cache.local.PARAM_CHECKSUM]
cache = self.dvc.cache.local.get(checksum)
os.chmod(cache, 0o644)
with open(cache, "w+") as fobj:
fobj.write("1")
@@ -502,7 +504,6 @@ def test_checkout_relink(tmp_dir, dvc, link, link_test_func):
@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
def test_checkout_relink_protected(tmp_dir, dvc, link):
dvc.cache.local.cache_types = [link]
dvc.cache.local.protected = True
tmp_dir.dvc_gen("foo", "foo")
dvc.unprotect("foo")
@@ -510,7 +511,12 @@ def test_checkout_relink_protected(tmp_dir, dvc, link):
stats = dvc.checkout(["foo.dvc"], relink=True)
assert stats == empty_checkout
assert not os.access("foo", os.W_OK)
# NOTE: Windows symlink perms don't propagate to the target
if link == "copy" or (link == "symlink" and os.name == "nt"):
assert os.access("foo", os.W_OK)
else:
assert not os.access("foo", os.W_OK)
@pytest.mark.parametrize(
@@ -582,6 +588,7 @@ def test_checkout_stats_on_failure(tmp_dir, dvc, scm):
# corrupt cache
cache = stage.outs[0].cache_path
os.chmod(cache, 0o644)
with open(cache, "a") as fd:
fd.write("destroy cache")
+9 -8
View File
@@ -22,6 +22,7 @@ from dvc.remote import RemoteS3
from dvc.remote import RemoteSSH
from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
from dvc.utils import file_md5
from dvc.utils.fs import remove
from dvc.utils.stage import dump_stage_file, load_stage_file
from tests.basic_env import TestDvc
@@ -149,7 +150,7 @@ class TestDataCloudBase(TestDvc):
self.assertEqual(status_dir, expected)
# Remove and check status
shutil.rmtree(self.dvc.cache.local.cache_dir)
remove(self.dvc.cache.local.cache_dir)
status = self.cloud.status(info, show_checksums=True)
expected = {md5: {"name": md5, "status": STATUS_DELETED}}
@@ -348,7 +349,7 @@ class TestDataCloudCLIBase(TestDvc):
self.assertTrue(os.path.isfile(cache))
self.assertTrue(os.path.isfile(cache_dir))
shutil.rmtree(self.dvc.cache.local.cache_dir)
remove(self.dvc.cache.local.cache_dir)
self.main(["fetch"] + args)
self.assertTrue(os.path.exists(cache))
@@ -572,7 +573,7 @@ class TestRecursiveSyncOperations(Local, TestDataCloudBase):
self.assertTrue(os.path.exists(self.DATA_SUB))
def _clear_local_cache(self):
shutil.rmtree(self.dvc.cache.local.cache_dir)
remove(self.dvc.cache.local.cache_dir)
def _test_recursive_fetch(self, data_md5, data_sub_md5):
self._clear_local_cache()
@@ -644,7 +645,7 @@ class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc):
self.assertEqual(0, ret)
# purge cache
shutil.rmtree(self.dvc.cache.local.cache_dir)
remove(self.dvc.cache.local.cache_dir)
ret = main(["remote", "add", "remote_name", "-d", cache_dir])
self.assertEqual(0, ret)
@@ -693,9 +694,9 @@ def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory):
dvc.push()
# remove artifacts and cache to trigger fetching
os.remove("file")
shutil.rmtree("dir")
shutil.rmtree(dvc.cache.local.cache_dir)
remove("file")
remove("dir")
remove(dvc.cache.local.cache_dir)
checksum_spy = mocker.spy(dvc.cache.local, "get_file_checksum")
@@ -703,7 +704,7 @@ def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory):
assert checksum_spy.call_count == 0
# Removing cache will invalidate existing state entries
shutil.rmtree(dvc.cache.local.cache_dir)
remove(dvc.cache.local.cache_dir)
dvc.config["remote"]["local_remote"]["verify"] = True
+4 -4
View File
@@ -1,11 +1,11 @@
import hashlib
import os
import pytest
import shutil
from funcy import first
from dvc.compat import fspath
from dvc.utils.fs import remove
from dvc.exceptions import DvcException
@@ -36,7 +36,7 @@ def test_no_cache_entry(tmp_dir, scm, dvc):
tmp_dir.dvc_gen({"dir": {"1": "1", "2": "2"}})
tmp_dir.dvc_gen("file", "second")
shutil.rmtree(fspath(tmp_dir / ".dvc" / "cache"))
remove(fspath(tmp_dir / ".dvc" / "cache"))
(tmp_dir / ".dvc" / "state").unlink()
dir_checksum = "5fb6b29836c388e093ca0715c872fe2a.dir"
@@ -167,8 +167,8 @@ def test_diff_no_cache(tmp_dir, scm, dvc):
{"dir": {"file": "modified file content"}}, commit="second"
)
os.remove(first(stage.outs).cache_path)
shutil.rmtree("dir")
remove(first(stage.outs).cache_path)
remove("dir")
# invalidate_dir_info to force cache loading
dvc.cache.local._dir_info = {}
+2 -2
View File
@@ -1,5 +1,4 @@
import os
import shutil
from mock import patch
from dvc.compat import fspath
@@ -8,6 +7,7 @@ from dvc.external_repo import external_repo
from dvc.scm.git import Git
from dvc.remote import RemoteLOCAL
from dvc.utils import relpath
from dvc.utils.fs import remove
def test_external_repo(erepo_dir):
@@ -110,7 +110,7 @@ def test_relative_remote(erepo_dir, tmp_dir):
erepo_dir.dvc.push()
(erepo_dir / "file").unlink()
shutil.rmtree(erepo_dir.dvc.cache.local.cache_dir)
remove(erepo_dir.dvc.cache.local.cache_dir)
url = fspath(erepo_dir)
+2 -1
View File
@@ -10,6 +10,7 @@ from dvc.exceptions import CollectCacheError
from dvc.main import main
from dvc.repo import Repo as DvcRepo
from dvc.remote.local import RemoteLOCAL
from dvc.utils.fs import remove
from tests.basic_env import TestDir, TestDvcGit
@@ -195,7 +196,7 @@ def test_gc_no_dir_cache(tmp_dir, dvc):
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
(dir_stage,) = tmp_dir.dvc_gen({"dir": {"x": "x", "subdir": {"y": "y"}}})
os.unlink(dir_stage.outs[0].cache_path)
remove(dir_stage.outs[0].cache_path)
with pytest.raises(CollectCacheError):
dvc.gc(workspace=True)
+9 -10
View File
@@ -1,6 +1,5 @@
import filecmp
import os
import shutil
from dvc.compat import fspath
import pytest
@@ -11,7 +10,7 @@ from dvc.exceptions import DownloadError, PathMissingError
from dvc.config import NoRemoteError
from dvc.stage import Stage
from dvc.system import System
from dvc.utils.fs import makedirs
from dvc.utils.fs import makedirs, remove
import dvc.data_cloud as cloud
from tests.utils import trees_equal
@@ -187,8 +186,8 @@ def test_pull_imported_stage(tmp_dir, dvc, erepo_dir):
dst_stage = Stage.load(dvc, "foo_imported.dvc")
dst_cache = dst_stage.outs[0].cache_path
os.remove("foo_imported")
os.remove(dst_cache)
remove("foo_imported")
remove(dst_cache)
dvc.pull(["foo_imported.dvc"])
assert os.path.isfile("foo_imported")
@@ -220,8 +219,8 @@ def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir):
dvc.imp(fspath(erepo_dir), "dir", "dir_imported")
shutil.rmtree("dir_imported")
shutil.rmtree(dvc.cache.local.cache_dir)
remove("dir_imported")
remove(dvc.cache.local.cache_dir)
dvc.pull(["dir_imported.dvc"])
@@ -237,8 +236,8 @@ def test_download_error_pulling_imported_stage(tmp_dir, dvc, erepo_dir):
dst_stage = Stage.load(dvc, "foo_imported.dvc")
dst_cache = dst_stage.outs[0].cache_path
os.remove("foo_imported")
os.remove(dst_cache)
remove("foo_imported")
remove(dst_cache)
with patch(
"dvc.remote.RemoteLOCAL._download", side_effect=Exception
@@ -276,7 +275,7 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir):
# Overwrite via import
dvc.imp(fspath(erepo_dir), "foo", "foo_imported", rev="master")
os.remove(stage.outs[0].cache_path)
remove(stage.outs[0].cache_path)
dvc.fetch(all_tags=True)
assert os.path.exists(stage.outs[0].cache_path)
@@ -299,7 +298,7 @@ def test_pull_no_rev_lock(erepo_dir, tmp_dir, dvc):
stage.deps[0].def_repo.pop("rev_lock")
stage.dump()
os.remove(stage.outs[0].cache_path)
remove(stage.outs[0].cache_path)
(tmp_dir / "foo_imported").unlink()
dvc.pull([stage.path])
+2 -2
View File
@@ -1,6 +1,5 @@
import errno
import os
import shutil
import configobj
import pytest
@@ -13,6 +12,7 @@ from dvc.path_info import PathInfo
from dvc.remote import RemoteLOCAL
from dvc.remote.base import RemoteBASE, RemoteCacheRequiredError
from dvc.compat import fspath
from dvc.utils.fs import remove
from tests.basic_env import TestDvc
from tests.remotes import Local
@@ -197,7 +197,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory):
# Push everything and delete local cache
dvc.push()
shutil.rmtree(dvc.cache.local.cache_dir)
remove(dvc.cache.local.cache_dir)
with patch.object(RemoteLOCAL, "_download", side_effect=Exception):
with pytest.raises(DownloadError) as download_error_info:
+2 -2
View File
@@ -1,5 +1,4 @@
import os
import shutil
from mock import patch
@@ -8,6 +7,7 @@ from dvc.main import main
from dvc.stage import Stage
from dvc.stage import StageFileDoesNotExistError
from dvc.system import System
from dvc.utils.fs import remove
from tests.basic_env import TestDvc
@@ -44,7 +44,7 @@ class TestRemoveBrokenSymlink(TestDvc):
ret = main(["add", self.FOO])
self.assertEqual(ret, 0)
shutil.rmtree(self.dvc.cache.local.cache_dir)
remove(self.dvc.cache.local.cache_dir)
self.assertTrue(System.is_symlink(self.FOO))
+2 -1
View File
@@ -34,6 +34,7 @@ from dvc.utils import file_md5
from dvc.utils import relpath
from dvc.utils.stage import dump_stage_file
from dvc.utils.stage import load_stage_file
from dvc.utils.fs import remove
from tests.basic_env import TestDvc
from tests.remotes import (
GCP,
@@ -1274,7 +1275,7 @@ class TestReproAllPipelines(TestDvc):
class TestReproNoCommit(TestRepro):
def test(self):
shutil.rmtree(self.dvc.cache.local.cache_dir)
remove(self.dvc.cache.local.cache_dir)
ret = main(["repro", self.file1_stage, "--no-commit"])
self.assertEqual(ret, 0)
self.assertFalse(os.path.exists(self.dvc.cache.local.cache_dir))
+17 -13
View File
@@ -304,9 +304,6 @@ class TestRunUnprotectOutsCopy(TestDvc):
fobj.write("with open(sys.argv[1], 'a+') as fobj:\n")
fobj.write(" fobj.write('foo')\n")
ret = main(["config", "cache.protected", "true"])
self.assertEqual(ret, 0)
ret = main(["config", "cache.type", "copy"])
self.assertEqual(ret, 0)
@@ -323,7 +320,7 @@ class TestRunUnprotectOutsCopy(TestDvc):
]
)
self.assertEqual(ret, 0)
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(os.access(self.FOO, os.W_OK))
with open(self.FOO, "r") as fd:
self.assertEqual(fd.read(), "foo")
@@ -342,7 +339,7 @@ class TestRunUnprotectOutsCopy(TestDvc):
]
)
self.assertEqual(ret, 0)
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(os.access(self.FOO, os.W_OK))
with open(self.FOO, "r") as fd:
self.assertEqual(fd.read(), "foo")
@@ -355,9 +352,6 @@ class TestRunUnprotectOutsSymlink(TestDvc):
fobj.write("with open(sys.argv[1], 'a+') as fobj:\n")
fobj.write(" fobj.write('foo')\n")
ret = main(["config", "cache.protected", "true"])
self.assertEqual(ret, 0)
ret = main(["config", "cache.type", "symlink"])
self.assertEqual(ret, 0)
@@ -375,7 +369,13 @@ class TestRunUnprotectOutsSymlink(TestDvc):
]
)
self.assertEqual(ret, 0)
self.assertFalse(os.access(self.FOO, os.W_OK))
if os.name == "nt":
# NOTE: Windows symlink perms don't propagate to the target
self.assertTrue(os.access(self.FOO, os.W_OK))
else:
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(System.is_symlink(self.FOO))
with open(self.FOO, "r") as fd:
self.assertEqual(fd.read(), "foo")
@@ -395,7 +395,13 @@ class TestRunUnprotectOutsSymlink(TestDvc):
]
)
self.assertEqual(ret, 0)
self.assertFalse(os.access(self.FOO, os.W_OK))
if os.name == "nt":
# NOTE: Windows symlink perms don't propagate to the target
self.assertTrue(os.access(self.FOO, os.W_OK))
else:
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(System.is_symlink(self.FOO))
with open(self.FOO, "r") as fd:
self.assertEqual(fd.read(), "foo")
@@ -409,9 +415,6 @@ class TestRunUnprotectOutsHardlink(TestDvc):
fobj.write("with open(sys.argv[1], 'a+') as fobj:\n")
fobj.write(" fobj.write('foo')\n")
ret = main(["config", "cache.protected", "true"])
self.assertEqual(ret, 0)
ret = main(["config", "cache.type", "hardlink"])
self.assertEqual(ret, 0)
@@ -848,6 +851,7 @@ class TestShouldNotCheckoutUponCorruptedLocalHardlinkCache(TestDvc):
cmd = "python {} {} {}".format(self.CODE, self.FOO, self.BAR)
stage = self.dvc.run(deps=[self.FOO], outs=[self.BAR], cmd=cmd)
os.chmod(self.BAR, 0o644)
with open(self.BAR, "w") as fd:
fd.write("corrupting the output cache")
+16 -16
View File
@@ -1,5 +1,4 @@
import os
import stat
from dvc.main import main
from tests.basic_env import TestDvc
@@ -7,28 +6,29 @@ from tests.basic_env import TestDvc
class TestUnprotect(TestDvc):
def test(self):
ret = main(["config", "cache.protected", "true"])
ret = main(["config", "cache.type", "hardlink"])
self.assertEqual(ret, 0)
ret = main(["add", self.FOO])
self.assertEqual(ret, 0)
self.assertNotEqual(
os.stat(self.FOO).st_mode
& (stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH),
0,
)
self.assertEqual(
os.stat(self.FOO).st_mode
& (stat.S_IWRITE | stat.S_IWGRP | stat.S_IWOTH),
0,
cache = os.path.join(
".dvc", "cache", "ac", "bd18db4cc2f85cedef654fccc4a4d8"
)
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertFalse(os.access(cache, os.W_OK))
ret = main(["unprotect", self.FOO])
self.assertEqual(ret, 0)
self.assertNotEqual(
os.stat(self.FOO).st_mode
& (stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH | stat.S_IWRITE),
0,
)
self.assertTrue(os.access(self.FOO, os.W_OK))
# NOTE: cache is now unprotected, bceause we try to set write perms
# on files that we try to delete, as some filesystems require that
# (e.g. NTFS). But it should be restored after the next cache check,
# hence why we call `dvc status` here.
self.assertTrue(os.access(cache, os.W_OK))
ret = main(["status"])
self.assertEqual(ret, 0)
self.assertFalse(os.access(cache, os.W_OK))
+35
View File
@@ -1,4 +1,9 @@
import os
import pytest
from dvc.cache import NamedCache
from dvc.path_info import PathInfo
from dvc.remote.local import RemoteLOCAL
@@ -23,3 +28,33 @@ def test_status_download_optimization(mocker):
remote.status(infos, other_remote, download=True)
assert other_remote.cache_exists.call_count == 0
@pytest.mark.parametrize("link_name", ["hardlink", "symlink"])
def test_is_protected(tmp_dir, link_name):
remote = RemoteLOCAL(None, {})
link_method = getattr(remote, link_name)
(tmp_dir / "foo").write_text("foo")
foo = PathInfo(tmp_dir / "foo")
link = PathInfo(tmp_dir / "link")
link_method(foo, link)
assert not remote.is_protected(foo)
assert not remote.is_protected(link)
remote.protect(foo)
assert remote.is_protected(foo)
assert remote.is_protected(link)
remote.unprotect(link)
assert not remote.is_protected(link)
if link_name == "symlink" and os.name == "nt":
# NOTE: Windows symlink perms don't propagate to the target
assert remote.is_protected(foo)
else:
assert not remote.is_protected(foo)