dvc: introduce protected mode

This is an opt-in mode, that will make data in the workspace
read-only, thus protecting cache from corruption when hardlinks or
symlinks are used. User can use `dvc unprotect file` command, to
replace read-only link to cache with an editable copy. We can use
this for now to decide if we want to go with a `unified workflow`
in the future or if we want to set unprotected `reflink,copy`
by default.

Fixes #799

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/cache.py

@@ -27,6 +27,9 @@ class Cache(object):
             t = config.get(Config.SECTION_CACHE_TYPE, None)
             if t:
                 sect[Config.SECTION_CACHE_TYPE] = t
+            protected = config.get(Config.SECTION_CACHE_PROTECTED, None)
+            if protected is not None:
+                sect[Config.SECTION_CACHE_PROTECTED] = protected
 
         self.local = Remote(project, sect)
 

dvc/cli.py

@@ -8,6 +8,7 @@ from dvc.command.init import CmdInit
 from dvc.command.destroy import CmdDestroy
 from dvc.command.remove import CmdRemove
 from dvc.command.move import CmdMove
+from dvc.command.unprotect import CmdUnprotect
 from dvc.command.run import CmdRun
 from dvc.command.repro import CmdRepro
 from dvc.command.data_sync import CmdDataPush, CmdDataPull, CmdDataFetch
@@ -521,6 +522,19 @@ def parse_args(argv=None):
                         help='Destination.')
     move_parser.set_defaults(func=CmdMove)
 
+    # Unprotect
+    UNPROTECT_HELP = 'Unprotect data file/directory.'
+    unprotect_parser = subparsers.add_parser(
+                        'unprotect',
+                        parents=[parent_parser],
+                        description=UNPROTECT_HELP,
+                        help=UNPROTECT_HELP)
+    unprotect_parser.add_argument(
+                        'targets',
+                        nargs='+',
+                        help='Data files/directory.')
+    unprotect_parser.set_defaults(func=CmdUnprotect)
+
     # Garbage collector
     GC_HELP = 'Collect garbage.'
     gc_parser = subparsers.add_parser(

dvc/command/unprotect.py

@@ -0,0 +1,14 @@
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+
+
+class CmdUnprotect(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                self.project.unprotect(target)
+            except DvcException as ex:
+                msg = "Failed to unprotect '{}'".format(target)
+                self.project.logger.error(msg, ex)
+                return 1
+        return 0

dvc/config.py

@@ -68,6 +68,7 @@ class Config(object):
     SECTION_CACHE_DIR = 'dir'
     SECTION_CACHE_TYPE = 'type'
     SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
+    SECTION_CACHE_PROTECTED = 'protected'
     SECTION_CACHE_LOCAL = 'local'
     SECTION_CACHE_S3 = 's3'
     SECTION_CACHE_GS = 'gs'
@@ -82,9 +83,10 @@ class Config(object):
         Optional(SECTION_CACHE_SSH): str,
         Optional(SECTION_CACHE_AZURE): str,
 
-        # backward compatibility
         Optional(SECTION_CACHE_DIR, default='cache'): str,
         Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
+        Optional(SECTION_CACHE_PROTECTED,
+                 default=False): And(str, is_bool, Use(to_bool)),
     }
 
     # backward compatibility
@@ -153,6 +155,8 @@ class Config(object):
         Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
+        Optional(SECTION_CACHE_PROTECTED,
+                 default=False): And(str, is_bool, Use(to_bool)),
         Optional(SECTION_REMOTE_USER): str,
         Optional(SECTION_REMOTE_PORT): Use(int),
         Optional(SECTION_REMOTE_KEY_FILE): str,

dvc/project.py

@@ -245,6 +245,38 @@ class Project(object):
             msg = 'Unable to find dvcfile with output \'{}\''
             raise DvcException(msg.format(from_path))
 
+    def _unprotect_file(self, path):
+        import stat
+        import uuid
+        from dvc.utils import copyfile, move, remove
+
+        self.logger.debug("Unprotecting '{}'".format(path))
+
+        tmp = os.path.join(os.path.dirname(path), '.' + str(uuid.uuid4()))
+        move(path, tmp)
+
+        copyfile(tmp, path)
+
+        remove(tmp)
+
+        os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
+
+    def _unprotect_dir(self, path):
+        for root, dirs, files in os.walk(path):
+            for f in files:
+                path = os.path.join(root, f)
+                self._unprotect_file(path)
+
+    def unprotect(self, path):
+        if not os.path.exists(path):
+            raise DvcException("Can't unprotect non-existing "
+                               "data '{}'".format(path))
+
+        if os.path.isdir(path):
+            self._unprotect_dir(path)
+        else:
+            self._unprotect_file(path)
+
     def run(self,
             cmd=None,
             deps=[],

dvc/remote/local.py

@@ -1,4 +1,5 @@
 import os
+import stat
 import uuid
 import json
 import ntpath
@@ -36,6 +37,7 @@ class RemoteLOCAL(RemoteBase):
     def __init__(self, project, config):
         self.project = project
         self.state = self.project.state
+        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
         storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
         self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
 
@@ -139,8 +141,13 @@ class RemoteLOCAL(RemoteBase):
         while i > 0:
             try:
                 self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
-                msg = "Created '{}': {} -> {}"
-                Logger.info(msg.format(self.cache_types[0],
+
+                if self.protected:
+                    os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
+
+                msg = "Created {}'{}': {} -> {}"
+                Logger.info(msg.format('protected ' if self.protected else '',
+                                       self.cache_types[0],
                                        os.path.relpath(cache),
                                        os.path.relpath(path)))
                 return

dvc/utils.py

@@ -3,6 +3,7 @@ Helpers for other modules.
 """
 import os
 import sys
+import stat
 import math
 import json
 import shutil
@@ -102,6 +103,7 @@ def copyfile(src, dest, no_progress_bar=False, name=None):
 
 
 def move(src, dst):
+    dst = os.path.abspath(dst)
     dname = os.path.dirname(dst)
     if not os.path.exists(dname):
         os.makedirs(dname)
@@ -119,10 +121,17 @@ def remove(path):
         return
 
     Logger.debug(u'Removing \'{}\''.format(os.path.relpath(path)))
+
+    def _chmod(func, p, excinfo):
+        perm = os.stat(p).st_mode
+        perm |= stat.S_IWRITE
+        os.chmod(p, perm)
+        func(p)
+
     if os.path.isfile(path):
-        os.unlink(path)
+        _chmod(os.unlink, path, None)
     else:
-        shutil.rmtree(path)
+        shutil.rmtree(path, onerror=_chmod)
 
 
 def to_chunks(l, jobs):

tests/test_unprotect.py

@@ -0,0 +1,31 @@
+import os
+import stat
+
+from dvc.main import main
+from dvc.project import Project
+
+from tests.basic_env import TestDvc
+
+
+class TestUnprotect(TestDvc):
+    def test(self):
+        ret = main(['config', 'cache.protected', 'true'])
+        self.assertEqual(ret, 0)
+
+        ret = main(['add', self.FOO])
+        self.assertEqual(ret, 0)
+
+        self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD|
+                                                         stat.S_IRGRP|
+                                                         stat.S_IROTH), 0)
+        self.assertEqual(os.stat(self.FOO).st_mode & (stat.S_IWRITE|
+                                                      stat.S_IWGRP|
+                                                      stat.S_IWOTH), 0)
+
+        ret = main(['unprotect', self.FOO])
+        self.assertEqual(ret, 0)
+
+        self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD|
+                                                         stat.S_IRGRP|
+                                                         stat.S_IROTH|
+                                                         stat.S_IWRITE), 0)