diff --git a/dvc/cache.py b/dvc/cache.py index fdaad2db8..821249689 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -27,6 +27,9 @@ class Cache(object): t = config.get(Config.SECTION_CACHE_TYPE, None) if t: sect[Config.SECTION_CACHE_TYPE] = t + protected = config.get(Config.SECTION_CACHE_PROTECTED, None) + if protected is not None: + sect[Config.SECTION_CACHE_PROTECTED] = protected self.local = Remote(project, sect) diff --git a/dvc/cli.py b/dvc/cli.py index b2469ee3c..495dca064 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -8,6 +8,7 @@ from dvc.command.init import CmdInit from dvc.command.destroy import CmdDestroy from dvc.command.remove import CmdRemove from dvc.command.move import CmdMove +from dvc.command.unprotect import CmdUnprotect from dvc.command.run import CmdRun from dvc.command.repro import CmdRepro from dvc.command.data_sync import CmdDataPush, CmdDataPull, CmdDataFetch @@ -521,6 +522,19 @@ def parse_args(argv=None): help='Destination.') move_parser.set_defaults(func=CmdMove) + # Unprotect + UNPROTECT_HELP = 'Unprotect data file/directory.' + unprotect_parser = subparsers.add_parser( + 'unprotect', + parents=[parent_parser], + description=UNPROTECT_HELP, + help=UNPROTECT_HELP) + unprotect_parser.add_argument( + 'targets', + nargs='+', + help='Data files/directory.') + unprotect_parser.set_defaults(func=CmdUnprotect) + # Garbage collector GC_HELP = 'Collect garbage.' gc_parser = subparsers.add_parser( diff --git a/dvc/command/unprotect.py b/dvc/command/unprotect.py new file mode 100644 index 000000000..4b8210df9 --- /dev/null +++ b/dvc/command/unprotect.py @@ -0,0 +1,14 @@ +from dvc.exceptions import DvcException +from dvc.command.base import CmdBase + + +class CmdUnprotect(CmdBase): + def run(self): + for target in self.args.targets: + try: + self.project.unprotect(target) + except DvcException as ex: + msg = "Failed to unprotect '{}'".format(target) + self.project.logger.error(msg, ex) + return 1 + return 0 diff --git a/dvc/config.py b/dvc/config.py index 6a1fff22b..44a96275d 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -68,6 +68,7 @@ class Config(object): SECTION_CACHE_DIR = 'dir' SECTION_CACHE_TYPE = 'type' SECTION_CACHE_TYPE_SCHEMA = supported_cache_type + SECTION_CACHE_PROTECTED = 'protected' SECTION_CACHE_LOCAL = 'local' SECTION_CACHE_S3 = 's3' SECTION_CACHE_GS = 'gs' @@ -82,9 +83,10 @@ class Config(object): Optional(SECTION_CACHE_SSH): str, Optional(SECTION_CACHE_AZURE): str, - # backward compatibility Optional(SECTION_CACHE_DIR, default='cache'): str, Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA, + Optional(SECTION_CACHE_PROTECTED, + default=False): And(str, is_bool, Use(to_bool)), } # backward compatibility @@ -153,6 +155,8 @@ class Config(object): Optional(SECTION_AWS_ENDPOINT_URL, default=None): str, Optional(SECTION_GCP_PROJECTNAME): str, Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA, + Optional(SECTION_CACHE_PROTECTED, + default=False): And(str, is_bool, Use(to_bool)), Optional(SECTION_REMOTE_USER): str, Optional(SECTION_REMOTE_PORT): Use(int), Optional(SECTION_REMOTE_KEY_FILE): str, diff --git a/dvc/project.py b/dvc/project.py index bcb40e411..542fb32bc 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -245,6 +245,38 @@ class Project(object): msg = 'Unable to find dvcfile with output \'{}\'' raise DvcException(msg.format(from_path)) + def _unprotect_file(self, path): + import stat + import uuid + from dvc.utils import copyfile, move, remove + + self.logger.debug("Unprotecting '{}'".format(path)) + + tmp = os.path.join(os.path.dirname(path), '.' + str(uuid.uuid4())) + move(path, tmp) + + copyfile(tmp, path) + + remove(tmp) + + os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE) + + def _unprotect_dir(self, path): + for root, dirs, files in os.walk(path): + for f in files: + path = os.path.join(root, f) + self._unprotect_file(path) + + def unprotect(self, path): + if not os.path.exists(path): + raise DvcException("Can't unprotect non-existing " + "data '{}'".format(path)) + + if os.path.isdir(path): + self._unprotect_dir(path) + else: + self._unprotect_file(path) + def run(self, cmd=None, deps=[], diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 7e1c2d2d1..44dc4c2a3 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -1,4 +1,5 @@ import os +import stat import uuid import json import ntpath @@ -36,6 +37,7 @@ class RemoteLOCAL(RemoteBase): def __init__(self, project, config): self.project = project self.state = self.project.state + self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False) storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None) self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath) @@ -139,8 +141,13 @@ class RemoteLOCAL(RemoteBase): while i > 0: try: self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path) - msg = "Created '{}': {} -> {}" - Logger.info(msg.format(self.cache_types[0], + + if self.protected: + os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH) + + msg = "Created {}'{}': {} -> {}" + Logger.info(msg.format('protected ' if self.protected else '', + self.cache_types[0], os.path.relpath(cache), os.path.relpath(path))) return diff --git a/dvc/utils.py b/dvc/utils.py index 822f93d98..060d3b62f 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -3,6 +3,7 @@ Helpers for other modules. """ import os import sys +import stat import math import json import shutil @@ -102,6 +103,7 @@ def copyfile(src, dest, no_progress_bar=False, name=None): def move(src, dst): + dst = os.path.abspath(dst) dname = os.path.dirname(dst) if not os.path.exists(dname): os.makedirs(dname) @@ -119,10 +121,17 @@ def remove(path): return Logger.debug(u'Removing \'{}\''.format(os.path.relpath(path))) + + def _chmod(func, p, excinfo): + perm = os.stat(p).st_mode + perm |= stat.S_IWRITE + os.chmod(p, perm) + func(p) + if os.path.isfile(path): - os.unlink(path) + _chmod(os.unlink, path, None) else: - shutil.rmtree(path) + shutil.rmtree(path, onerror=_chmod) def to_chunks(l, jobs): diff --git a/tests/test_unprotect.py b/tests/test_unprotect.py new file mode 100644 index 000000000..b62769357 --- /dev/null +++ b/tests/test_unprotect.py @@ -0,0 +1,31 @@ +import os +import stat + +from dvc.main import main +from dvc.project import Project + +from tests.basic_env import TestDvc + + +class TestUnprotect(TestDvc): + def test(self): + ret = main(['config', 'cache.protected', 'true']) + self.assertEqual(ret, 0) + + ret = main(['add', self.FOO]) + self.assertEqual(ret, 0) + + self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD| + stat.S_IRGRP| + stat.S_IROTH), 0) + self.assertEqual(os.stat(self.FOO).st_mode & (stat.S_IWRITE| + stat.S_IWGRP| + stat.S_IWOTH), 0) + + ret = main(['unprotect', self.FOO]) + self.assertEqual(ret, 0) + + self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD| + stat.S_IRGRP| + stat.S_IROTH| + stat.S_IWRITE), 0)