diff --git a/dvc/cache.py b/dvc/cache.py index fa6711150..dfa120d80 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -1,18 +1,17 @@ import os import json +import uuid import shutil from dvc.state import State from dvc.system import System from dvc.logger import Logger from dvc.utils import move, remove -from dvc.lock import Lock from dvc.exceptions import DvcException class Cache(object): CACHE_DIR = 'cache' - CACHE_DIR_LOCK = 'cache.lock' CACHE_TYPES = ['reflink', 'hardlink', 'symlink', 'copy'] CACHE_TYPE_MAP = { 'copy': shutil.copyfile, @@ -35,25 +34,23 @@ class Cache(object): self.state = State(self.cache_dir) self.link_state = link_state - self.lock = Lock(self.cache_dir, name=self.CACHE_DIR_LOCK) @staticmethod def init(root_dir, dvc_dir, cache_dir=None): return Cache(root_dir, dvc_dir, cache_dir=None) def all(self): - with self.lock: - clist = [] - for entry in os.listdir(self.cache_dir): - subdir = os.path.join(self.cache_dir, entry) - if not os.path.isdir(subdir): - continue + clist = [] + for entry in os.listdir(self.cache_dir): + subdir = os.path.join(self.cache_dir, entry) + if not os.path.isdir(subdir): + continue - for cache in os.listdir(subdir): - path = os.path.join(subdir, cache) - clist.append(path) + for cache in os.listdir(subdir): + path = os.path.join(subdir, cache) + clist.append(path) - return clist + return clist def get(self, md5): if not md5: @@ -65,7 +62,7 @@ class Cache(object): relpath = os.path.relpath(path, self.cache_dir) return os.path.dirname(relpath) + os.path.basename(relpath) - def _changed(self, md5): + def changed(self, md5): cache = self.get(md5) if self.state.changed(cache, md5=md5): if os.path.exists(cache): @@ -75,10 +72,6 @@ class Cache(object): return False - def changed(self, md5): - with self.lock: - return self._changed(md5) - def link(self, src, link): dname = os.path.dirname(link) if not os.path.exists(dname): @@ -145,10 +138,10 @@ class Cache(object): def is_dir_cache(cache): return cache.endswith(State.MD5_DIR_SUFFIX) - def _checkout(self, path, md5): + def checkout(self, path, md5): cache = self.get(md5) - if not cache or not os.path.exists(cache) or self._changed(md5): + if not cache or not os.path.exists(cache) or self.changed(md5): if cache: Logger.warn(u'\'{}({})\': cache file not found'.format(os.path.relpath(cache), os.path.relpath(path))) @@ -172,17 +165,20 @@ class Cache(object): p = os.path.join(path, relpath) self.link(c, p) - def checkout(self, path, md5): - with self.lock: - return self._checkout(path, md5) + def _move(self, inp, outp): + # moving in two stages to make last the move atomic in + # case inp and outp are in different filesystems + tmp = '{}.{}'.format(outp, str(uuid.uuid4())) + move(inp, tmp) + move(tmp, outp) def _save_file(self, path): md5 = self.state.update(path) cache = self.get(md5) - if self._changed(md5): - move(path, cache) + if self.changed(md5): + self._move(path, cache) self.state.update(cache) - self._checkout(path, md5) + self.checkout(path, md5) def _save_dir(self, path): md5 = self.state.update(path) @@ -203,8 +199,7 @@ class Cache(object): json.dump(dir_info, fd, sort_keys=True) def save(self, path): - with self.lock: - if os.path.isdir(path): - self._save_dir(path) - else: - self._save_file(path) + if os.path.isdir(path): + self._save_dir(path) + else: + self._save_file(path)