dvc: fix locks in Python2.7/Windows

Return to zc.lockfile lock in Python 2, NFS is not supported there
anyway since sqlite3 doesn't support URIs.

dvc/lock.py

@@ -3,61 +3,118 @@
 from __future__ import unicode_literals
 
 import os
-from datetime import timedelta
 import hashlib
-from flufl.lock import Lock as _Lock, TimeOutError
+import time
+from datetime import timedelta
+
+from funcy.py3 import lkeep
 
 from dvc.exceptions import DvcException
 from dvc.utils import makedirs
+from dvc.utils.compat import is_py3
+
+
+DEFAULT_TIMEOUT = 5
 
 
 class LockError(DvcException):
     """Thrown when unable to acquire the lock for dvc repo."""
 
 
-class Lock(_Lock):
-    """Class for dvc repo lock.
+if is_py3:
+    import flufl.lock
 
-    Args:
-        lockfile (str): the lock filename
-            in.
-        lifetime (int | timedelta): hold the lock for so long.
-        tmp_dir (str): a directory to store claim files.
-    """
+    class Lock(flufl.lock.Lock):
+        """Class for dvc repo lock.
 
-    TIMEOUT = 5
+        Args:
+            lockfile (str): the lock filename
+                in.
+            lifetime (int | timedelta): hold the lock for so long.
+            tmp_dir (str): a directory to store claim files.
+        """
 
-    def __init__(self, lockfile, lifetime=None, tmp_dir=None):
-        if isinstance(lifetime, int):
-            lifetime = timedelta(seconds=lifetime)
+        def __init__(self, lockfile, lifetime=None, tmp_dir=None):
+            if isinstance(lifetime, int):
+                lifetime = timedelta(seconds=lifetime)
 
-        self._tmp_dir = tmp_dir
-        if self._tmp_dir is not None:
-            makedirs(self._tmp_dir, exist_ok=True)
+            self._tmp_dir = tmp_dir
+            if self._tmp_dir is not None:
+                makedirs(self._tmp_dir, exist_ok=True)
 
-        super(Lock, self).__init__(lockfile, lifetime=lifetime)
+            super(Lock, self).__init__(lockfile, lifetime=lifetime)
 
-    @property
-    def lockfile(self):
-        return self._lockfile
+        @property
+        def lockfile(self):
+            return self._lockfile
 
-    @property
-    def tmp_dir(self):
-        return self._tmp_dir
+        @property
+        def files(self):
+            return lkeep([self._lockfile, self._tmp_dir])
 
-    def lock(self):
-        try:
-            super(Lock, self).lock(timedelta(seconds=self.TIMEOUT))
-        except TimeOutError:
-            raise LockError(
-                "cannot perform the cmd since DVC is busy and "
-                "locked. Please retry the cmd later."
-            )
+        def lock(self):
+            try:
+                super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
+            except flufl.lock.TimeOutError:
+                raise LockError(
+                    "cannot perform the cmd since DVC is busy and "
+                    "locked. Please retry the cmd later."
+                )
 
-    def _set_claimfile(self, pid=None):
-        super(Lock, self)._set_claimfile(pid)
+        def _set_claimfile(self, pid=None):
+            super(Lock, self)._set_claimfile(pid)
 
-        if self._tmp_dir is not None:
-            # Under Windows file path length is limited so we hash it
-            filename = hashlib.md5(self._claimfile.encode()).hexdigest()
-            self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
+            if self._tmp_dir is not None:
+                # Under Windows file path length is limited so we hash it
+                filename = hashlib.md5(self._claimfile.encode()).hexdigest()
+                self._claimfile = os.path.join(
+                    self._tmp_dir, filename + ".lock"
+                )
+
+
+else:
+    import zc.lockfile
+
+    class Lock(object):
+        """Class for dvc repo lock.
+
+        Uses zc.lockfile as backend.
+        """
+
+        TIMEOUT = 5
+
+        def __init__(self, lockfile, lifetime=None, tmp_dir=None):
+            self.lockfile = lockfile
+            self._lock = None
+
+        @property
+        def files(self):
+            return [self.lockfile]
+
+        def _do_lock(self):
+            try:
+                self._lock = zc.lockfile.LockFile(self.lockfile)
+            except zc.lockfile.LockError:
+                raise LockError(
+                    "cannot perform the cmd since DVC is busy and "
+                    "locked. Please retry the cmd later."
+                )
+
+        def lock(self):
+            try:
+                self._do_lock()
+                return
+            except LockError:
+                time.sleep(DEFAULT_TIMEOUT)
+
+            self._do_lock()
+
+        def unlock(self):
+            self._lock.close()
+            self._lock = None
+
+        def __enter__(self):
+            self.lock()
+
+        def __exit__(self, typ, value, tbck):
+            self.unlock()

dvc/repo/__init__.py

@@ -133,13 +133,12 @@ class Repo(object):
 
         updater = Updater(self.dvc_dir)
 
-        flist = [
-            self.lock.lockfile,
-            self.lock.tmp_dir,
-            self.config.config_local_file,
-            updater.updater_file,
-            updater.lock.lockfile,
-        ] + self.state.files
+        flist = (
+            [self.config.config_local_file, updater.updater_file]
+            + self.state.files
+            + self.lock.files
+            + updater.lock.files
+        )
 
         if self.cache.local.cache_dir.startswith(self.root_dir):
             flist += [self.cache.local.cache_dir]

setup.py

@@ -138,7 +138,11 @@ setup(
         "ssh_gssapi": ssh_gssapi,
         "hdfs": hdfs,
         # NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1
-        ":python_version=='2.7'": ["futures", "pathlib2", "flufl.lock==2.4.1"],
+        ":python_version=='2.7'": [
+            "futures",
+            "pathlib2",
+            "zc.lockfile>=1.2.1",
+        ],
         ":python_version>='3.0'": ["flufl.lock>=3.2"],
         "tests": tests_requirements,
     },