From d9d85f3cb0d07757742b248c4e129d58c2ae640b Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Mon, 9 Sep 2019 14:35:00 +0700 Subject: [PATCH] dvc: fix locks in Python2.7/Windows Return to zc.lockfile lock in Python 2, NFS is not supported there anyway since sqlite3 doesn't support URIs. --- dvc/lock.py | 133 ++++++++++++++++++++++++++++++------------- dvc/repo/__init__.py | 13 ++--- setup.py | 6 +- 3 files changed, 106 insertions(+), 46 deletions(-) diff --git a/dvc/lock.py b/dvc/lock.py index 32a4f460c..eb691f33d 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -3,61 +3,118 @@ from __future__ import unicode_literals import os -from datetime import timedelta import hashlib -from flufl.lock import Lock as _Lock, TimeOutError +import time +from datetime import timedelta + +from funcy.py3 import lkeep from dvc.exceptions import DvcException from dvc.utils import makedirs +from dvc.utils.compat import is_py3 + + +DEFAULT_TIMEOUT = 5 class LockError(DvcException): """Thrown when unable to acquire the lock for dvc repo.""" -class Lock(_Lock): - """Class for dvc repo lock. +if is_py3: + import flufl.lock - Args: - lockfile (str): the lock filename - in. - lifetime (int | timedelta): hold the lock for so long. - tmp_dir (str): a directory to store claim files. - """ + class Lock(flufl.lock.Lock): + """Class for dvc repo lock. - TIMEOUT = 5 + Args: + lockfile (str): the lock filename + in. + lifetime (int | timedelta): hold the lock for so long. + tmp_dir (str): a directory to store claim files. + """ - def __init__(self, lockfile, lifetime=None, tmp_dir=None): - if isinstance(lifetime, int): - lifetime = timedelta(seconds=lifetime) + def __init__(self, lockfile, lifetime=None, tmp_dir=None): + if isinstance(lifetime, int): + lifetime = timedelta(seconds=lifetime) - self._tmp_dir = tmp_dir - if self._tmp_dir is not None: - makedirs(self._tmp_dir, exist_ok=True) + self._tmp_dir = tmp_dir + if self._tmp_dir is not None: + makedirs(self._tmp_dir, exist_ok=True) - super(Lock, self).__init__(lockfile, lifetime=lifetime) + super(Lock, self).__init__(lockfile, lifetime=lifetime) - @property - def lockfile(self): - return self._lockfile + @property + def lockfile(self): + return self._lockfile - @property - def tmp_dir(self): - return self._tmp_dir + @property + def files(self): + return lkeep([self._lockfile, self._tmp_dir]) - def lock(self): - try: - super(Lock, self).lock(timedelta(seconds=self.TIMEOUT)) - except TimeOutError: - raise LockError( - "cannot perform the cmd since DVC is busy and " - "locked. Please retry the cmd later." - ) + def lock(self): + try: + super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT)) + except flufl.lock.TimeOutError: + raise LockError( + "cannot perform the cmd since DVC is busy and " + "locked. Please retry the cmd later." + ) - def _set_claimfile(self, pid=None): - super(Lock, self)._set_claimfile(pid) + def _set_claimfile(self, pid=None): + super(Lock, self)._set_claimfile(pid) - if self._tmp_dir is not None: - # Under Windows file path length is limited so we hash it - filename = hashlib.md5(self._claimfile.encode()).hexdigest() - self._claimfile = os.path.join(self._tmp_dir, filename + ".lock") + if self._tmp_dir is not None: + # Under Windows file path length is limited so we hash it + filename = hashlib.md5(self._claimfile.encode()).hexdigest() + self._claimfile = os.path.join( + self._tmp_dir, filename + ".lock" + ) + + +else: + import zc.lockfile + + class Lock(object): + """Class for dvc repo lock. + + Uses zc.lockfile as backend. + """ + + TIMEOUT = 5 + + def __init__(self, lockfile, lifetime=None, tmp_dir=None): + self.lockfile = lockfile + self._lock = None + + @property + def files(self): + return [self.lockfile] + + def _do_lock(self): + try: + self._lock = zc.lockfile.LockFile(self.lockfile) + except zc.lockfile.LockError: + raise LockError( + "cannot perform the cmd since DVC is busy and " + "locked. Please retry the cmd later." + ) + + def lock(self): + try: + self._do_lock() + return + except LockError: + time.sleep(DEFAULT_TIMEOUT) + + self._do_lock() + + def unlock(self): + self._lock.close() + self._lock = None + + def __enter__(self): + self.lock() + + def __exit__(self, typ, value, tbck): + self.unlock() diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 0502adbcc..70acdabc0 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -133,13 +133,12 @@ class Repo(object): updater = Updater(self.dvc_dir) - flist = [ - self.lock.lockfile, - self.lock.tmp_dir, - self.config.config_local_file, - updater.updater_file, - updater.lock.lockfile, - ] + self.state.files + flist = ( + [self.config.config_local_file, updater.updater_file] + + self.state.files + + self.lock.files + + updater.lock.files + ) if self.cache.local.cache_dir.startswith(self.root_dir): flist += [self.cache.local.cache_dir] diff --git a/setup.py b/setup.py index c80bb2ac2..cc811fa86 100644 --- a/setup.py +++ b/setup.py @@ -138,7 +138,11 @@ setup( "ssh_gssapi": ssh_gssapi, "hdfs": hdfs, # NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1 - ":python_version=='2.7'": ["futures", "pathlib2", "flufl.lock==2.4.1"], + ":python_version=='2.7'": [ + "futures", + "pathlib2", + "zc.lockfile>=1.2.1", + ], ":python_version>='3.0'": ["flufl.lock>=3.2"], "tests": tests_requirements, },