mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 23:11:44 +01:00
Merge pull request #2918 from efiop/lock
dvc: make flufl.lock opt-in and use zc.lockfile
This commit is contained in:
+6
-9
@@ -2,8 +2,6 @@ from __future__ import unicode_literals
|
||||
|
||||
import logging
|
||||
|
||||
import colorama
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -24,15 +22,13 @@ def fix_subparsers(subparsers):
|
||||
|
||||
|
||||
def append_doc_link(help_message, path):
|
||||
from dvc.utils import format_link
|
||||
|
||||
if not path:
|
||||
return help_message
|
||||
doc_base = "https://man.dvc.org/"
|
||||
return "{message}\nDocumentation: <{blue}{base}{path}{nc}>".format(
|
||||
message=help_message,
|
||||
base=doc_base,
|
||||
path=path,
|
||||
blue=colorama.Fore.CYAN,
|
||||
nc=colorama.Fore.RESET,
|
||||
return "{message}\nDocumentation: {link}".format(
|
||||
message=help_message, link=format_link(doc_base + path)
|
||||
)
|
||||
|
||||
|
||||
@@ -44,7 +40,8 @@ class CmdBase(object):
|
||||
self.repo = Repo()
|
||||
self.config = self.repo.config
|
||||
self.args = args
|
||||
updater = Updater(self.repo.dvc_dir)
|
||||
hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
|
||||
updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
|
||||
updater.check()
|
||||
|
||||
@property
|
||||
|
||||
@@ -12,11 +12,16 @@ class CmdDaemonUpdater(CmdDaemonBase):
|
||||
def run(self):
|
||||
import os
|
||||
from dvc.repo import Repo
|
||||
from dvc.config import Config
|
||||
from dvc.updater import Updater
|
||||
|
||||
root_dir = Repo.find_root()
|
||||
dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
|
||||
updater = Updater(dvc_dir)
|
||||
config = Config(dvc_dir, verify=False)
|
||||
hardlink_lock = config.config.get("core", {}).get(
|
||||
"hardlink_lock", False
|
||||
)
|
||||
updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
|
||||
updater.fetch(detach=False)
|
||||
|
||||
return 0
|
||||
|
||||
@@ -125,6 +125,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes
|
||||
SECTION_CORE_INTERACTIVE = "interactive"
|
||||
SECTION_CORE_ANALYTICS = "analytics"
|
||||
SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
|
||||
SECTION_CORE_HARDLINK_LOCK = "hardlink_lock"
|
||||
|
||||
SECTION_CACHE = "cache"
|
||||
SECTION_CACHE_DIR = "dir"
|
||||
@@ -160,6 +161,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes
|
||||
Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
|
||||
Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
|
||||
SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
|
||||
Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool,
|
||||
}
|
||||
|
||||
# aws specific options
|
||||
|
||||
+66
-45
@@ -6,12 +6,13 @@ import os
|
||||
import time
|
||||
from datetime import timedelta
|
||||
|
||||
import zc.lockfile
|
||||
from funcy.py3 import lkeep
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils import makedirs
|
||||
from dvc.utils.compat import is_py3
|
||||
|
||||
from dvc.utils import makedirs, format_link
|
||||
from dvc.utils.compat import is_py3, is_py2
|
||||
from dvc.progress import Tqdm
|
||||
|
||||
DEFAULT_TIMEOUT = 5
|
||||
|
||||
@@ -26,10 +27,60 @@ class LockError(DvcException):
|
||||
"""Thrown when unable to acquire the lock for dvc repo."""
|
||||
|
||||
|
||||
class Lock(object):
|
||||
"""Class for dvc repo lock.
|
||||
|
||||
Uses zc.lockfile as backend.
|
||||
"""
|
||||
|
||||
def __init__(self, lockfile, friendly=False, **kwargs):
|
||||
self._friendly = friendly
|
||||
self.lockfile = lockfile
|
||||
self._lock = None
|
||||
|
||||
@property
|
||||
def files(self):
|
||||
return [self.lockfile]
|
||||
|
||||
def _do_lock(self):
|
||||
try:
|
||||
with Tqdm(
|
||||
bar_format="{desc}",
|
||||
disable=not self._friendly,
|
||||
desc=(
|
||||
"If DVC froze, see `hardlink_lock` in {}".format(
|
||||
format_link("man.dvc.org/config#core")
|
||||
)
|
||||
),
|
||||
):
|
||||
self._lock = zc.lockfile.LockFile(self.lockfile)
|
||||
except zc.lockfile.LockError:
|
||||
raise LockError(FAILED_TO_LOCK_MESSAGE)
|
||||
|
||||
def lock(self):
|
||||
try:
|
||||
self._do_lock()
|
||||
return
|
||||
except LockError:
|
||||
time.sleep(DEFAULT_TIMEOUT)
|
||||
|
||||
self._do_lock()
|
||||
|
||||
def unlock(self):
|
||||
self._lock.close()
|
||||
self._lock = None
|
||||
|
||||
def __enter__(self):
|
||||
self.lock()
|
||||
|
||||
def __exit__(self, typ, value, tbck):
|
||||
self.unlock()
|
||||
|
||||
|
||||
if is_py3:
|
||||
import flufl.lock
|
||||
|
||||
class Lock(flufl.lock.Lock):
|
||||
class HardlinkLock(flufl.lock.Lock):
|
||||
"""Class for dvc repo lock.
|
||||
|
||||
Args:
|
||||
@@ -38,7 +89,7 @@ if is_py3:
|
||||
tmp_dir (str): a directory to store claim files.
|
||||
"""
|
||||
|
||||
def __init__(self, lockfile, tmp_dir=None):
|
||||
def __init__(self, lockfile, tmp_dir=None, **kwargs):
|
||||
import socket
|
||||
|
||||
self._tmp_dir = tmp_dir
|
||||
@@ -101,44 +152,14 @@ if is_py3:
|
||||
pass
|
||||
|
||||
|
||||
else:
|
||||
import zc.lockfile
|
||||
def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False):
|
||||
if hardlink_lock and is_py2:
|
||||
raise DvcException(
|
||||
"Hardlink locks are not supported on Python <3.5. "
|
||||
"See `hardlink_lock` in {}".format(
|
||||
format_link("man.dvc.org/config#core")
|
||||
)
|
||||
)
|
||||
|
||||
class Lock(object):
|
||||
"""Class for dvc repo lock.
|
||||
|
||||
Uses zc.lockfile as backend.
|
||||
"""
|
||||
|
||||
def __init__(self, lockfile, tmp_dir=None):
|
||||
self.lockfile = lockfile
|
||||
self._lock = None
|
||||
|
||||
@property
|
||||
def files(self):
|
||||
return [self.lockfile]
|
||||
|
||||
def _do_lock(self):
|
||||
try:
|
||||
self._lock = zc.lockfile.LockFile(self.lockfile)
|
||||
except zc.lockfile.LockError:
|
||||
raise LockError(FAILED_TO_LOCK_MESSAGE)
|
||||
|
||||
def lock(self):
|
||||
try:
|
||||
self._do_lock()
|
||||
return
|
||||
except LockError:
|
||||
time.sleep(DEFAULT_TIMEOUT)
|
||||
|
||||
self._do_lock()
|
||||
|
||||
def unlock(self):
|
||||
self._lock.close()
|
||||
self._lock = None
|
||||
|
||||
def __enter__(self):
|
||||
self.lock()
|
||||
|
||||
def __exit__(self, typ, value, tbck):
|
||||
self.unlock()
|
||||
cls = HardlinkLock if hardlink_lock else Lock
|
||||
return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly)
|
||||
|
||||
@@ -69,7 +69,7 @@ class Repo(object):
|
||||
|
||||
def __init__(self, root_dir=None):
|
||||
from dvc.state import State
|
||||
from dvc.lock import Lock
|
||||
from dvc.lock import make_lock
|
||||
from dvc.scm import SCM
|
||||
from dvc.cache import Cache
|
||||
from dvc.data_cloud import DataCloud
|
||||
@@ -88,9 +88,12 @@ class Repo(object):
|
||||
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
|
||||
self.lock = Lock(
|
||||
hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
|
||||
self.lock = make_lock(
|
||||
os.path.join(self.dvc_dir, "lock"),
|
||||
tmp_dir=os.path.join(self.dvc_dir, "tmp"),
|
||||
hardlink_lock=hardlink_lock,
|
||||
friendly=True,
|
||||
)
|
||||
# NOTE: storing state and link_state in the repository itself to avoid
|
||||
# any possible state corruption in 'shared cache dir' scenario.
|
||||
|
||||
+7
-5
@@ -9,8 +9,7 @@ import colorama
|
||||
from packaging import version
|
||||
|
||||
from dvc import __version__
|
||||
from dvc.lock import Lock
|
||||
from dvc.lock import LockError
|
||||
from dvc.lock import make_lock, LockError
|
||||
from dvc.utils import boxify
|
||||
from dvc.utils import env2bool
|
||||
from dvc.utils.pkg import PKG
|
||||
@@ -24,11 +23,14 @@ class Updater(object): # pragma: no cover
|
||||
TIMEOUT = 24 * 60 * 60 # every day
|
||||
TIMEOUT_GET = 10
|
||||
|
||||
def __init__(self, dvc_dir):
|
||||
def __init__(self, dvc_dir, friendly=False, hardlink_lock=False):
|
||||
self.dvc_dir = dvc_dir
|
||||
self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
|
||||
self.lock = Lock(
|
||||
self.updater_file + ".lock", tmp_dir=os.path.join(dvc_dir, "tmp")
|
||||
self.lock = make_lock(
|
||||
self.updater_file + ".lock",
|
||||
tmp_dir=os.path.join(dvc_dir, "tmp"),
|
||||
friendly=friendly,
|
||||
hardlink_lock=hardlink_lock,
|
||||
)
|
||||
self.current = version.parse(__version__).base_version
|
||||
|
||||
|
||||
@@ -420,3 +420,11 @@ def resolve_output(inp, out):
|
||||
if os.path.isdir(out):
|
||||
return os.path.join(out, name)
|
||||
return out
|
||||
|
||||
|
||||
def format_link(link):
|
||||
import colorama
|
||||
|
||||
return "<{blue}{link}{nc}>".format(
|
||||
blue=colorama.Fore.CYAN, link=link, nc=colorama.Fore.RESET
|
||||
)
|
||||
|
||||
@@ -79,6 +79,7 @@ install_requires = [
|
||||
"shortuuid>=0.5.0",
|
||||
"tqdm>=4.40.0,<5",
|
||||
"packaging>=19.0",
|
||||
"zc.lockfile>=1.2.1",
|
||||
"win-unicode-console>=0.5; sys_platform == 'win32'",
|
||||
"pywin32>=225; sys_platform == 'win32'",
|
||||
]
|
||||
@@ -163,12 +164,7 @@ setup(
|
||||
"ssh_gssapi": ssh_gssapi,
|
||||
"hdfs": hdfs,
|
||||
# NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1
|
||||
":python_version=='2.7'": [
|
||||
"futures",
|
||||
"pathlib2",
|
||||
"contextlib2",
|
||||
"zc.lockfile>=1.2.1",
|
||||
],
|
||||
":python_version=='2.7'": ["futures", "pathlib2", "contextlib2"],
|
||||
":python_version>='3.0'": ["flufl.lock>=3.2"],
|
||||
"tests": tests_requirements,
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user