mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-07 23:09:21 +01:00
Ruslan Kuprieiev
e384685715
Currently only one instance of dvc could run in a particular repo because of the `.dvc/lock`. That gets furstrating when you are running some long-running command and blocked from doing anything else. This PR solved that issue for `dvc run` by releasing the repo lock while running the command. To protect against conflicts, we now use rwlock, which keeps the list of readers and writers for particular paths. The process looks like this: 1) aquire read locks for deps; 2) aquire write locks for outs; 3) release repo lock; 4) run cmd; 5) aquire repo lock; 6) release write locks for outs; 7) release read locks for outs; We also take rwlocks for operations such as `import`, `add` and `checkout` to ensure that we won't hit the same files that are being used by another `dvc run` in the background. We don't aquire rwlocks for the stage file itself, as we don't really care about anyone overwriting a stage file, as there is a very little difference here between parallel runs and sequential runs that overwrite the same stage file. Next steps might be to release repo lock on `import`(when dowloading stuff) and when computing checksums or copying files. Fixes #755
99 lines
2.6 KiB
Python
99 lines
2.6 KiB
Python
from __future__ import unicode_literals
|
|
|
|
import os
|
|
|
|
import pytest
|
|
|
|
from dvc.rwlock import (
|
|
rwlock,
|
|
_edit_rwlock,
|
|
RWLockFileFormatError,
|
|
RWLockFileCorruptedError,
|
|
)
|
|
from dvc.lock import LockError
|
|
from dvc.utils.compat import fspath
|
|
from dvc.path_info import PathInfo
|
|
|
|
|
|
def test_rwlock(tmp_path):
|
|
path = fspath(tmp_path)
|
|
foo = PathInfo("foo")
|
|
|
|
with rwlock(path, "cmd1", [foo], []):
|
|
with pytest.raises(LockError):
|
|
with rwlock(path, "cmd2", [], [foo]):
|
|
pass
|
|
|
|
with rwlock(path, "cmd1", [], [foo]):
|
|
with pytest.raises(LockError):
|
|
with rwlock(path, "cmd2", [foo], []):
|
|
pass
|
|
|
|
with rwlock(path, "cmd1", [], [foo]):
|
|
with pytest.raises(LockError):
|
|
with rwlock(path, "cmd2", [], [foo]):
|
|
pass
|
|
|
|
|
|
def test_rwlock_reentrant(tmp_path):
|
|
path = fspath(tmp_path)
|
|
foo = PathInfo("foo")
|
|
|
|
with rwlock(path, "cmd1", [], [foo]):
|
|
with rwlock(path, "cmd1", [], [foo]):
|
|
pass
|
|
with _edit_rwlock(path) as lock:
|
|
assert lock == {
|
|
"read": {},
|
|
"write": {"foo": {"cmd": "cmd1", "pid": os.getpid()}},
|
|
}
|
|
|
|
with rwlock(path, "cmd", [foo], []):
|
|
with rwlock(path, "cmd", [foo], []):
|
|
pass
|
|
with _edit_rwlock(path) as lock:
|
|
assert lock == {
|
|
"read": {"foo": [{"cmd": "cmd", "pid": os.getpid()}]},
|
|
"write": {},
|
|
}
|
|
|
|
|
|
def test_rwlock_subdirs(tmp_path):
|
|
path = fspath(tmp_path)
|
|
foo = PathInfo("foo")
|
|
subfoo = PathInfo("foo/subfoo")
|
|
|
|
with rwlock(path, "cmd1", [foo], []):
|
|
with pytest.raises(LockError):
|
|
with rwlock(path, "cmd2", [], [subfoo]):
|
|
pass
|
|
|
|
with rwlock(path, "cmd1", [], [subfoo]):
|
|
with pytest.raises(LockError):
|
|
with rwlock(path, "cmd2", [foo], []):
|
|
pass
|
|
|
|
with rwlock(path, "cmd1", [], [subfoo]):
|
|
with pytest.raises(LockError):
|
|
with rwlock(path, "cmd2", [], [foo]):
|
|
pass
|
|
|
|
with rwlock(path, "cmd1", [subfoo], []):
|
|
with rwlock(path, "cmd2", [foo], []):
|
|
pass
|
|
|
|
|
|
def test_broken_rwlock(tmp_path):
|
|
dir_path = fspath(tmp_path)
|
|
path = tmp_path / "rwlock"
|
|
|
|
path.write_text('{"broken": "format"}', encoding="utf-8")
|
|
with pytest.raises(RWLockFileFormatError):
|
|
with _edit_rwlock(dir_path):
|
|
pass
|
|
|
|
path.write_text("{broken json", encoding="utf-8")
|
|
with pytest.raises(RWLockFileCorruptedError):
|
|
with _edit_rwlock(dir_path):
|
|
pass
|