mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
Ruslan Kuprieiev
e384685715
Currently only one instance of dvc could run in a particular repo because of the `.dvc/lock`. That gets furstrating when you are running some long-running command and blocked from doing anything else. This PR solved that issue for `dvc run` by releasing the repo lock while running the command. To protect against conflicts, we now use rwlock, which keeps the list of readers and writers for particular paths. The process looks like this: 1) aquire read locks for deps; 2) aquire write locks for outs; 3) release repo lock; 4) run cmd; 5) aquire repo lock; 6) release write locks for outs; 7) release read locks for outs; We also take rwlocks for operations such as `import`, `add` and `checkout` to ensure that we won't hit the same files that are being used by another `dvc run` in the background. We don't aquire rwlocks for the stage file itself, as we don't really care about anyone overwriting a stage file, as there is a very little difference here between parallel runs and sequential runs that overwrite the same stage file. Next steps might be to release repo lock on `import`(when dowloading stuff) and when computing checksums or copying files. Fixes #755
284 lines
8.0 KiB
Python
284 lines
8.0 KiB
Python
from __future__ import absolute_import
|
|
from __future__ import unicode_literals
|
|
|
|
import os
|
|
import posixpath
|
|
import sys
|
|
|
|
from funcy import cached_property
|
|
|
|
from dvc.utils import relpath
|
|
from dvc.utils.compat import basestring
|
|
from dvc.utils.compat import builtin_str
|
|
from dvc.utils.compat import is_py2
|
|
from dvc.utils.compat import pathlib
|
|
from dvc.utils.compat import str
|
|
from dvc.utils.compat import urlparse
|
|
|
|
|
|
# On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
|
|
# which is lossy, thus we can't use that,
|
|
# see https://github.com/mcmtroffaes/pathlib2/issues/56.
|
|
if is_py2:
|
|
fs_encoding = "utf-8"
|
|
|
|
|
|
class _BasePath(object):
|
|
def overlaps(self, other):
|
|
if isinstance(other, basestring):
|
|
other = self.__class__(other)
|
|
elif self.__class__ != other.__class__:
|
|
return False
|
|
return self == other or self.isin(other) or other.isin(self)
|
|
|
|
|
|
class PathInfo(pathlib.PurePath, _BasePath):
|
|
# Use __slots__ in PathInfo objects following PurePath implementation.
|
|
# This makes objects smaller and speeds up attribute access.
|
|
# We don't add any fields so it's empty.
|
|
__slots__ = ()
|
|
scheme = "local"
|
|
|
|
def __new__(cls, *args):
|
|
# Construct a proper subclass depending on current os
|
|
if cls is PathInfo:
|
|
cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo
|
|
return cls._from_parts(args)
|
|
|
|
@classmethod
|
|
def from_posix(cls, s):
|
|
return cls(PosixPathInfo(s))
|
|
|
|
def as_posix(self):
|
|
f = self._flavour
|
|
# Unlike original implementation [1] that uses `str()` we actually need
|
|
# to use `fspath`, because we've overridden `__str__` method to return
|
|
# relative paths, which will break original `as_posix`.
|
|
#
|
|
# [1] https://github.com/python/cpython/blob/v3.7.0/Lib/pathlib.py#L692
|
|
return self.fspath.replace(f.sep, "/")
|
|
|
|
def __str__(self):
|
|
path = self.__fspath__()
|
|
return relpath(path)
|
|
|
|
def __repr__(self):
|
|
return builtin_str("{}: '{}'").format(type(self).__name__, self)
|
|
|
|
# This permits passing it to file utils directly in Python 3.6+
|
|
# With Python 2.7, Python 3.5+ we are stuck with path_info.fspath for now
|
|
def __fspath__(self):
|
|
return pathlib.PurePath.__str__(self)
|
|
|
|
@property
|
|
def fspath(self):
|
|
return self.__fspath__()
|
|
|
|
url = fspath
|
|
|
|
def relpath(self, other):
|
|
return self.__class__(relpath(self, other))
|
|
|
|
def isin(self, other):
|
|
if isinstance(other, basestring):
|
|
other = self.__class__(other)
|
|
elif self.__class__ != other.__class__:
|
|
return False
|
|
# Use cached casefolded parts to compare paths
|
|
n = len(other._cparts)
|
|
return len(self._cparts) > n and self._cparts[:n] == other._cparts
|
|
|
|
# pathlib2 uses bytes internally in Python 2, and we use unicode everywhere
|
|
# for paths in both pythons, thus we need this glue.
|
|
if is_py2:
|
|
__unicode__ = __str__
|
|
|
|
def __str__(self):
|
|
return self.__unicode__().encode(sys.getfilesystemencoding())
|
|
|
|
@classmethod
|
|
def _parse_args(cls, args):
|
|
args = [
|
|
a.encode(fs_encoding)
|
|
if isinstance(a, unicode) # noqa: F821
|
|
else a
|
|
for a in args
|
|
]
|
|
return super(PathInfo, cls)._parse_args(args)
|
|
|
|
@property
|
|
def name(self):
|
|
return super(PathInfo, self).name.decode(fs_encoding)
|
|
|
|
def __fspath__(self): # noqa: F811
|
|
return pathlib.PurePath.__str__(self).decode(fs_encoding)
|
|
|
|
def with_name(self, name):
|
|
return pathlib.PurePath.with_name(self, name.encode(fs_encoding))
|
|
|
|
|
|
class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
|
|
pass
|
|
|
|
|
|
class PosixPathInfo(PathInfo, pathlib.PurePosixPath):
|
|
pass
|
|
|
|
|
|
class _URLPathInfo(PosixPathInfo):
|
|
def __str__(self):
|
|
return self.__fspath__()
|
|
|
|
__unicode__ = __str__
|
|
|
|
|
|
class _URLPathParents(object):
|
|
def __init__(self, src):
|
|
self.src = src
|
|
self._parents = self.src._path.parents
|
|
|
|
def __len__(self):
|
|
return len(self._parents)
|
|
|
|
def __getitem__(self, idx):
|
|
return self.src.replace(path=self._parents[idx])
|
|
|
|
def __repr__(self):
|
|
return "<{}.parents>".format(self.src)
|
|
|
|
|
|
class URLInfo(_BasePath):
|
|
DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22, "hdfs": 0}
|
|
|
|
def __init__(self, url):
|
|
p = urlparse(url)
|
|
assert not p.query and not p.params and not p.fragment
|
|
assert p.password is None
|
|
|
|
self.fill_parts(p.scheme, p.hostname, p.username, p.port, p.path)
|
|
|
|
@classmethod
|
|
def from_parts(
|
|
cls, scheme=None, host=None, user=None, port=None, path="", netloc=None
|
|
):
|
|
assert bool(host) ^ bool(netloc)
|
|
|
|
if netloc is not None:
|
|
return cls("{}://{}{}".format(scheme, netloc, path))
|
|
|
|
obj = cls.__new__(cls)
|
|
obj.fill_parts(scheme, host, user, port, path)
|
|
return obj
|
|
|
|
def fill_parts(self, scheme, host, user, port, path):
|
|
assert scheme != "remote"
|
|
assert isinstance(path, (basestring, _URLPathInfo))
|
|
|
|
self.scheme, self.host, self.user = scheme, host, user
|
|
self.port = int(port) if port else self.DEFAULT_PORTS.get(self.scheme)
|
|
|
|
if isinstance(path, _URLPathInfo):
|
|
self._spath = builtin_str(path)
|
|
self._path = path
|
|
else:
|
|
if path and path[0] != "/":
|
|
path = "/" + path
|
|
self._spath = path
|
|
|
|
@property
|
|
def _base_parts(self):
|
|
return (self.scheme, self.host, self.user, self.port)
|
|
|
|
@property
|
|
def parts(self):
|
|
return self._base_parts + self._path.parts
|
|
|
|
def replace(self, path=None):
|
|
return self.from_parts(*self._base_parts, path=path)
|
|
|
|
@cached_property
|
|
def url(self):
|
|
return "{}://{}{}".format(self.scheme, self.netloc, self._spath)
|
|
|
|
def __str__(self):
|
|
return self.url
|
|
|
|
def __repr__(self):
|
|
return "{}: '{}'".format(type(self).__name__, self)
|
|
|
|
def __eq__(self, other):
|
|
if isinstance(other, basestring):
|
|
other = self.__class__(other)
|
|
return (
|
|
self.__class__ == other.__class__
|
|
and self._base_parts == other._base_parts
|
|
and self._path == other._path
|
|
)
|
|
|
|
def __hash__(self):
|
|
return hash(self.parts)
|
|
|
|
def __div__(self, other):
|
|
return self.replace(path=posixpath.join(self._spath, other))
|
|
|
|
__truediv__ = __div__
|
|
|
|
@property
|
|
def path(self):
|
|
return self._spath
|
|
|
|
@cached_property
|
|
def _path(self):
|
|
return _URLPathInfo(self._spath)
|
|
|
|
@property
|
|
def name(self):
|
|
return self._path.name
|
|
|
|
@cached_property
|
|
def netloc(self):
|
|
netloc = self.host
|
|
if self.user:
|
|
netloc = self.user + "@" + netloc
|
|
if self.port and int(self.port) != self.DEFAULT_PORTS.get(self.scheme):
|
|
netloc += ":" + str(self.port)
|
|
return netloc
|
|
|
|
@property
|
|
def bucket(self):
|
|
return self.netloc
|
|
|
|
@property
|
|
def parent(self):
|
|
return self.replace(path=self._path.parent)
|
|
|
|
@property
|
|
def parents(self):
|
|
return _URLPathParents(self)
|
|
|
|
def relative_to(self, other):
|
|
if isinstance(other, basestring):
|
|
other = self.__class__(other)
|
|
if self.__class__ != other.__class__:
|
|
msg = "'{}' has incompatible class with '{}'".format(self, other)
|
|
raise ValueError(msg)
|
|
if self._base_parts != other._base_parts:
|
|
msg = "'{}' does not start with '{}'".format(self, other)
|
|
raise ValueError(msg)
|
|
return self._path.relative_to(other._path)
|
|
|
|
def isin(self, other):
|
|
if isinstance(other, basestring):
|
|
other = self.__class__(other)
|
|
elif self.__class__ != other.__class__:
|
|
return False
|
|
return self._base_parts == other._base_parts and self._path.isin(
|
|
other._path
|
|
)
|
|
|
|
|
|
class CloudURLInfo(URLInfo):
|
|
@property
|
|
def path(self):
|
|
return self._spath.lstrip("/")
|