state: use diskcache (#5505)

* state: only load when needed

* state: don't use separate load/dump

* checkout: group link removal logic together

* state: use diskcache.Cache

* staging: use pbar and save to state

* state: don't pass the whole repo to __init__

* staging: explicitly pass state object

* fs: get rid of odb property

* state: don't close the connection to cache

* repo: close: close state
This commit is contained in:
Ruslan Kuprieiev
2021-02-23 20:51:34 +02:00
committed by GitHub
parent 2e01ce1974
commit 8f568ab197
32 changed files with 245 additions and 589 deletions
+1 -1
View File
@@ -29,7 +29,7 @@ def get_url(path, repo=None, rev=None, remote=None):
raise OutputNotFoundError(path, repo)
cloud = metadata.repo.cloud
hash_info = get_hash(path_info, _repo.repo_fs, "md5")
hash_info = get_hash(path_info, _repo.repo_fs, "md5", _repo.odb)
return cloud.get_url_for(remote, checksum=hash_info.value)
+2 -2
View File
@@ -30,7 +30,7 @@ def _changed(path_info, fs, obj, cache):
return True
try:
actual = get_hash(path_info, fs, obj.hash_info.name)
actual = get_hash(path_info, fs, obj.hash_info.name, cache)
except FileNotFoundError:
logger.debug("'%s' doesn't exist.", path_info)
return True
@@ -56,7 +56,7 @@ def _remove(path_info, fs, cache, force=False):
fs.remove(path_info)
return
current = get_hash(path_info, fs, fs.PARAM_CHECKSUM)
current = get_hash(path_info, fs, fs.PARAM_CHECKSUM, cache)
try:
obj = load(cache, current)
check(cache, obj)
+5 -6
View File
@@ -69,14 +69,13 @@ class CmdMergeDriver(CmdHookBase):
dvc = Repo()
try:
with dvc.state:
ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
our = Dvcfile(dvc, self.args.our, verify=False)
their = Dvcfile(dvc, self.args.their, verify=False)
ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
our = Dvcfile(dvc, self.args.our, verify=False)
their = Dvcfile(dvc, self.args.their, verify=False)
our.merge(ancestor, their)
our.merge(ancestor, their)
return 0
return 0
finally:
dvc.close()
+2 -2
View File
@@ -213,8 +213,8 @@ SCHEMA = {
)
},
"state": {
"row_limit": All(Coerce(int), Range(1)),
"row_cleanup_quota": All(Coerce(int), Range(0, 100)),
"row_limit": All(Coerce(int), Range(1)), # obsoleted
"row_cleanup_quota": All(Coerce(int), Range(0, 100)), # obsoleted
},
# section for experimental features
"feature": {
+7 -4
View File
@@ -47,13 +47,16 @@ class RepoDependency(LocalDependency):
return external_repo(d["url"], rev=rev, **kwargs)
def _get_hash(self, locked=True):
from dvc.objects.stage import get_hash
from dvc.objects.stage import stage
with self._make_repo(locked=locked) as repo:
path_info = PathInfo(repo.root_dir) / self.def_path
return get_hash(
path_info, repo.repo_fs, "md5", follow_subrepos=False
)
return stage(
self.repo.odb.local,
path_info,
repo.repo_fs,
follow_subrepos=False,
).hash_info
def workspace_status(self):
current = self._get_hash(locked=True)
-4
View File
@@ -130,10 +130,6 @@ class BaseFileSystem:
f"dependencies: {missing}. {hint}"
)
@property
def odb(self):
return getattr(self.repo.odb, self.scheme)
def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
if hasattr(self, "_generate_download_url"):
# pylint:disable=no-member
+8 -4
View File
@@ -36,7 +36,7 @@ class HashFile:
return bool(self.hash_info)
def check(self, odb):
actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM)
actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM, odb)
logger.trace(
"cache '%s' expected '%s' actual '%s'",
@@ -64,7 +64,9 @@ class File(HashFile):
@classmethod
def stage(cls, odb, path_info, fs, **kwargs):
hash_info = get_hash(path_info, fs, odb.fs.PARAM_CHECKSUM, **kwargs)
hash_info = get_hash(
path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs
)
raw = odb.get(hash_info)
obj = cls(raw.path_info, raw.fs, hash_info)
obj.src = HashFile(path_info, fs, hash_info)
@@ -114,7 +116,7 @@ class Tree(HashFile):
with fs.open(path_info, "rb") as fobj:
odb.fs.upload_fobj(fobj, tmp_info)
hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM)
hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM, odb)
hash_info.value += odb.fs.CHECKSUM_DIR_SUFFIX
hash_info.dir_info = dir_info
hash_info.nfiles = dir_info.nfiles
@@ -125,7 +127,9 @@ class Tree(HashFile):
@classmethod
def stage(cls, odb, path_info, fs, **kwargs):
hash_info = get_hash(path_info, fs, odb.fs.PARAM_CHECKSUM, **kwargs)
hash_info = get_hash(
path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs
)
hi = cls.save_dir_info(odb, hash_info.dir_info, hash_info)
hi.size = hash_info.size
raw = odb.get(hi)
+1 -2
View File
@@ -58,8 +58,7 @@ class ObjectDB:
with fs.open(path_info, mode="rb") as fobj:
self.fs.upload_fobj(fobj, cache_info)
self.protect(cache_info)
with self.fs.repo.state:
self.fs.repo.state.save(cache_info, self.fs, hash_info)
self.fs.repo.state.save(cache_info, self.fs, hash_info)
callback = kwargs.get("download_callback")
if callback:
+44 -51
View File
@@ -27,22 +27,26 @@ def get_file_hash(path_info, fs, name):
raise NotImplementedError
def _calculate_hashes(file_infos, fs, name):
def _get_file_hash(path_info):
return get_file_hash(path_info, fs, name)
def _calculate_hashes(path_info, fs, name, state, **kwargs):
def _get_file_hash(file_info):
hash_info = state.get( # pylint: disable=assignment-from-none
file_info, fs,
)
if not hash_info:
hash_info = get_file_hash(file_info, fs, name)
state.save(file_info, fs, hash_info)
return file_info, hash_info
with Tqdm(
total=len(file_infos),
unit="md5",
desc="Computing file/dir hashes (only done once)",
unit="md5", desc="Computing file/dir hashes (only done once)",
) as pbar:
worker = pbar.wrap_fn(_get_file_hash)
with ThreadPoolExecutor(max_workers=fs.hash_jobs) as executor:
hash_infos = executor.map(worker, file_infos)
return dict(zip(file_infos, hash_infos))
pairs = executor.map(worker, fs.walk_files(path_info, **kwargs))
return dict(pairs)
def _iter_hashes(path_info, fs, name, **kwargs):
def _iter_hashes(path_info, fs, name, state, **kwargs):
if name in fs.DETAIL_FIELDS:
for details in fs.ls(path_info, recursive=True, detail=True):
file_info = path_info.replace(path=details["name"])
@@ -53,25 +57,14 @@ def _iter_hashes(path_info, fs, name, **kwargs):
return None
file_infos = []
for file_info in fs.walk_files(path_info, **kwargs):
hash_info = fs.repo.state.get( # pylint: disable=assignment-from-none
file_info, fs,
)
if not hash_info:
file_infos.append(file_info)
continue
assert hash_info.name == name
yield file_info, hash_info
yield from _calculate_hashes(file_infos, fs, name).items()
yield from _calculate_hashes(path_info, fs, name, state, **kwargs).items()
def _collect_dir(path_info, fs, name, **kwargs):
def _collect_dir(path_info, fs, name, state, **kwargs):
from dvc.dir_info import DirInfo
dir_info = DirInfo()
for fi, hi in _iter_hashes(path_info, fs, name, **kwargs):
for fi, hi in _iter_hashes(path_info, fs, name, state, **kwargs):
if DvcIgnore.DVCIGNORE_FILE == fi.name:
raise DvcIgnoreInCollectedDirError(fi.parent)
@@ -88,17 +81,17 @@ def _collect_dir(path_info, fs, name, **kwargs):
return dir_info
def get_dir_hash(path_info, fs, name, **kwargs):
def get_dir_hash(path_info, fs, name, state, **kwargs):
from . import Tree
dir_info = _collect_dir(path_info, fs, name, **kwargs)
dir_info = _collect_dir(path_info, fs, name, state, **kwargs)
hash_info = Tree.save_dir_info(fs.repo.odb.local, dir_info)
hash_info.size = dir_info.size
hash_info.dir_info = dir_info
return hash_info
def get_hash(path_info, fs, name, **kwargs):
def get_hash(path_info, fs, name, odb, **kwargs):
assert path_info and (
isinstance(path_info, str) or path_info.scheme == fs.scheme
)
@@ -108,36 +101,36 @@ def get_hash(path_info, fs, name, **kwargs):
errno.ENOENT, os.strerror(errno.ENOENT), path_info
)
with fs.repo.state:
# pylint: disable=assignment-from-none
hash_info = fs.repo.state.get(path_info, fs)
state = odb.repo.state
# pylint: disable=assignment-from-none
hash_info = state.get(path_info, fs)
# If we have dir hash in state db, but dir cache file is lost,
# then we need to recollect the dir via .get_dir_hash() call below,
# see https://github.com/iterative/dvc/issues/2219 for context
if (
hash_info
and hash_info.isdir
and not fs.odb.fs.exists(fs.odb.hash_to_path_info(hash_info.value))
):
hash_info = None
# If we have dir hash in state db, but dir cache file is lost,
# then we need to recollect the dir via .get_dir_hash() call below,
# see https://github.com/iterative/dvc/issues/2219 for context
if (
hash_info
and hash_info.isdir
and not odb.fs.exists(odb.hash_to_path_info(hash_info.value))
):
hash_info = None
if hash_info:
if hash_info.isdir:
from . import Tree
if hash_info:
if hash_info.isdir:
from . import Tree
# NOTE: loading the fs will restore hash_info.dir_info
Tree.load(fs.odb, hash_info)
assert hash_info.name == name
return hash_info
# NOTE: loading the fs will restore hash_info.dir_info
Tree.load(odb, hash_info)
assert hash_info.name == name
return hash_info
if fs.isdir(path_info):
hash_info = get_dir_hash(path_info, fs, name, **kwargs)
else:
hash_info = get_file_hash(path_info, fs, name)
if fs.isdir(path_info):
hash_info = get_dir_hash(path_info, fs, name, state, **kwargs)
else:
hash_info = get_file_hash(path_info, fs, name)
if hash_info and fs.exists(path_info):
fs.repo.state.save(path_info, fs, hash_info)
if hash_info and fs.exists(path_info):
state.save(path_info, fs, hash_info)
return hash_info
+6 -1
View File
@@ -193,7 +193,12 @@ class BaseOutput:
def get_hash(self):
if not self.use_cache:
return get_hash(self.path_info, self.fs, self.fs.PARAM_CHECKSUM)
return get_hash(
self.path_info,
self.fs,
self.fs.PARAM_CHECKSUM,
self.repo.odb.local,
)
return ostage(self.odb, self.path_info, self.fs).hash_info
@property
+16 -20
View File
@@ -472,13 +472,12 @@ class Remote:
)
if self.fs.scheme == "local":
with self.fs.repo.state:
for checksum in named_cache.scheme_keys("local"):
cache_file = self.odb.hash_to_path_info(checksum)
if self.fs.exists(cache_file):
hash_info = HashInfo(self.fs.PARAM_CHECKSUM, checksum)
self.fs.repo.state.save(cache_file, self.fs, hash_info)
self.odb.protect(cache_file)
for checksum in named_cache.scheme_keys("local"):
cache_file = self.odb.hash_to_path_info(checksum)
if self.fs.exists(cache_file):
hash_info = HashInfo(self.fs.PARAM_CHECKSUM, checksum)
self.fs.repo.state.save(cache_file, self.fs, hash_info)
self.odb.protect(cache_file)
return ret
@@ -493,19 +492,16 @@ class Remote:
)
if not self.odb.verify:
with cache.fs.repo.state:
for checksum in named_cache.scheme_keys("local"):
cache_file = cache.hash_to_path_info(checksum)
if cache.fs.exists(cache_file):
# We can safely save here, as existing corrupted files
# will be removed upon status, while files corrupted
# during download will not be moved from tmp_file
# (see `BaseFileSystem.download()`)
hash_info = HashInfo(cache.fs.PARAM_CHECKSUM, checksum)
cache.fs.repo.state.save(
cache_file, cache.fs, hash_info
)
cache.protect(cache_file)
for checksum in named_cache.scheme_keys("local"):
cache_file = cache.hash_to_path_info(checksum)
if cache.fs.exists(cache_file):
# We can safely save here, as existing corrupted files
# will be removed upon status, while files corrupted
# during download will not be moved from tmp_file
# (see `BaseFileSystem.download()`)
hash_info = HashInfo(cache.fs.PARAM_CHECKSUM, checksum)
cache.fs.repo.state.save(cache_file, cache.fs, hash_info)
cache.protect(cache_file)
return ret
+8 -7
View File
@@ -33,7 +33,7 @@ def lock_repo(repo: "Repo"):
if depth > 0:
yield
else:
with repo.lock, repo.state:
with repo.lock:
repo._reset()
yield
# Graph cache is no longer valid after we release the repo.lock
@@ -186,7 +186,7 @@ class Repo:
# NOTE: storing state and link_state in the repository itself to
# avoid any possible state corruption in 'shared cache dir'
# scenario.
self.state = State(self)
self.state = State(self.root_dir, self.tmp_dir)
self.stage_cache = StageCache(self)
self._ignore()
@@ -477,11 +477,10 @@ class Repo:
fs = RepoFileSystem(self, subrepos=True)
path = PathInfo(self.root_dir) / path
try:
with self.state:
with fs.open(
path, mode=mode, encoding=encoding, remote=remote,
) as fobj:
yield fobj
with fs.open(
path, mode=mode, encoding=encoding, remote=remote,
) as fobj:
yield fobj
except FileNotFoundError as exc:
raise FileMissingError(path) from exc
except IsADirectoryError as exc:
@@ -489,8 +488,10 @@ class Repo:
def close(self):
self.scm.close()
self.state.close()
def _reset(self):
self.state.close()
self.scm._reset() # pylint: disable=protected-access
self.__dict__.pop("outs_trie", None)
self.__dict__.pop("outs_graph", None)
+15 -15
View File
@@ -19,16 +19,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
def _get_unused_links(repo):
used = [
out.fspath
for stage in repo.stages
for out in stage.outs
if out.scheme == "local"
]
return repo.state.get_unused_links(used, repo.fs)
def _fspath_dir(path):
if not os.path.exists(str(path)):
return str(path)
@@ -37,6 +27,20 @@ def _fspath_dir(path):
return os.path.join(path, "") if os.path.isdir(path) else path
def _remove_unused_links(repo):
used = [
out.fspath
for stage in repo.stages
for out in stage.outs
if out.scheme == "local"
]
unused = repo.state.get_unused_links(used, repo.fs)
ret = [_fspath_dir(u) for u in unused]
repo.state.remove_links(unused, repo.fs)
return ret
def get_all_files_numbers(pairs):
return sum(
stage.get_all_files_number(filter_info) for stage, filter_info in pairs
@@ -83,7 +87,6 @@ def checkout(
**kwargs,
):
unused = []
stats = {
"added": [],
"deleted": [],
@@ -92,10 +95,7 @@ def checkout(
}
if not targets:
targets = [None]
unused = _get_unused_links(self)
stats["deleted"] = [_fspath_dir(u) for u in unused]
self.state.remove_links(unused, self.fs)
stats["deleted"] = _remove_unused_links(self)
if isinstance(targets, str):
targets = [targets]
+1
View File
@@ -14,4 +14,5 @@ def _destroy_stages(repo):
# which will cause issues on Windows, as `.dvc/lock` will be busy.
def destroy(repo):
_destroy_stages(repo)
repo.close()
remove(repo.dvc_dir)
+5 -2
View File
@@ -2,7 +2,8 @@ import logging
import os
from dvc.exceptions import PathMissingError
from dvc.objects.stage import get_file_hash, get_hash
from dvc.objects.stage import get_file_hash
from dvc.objects.stage import stage as ostage
from dvc.repo import locked
logger = logging.getLogger(__name__)
@@ -128,7 +129,9 @@ def _output_paths(repo, repo_fs, targets):
def _to_checksum(output):
if on_working_fs:
return get_hash(output.path_info, repo.odb.local.fs, "md5").value
return ostage(
repo.odb.local, output.path_info, repo.odb.local.fs,
).hash_info.value
return output.hash_info.value
for stage in repo.stages:
-2
View File
@@ -51,14 +51,12 @@ def scm_locked(f):
def unlocked_repo(f):
@wraps(f)
def wrapper(exp, *args, **kwargs):
exp.repo.state.dump()
exp.repo.lock.unlock()
exp.repo._reset() # pylint: disable=protected-access
try:
ret = f(exp, *args, **kwargs)
finally:
exp.repo.lock.lock()
exp.repo.state.load()
return ret
return wrapper
+1 -1
View File
@@ -337,7 +337,7 @@ class BaseExecutor(ABC):
logger.exception("unexpected error")
raise
finally:
dvc.scm.close()
dvc.close()
if old_cwd:
os.chdir(old_cwd)
-1
View File
@@ -55,7 +55,6 @@ def gc(
with ExitStack() as stack:
for repo in all_repos:
stack.enter_context(repo.lock)
stack.enter_context(repo.state)
used = NamedCache()
for repo in all_repos + [self]:
-4
View File
@@ -39,14 +39,12 @@ def rwlocked(call, read=None, write=None):
def unlocked_repo(f):
@wraps(f)
def wrapper(stage, *args, **kwargs):
stage.repo.state.dump()
stage.repo.lock.unlock()
stage.repo._reset() # pylint: disable=protected-access
try:
ret = f(stage, *args, **kwargs)
finally:
stage.repo.lock.lock()
stage.repo.state.load()
return ret
return wrapper
@@ -56,11 +54,9 @@ def relock_repo(f):
@wraps(f)
def wrapper(stage, *args, **kwargs):
stage.repo.lock.lock()
stage.repo.state.load()
try:
ret = f(stage, *args, **kwargs)
finally:
stage.repo.state.dump()
stage.repo.lock.unlock()
stage.repo._reset() # pylint: disable=protected-access
return ret
+46 -358
View File
@@ -9,7 +9,7 @@ from urllib.parse import urlencode, urlunparse
from dvc.exceptions import DvcException
from dvc.fs.local import LocalFileSystem
from dvc.hash_info import HashInfo
from dvc.utils import current_timestamp, relpath, to_chunks
from dvc.utils import relpath
from dvc.utils.fs import get_inode, get_mtime_and_size, remove
SQLITE_MAX_VARIABLES_NUMBER = 999
@@ -32,8 +32,9 @@ class StateVersionTooNewError(DvcException):
class StateBase(ABC):
def __init__(self):
self.count = 0
@abstractmethod
def close(self):
pass
@abstractmethod
def save(self, path_info, fs, hash_info):
@@ -47,27 +48,11 @@ class StateBase(ABC):
def save_link(self, path_info, fs):
pass
@abstractmethod
def load(self):
pass
@abstractmethod
def dump(self):
pass
def __enter__(self):
if self.count <= 0:
self.load()
self.count += 1
def __exit__(self, typ, value, tbck):
self.count -= 1
if self.count > 0:
return
self.dump()
class StateNoop(StateBase):
def close(self):
pass
def save(self, path_info, fs, hash_info):
pass
@@ -77,307 +62,27 @@ class StateNoop(StateBase):
def save_link(self, path_info, fs):
pass
def load(self):
pass
def dump(self):
pass
class State(StateBase): # pylint: disable=too-many-instance-attributes
"""Class for the state database.
Args:
repo (dvc.repo.Repo): repo instance that this state belongs to.
config (configobj.ConfigObj): config for the state.
Raises:
StateVersionTooNewError: thrown when dvc version is older than the
state database version.
"""
VERSION = 3
STATE_FILE = "state"
STATE_TABLE = "state"
STATE_TABLE_LAYOUT = (
"inode INTEGER PRIMARY KEY, "
"mtime TEXT NOT NULL, "
"size TEXT NOT NULL, "
"md5 TEXT NOT NULL, "
"timestamp TEXT NOT NULL"
)
STATE_INFO_TABLE = "state_info"
STATE_INFO_TABLE_LAYOUT = "count INTEGER"
STATE_INFO_ROW = 1
LINK_STATE_TABLE = "link_state"
LINK_STATE_TABLE_LAYOUT = (
"path TEXT PRIMARY KEY, "
"inode INTEGER NOT NULL, "
"mtime TEXT NOT NULL"
)
STATE_ROW_LIMIT = 100000000
STATE_ROW_CLEANUP_QUOTA = 50
MAX_INT = 2 ** 63 - 1
MAX_UINT = 2 ** 64 - 2
def __init__(self, repo):
def __init__(self, root_dir=None, tmp_dir=None):
from diskcache import Cache
super().__init__()
self.repo = repo
self.root_dir = repo.root_dir
self.tmp_dir = tmp_dir
self.root_dir = root_dir
self.fs = LocalFileSystem(None, {"url": self.root_dir})
state_config = repo.config.get("state", {})
self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
self.row_cleanup_quota = state_config.get(
"row_cleanup_quota", self.STATE_ROW_CLEANUP_QUOTA
)
if not repo.tmp_dir:
self.state_file = None
if not tmp_dir:
return
self.state_file = os.path.join(repo.tmp_dir, self.STATE_FILE)
config = {"eviction_policy": "least-recently-used"}
self.links = Cache(directory=os.path.join(tmp_dir, "links"), **config)
self.md5s = Cache(directory=os.path.join(tmp_dir, "md5s"), **config)
self.database = None
self.cursor = None
self.inserts = 0
def _execute(self, cmd, parameters=()):
logger.trace(cmd)
return self.cursor.execute(cmd, parameters)
def _fetchall(self):
ret = self.cursor.fetchall()
logger.trace("fetched: %s", ret)
return ret
def _to_sqlite(self, num):
assert num >= 0
assert num < self.MAX_UINT
# NOTE: sqlite stores unit as signed ints, so maximum uint is 2^63-1
# see http://jakegoulding.com/blog/2011/02/06/sqlite-64-bit-integers/
if num > self.MAX_INT:
ret = -(num - self.MAX_INT)
else:
ret = num
assert self._from_sqlite(ret) == num
return ret
def _from_sqlite(self, num):
assert abs(num) <= self.MAX_INT
if num < 0:
return abs(num) + self.MAX_INT
assert num < self.MAX_UINT
assert num >= 0
return num
def _prepare_db(self, empty=False):
from dvc import __version__
if not empty:
cmd = "PRAGMA user_version;"
self._execute(cmd)
ret = self._fetchall()
assert len(ret) == 1
assert len(ret[0]) == 1
assert isinstance(ret[0][0], int)
version = ret[0][0]
if version > self.VERSION:
raise StateVersionTooNewError(
__version__, self.VERSION, version
)
elif version < self.VERSION:
logger.warning(
"State file version '%d' is too old. "
"Reformatting to the current version '%d'.",
version,
self.VERSION,
)
cmd = "DROP TABLE IF EXISTS {};"
self._execute(cmd.format(self.STATE_TABLE))
self._execute(cmd.format(self.STATE_INFO_TABLE))
self._execute(cmd.format(self.LINK_STATE_TABLE))
# Check that the state file is indeed a database
cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
self._execute(cmd.format(self.STATE_TABLE, self.STATE_TABLE_LAYOUT))
self._execute(
cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE_LAYOUT)
)
self._execute(
cmd.format(self.LINK_STATE_TABLE, self.LINK_STATE_TABLE_LAYOUT)
)
cmd = (
"INSERT OR IGNORE INTO {} (count) SELECT 0 "
"WHERE NOT EXISTS (SELECT * FROM {})"
)
self._execute(cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE))
cmd = "PRAGMA user_version = {};"
self._execute(cmd.format(self.VERSION))
def load(self):
"""Loads state database."""
retries = 1
while True:
assert self.database is None
assert self.cursor is None
assert self.inserts == 0
empty = not os.path.exists(self.state_file)
# NOTE: we use nolock option because fcntl() lock sqlite uses
# doesn't work on some older NFS/CIFS filesystems.
# This opens a possibility of data corruption by concurrent writes,
# which is prevented by repo lock.
self.database = _connect_sqlite(self.state_file, {"nolock": 1})
self.cursor = self.database.cursor()
from sqlite3 import DatabaseError
# Try loading once to check that the file is indeed a database
# and reformat it if it is not.
try:
self._prepare_db(empty=empty)
return
except DatabaseError:
self.cursor.close()
self.database.close()
self.database = None
self.cursor = None
self.inserts = 0
if retries > 0:
os.unlink(self.state_file)
retries -= 1
else:
raise
def _vacuum(self):
# NOTE: see https://bugs.python.org/issue28518
self.database.isolation_level = None
self._execute("VACUUM")
self.database.isolation_level = ""
def dump(self):
"""Saves state database."""
assert self.database is not None
cmd = "SELECT count from {} WHERE rowid=?".format(
self.STATE_INFO_TABLE
)
self._execute(cmd, (self.STATE_INFO_ROW,))
ret = self._fetchall()
assert len(ret) == 1
assert len(ret[0]) == 1
count = self._from_sqlite(ret[0][0]) + self.inserts
if count > self.row_limit:
msg = "cleaning up state, this might take a while."
logger.warning(msg)
delete = count - self.row_limit
delete += int(self.row_limit * (self.row_cleanup_quota / 100.0))
cmd = (
"DELETE FROM {} WHERE timestamp IN ("
"SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
)
self._execute(
cmd.format(self.STATE_TABLE, self.STATE_TABLE, delete)
)
self._vacuum()
cmd = "SELECT COUNT(*) FROM {}"
self._execute(cmd.format(self.STATE_TABLE))
ret = self._fetchall()
assert len(ret) == 1
assert len(ret[0]) == 1
count = ret[0][0]
cmd = "UPDATE {} SET count = ? WHERE rowid = ?".format(
self.STATE_INFO_TABLE
)
self._execute(cmd, (self._to_sqlite(count), self.STATE_INFO_ROW))
self.database.commit()
self.cursor.close()
self.database.close()
self.database = None
self.cursor = None
self.inserts = 0
@staticmethod
def _file_metadata_changed(actual_mtime, mtime, actual_size, size):
return actual_mtime != mtime or actual_size != size
def _update_state_record_timestamp_for_inode(self, actual_inode):
cmd = "UPDATE {} SET timestamp = ? WHERE inode = ?".format(
self.STATE_TABLE
)
self._execute(
cmd, (current_timestamp(), self._to_sqlite(actual_inode))
)
def _update_state_for_path_changed(
self, actual_inode, actual_mtime, actual_size, checksum
):
cmd = (
"UPDATE {} SET "
"mtime = ?, size = ?, "
"md5 = ?, timestamp = ? "
"WHERE inode = ?"
).format(self.STATE_TABLE)
self._execute(
cmd,
(
actual_mtime,
actual_size,
checksum,
current_timestamp(),
self._to_sqlite(actual_inode),
),
)
def _insert_new_state_record(
self, actual_inode, actual_mtime, actual_size, checksum
):
assert checksum is not None
cmd = (
"INSERT INTO {}(inode, mtime, size, md5, timestamp) "
"VALUES (?, ?, ?, ?, ?)"
).format(self.STATE_TABLE)
self._execute(
cmd,
(
self._to_sqlite(actual_inode),
actual_mtime,
actual_size,
checksum,
current_timestamp(),
),
)
self.inserts += 1
def get_state_record_for_inode(self, inode):
cmd = (
"SELECT mtime, size, md5, timestamp from {} WHERE "
"inode=?".format(self.STATE_TABLE)
)
self._execute(cmd, (self._to_sqlite(inode),))
results = self._fetchall()
if results:
# uniqueness constrain on inode
assert len(results) == 1
return results[0]
return None
def close(self):
self.md5s.close()
self.links.close()
def save(self, path_info, fs, hash_info):
"""Save hash for the specified path info.
@@ -395,20 +100,15 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
assert isinstance(hash_info, HashInfo)
assert os.path.exists(path_info)
actual_mtime, actual_size = get_mtime_and_size(path_info, self.fs)
actual_inode = get_inode(path_info)
mtime, size = get_mtime_and_size(path_info, self.fs)
inode = get_inode(path_info)
existing_record = self.get_state_record_for_inode(actual_inode)
if not existing_record:
self._insert_new_state_record(
actual_inode, actual_mtime, actual_size, hash_info.value
)
return
self._update_state_for_path_changed(
actual_inode, actual_mtime, actual_size, hash_info.value
logger.debug(
"state save (%s, %s, %s) %s", inode, mtime, size, hash_info.value
)
self.md5s[inode] = (mtime, size, hash_info.value)
def get(self, path_info, fs):
"""Gets the hash for the specified path info. Hash will be
retrieved from the state database if available.
@@ -432,19 +132,15 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
if not os.path.exists(path):
return None
actual_mtime, actual_size = get_mtime_and_size(path, self.fs)
actual_inode = get_inode(path)
mtime, size = get_mtime_and_size(path, self.fs)
inode = get_inode(path)
existing_record = self.get_state_record_for_inode(actual_inode)
if not existing_record:
value = self.md5s.get(inode)
if not value or value[0] != mtime or value[1] != size:
return None
mtime, size, value, _ = existing_record
if self._file_metadata_changed(actual_mtime, mtime, actual_size, size):
return None
self._update_state_record_timestamp_for_inode(actual_inode)
return HashInfo("md5", value, size=int(actual_size))
return HashInfo("md5", value[2], size=int(size))
def save_link(self, path_info, fs):
"""Adds the specified path to the list of links created by dvc. This
@@ -465,10 +161,8 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
inode = get_inode(path_info)
relative_path = relpath(path_info, self.root_dir)
cmd = "REPLACE INTO {}(path, inode, mtime) " "VALUES (?, ?, ?)".format(
self.LINK_STATE_TABLE
)
self._execute(cmd, (relative_path, self._to_sqlite(inode), mtime))
with self.links as ref:
ref[relative_path] = (inode, mtime)
def get_unused_links(self, used, fs):
"""Removes all saved links except the ones that are used.
@@ -481,21 +175,19 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
unused = []
self._execute(f"SELECT * FROM {self.LINK_STATE_TABLE}")
for row in self.cursor:
relative_path, inode, mtime = row
inode = self._from_sqlite(inode)
path = os.path.join(self.root_dir, relative_path)
with self.links as ref:
for relative_path in ref:
path = os.path.join(self.root_dir, relative_path)
if path in used or not self.fs.exists(path):
continue
if path in used or not self.fs.exists(path):
continue
actual_inode = get_inode(path)
actual_mtime, _ = get_mtime_and_size(path, self.fs)
inode = get_inode(path)
mtime, _ = get_mtime_and_size(path, self.fs)
if (inode, mtime) == (actual_inode, actual_mtime):
logger.debug("Removing '%s' as unused link.", path)
unused.append(relative_path)
if ref[relative_path] == (inode, mtime):
logger.debug("Removing '%s' as unused link.", path)
unused.append(relative_path)
return unused
@@ -504,15 +196,11 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
return
for path in unused:
remove(path)
remove(os.path.join(self.root_dir, path))
for chunk_unused in to_chunks(
unused, chunk_size=SQLITE_MAX_VARIABLES_NUMBER
):
cmd = "DELETE FROM {} WHERE path IN ({})".format(
self.LINK_STATE_TABLE, ",".join(["?"] * len(chunk_unused))
)
self._execute(cmd, tuple(chunk_unused))
with self.links as ref:
for path in unused:
del ref[path]
def _connect_sqlite(filename, options):
+1
View File
@@ -89,6 +89,7 @@ install_requires = [
"typing_extensions>=3.7.4",
"fsspec>=0.8.5",
"dvclive>=0.0.1a0",
"diskcache>=5.2.1",
]
+2
View File
@@ -114,6 +114,8 @@ class TmpDir(pathlib.Path):
def close(self):
if hasattr(self, "scm"):
self.scm.close()
if hasattr(self, "dvc"):
self.dvc.close()
def _require(self, name):
if not hasattr(self, name):
+1 -1
View File
@@ -40,7 +40,6 @@ def test_no_cache_entry(tmp_dir, scm, dvc):
tmp_dir.dvc_gen("file", "second")
remove(tmp_dir / ".dvc" / "cache")
(tmp_dir / ".dvc" / "tmp" / "state").unlink()
dir_checksum = "5fb6b29836c388e093ca0715c872fe2a.dir"
@@ -131,6 +130,7 @@ def test_directories(tmp_dir, scm, dvc):
tmp_dir.dvc_gen({"dir": {"2": "two"}}, commit="modify a file")
(tmp_dir / "dir" / "2").unlink()
assert dvc.status() != {} # sanity check
dvc.add("dir")
scm.add(["dir.dvc"])
scm.commit("delete a file")
+1 -1
View File
@@ -14,7 +14,7 @@ class TestInit(TestGit):
self.assertTrue(os.path.isdir(DvcRepo.DVC_DIR))
def test_api(self):
DvcRepo.init()
DvcRepo.init().close()
self._test_init()
+3 -3
View File
@@ -146,7 +146,7 @@ class TestRemoteShouldHandleUppercaseRemoteName(TestDvc):
def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
from dvc.objects.stage import get_hash
from dvc.objects.stage import stage
tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}})
@@ -158,7 +158,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
with patch(
"dvc.objects.stage._collect_dir", return_value=dir_info,
):
hash1 = get_hash(path_info, dvc.odb.local.fs, "md5")
hash1 = stage(dvc.odb.local, path_info, dvc.odb.local.fs).hash_info
dir_info = DirInfo.from_list(
[{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}]
@@ -166,7 +166,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
with patch(
"dvc.objects.stage._collect_dir", return_value=dir_info,
):
hash2 = get_hash(path_info, dvc.odb.local.fs, "md5")
hash2 = stage(dvc.odb.local, path_info, dvc.odb.local.fs).hash_info
assert hash1 == hash2
+1 -1
View File
@@ -917,7 +917,7 @@ class TestShouldNotCheckoutUponCorruptedLocalHardlinkCache(TestDvc):
patch_run = mock.patch("dvc.stage.run.cmd_run", wraps=cmd_run)
with self.dvc.lock, self.dvc.state:
with self.dvc.lock:
with patch_checkout as mock_checkout:
with patch_run as mock_run:
stage.run()
+1 -1
View File
@@ -107,7 +107,7 @@ class TestDefaultWorkingDirectory(TestDvc):
d = load_yaml(stage.relpath)
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
with self.dvc.lock, self.dvc.state:
with self.dvc.lock:
stage = SingleStageFile(self.dvc, stage.relpath).stage
self.assertFalse(stage.changed())
+22 -53
View File
@@ -1,7 +1,5 @@
import os
import mock
from dvc.hash_info import HashInfo
from dvc.path_info import PathInfo
from dvc.state import State
@@ -14,21 +12,20 @@ def test_state(tmp_dir, dvc):
path_info = PathInfo(path)
hash_info = HashInfo("md5", file_md5(path, dvc.fs))
state = State(dvc)
state = State(dvc.root_dir, dvc.tmp_dir)
with state:
state.save(path_info, dvc.fs, hash_info)
assert state.get(path_info, dvc.fs) == hash_info
state.save(path_info, dvc.fs, hash_info)
assert state.get(path_info, dvc.fs) == hash_info
path.unlink()
path.write_text("1")
path.unlink()
path.write_text("1")
assert state.get(path_info, dvc.fs) is None
assert state.get(path_info, dvc.fs) is None
hash_info = HashInfo("md5", file_md5(path, dvc.fs))
state.save(path_info, dvc.fs, hash_info)
hash_info = HashInfo("md5", file_md5(path, dvc.fs))
state.save(path_info, dvc.fs, hash_info)
assert state.get(path_info, dvc.fs) == hash_info
assert state.get(path_info, dvc.fs) == hash_info
def test_state_overflow(tmp_dir, dvc):
@@ -51,54 +48,26 @@ def mock_get_inode(inode):
return get_inode_mocked
@mock.patch("dvc.state.get_inode", autospec=True)
def test_get_state_record_for_inode(get_inode_mock, tmp_dir, dvc):
tmp_dir.gen("foo", "foo content")
state = State(dvc)
inode = state.MAX_INT + 2
assert inode != state._to_sqlite(inode)
foo = tmp_dir / "foo"
md5 = file_md5(foo, dvc.fs)
get_inode_mock.side_effect = mock_get_inode(inode)
with state:
state.save(PathInfo(foo), dvc.fs, HashInfo("md5", md5))
ret = state.get_state_record_for_inode(inode)
assert ret is not None
def test_remove_links(tmp_dir, dvc):
tmp_dir.dvc_gen({"foo": "foo_content", "bar": "bar_content"})
with dvc.state:
cmd_count_links = "SELECT count(*) FROM {}".format(
State.LINK_STATE_TABLE
)
result = dvc.state._execute(cmd_count_links).fetchone()[0]
assert result == 2
assert len(dvc.state.links) == 2
dvc.state.remove_links(["foo", "bar"], dvc.fs)
dvc.state.remove_links(["foo", "bar"], dvc.fs)
result = dvc.state._execute(cmd_count_links).fetchone()[0]
assert result == 0
assert len(dvc.state.links) == 0
def test_get_unused_links(tmp_dir, dvc):
tmp_dir.dvc_gen({"foo": "foo_content", "bar": "bar_content"})
with dvc.state:
links = [os.path.join(dvc.root_dir, link) for link in ["foo", "bar"]]
assert set(dvc.state.get_unused_links([], dvc.fs)) == {"foo", "bar"}
assert set(dvc.state.get_unused_links(links[:1], dvc.fs)) == {"bar"}
assert set(dvc.state.get_unused_links(links, dvc.fs)) == set()
assert set(
dvc.state.get_unused_links(
(
links[:1]
+ [os.path.join(dvc.root_dir, "not-existing-file")]
),
dvc.fs,
)
) == {"bar"}
links = [os.path.join(dvc.root_dir, link) for link in ["foo", "bar"]]
assert set(dvc.state.get_unused_links([], dvc.fs)) == {"foo", "bar"}
assert set(dvc.state.get_unused_links(links[:1], dvc.fs)) == {"bar"}
assert set(dvc.state.get_unused_links(links, dvc.fs)) == set()
assert set(
dvc.state.get_unused_links(
(links[:1] + [os.path.join(dvc.root_dir, "not-existing-file")]),
dvc.fs,
)
) == {"bar"}
+9 -9
View File
@@ -5,7 +5,7 @@ import pytest
from dvc.fs.dvc import DvcFileSystem
from dvc.hash_info import HashInfo
from dvc.objects.stage import get_hash
from dvc.objects.stage import stage
from dvc.path_info import PathInfo
@@ -229,13 +229,13 @@ def test_get_hash_granular(tmp_dir, dvc):
fs = DvcFileSystem(dvc)
subdir = PathInfo(tmp_dir) / "dir" / "subdir"
assert fs.info(subdir).get("md5") is None
assert get_hash(subdir, fs, "md5") == HashInfo(
assert stage(dvc.odb.local, subdir, fs).hash_info == HashInfo(
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir",
)
assert (
fs.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc"
)
assert get_hash(subdir / "data", fs, "md5") == HashInfo(
assert stage(dvc.odb.local, subdir / "data", fs).hash_info == HashInfo(
"md5", "8d777f385d3dfec8815d20f7496026dc",
)
@@ -247,9 +247,9 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
fs = DvcFileSystem(dvc)
expected = "8c7dd922ad47494fc02c388e12c00eac"
assert fs.info(PathInfo(tmp_dir) / "file").get("md5") == expected
assert get_hash(PathInfo(tmp_dir) / "file", fs, "md5") == HashInfo(
"md5", expected
)
assert stage(
dvc.odb.local, PathInfo(tmp_dir) / "file", fs
).hash_info == HashInfo("md5", expected)
def test_get_hash_dirty_dir(tmp_dir, dvc):
@@ -259,6 +259,6 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
fs = DvcFileSystem(dvc)
expected = "5ea40360f5b4ec688df672a4db9c17d1.dir"
assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") == expected
assert get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") == HashInfo(
"md5", expected
)
assert stage(
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs
).hash_info == HashInfo("md5", expected)
+16 -16
View File
@@ -6,7 +6,7 @@ import pytest
from dvc.fs.repo import RepoFileSystem
from dvc.hash_info import HashInfo
from dvc.objects.stage import get_hash
from dvc.objects.stage import stage
from dvc.path_info import PathInfo
@@ -482,9 +482,9 @@ def test_get_hash_cached_file(tmp_dir, dvc, mocker):
fs = RepoFileSystem(dvc)
expected = "acbd18db4cc2f85cedef654fccc4a4d8"
assert fs.info(PathInfo(tmp_dir) / "foo").get("md5") is None
assert get_hash(PathInfo(tmp_dir) / "foo", fs, "md5") == HashInfo(
"md5", expected,
)
assert stage(
dvc.odb.local, PathInfo(tmp_dir) / "foo", fs
).hash_info == HashInfo("md5", expected,)
(tmp_dir / "foo").unlink()
assert fs.info(PathInfo(tmp_dir) / "foo")["md5"] == expected
@@ -496,15 +496,15 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker):
fs = RepoFileSystem(dvc)
expected = "8761c4e9acad696bee718615e23e22db.dir"
assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") is None
assert get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") == HashInfo(
"md5", "8761c4e9acad696bee718615e23e22db.dir",
)
assert stage(
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs
).hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir",)
shutil.rmtree(tmp_dir / "dir")
assert fs.info(PathInfo(tmp_dir) / "dir")["md5"] == expected
assert get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") == HashInfo(
"md5", "8761c4e9acad696bee718615e23e22db.dir",
)
assert stage(
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs
).hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir",)
def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
@@ -514,11 +514,11 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
fs = RepoFileSystem(dvc)
subdir = PathInfo(tmp_dir) / "dir" / "subdir"
assert fs.info(subdir).get("md5") is None
assert get_hash(subdir, fs, "md5") == HashInfo(
assert stage(dvc.odb.local, subdir, fs).hash_info == HashInfo(
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir",
)
assert fs.info(subdir / "data").get("md5") is None
assert get_hash(subdir / "data", fs, "md5") == HashInfo(
assert stage(dvc.odb.local, subdir / "data", fs).hash_info == HashInfo(
"md5", "8d777f385d3dfec8815d20f7496026dc",
)
(tmp_dir / "dir" / "subdir" / "data").unlink()
@@ -540,7 +540,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc):
tmp_dir.scm.commit("add dir")
fs = RepoFileSystem(dvc)
actual = get_hash(PathInfo(tmp_dir) / "dir", fs, "md5")
actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs).hash_info
expected = HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir")
assert actual == expected
@@ -551,7 +551,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
fs = RepoFileSystem(dvc)
assert fs.info(PathInfo(tmp_dir) / "file").get("md5") is None
actual = get_hash(PathInfo(tmp_dir) / "file", fs, "md5")
actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs).hash_info
expected = HashInfo("md5", "437b930db84b8079c2dd804a71936b5f")
assert actual == expected
@@ -560,7 +560,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
fs.info(PathInfo(tmp_dir) / "file")["md5"]
== "8c7dd922ad47494fc02c388e12c00eac"
)
actual = get_hash(PathInfo(tmp_dir) / "file", fs, "md5")
actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs).hash_info
expected = HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac")
assert actual == expected
@@ -570,7 +570,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
(tmp_dir / "dir" / "baz").write_text("baz")
fs = RepoFileSystem(dvc)
actual = get_hash(PathInfo(tmp_dir) / "dir", fs, "md5")
actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs).hash_info
expected = HashInfo("md5", "ba75a2162ca9c29acecb7957105a0bc2.dir")
assert actual == expected
assert actual.dir_info.nfiles == 3
+14 -11
View File
@@ -5,7 +5,6 @@ import pytest
from dvc.exceptions import OutputDuplicationError
from dvc.repo import NotDvcRepoError, Repo, locked
from dvc.utils.fs import remove
def test_is_dvc_internal(dvc):
@@ -113,25 +112,28 @@ def test_skip_graph_checks(tmp_dir, dvc, mocker, run_copy):
def test_branch_config(tmp_dir, scm):
tmp_dir.scm_gen("foo", "foo", commit="init")
# sanity check
with pytest.raises(NotDvcRepoError):
Repo().close()
scm.checkout("branch", create_new=True)
dvc = Repo.init()
with dvc.config.edit() as conf:
conf["remote"]["branch"] = {"url": "/some/path"}
scm.add([".dvc"])
dvc.close()
scm.add([os.path.join(".dvc", "config")])
scm.commit("init dvc")
scm.checkout("master")
remove(".dvc")
# sanity check
with pytest.raises(NotDvcRepoError):
Repo()
with pytest.raises(NotDvcRepoError):
Repo(scm=scm, rev="master")
Repo(scm=scm, rev="master").close()
dvc = Repo(scm=scm, rev="branch")
assert dvc.config["remote"]["branch"]["url"] == "/some/path"
try:
assert dvc.config["remote"]["branch"]["url"] == "/some/path"
finally:
dvc.close()
def test_dynamic_cache_initalization(tmp_dir, scm):
@@ -139,5 +141,6 @@ def test_dynamic_cache_initalization(tmp_dir, scm):
with dvc.config.edit() as conf:
conf["cache"]["ssh"] = "foo"
conf["remote"]["foo"] = {"url": "remote://bar/baz"}
dvc.close()
Repo(str(tmp_dir))
Repo(str(tmp_dir)).close()
+6 -6
View File
@@ -22,7 +22,7 @@ def test_stage_cache(tmp_dir, dvc, mocker):
single_stage=True,
)
with dvc.lock, dvc.state:
with dvc.lock:
stage.remove(remove_outs=True, force=True)
assert not (tmp_dir / "out").exists()
@@ -45,7 +45,7 @@ def test_stage_cache(tmp_dir, dvc, mocker):
run_spy = mocker.patch("dvc.stage.run.cmd_run")
checkout_spy = mocker.spy(base, "checkout")
with dvc.lock, dvc.state:
with dvc.lock:
stage.run()
assert not run_spy.called
@@ -75,7 +75,7 @@ def test_stage_cache_params(tmp_dir, dvc, mocker):
single_stage=True,
)
with dvc.lock, dvc.state:
with dvc.lock:
stage.remove(remove_outs=True, force=True)
assert not (tmp_dir / "out").exists()
@@ -98,7 +98,7 @@ def test_stage_cache_params(tmp_dir, dvc, mocker):
run_spy = mocker.patch("dvc.stage.run.cmd_run")
checkout_spy = mocker.spy(base, "checkout")
with dvc.lock, dvc.state:
with dvc.lock:
stage.run()
assert not run_spy.called
@@ -129,7 +129,7 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker):
wdir="wdir",
)
with dvc.lock, dvc.state:
with dvc.lock:
stage.remove(remove_outs=True, force=True)
assert not (tmp_dir / "wdir" / "out").exists()
@@ -152,7 +152,7 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker):
run_spy = mocker.patch("dvc.stage.run.cmd_run")
checkout_spy = mocker.spy(base, "checkout")
with dvc.lock, dvc.state:
with dvc.lock:
stage.run()
assert not run_spy.called