mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 23:11:44 +01:00
state: use diskcache (#5505)
* state: only load when needed * state: don't use separate load/dump * checkout: group link removal logic together * state: use diskcache.Cache * staging: use pbar and save to state * state: don't pass the whole repo to __init__ * staging: explicitly pass state object * fs: get rid of odb property * state: don't close the connection to cache * repo: close: close state
This commit is contained in:
+1
-1
@@ -29,7 +29,7 @@ def get_url(path, repo=None, rev=None, remote=None):
|
||||
raise OutputNotFoundError(path, repo)
|
||||
|
||||
cloud = metadata.repo.cloud
|
||||
hash_info = get_hash(path_info, _repo.repo_fs, "md5")
|
||||
hash_info = get_hash(path_info, _repo.repo_fs, "md5", _repo.odb)
|
||||
return cloud.get_url_for(remote, checksum=hash_info.value)
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -30,7 +30,7 @@ def _changed(path_info, fs, obj, cache):
|
||||
return True
|
||||
|
||||
try:
|
||||
actual = get_hash(path_info, fs, obj.hash_info.name)
|
||||
actual = get_hash(path_info, fs, obj.hash_info.name, cache)
|
||||
except FileNotFoundError:
|
||||
logger.debug("'%s' doesn't exist.", path_info)
|
||||
return True
|
||||
@@ -56,7 +56,7 @@ def _remove(path_info, fs, cache, force=False):
|
||||
fs.remove(path_info)
|
||||
return
|
||||
|
||||
current = get_hash(path_info, fs, fs.PARAM_CHECKSUM)
|
||||
current = get_hash(path_info, fs, fs.PARAM_CHECKSUM, cache)
|
||||
try:
|
||||
obj = load(cache, current)
|
||||
check(cache, obj)
|
||||
|
||||
@@ -69,14 +69,13 @@ class CmdMergeDriver(CmdHookBase):
|
||||
dvc = Repo()
|
||||
|
||||
try:
|
||||
with dvc.state:
|
||||
ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
|
||||
our = Dvcfile(dvc, self.args.our, verify=False)
|
||||
their = Dvcfile(dvc, self.args.their, verify=False)
|
||||
ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
|
||||
our = Dvcfile(dvc, self.args.our, verify=False)
|
||||
their = Dvcfile(dvc, self.args.their, verify=False)
|
||||
|
||||
our.merge(ancestor, their)
|
||||
our.merge(ancestor, their)
|
||||
|
||||
return 0
|
||||
return 0
|
||||
finally:
|
||||
dvc.close()
|
||||
|
||||
|
||||
@@ -213,8 +213,8 @@ SCHEMA = {
|
||||
)
|
||||
},
|
||||
"state": {
|
||||
"row_limit": All(Coerce(int), Range(1)),
|
||||
"row_cleanup_quota": All(Coerce(int), Range(0, 100)),
|
||||
"row_limit": All(Coerce(int), Range(1)), # obsoleted
|
||||
"row_cleanup_quota": All(Coerce(int), Range(0, 100)), # obsoleted
|
||||
},
|
||||
# section for experimental features
|
||||
"feature": {
|
||||
|
||||
@@ -47,13 +47,16 @@ class RepoDependency(LocalDependency):
|
||||
return external_repo(d["url"], rev=rev, **kwargs)
|
||||
|
||||
def _get_hash(self, locked=True):
|
||||
from dvc.objects.stage import get_hash
|
||||
from dvc.objects.stage import stage
|
||||
|
||||
with self._make_repo(locked=locked) as repo:
|
||||
path_info = PathInfo(repo.root_dir) / self.def_path
|
||||
return get_hash(
|
||||
path_info, repo.repo_fs, "md5", follow_subrepos=False
|
||||
)
|
||||
return stage(
|
||||
self.repo.odb.local,
|
||||
path_info,
|
||||
repo.repo_fs,
|
||||
follow_subrepos=False,
|
||||
).hash_info
|
||||
|
||||
def workspace_status(self):
|
||||
current = self._get_hash(locked=True)
|
||||
|
||||
@@ -130,10 +130,6 @@ class BaseFileSystem:
|
||||
f"dependencies: {missing}. {hint}"
|
||||
)
|
||||
|
||||
@property
|
||||
def odb(self):
|
||||
return getattr(self.repo.odb, self.scheme)
|
||||
|
||||
def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
|
||||
if hasattr(self, "_generate_download_url"):
|
||||
# pylint:disable=no-member
|
||||
|
||||
@@ -36,7 +36,7 @@ class HashFile:
|
||||
return bool(self.hash_info)
|
||||
|
||||
def check(self, odb):
|
||||
actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM)
|
||||
actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM, odb)
|
||||
|
||||
logger.trace(
|
||||
"cache '%s' expected '%s' actual '%s'",
|
||||
@@ -64,7 +64,9 @@ class File(HashFile):
|
||||
|
||||
@classmethod
|
||||
def stage(cls, odb, path_info, fs, **kwargs):
|
||||
hash_info = get_hash(path_info, fs, odb.fs.PARAM_CHECKSUM, **kwargs)
|
||||
hash_info = get_hash(
|
||||
path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs
|
||||
)
|
||||
raw = odb.get(hash_info)
|
||||
obj = cls(raw.path_info, raw.fs, hash_info)
|
||||
obj.src = HashFile(path_info, fs, hash_info)
|
||||
@@ -114,7 +116,7 @@ class Tree(HashFile):
|
||||
with fs.open(path_info, "rb") as fobj:
|
||||
odb.fs.upload_fobj(fobj, tmp_info)
|
||||
|
||||
hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM)
|
||||
hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM, odb)
|
||||
hash_info.value += odb.fs.CHECKSUM_DIR_SUFFIX
|
||||
hash_info.dir_info = dir_info
|
||||
hash_info.nfiles = dir_info.nfiles
|
||||
@@ -125,7 +127,9 @@ class Tree(HashFile):
|
||||
|
||||
@classmethod
|
||||
def stage(cls, odb, path_info, fs, **kwargs):
|
||||
hash_info = get_hash(path_info, fs, odb.fs.PARAM_CHECKSUM, **kwargs)
|
||||
hash_info = get_hash(
|
||||
path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs
|
||||
)
|
||||
hi = cls.save_dir_info(odb, hash_info.dir_info, hash_info)
|
||||
hi.size = hash_info.size
|
||||
raw = odb.get(hi)
|
||||
|
||||
@@ -58,8 +58,7 @@ class ObjectDB:
|
||||
with fs.open(path_info, mode="rb") as fobj:
|
||||
self.fs.upload_fobj(fobj, cache_info)
|
||||
self.protect(cache_info)
|
||||
with self.fs.repo.state:
|
||||
self.fs.repo.state.save(cache_info, self.fs, hash_info)
|
||||
self.fs.repo.state.save(cache_info, self.fs, hash_info)
|
||||
|
||||
callback = kwargs.get("download_callback")
|
||||
if callback:
|
||||
|
||||
+44
-51
@@ -27,22 +27,26 @@ def get_file_hash(path_info, fs, name):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _calculate_hashes(file_infos, fs, name):
|
||||
def _get_file_hash(path_info):
|
||||
return get_file_hash(path_info, fs, name)
|
||||
def _calculate_hashes(path_info, fs, name, state, **kwargs):
|
||||
def _get_file_hash(file_info):
|
||||
hash_info = state.get( # pylint: disable=assignment-from-none
|
||||
file_info, fs,
|
||||
)
|
||||
if not hash_info:
|
||||
hash_info = get_file_hash(file_info, fs, name)
|
||||
state.save(file_info, fs, hash_info)
|
||||
return file_info, hash_info
|
||||
|
||||
with Tqdm(
|
||||
total=len(file_infos),
|
||||
unit="md5",
|
||||
desc="Computing file/dir hashes (only done once)",
|
||||
unit="md5", desc="Computing file/dir hashes (only done once)",
|
||||
) as pbar:
|
||||
worker = pbar.wrap_fn(_get_file_hash)
|
||||
with ThreadPoolExecutor(max_workers=fs.hash_jobs) as executor:
|
||||
hash_infos = executor.map(worker, file_infos)
|
||||
return dict(zip(file_infos, hash_infos))
|
||||
pairs = executor.map(worker, fs.walk_files(path_info, **kwargs))
|
||||
return dict(pairs)
|
||||
|
||||
|
||||
def _iter_hashes(path_info, fs, name, **kwargs):
|
||||
def _iter_hashes(path_info, fs, name, state, **kwargs):
|
||||
if name in fs.DETAIL_FIELDS:
|
||||
for details in fs.ls(path_info, recursive=True, detail=True):
|
||||
file_info = path_info.replace(path=details["name"])
|
||||
@@ -53,25 +57,14 @@ def _iter_hashes(path_info, fs, name, **kwargs):
|
||||
|
||||
return None
|
||||
|
||||
file_infos = []
|
||||
for file_info in fs.walk_files(path_info, **kwargs):
|
||||
hash_info = fs.repo.state.get( # pylint: disable=assignment-from-none
|
||||
file_info, fs,
|
||||
)
|
||||
if not hash_info:
|
||||
file_infos.append(file_info)
|
||||
continue
|
||||
assert hash_info.name == name
|
||||
yield file_info, hash_info
|
||||
|
||||
yield from _calculate_hashes(file_infos, fs, name).items()
|
||||
yield from _calculate_hashes(path_info, fs, name, state, **kwargs).items()
|
||||
|
||||
|
||||
def _collect_dir(path_info, fs, name, **kwargs):
|
||||
def _collect_dir(path_info, fs, name, state, **kwargs):
|
||||
from dvc.dir_info import DirInfo
|
||||
|
||||
dir_info = DirInfo()
|
||||
for fi, hi in _iter_hashes(path_info, fs, name, **kwargs):
|
||||
for fi, hi in _iter_hashes(path_info, fs, name, state, **kwargs):
|
||||
if DvcIgnore.DVCIGNORE_FILE == fi.name:
|
||||
raise DvcIgnoreInCollectedDirError(fi.parent)
|
||||
|
||||
@@ -88,17 +81,17 @@ def _collect_dir(path_info, fs, name, **kwargs):
|
||||
return dir_info
|
||||
|
||||
|
||||
def get_dir_hash(path_info, fs, name, **kwargs):
|
||||
def get_dir_hash(path_info, fs, name, state, **kwargs):
|
||||
from . import Tree
|
||||
|
||||
dir_info = _collect_dir(path_info, fs, name, **kwargs)
|
||||
dir_info = _collect_dir(path_info, fs, name, state, **kwargs)
|
||||
hash_info = Tree.save_dir_info(fs.repo.odb.local, dir_info)
|
||||
hash_info.size = dir_info.size
|
||||
hash_info.dir_info = dir_info
|
||||
return hash_info
|
||||
|
||||
|
||||
def get_hash(path_info, fs, name, **kwargs):
|
||||
def get_hash(path_info, fs, name, odb, **kwargs):
|
||||
assert path_info and (
|
||||
isinstance(path_info, str) or path_info.scheme == fs.scheme
|
||||
)
|
||||
@@ -108,36 +101,36 @@ def get_hash(path_info, fs, name, **kwargs):
|
||||
errno.ENOENT, os.strerror(errno.ENOENT), path_info
|
||||
)
|
||||
|
||||
with fs.repo.state:
|
||||
# pylint: disable=assignment-from-none
|
||||
hash_info = fs.repo.state.get(path_info, fs)
|
||||
state = odb.repo.state
|
||||
# pylint: disable=assignment-from-none
|
||||
hash_info = state.get(path_info, fs)
|
||||
|
||||
# If we have dir hash in state db, but dir cache file is lost,
|
||||
# then we need to recollect the dir via .get_dir_hash() call below,
|
||||
# see https://github.com/iterative/dvc/issues/2219 for context
|
||||
if (
|
||||
hash_info
|
||||
and hash_info.isdir
|
||||
and not fs.odb.fs.exists(fs.odb.hash_to_path_info(hash_info.value))
|
||||
):
|
||||
hash_info = None
|
||||
# If we have dir hash in state db, but dir cache file is lost,
|
||||
# then we need to recollect the dir via .get_dir_hash() call below,
|
||||
# see https://github.com/iterative/dvc/issues/2219 for context
|
||||
if (
|
||||
hash_info
|
||||
and hash_info.isdir
|
||||
and not odb.fs.exists(odb.hash_to_path_info(hash_info.value))
|
||||
):
|
||||
hash_info = None
|
||||
|
||||
if hash_info:
|
||||
if hash_info.isdir:
|
||||
from . import Tree
|
||||
if hash_info:
|
||||
if hash_info.isdir:
|
||||
from . import Tree
|
||||
|
||||
# NOTE: loading the fs will restore hash_info.dir_info
|
||||
Tree.load(fs.odb, hash_info)
|
||||
assert hash_info.name == name
|
||||
return hash_info
|
||||
# NOTE: loading the fs will restore hash_info.dir_info
|
||||
Tree.load(odb, hash_info)
|
||||
assert hash_info.name == name
|
||||
return hash_info
|
||||
|
||||
if fs.isdir(path_info):
|
||||
hash_info = get_dir_hash(path_info, fs, name, **kwargs)
|
||||
else:
|
||||
hash_info = get_file_hash(path_info, fs, name)
|
||||
if fs.isdir(path_info):
|
||||
hash_info = get_dir_hash(path_info, fs, name, state, **kwargs)
|
||||
else:
|
||||
hash_info = get_file_hash(path_info, fs, name)
|
||||
|
||||
if hash_info and fs.exists(path_info):
|
||||
fs.repo.state.save(path_info, fs, hash_info)
|
||||
if hash_info and fs.exists(path_info):
|
||||
state.save(path_info, fs, hash_info)
|
||||
|
||||
return hash_info
|
||||
|
||||
|
||||
+6
-1
@@ -193,7 +193,12 @@ class BaseOutput:
|
||||
|
||||
def get_hash(self):
|
||||
if not self.use_cache:
|
||||
return get_hash(self.path_info, self.fs, self.fs.PARAM_CHECKSUM)
|
||||
return get_hash(
|
||||
self.path_info,
|
||||
self.fs,
|
||||
self.fs.PARAM_CHECKSUM,
|
||||
self.repo.odb.local,
|
||||
)
|
||||
return ostage(self.odb, self.path_info, self.fs).hash_info
|
||||
|
||||
@property
|
||||
|
||||
+16
-20
@@ -472,13 +472,12 @@ class Remote:
|
||||
)
|
||||
|
||||
if self.fs.scheme == "local":
|
||||
with self.fs.repo.state:
|
||||
for checksum in named_cache.scheme_keys("local"):
|
||||
cache_file = self.odb.hash_to_path_info(checksum)
|
||||
if self.fs.exists(cache_file):
|
||||
hash_info = HashInfo(self.fs.PARAM_CHECKSUM, checksum)
|
||||
self.fs.repo.state.save(cache_file, self.fs, hash_info)
|
||||
self.odb.protect(cache_file)
|
||||
for checksum in named_cache.scheme_keys("local"):
|
||||
cache_file = self.odb.hash_to_path_info(checksum)
|
||||
if self.fs.exists(cache_file):
|
||||
hash_info = HashInfo(self.fs.PARAM_CHECKSUM, checksum)
|
||||
self.fs.repo.state.save(cache_file, self.fs, hash_info)
|
||||
self.odb.protect(cache_file)
|
||||
|
||||
return ret
|
||||
|
||||
@@ -493,19 +492,16 @@ class Remote:
|
||||
)
|
||||
|
||||
if not self.odb.verify:
|
||||
with cache.fs.repo.state:
|
||||
for checksum in named_cache.scheme_keys("local"):
|
||||
cache_file = cache.hash_to_path_info(checksum)
|
||||
if cache.fs.exists(cache_file):
|
||||
# We can safely save here, as existing corrupted files
|
||||
# will be removed upon status, while files corrupted
|
||||
# during download will not be moved from tmp_file
|
||||
# (see `BaseFileSystem.download()`)
|
||||
hash_info = HashInfo(cache.fs.PARAM_CHECKSUM, checksum)
|
||||
cache.fs.repo.state.save(
|
||||
cache_file, cache.fs, hash_info
|
||||
)
|
||||
cache.protect(cache_file)
|
||||
for checksum in named_cache.scheme_keys("local"):
|
||||
cache_file = cache.hash_to_path_info(checksum)
|
||||
if cache.fs.exists(cache_file):
|
||||
# We can safely save here, as existing corrupted files
|
||||
# will be removed upon status, while files corrupted
|
||||
# during download will not be moved from tmp_file
|
||||
# (see `BaseFileSystem.download()`)
|
||||
hash_info = HashInfo(cache.fs.PARAM_CHECKSUM, checksum)
|
||||
cache.fs.repo.state.save(cache_file, cache.fs, hash_info)
|
||||
cache.protect(cache_file)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ def lock_repo(repo: "Repo"):
|
||||
if depth > 0:
|
||||
yield
|
||||
else:
|
||||
with repo.lock, repo.state:
|
||||
with repo.lock:
|
||||
repo._reset()
|
||||
yield
|
||||
# Graph cache is no longer valid after we release the repo.lock
|
||||
@@ -186,7 +186,7 @@ class Repo:
|
||||
# NOTE: storing state and link_state in the repository itself to
|
||||
# avoid any possible state corruption in 'shared cache dir'
|
||||
# scenario.
|
||||
self.state = State(self)
|
||||
self.state = State(self.root_dir, self.tmp_dir)
|
||||
self.stage_cache = StageCache(self)
|
||||
|
||||
self._ignore()
|
||||
@@ -477,11 +477,10 @@ class Repo:
|
||||
fs = RepoFileSystem(self, subrepos=True)
|
||||
path = PathInfo(self.root_dir) / path
|
||||
try:
|
||||
with self.state:
|
||||
with fs.open(
|
||||
path, mode=mode, encoding=encoding, remote=remote,
|
||||
) as fobj:
|
||||
yield fobj
|
||||
with fs.open(
|
||||
path, mode=mode, encoding=encoding, remote=remote,
|
||||
) as fobj:
|
||||
yield fobj
|
||||
except FileNotFoundError as exc:
|
||||
raise FileMissingError(path) from exc
|
||||
except IsADirectoryError as exc:
|
||||
@@ -489,8 +488,10 @@ class Repo:
|
||||
|
||||
def close(self):
|
||||
self.scm.close()
|
||||
self.state.close()
|
||||
|
||||
def _reset(self):
|
||||
self.state.close()
|
||||
self.scm._reset() # pylint: disable=protected-access
|
||||
self.__dict__.pop("outs_trie", None)
|
||||
self.__dict__.pop("outs_graph", None)
|
||||
|
||||
+15
-15
@@ -19,16 +19,6 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_unused_links(repo):
|
||||
used = [
|
||||
out.fspath
|
||||
for stage in repo.stages
|
||||
for out in stage.outs
|
||||
if out.scheme == "local"
|
||||
]
|
||||
return repo.state.get_unused_links(used, repo.fs)
|
||||
|
||||
|
||||
def _fspath_dir(path):
|
||||
if not os.path.exists(str(path)):
|
||||
return str(path)
|
||||
@@ -37,6 +27,20 @@ def _fspath_dir(path):
|
||||
return os.path.join(path, "") if os.path.isdir(path) else path
|
||||
|
||||
|
||||
def _remove_unused_links(repo):
|
||||
used = [
|
||||
out.fspath
|
||||
for stage in repo.stages
|
||||
for out in stage.outs
|
||||
if out.scheme == "local"
|
||||
]
|
||||
|
||||
unused = repo.state.get_unused_links(used, repo.fs)
|
||||
ret = [_fspath_dir(u) for u in unused]
|
||||
repo.state.remove_links(unused, repo.fs)
|
||||
return ret
|
||||
|
||||
|
||||
def get_all_files_numbers(pairs):
|
||||
return sum(
|
||||
stage.get_all_files_number(filter_info) for stage, filter_info in pairs
|
||||
@@ -83,7 +87,6 @@ def checkout(
|
||||
**kwargs,
|
||||
):
|
||||
|
||||
unused = []
|
||||
stats = {
|
||||
"added": [],
|
||||
"deleted": [],
|
||||
@@ -92,10 +95,7 @@ def checkout(
|
||||
}
|
||||
if not targets:
|
||||
targets = [None]
|
||||
unused = _get_unused_links(self)
|
||||
|
||||
stats["deleted"] = [_fspath_dir(u) for u in unused]
|
||||
self.state.remove_links(unused, self.fs)
|
||||
stats["deleted"] = _remove_unused_links(self)
|
||||
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
|
||||
@@ -14,4 +14,5 @@ def _destroy_stages(repo):
|
||||
# which will cause issues on Windows, as `.dvc/lock` will be busy.
|
||||
def destroy(repo):
|
||||
_destroy_stages(repo)
|
||||
repo.close()
|
||||
remove(repo.dvc_dir)
|
||||
|
||||
+5
-2
@@ -2,7 +2,8 @@ import logging
|
||||
import os
|
||||
|
||||
from dvc.exceptions import PathMissingError
|
||||
from dvc.objects.stage import get_file_hash, get_hash
|
||||
from dvc.objects.stage import get_file_hash
|
||||
from dvc.objects.stage import stage as ostage
|
||||
from dvc.repo import locked
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -128,7 +129,9 @@ def _output_paths(repo, repo_fs, targets):
|
||||
|
||||
def _to_checksum(output):
|
||||
if on_working_fs:
|
||||
return get_hash(output.path_info, repo.odb.local.fs, "md5").value
|
||||
return ostage(
|
||||
repo.odb.local, output.path_info, repo.odb.local.fs,
|
||||
).hash_info.value
|
||||
return output.hash_info.value
|
||||
|
||||
for stage in repo.stages:
|
||||
|
||||
@@ -51,14 +51,12 @@ def scm_locked(f):
|
||||
def unlocked_repo(f):
|
||||
@wraps(f)
|
||||
def wrapper(exp, *args, **kwargs):
|
||||
exp.repo.state.dump()
|
||||
exp.repo.lock.unlock()
|
||||
exp.repo._reset() # pylint: disable=protected-access
|
||||
try:
|
||||
ret = f(exp, *args, **kwargs)
|
||||
finally:
|
||||
exp.repo.lock.lock()
|
||||
exp.repo.state.load()
|
||||
return ret
|
||||
|
||||
return wrapper
|
||||
|
||||
@@ -337,7 +337,7 @@ class BaseExecutor(ABC):
|
||||
logger.exception("unexpected error")
|
||||
raise
|
||||
finally:
|
||||
dvc.scm.close()
|
||||
dvc.close()
|
||||
if old_cwd:
|
||||
os.chdir(old_cwd)
|
||||
|
||||
|
||||
@@ -55,7 +55,6 @@ def gc(
|
||||
with ExitStack() as stack:
|
||||
for repo in all_repos:
|
||||
stack.enter_context(repo.lock)
|
||||
stack.enter_context(repo.state)
|
||||
|
||||
used = NamedCache()
|
||||
for repo in all_repos + [self]:
|
||||
|
||||
@@ -39,14 +39,12 @@ def rwlocked(call, read=None, write=None):
|
||||
def unlocked_repo(f):
|
||||
@wraps(f)
|
||||
def wrapper(stage, *args, **kwargs):
|
||||
stage.repo.state.dump()
|
||||
stage.repo.lock.unlock()
|
||||
stage.repo._reset() # pylint: disable=protected-access
|
||||
try:
|
||||
ret = f(stage, *args, **kwargs)
|
||||
finally:
|
||||
stage.repo.lock.lock()
|
||||
stage.repo.state.load()
|
||||
return ret
|
||||
|
||||
return wrapper
|
||||
@@ -56,11 +54,9 @@ def relock_repo(f):
|
||||
@wraps(f)
|
||||
def wrapper(stage, *args, **kwargs):
|
||||
stage.repo.lock.lock()
|
||||
stage.repo.state.load()
|
||||
try:
|
||||
ret = f(stage, *args, **kwargs)
|
||||
finally:
|
||||
stage.repo.state.dump()
|
||||
stage.repo.lock.unlock()
|
||||
stage.repo._reset() # pylint: disable=protected-access
|
||||
return ret
|
||||
|
||||
+46
-358
@@ -9,7 +9,7 @@ from urllib.parse import urlencode, urlunparse
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
from dvc.hash_info import HashInfo
|
||||
from dvc.utils import current_timestamp, relpath, to_chunks
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import get_inode, get_mtime_and_size, remove
|
||||
|
||||
SQLITE_MAX_VARIABLES_NUMBER = 999
|
||||
@@ -32,8 +32,9 @@ class StateVersionTooNewError(DvcException):
|
||||
|
||||
|
||||
class StateBase(ABC):
|
||||
def __init__(self):
|
||||
self.count = 0
|
||||
@abstractmethod
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def save(self, path_info, fs, hash_info):
|
||||
@@ -47,27 +48,11 @@ class StateBase(ABC):
|
||||
def save_link(self, path_info, fs):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def load(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def dump(self):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
if self.count <= 0:
|
||||
self.load()
|
||||
self.count += 1
|
||||
|
||||
def __exit__(self, typ, value, tbck):
|
||||
self.count -= 1
|
||||
if self.count > 0:
|
||||
return
|
||||
self.dump()
|
||||
|
||||
|
||||
class StateNoop(StateBase):
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
def save(self, path_info, fs, hash_info):
|
||||
pass
|
||||
|
||||
@@ -77,307 +62,27 @@ class StateNoop(StateBase):
|
||||
def save_link(self, path_info, fs):
|
||||
pass
|
||||
|
||||
def load(self):
|
||||
pass
|
||||
|
||||
def dump(self):
|
||||
pass
|
||||
|
||||
|
||||
class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
"""Class for the state database.
|
||||
|
||||
Args:
|
||||
repo (dvc.repo.Repo): repo instance that this state belongs to.
|
||||
config (configobj.ConfigObj): config for the state.
|
||||
|
||||
Raises:
|
||||
StateVersionTooNewError: thrown when dvc version is older than the
|
||||
state database version.
|
||||
"""
|
||||
|
||||
VERSION = 3
|
||||
STATE_FILE = "state"
|
||||
STATE_TABLE = "state"
|
||||
STATE_TABLE_LAYOUT = (
|
||||
"inode INTEGER PRIMARY KEY, "
|
||||
"mtime TEXT NOT NULL, "
|
||||
"size TEXT NOT NULL, "
|
||||
"md5 TEXT NOT NULL, "
|
||||
"timestamp TEXT NOT NULL"
|
||||
)
|
||||
|
||||
STATE_INFO_TABLE = "state_info"
|
||||
STATE_INFO_TABLE_LAYOUT = "count INTEGER"
|
||||
STATE_INFO_ROW = 1
|
||||
|
||||
LINK_STATE_TABLE = "link_state"
|
||||
LINK_STATE_TABLE_LAYOUT = (
|
||||
"path TEXT PRIMARY KEY, "
|
||||
"inode INTEGER NOT NULL, "
|
||||
"mtime TEXT NOT NULL"
|
||||
)
|
||||
|
||||
STATE_ROW_LIMIT = 100000000
|
||||
STATE_ROW_CLEANUP_QUOTA = 50
|
||||
|
||||
MAX_INT = 2 ** 63 - 1
|
||||
MAX_UINT = 2 ** 64 - 2
|
||||
|
||||
def __init__(self, repo):
|
||||
def __init__(self, root_dir=None, tmp_dir=None):
|
||||
from diskcache import Cache
|
||||
|
||||
super().__init__()
|
||||
|
||||
self.repo = repo
|
||||
self.root_dir = repo.root_dir
|
||||
self.tmp_dir = tmp_dir
|
||||
self.root_dir = root_dir
|
||||
self.fs = LocalFileSystem(None, {"url": self.root_dir})
|
||||
|
||||
state_config = repo.config.get("state", {})
|
||||
self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
|
||||
self.row_cleanup_quota = state_config.get(
|
||||
"row_cleanup_quota", self.STATE_ROW_CLEANUP_QUOTA
|
||||
)
|
||||
|
||||
if not repo.tmp_dir:
|
||||
self.state_file = None
|
||||
if not tmp_dir:
|
||||
return
|
||||
|
||||
self.state_file = os.path.join(repo.tmp_dir, self.STATE_FILE)
|
||||
config = {"eviction_policy": "least-recently-used"}
|
||||
self.links = Cache(directory=os.path.join(tmp_dir, "links"), **config)
|
||||
self.md5s = Cache(directory=os.path.join(tmp_dir, "md5s"), **config)
|
||||
|
||||
self.database = None
|
||||
self.cursor = None
|
||||
self.inserts = 0
|
||||
|
||||
def _execute(self, cmd, parameters=()):
|
||||
logger.trace(cmd)
|
||||
return self.cursor.execute(cmd, parameters)
|
||||
|
||||
def _fetchall(self):
|
||||
ret = self.cursor.fetchall()
|
||||
logger.trace("fetched: %s", ret)
|
||||
return ret
|
||||
|
||||
def _to_sqlite(self, num):
|
||||
assert num >= 0
|
||||
assert num < self.MAX_UINT
|
||||
# NOTE: sqlite stores unit as signed ints, so maximum uint is 2^63-1
|
||||
# see http://jakegoulding.com/blog/2011/02/06/sqlite-64-bit-integers/
|
||||
if num > self.MAX_INT:
|
||||
ret = -(num - self.MAX_INT)
|
||||
else:
|
||||
ret = num
|
||||
assert self._from_sqlite(ret) == num
|
||||
return ret
|
||||
|
||||
def _from_sqlite(self, num):
|
||||
assert abs(num) <= self.MAX_INT
|
||||
if num < 0:
|
||||
return abs(num) + self.MAX_INT
|
||||
assert num < self.MAX_UINT
|
||||
assert num >= 0
|
||||
return num
|
||||
|
||||
def _prepare_db(self, empty=False):
|
||||
from dvc import __version__
|
||||
|
||||
if not empty:
|
||||
cmd = "PRAGMA user_version;"
|
||||
self._execute(cmd)
|
||||
ret = self._fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
assert isinstance(ret[0][0], int)
|
||||
version = ret[0][0]
|
||||
|
||||
if version > self.VERSION:
|
||||
raise StateVersionTooNewError(
|
||||
__version__, self.VERSION, version
|
||||
)
|
||||
elif version < self.VERSION:
|
||||
logger.warning(
|
||||
"State file version '%d' is too old. "
|
||||
"Reformatting to the current version '%d'.",
|
||||
version,
|
||||
self.VERSION,
|
||||
)
|
||||
cmd = "DROP TABLE IF EXISTS {};"
|
||||
self._execute(cmd.format(self.STATE_TABLE))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE))
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE))
|
||||
|
||||
# Check that the state file is indeed a database
|
||||
cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
|
||||
self._execute(cmd.format(self.STATE_TABLE, self.STATE_TABLE_LAYOUT))
|
||||
self._execute(
|
||||
cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE_LAYOUT)
|
||||
)
|
||||
self._execute(
|
||||
cmd.format(self.LINK_STATE_TABLE, self.LINK_STATE_TABLE_LAYOUT)
|
||||
)
|
||||
|
||||
cmd = (
|
||||
"INSERT OR IGNORE INTO {} (count) SELECT 0 "
|
||||
"WHERE NOT EXISTS (SELECT * FROM {})"
|
||||
)
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE))
|
||||
|
||||
cmd = "PRAGMA user_version = {};"
|
||||
self._execute(cmd.format(self.VERSION))
|
||||
|
||||
def load(self):
|
||||
"""Loads state database."""
|
||||
retries = 1
|
||||
while True:
|
||||
assert self.database is None
|
||||
assert self.cursor is None
|
||||
assert self.inserts == 0
|
||||
empty = not os.path.exists(self.state_file)
|
||||
# NOTE: we use nolock option because fcntl() lock sqlite uses
|
||||
# doesn't work on some older NFS/CIFS filesystems.
|
||||
# This opens a possibility of data corruption by concurrent writes,
|
||||
# which is prevented by repo lock.
|
||||
self.database = _connect_sqlite(self.state_file, {"nolock": 1})
|
||||
self.cursor = self.database.cursor()
|
||||
|
||||
from sqlite3 import DatabaseError
|
||||
|
||||
# Try loading once to check that the file is indeed a database
|
||||
# and reformat it if it is not.
|
||||
try:
|
||||
self._prepare_db(empty=empty)
|
||||
return
|
||||
except DatabaseError:
|
||||
self.cursor.close()
|
||||
self.database.close()
|
||||
self.database = None
|
||||
self.cursor = None
|
||||
self.inserts = 0
|
||||
if retries > 0:
|
||||
os.unlink(self.state_file)
|
||||
retries -= 1
|
||||
else:
|
||||
raise
|
||||
|
||||
def _vacuum(self):
|
||||
# NOTE: see https://bugs.python.org/issue28518
|
||||
self.database.isolation_level = None
|
||||
self._execute("VACUUM")
|
||||
self.database.isolation_level = ""
|
||||
|
||||
def dump(self):
|
||||
"""Saves state database."""
|
||||
assert self.database is not None
|
||||
|
||||
cmd = "SELECT count from {} WHERE rowid=?".format(
|
||||
self.STATE_INFO_TABLE
|
||||
)
|
||||
self._execute(cmd, (self.STATE_INFO_ROW,))
|
||||
ret = self._fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
count = self._from_sqlite(ret[0][0]) + self.inserts
|
||||
|
||||
if count > self.row_limit:
|
||||
msg = "cleaning up state, this might take a while."
|
||||
logger.warning(msg)
|
||||
|
||||
delete = count - self.row_limit
|
||||
delete += int(self.row_limit * (self.row_cleanup_quota / 100.0))
|
||||
cmd = (
|
||||
"DELETE FROM {} WHERE timestamp IN ("
|
||||
"SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
|
||||
)
|
||||
self._execute(
|
||||
cmd.format(self.STATE_TABLE, self.STATE_TABLE, delete)
|
||||
)
|
||||
|
||||
self._vacuum()
|
||||
|
||||
cmd = "SELECT COUNT(*) FROM {}"
|
||||
|
||||
self._execute(cmd.format(self.STATE_TABLE))
|
||||
ret = self._fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
count = ret[0][0]
|
||||
|
||||
cmd = "UPDATE {} SET count = ? WHERE rowid = ?".format(
|
||||
self.STATE_INFO_TABLE
|
||||
)
|
||||
self._execute(cmd, (self._to_sqlite(count), self.STATE_INFO_ROW))
|
||||
|
||||
self.database.commit()
|
||||
self.cursor.close()
|
||||
self.database.close()
|
||||
self.database = None
|
||||
self.cursor = None
|
||||
self.inserts = 0
|
||||
|
||||
@staticmethod
|
||||
def _file_metadata_changed(actual_mtime, mtime, actual_size, size):
|
||||
return actual_mtime != mtime or actual_size != size
|
||||
|
||||
def _update_state_record_timestamp_for_inode(self, actual_inode):
|
||||
cmd = "UPDATE {} SET timestamp = ? WHERE inode = ?".format(
|
||||
self.STATE_TABLE
|
||||
)
|
||||
self._execute(
|
||||
cmd, (current_timestamp(), self._to_sqlite(actual_inode))
|
||||
)
|
||||
|
||||
def _update_state_for_path_changed(
|
||||
self, actual_inode, actual_mtime, actual_size, checksum
|
||||
):
|
||||
cmd = (
|
||||
"UPDATE {} SET "
|
||||
"mtime = ?, size = ?, "
|
||||
"md5 = ?, timestamp = ? "
|
||||
"WHERE inode = ?"
|
||||
).format(self.STATE_TABLE)
|
||||
self._execute(
|
||||
cmd,
|
||||
(
|
||||
actual_mtime,
|
||||
actual_size,
|
||||
checksum,
|
||||
current_timestamp(),
|
||||
self._to_sqlite(actual_inode),
|
||||
),
|
||||
)
|
||||
|
||||
def _insert_new_state_record(
|
||||
self, actual_inode, actual_mtime, actual_size, checksum
|
||||
):
|
||||
assert checksum is not None
|
||||
|
||||
cmd = (
|
||||
"INSERT INTO {}(inode, mtime, size, md5, timestamp) "
|
||||
"VALUES (?, ?, ?, ?, ?)"
|
||||
).format(self.STATE_TABLE)
|
||||
self._execute(
|
||||
cmd,
|
||||
(
|
||||
self._to_sqlite(actual_inode),
|
||||
actual_mtime,
|
||||
actual_size,
|
||||
checksum,
|
||||
current_timestamp(),
|
||||
),
|
||||
)
|
||||
self.inserts += 1
|
||||
|
||||
def get_state_record_for_inode(self, inode):
|
||||
cmd = (
|
||||
"SELECT mtime, size, md5, timestamp from {} WHERE "
|
||||
"inode=?".format(self.STATE_TABLE)
|
||||
)
|
||||
self._execute(cmd, (self._to_sqlite(inode),))
|
||||
results = self._fetchall()
|
||||
if results:
|
||||
# uniqueness constrain on inode
|
||||
assert len(results) == 1
|
||||
return results[0]
|
||||
return None
|
||||
def close(self):
|
||||
self.md5s.close()
|
||||
self.links.close()
|
||||
|
||||
def save(self, path_info, fs, hash_info):
|
||||
"""Save hash for the specified path info.
|
||||
@@ -395,20 +100,15 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
assert isinstance(hash_info, HashInfo)
|
||||
assert os.path.exists(path_info)
|
||||
|
||||
actual_mtime, actual_size = get_mtime_and_size(path_info, self.fs)
|
||||
actual_inode = get_inode(path_info)
|
||||
mtime, size = get_mtime_and_size(path_info, self.fs)
|
||||
inode = get_inode(path_info)
|
||||
|
||||
existing_record = self.get_state_record_for_inode(actual_inode)
|
||||
if not existing_record:
|
||||
self._insert_new_state_record(
|
||||
actual_inode, actual_mtime, actual_size, hash_info.value
|
||||
)
|
||||
return
|
||||
|
||||
self._update_state_for_path_changed(
|
||||
actual_inode, actual_mtime, actual_size, hash_info.value
|
||||
logger.debug(
|
||||
"state save (%s, %s, %s) %s", inode, mtime, size, hash_info.value
|
||||
)
|
||||
|
||||
self.md5s[inode] = (mtime, size, hash_info.value)
|
||||
|
||||
def get(self, path_info, fs):
|
||||
"""Gets the hash for the specified path info. Hash will be
|
||||
retrieved from the state database if available.
|
||||
@@ -432,19 +132,15 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
|
||||
actual_mtime, actual_size = get_mtime_and_size(path, self.fs)
|
||||
actual_inode = get_inode(path)
|
||||
mtime, size = get_mtime_and_size(path, self.fs)
|
||||
inode = get_inode(path)
|
||||
|
||||
existing_record = self.get_state_record_for_inode(actual_inode)
|
||||
if not existing_record:
|
||||
value = self.md5s.get(inode)
|
||||
|
||||
if not value or value[0] != mtime or value[1] != size:
|
||||
return None
|
||||
|
||||
mtime, size, value, _ = existing_record
|
||||
if self._file_metadata_changed(actual_mtime, mtime, actual_size, size):
|
||||
return None
|
||||
|
||||
self._update_state_record_timestamp_for_inode(actual_inode)
|
||||
return HashInfo("md5", value, size=int(actual_size))
|
||||
return HashInfo("md5", value[2], size=int(size))
|
||||
|
||||
def save_link(self, path_info, fs):
|
||||
"""Adds the specified path to the list of links created by dvc. This
|
||||
@@ -465,10 +161,8 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
inode = get_inode(path_info)
|
||||
relative_path = relpath(path_info, self.root_dir)
|
||||
|
||||
cmd = "REPLACE INTO {}(path, inode, mtime) " "VALUES (?, ?, ?)".format(
|
||||
self.LINK_STATE_TABLE
|
||||
)
|
||||
self._execute(cmd, (relative_path, self._to_sqlite(inode), mtime))
|
||||
with self.links as ref:
|
||||
ref[relative_path] = (inode, mtime)
|
||||
|
||||
def get_unused_links(self, used, fs):
|
||||
"""Removes all saved links except the ones that are used.
|
||||
@@ -481,21 +175,19 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
|
||||
unused = []
|
||||
|
||||
self._execute(f"SELECT * FROM {self.LINK_STATE_TABLE}")
|
||||
for row in self.cursor:
|
||||
relative_path, inode, mtime = row
|
||||
inode = self._from_sqlite(inode)
|
||||
path = os.path.join(self.root_dir, relative_path)
|
||||
with self.links as ref:
|
||||
for relative_path in ref:
|
||||
path = os.path.join(self.root_dir, relative_path)
|
||||
|
||||
if path in used or not self.fs.exists(path):
|
||||
continue
|
||||
if path in used or not self.fs.exists(path):
|
||||
continue
|
||||
|
||||
actual_inode = get_inode(path)
|
||||
actual_mtime, _ = get_mtime_and_size(path, self.fs)
|
||||
inode = get_inode(path)
|
||||
mtime, _ = get_mtime_and_size(path, self.fs)
|
||||
|
||||
if (inode, mtime) == (actual_inode, actual_mtime):
|
||||
logger.debug("Removing '%s' as unused link.", path)
|
||||
unused.append(relative_path)
|
||||
if ref[relative_path] == (inode, mtime):
|
||||
logger.debug("Removing '%s' as unused link.", path)
|
||||
unused.append(relative_path)
|
||||
|
||||
return unused
|
||||
|
||||
@@ -504,15 +196,11 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
return
|
||||
|
||||
for path in unused:
|
||||
remove(path)
|
||||
remove(os.path.join(self.root_dir, path))
|
||||
|
||||
for chunk_unused in to_chunks(
|
||||
unused, chunk_size=SQLITE_MAX_VARIABLES_NUMBER
|
||||
):
|
||||
cmd = "DELETE FROM {} WHERE path IN ({})".format(
|
||||
self.LINK_STATE_TABLE, ",".join(["?"] * len(chunk_unused))
|
||||
)
|
||||
self._execute(cmd, tuple(chunk_unused))
|
||||
with self.links as ref:
|
||||
for path in unused:
|
||||
del ref[path]
|
||||
|
||||
|
||||
def _connect_sqlite(filename, options):
|
||||
|
||||
@@ -89,6 +89,7 @@ install_requires = [
|
||||
"typing_extensions>=3.7.4",
|
||||
"fsspec>=0.8.5",
|
||||
"dvclive>=0.0.1a0",
|
||||
"diskcache>=5.2.1",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -114,6 +114,8 @@ class TmpDir(pathlib.Path):
|
||||
def close(self):
|
||||
if hasattr(self, "scm"):
|
||||
self.scm.close()
|
||||
if hasattr(self, "dvc"):
|
||||
self.dvc.close()
|
||||
|
||||
def _require(self, name):
|
||||
if not hasattr(self, name):
|
||||
|
||||
@@ -40,7 +40,6 @@ def test_no_cache_entry(tmp_dir, scm, dvc):
|
||||
tmp_dir.dvc_gen("file", "second")
|
||||
|
||||
remove(tmp_dir / ".dvc" / "cache")
|
||||
(tmp_dir / ".dvc" / "tmp" / "state").unlink()
|
||||
|
||||
dir_checksum = "5fb6b29836c388e093ca0715c872fe2a.dir"
|
||||
|
||||
@@ -131,6 +130,7 @@ def test_directories(tmp_dir, scm, dvc):
|
||||
tmp_dir.dvc_gen({"dir": {"2": "two"}}, commit="modify a file")
|
||||
|
||||
(tmp_dir / "dir" / "2").unlink()
|
||||
assert dvc.status() != {} # sanity check
|
||||
dvc.add("dir")
|
||||
scm.add(["dir.dvc"])
|
||||
scm.commit("delete a file")
|
||||
|
||||
@@ -14,7 +14,7 @@ class TestInit(TestGit):
|
||||
self.assertTrue(os.path.isdir(DvcRepo.DVC_DIR))
|
||||
|
||||
def test_api(self):
|
||||
DvcRepo.init()
|
||||
DvcRepo.init().close()
|
||||
|
||||
self._test_init()
|
||||
|
||||
|
||||
@@ -146,7 +146,7 @@ class TestRemoteShouldHandleUppercaseRemoteName(TestDvc):
|
||||
|
||||
|
||||
def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
from dvc.objects.stage import get_hash
|
||||
from dvc.objects.stage import stage
|
||||
|
||||
tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}})
|
||||
|
||||
@@ -158,7 +158,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
with patch(
|
||||
"dvc.objects.stage._collect_dir", return_value=dir_info,
|
||||
):
|
||||
hash1 = get_hash(path_info, dvc.odb.local.fs, "md5")
|
||||
hash1 = stage(dvc.odb.local, path_info, dvc.odb.local.fs).hash_info
|
||||
|
||||
dir_info = DirInfo.from_list(
|
||||
[{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}]
|
||||
@@ -166,7 +166,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
with patch(
|
||||
"dvc.objects.stage._collect_dir", return_value=dir_info,
|
||||
):
|
||||
hash2 = get_hash(path_info, dvc.odb.local.fs, "md5")
|
||||
hash2 = stage(dvc.odb.local, path_info, dvc.odb.local.fs).hash_info
|
||||
|
||||
assert hash1 == hash2
|
||||
|
||||
|
||||
@@ -917,7 +917,7 @@ class TestShouldNotCheckoutUponCorruptedLocalHardlinkCache(TestDvc):
|
||||
|
||||
patch_run = mock.patch("dvc.stage.run.cmd_run", wraps=cmd_run)
|
||||
|
||||
with self.dvc.lock, self.dvc.state:
|
||||
with self.dvc.lock:
|
||||
with patch_checkout as mock_checkout:
|
||||
with patch_run as mock_run:
|
||||
stage.run()
|
||||
|
||||
@@ -107,7 +107,7 @@ class TestDefaultWorkingDirectory(TestDvc):
|
||||
d = load_yaml(stage.relpath)
|
||||
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
|
||||
|
||||
with self.dvc.lock, self.dvc.state:
|
||||
with self.dvc.lock:
|
||||
stage = SingleStageFile(self.dvc, stage.relpath).stage
|
||||
self.assertFalse(stage.changed())
|
||||
|
||||
|
||||
+22
-53
@@ -1,7 +1,5 @@
|
||||
import os
|
||||
|
||||
import mock
|
||||
|
||||
from dvc.hash_info import HashInfo
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.state import State
|
||||
@@ -14,21 +12,20 @@ def test_state(tmp_dir, dvc):
|
||||
path_info = PathInfo(path)
|
||||
hash_info = HashInfo("md5", file_md5(path, dvc.fs))
|
||||
|
||||
state = State(dvc)
|
||||
state = State(dvc.root_dir, dvc.tmp_dir)
|
||||
|
||||
with state:
|
||||
state.save(path_info, dvc.fs, hash_info)
|
||||
assert state.get(path_info, dvc.fs) == hash_info
|
||||
state.save(path_info, dvc.fs, hash_info)
|
||||
assert state.get(path_info, dvc.fs) == hash_info
|
||||
|
||||
path.unlink()
|
||||
path.write_text("1")
|
||||
path.unlink()
|
||||
path.write_text("1")
|
||||
|
||||
assert state.get(path_info, dvc.fs) is None
|
||||
assert state.get(path_info, dvc.fs) is None
|
||||
|
||||
hash_info = HashInfo("md5", file_md5(path, dvc.fs))
|
||||
state.save(path_info, dvc.fs, hash_info)
|
||||
hash_info = HashInfo("md5", file_md5(path, dvc.fs))
|
||||
state.save(path_info, dvc.fs, hash_info)
|
||||
|
||||
assert state.get(path_info, dvc.fs) == hash_info
|
||||
assert state.get(path_info, dvc.fs) == hash_info
|
||||
|
||||
|
||||
def test_state_overflow(tmp_dir, dvc):
|
||||
@@ -51,54 +48,26 @@ def mock_get_inode(inode):
|
||||
return get_inode_mocked
|
||||
|
||||
|
||||
@mock.patch("dvc.state.get_inode", autospec=True)
|
||||
def test_get_state_record_for_inode(get_inode_mock, tmp_dir, dvc):
|
||||
tmp_dir.gen("foo", "foo content")
|
||||
|
||||
state = State(dvc)
|
||||
inode = state.MAX_INT + 2
|
||||
assert inode != state._to_sqlite(inode)
|
||||
|
||||
foo = tmp_dir / "foo"
|
||||
md5 = file_md5(foo, dvc.fs)
|
||||
get_inode_mock.side_effect = mock_get_inode(inode)
|
||||
|
||||
with state:
|
||||
state.save(PathInfo(foo), dvc.fs, HashInfo("md5", md5))
|
||||
ret = state.get_state_record_for_inode(inode)
|
||||
assert ret is not None
|
||||
|
||||
|
||||
def test_remove_links(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen({"foo": "foo_content", "bar": "bar_content"})
|
||||
|
||||
with dvc.state:
|
||||
cmd_count_links = "SELECT count(*) FROM {}".format(
|
||||
State.LINK_STATE_TABLE
|
||||
)
|
||||
result = dvc.state._execute(cmd_count_links).fetchone()[0]
|
||||
assert result == 2
|
||||
assert len(dvc.state.links) == 2
|
||||
|
||||
dvc.state.remove_links(["foo", "bar"], dvc.fs)
|
||||
dvc.state.remove_links(["foo", "bar"], dvc.fs)
|
||||
|
||||
result = dvc.state._execute(cmd_count_links).fetchone()[0]
|
||||
assert result == 0
|
||||
assert len(dvc.state.links) == 0
|
||||
|
||||
|
||||
def test_get_unused_links(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen({"foo": "foo_content", "bar": "bar_content"})
|
||||
|
||||
with dvc.state:
|
||||
links = [os.path.join(dvc.root_dir, link) for link in ["foo", "bar"]]
|
||||
assert set(dvc.state.get_unused_links([], dvc.fs)) == {"foo", "bar"}
|
||||
assert set(dvc.state.get_unused_links(links[:1], dvc.fs)) == {"bar"}
|
||||
assert set(dvc.state.get_unused_links(links, dvc.fs)) == set()
|
||||
assert set(
|
||||
dvc.state.get_unused_links(
|
||||
(
|
||||
links[:1]
|
||||
+ [os.path.join(dvc.root_dir, "not-existing-file")]
|
||||
),
|
||||
dvc.fs,
|
||||
)
|
||||
) == {"bar"}
|
||||
links = [os.path.join(dvc.root_dir, link) for link in ["foo", "bar"]]
|
||||
assert set(dvc.state.get_unused_links([], dvc.fs)) == {"foo", "bar"}
|
||||
assert set(dvc.state.get_unused_links(links[:1], dvc.fs)) == {"bar"}
|
||||
assert set(dvc.state.get_unused_links(links, dvc.fs)) == set()
|
||||
assert set(
|
||||
dvc.state.get_unused_links(
|
||||
(links[:1] + [os.path.join(dvc.root_dir, "not-existing-file")]),
|
||||
dvc.fs,
|
||||
)
|
||||
) == {"bar"}
|
||||
|
||||
@@ -5,7 +5,7 @@ import pytest
|
||||
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
from dvc.hash_info import HashInfo
|
||||
from dvc.objects.stage import get_hash
|
||||
from dvc.objects.stage import stage
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
|
||||
@@ -229,13 +229,13 @@ def test_get_hash_granular(tmp_dir, dvc):
|
||||
fs = DvcFileSystem(dvc)
|
||||
subdir = PathInfo(tmp_dir) / "dir" / "subdir"
|
||||
assert fs.info(subdir).get("md5") is None
|
||||
assert get_hash(subdir, fs, "md5") == HashInfo(
|
||||
assert stage(dvc.odb.local, subdir, fs).hash_info == HashInfo(
|
||||
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir",
|
||||
)
|
||||
assert (
|
||||
fs.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc"
|
||||
)
|
||||
assert get_hash(subdir / "data", fs, "md5") == HashInfo(
|
||||
assert stage(dvc.odb.local, subdir / "data", fs).hash_info == HashInfo(
|
||||
"md5", "8d777f385d3dfec8815d20f7496026dc",
|
||||
)
|
||||
|
||||
@@ -247,9 +247,9 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
|
||||
fs = DvcFileSystem(dvc)
|
||||
expected = "8c7dd922ad47494fc02c388e12c00eac"
|
||||
assert fs.info(PathInfo(tmp_dir) / "file").get("md5") == expected
|
||||
assert get_hash(PathInfo(tmp_dir) / "file", fs, "md5") == HashInfo(
|
||||
"md5", expected
|
||||
)
|
||||
assert stage(
|
||||
dvc.odb.local, PathInfo(tmp_dir) / "file", fs
|
||||
).hash_info == HashInfo("md5", expected)
|
||||
|
||||
|
||||
def test_get_hash_dirty_dir(tmp_dir, dvc):
|
||||
@@ -259,6 +259,6 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
|
||||
fs = DvcFileSystem(dvc)
|
||||
expected = "5ea40360f5b4ec688df672a4db9c17d1.dir"
|
||||
assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") == expected
|
||||
assert get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") == HashInfo(
|
||||
"md5", expected
|
||||
)
|
||||
assert stage(
|
||||
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs
|
||||
).hash_info == HashInfo("md5", expected)
|
||||
|
||||
+16
-16
@@ -6,7 +6,7 @@ import pytest
|
||||
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.hash_info import HashInfo
|
||||
from dvc.objects.stage import get_hash
|
||||
from dvc.objects.stage import stage
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
|
||||
@@ -482,9 +482,9 @@ def test_get_hash_cached_file(tmp_dir, dvc, mocker):
|
||||
fs = RepoFileSystem(dvc)
|
||||
expected = "acbd18db4cc2f85cedef654fccc4a4d8"
|
||||
assert fs.info(PathInfo(tmp_dir) / "foo").get("md5") is None
|
||||
assert get_hash(PathInfo(tmp_dir) / "foo", fs, "md5") == HashInfo(
|
||||
"md5", expected,
|
||||
)
|
||||
assert stage(
|
||||
dvc.odb.local, PathInfo(tmp_dir) / "foo", fs
|
||||
).hash_info == HashInfo("md5", expected,)
|
||||
(tmp_dir / "foo").unlink()
|
||||
assert fs.info(PathInfo(tmp_dir) / "foo")["md5"] == expected
|
||||
|
||||
@@ -496,15 +496,15 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker):
|
||||
fs = RepoFileSystem(dvc)
|
||||
expected = "8761c4e9acad696bee718615e23e22db.dir"
|
||||
assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") is None
|
||||
assert get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") == HashInfo(
|
||||
"md5", "8761c4e9acad696bee718615e23e22db.dir",
|
||||
)
|
||||
assert stage(
|
||||
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs
|
||||
).hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir",)
|
||||
|
||||
shutil.rmtree(tmp_dir / "dir")
|
||||
assert fs.info(PathInfo(tmp_dir) / "dir")["md5"] == expected
|
||||
assert get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") == HashInfo(
|
||||
"md5", "8761c4e9acad696bee718615e23e22db.dir",
|
||||
)
|
||||
assert stage(
|
||||
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs
|
||||
).hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir",)
|
||||
|
||||
|
||||
def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
|
||||
@@ -514,11 +514,11 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
|
||||
fs = RepoFileSystem(dvc)
|
||||
subdir = PathInfo(tmp_dir) / "dir" / "subdir"
|
||||
assert fs.info(subdir).get("md5") is None
|
||||
assert get_hash(subdir, fs, "md5") == HashInfo(
|
||||
assert stage(dvc.odb.local, subdir, fs).hash_info == HashInfo(
|
||||
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir",
|
||||
)
|
||||
assert fs.info(subdir / "data").get("md5") is None
|
||||
assert get_hash(subdir / "data", fs, "md5") == HashInfo(
|
||||
assert stage(dvc.odb.local, subdir / "data", fs).hash_info == HashInfo(
|
||||
"md5", "8d777f385d3dfec8815d20f7496026dc",
|
||||
)
|
||||
(tmp_dir / "dir" / "subdir" / "data").unlink()
|
||||
@@ -540,7 +540,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc):
|
||||
tmp_dir.scm.commit("add dir")
|
||||
|
||||
fs = RepoFileSystem(dvc)
|
||||
actual = get_hash(PathInfo(tmp_dir) / "dir", fs, "md5")
|
||||
actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs).hash_info
|
||||
expected = HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir")
|
||||
assert actual == expected
|
||||
|
||||
@@ -551,7 +551,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
|
||||
|
||||
fs = RepoFileSystem(dvc)
|
||||
assert fs.info(PathInfo(tmp_dir) / "file").get("md5") is None
|
||||
actual = get_hash(PathInfo(tmp_dir) / "file", fs, "md5")
|
||||
actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs).hash_info
|
||||
expected = HashInfo("md5", "437b930db84b8079c2dd804a71936b5f")
|
||||
assert actual == expected
|
||||
|
||||
@@ -560,7 +560,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
|
||||
fs.info(PathInfo(tmp_dir) / "file")["md5"]
|
||||
== "8c7dd922ad47494fc02c388e12c00eac"
|
||||
)
|
||||
actual = get_hash(PathInfo(tmp_dir) / "file", fs, "md5")
|
||||
actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs).hash_info
|
||||
expected = HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac")
|
||||
assert actual == expected
|
||||
|
||||
@@ -570,7 +570,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
|
||||
(tmp_dir / "dir" / "baz").write_text("baz")
|
||||
|
||||
fs = RepoFileSystem(dvc)
|
||||
actual = get_hash(PathInfo(tmp_dir) / "dir", fs, "md5")
|
||||
actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs).hash_info
|
||||
expected = HashInfo("md5", "ba75a2162ca9c29acecb7957105a0bc2.dir")
|
||||
assert actual == expected
|
||||
assert actual.dir_info.nfiles == 3
|
||||
|
||||
@@ -5,7 +5,6 @@ import pytest
|
||||
|
||||
from dvc.exceptions import OutputDuplicationError
|
||||
from dvc.repo import NotDvcRepoError, Repo, locked
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
|
||||
def test_is_dvc_internal(dvc):
|
||||
@@ -113,25 +112,28 @@ def test_skip_graph_checks(tmp_dir, dvc, mocker, run_copy):
|
||||
def test_branch_config(tmp_dir, scm):
|
||||
tmp_dir.scm_gen("foo", "foo", commit="init")
|
||||
|
||||
# sanity check
|
||||
with pytest.raises(NotDvcRepoError):
|
||||
Repo().close()
|
||||
|
||||
scm.checkout("branch", create_new=True)
|
||||
dvc = Repo.init()
|
||||
with dvc.config.edit() as conf:
|
||||
conf["remote"]["branch"] = {"url": "/some/path"}
|
||||
scm.add([".dvc"])
|
||||
dvc.close()
|
||||
|
||||
scm.add([os.path.join(".dvc", "config")])
|
||||
scm.commit("init dvc")
|
||||
scm.checkout("master")
|
||||
|
||||
remove(".dvc")
|
||||
|
||||
# sanity check
|
||||
with pytest.raises(NotDvcRepoError):
|
||||
Repo()
|
||||
|
||||
with pytest.raises(NotDvcRepoError):
|
||||
Repo(scm=scm, rev="master")
|
||||
Repo(scm=scm, rev="master").close()
|
||||
|
||||
dvc = Repo(scm=scm, rev="branch")
|
||||
assert dvc.config["remote"]["branch"]["url"] == "/some/path"
|
||||
try:
|
||||
assert dvc.config["remote"]["branch"]["url"] == "/some/path"
|
||||
finally:
|
||||
dvc.close()
|
||||
|
||||
|
||||
def test_dynamic_cache_initalization(tmp_dir, scm):
|
||||
@@ -139,5 +141,6 @@ def test_dynamic_cache_initalization(tmp_dir, scm):
|
||||
with dvc.config.edit() as conf:
|
||||
conf["cache"]["ssh"] = "foo"
|
||||
conf["remote"]["foo"] = {"url": "remote://bar/baz"}
|
||||
dvc.close()
|
||||
|
||||
Repo(str(tmp_dir))
|
||||
Repo(str(tmp_dir)).close()
|
||||
|
||||
@@ -22,7 +22,7 @@ def test_stage_cache(tmp_dir, dvc, mocker):
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
with dvc.lock, dvc.state:
|
||||
with dvc.lock:
|
||||
stage.remove(remove_outs=True, force=True)
|
||||
|
||||
assert not (tmp_dir / "out").exists()
|
||||
@@ -45,7 +45,7 @@ def test_stage_cache(tmp_dir, dvc, mocker):
|
||||
|
||||
run_spy = mocker.patch("dvc.stage.run.cmd_run")
|
||||
checkout_spy = mocker.spy(base, "checkout")
|
||||
with dvc.lock, dvc.state:
|
||||
with dvc.lock:
|
||||
stage.run()
|
||||
|
||||
assert not run_spy.called
|
||||
@@ -75,7 +75,7 @@ def test_stage_cache_params(tmp_dir, dvc, mocker):
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
with dvc.lock, dvc.state:
|
||||
with dvc.lock:
|
||||
stage.remove(remove_outs=True, force=True)
|
||||
|
||||
assert not (tmp_dir / "out").exists()
|
||||
@@ -98,7 +98,7 @@ def test_stage_cache_params(tmp_dir, dvc, mocker):
|
||||
|
||||
run_spy = mocker.patch("dvc.stage.run.cmd_run")
|
||||
checkout_spy = mocker.spy(base, "checkout")
|
||||
with dvc.lock, dvc.state:
|
||||
with dvc.lock:
|
||||
stage.run()
|
||||
|
||||
assert not run_spy.called
|
||||
@@ -129,7 +129,7 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker):
|
||||
wdir="wdir",
|
||||
)
|
||||
|
||||
with dvc.lock, dvc.state:
|
||||
with dvc.lock:
|
||||
stage.remove(remove_outs=True, force=True)
|
||||
|
||||
assert not (tmp_dir / "wdir" / "out").exists()
|
||||
@@ -152,7 +152,7 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker):
|
||||
|
||||
run_spy = mocker.patch("dvc.stage.run.cmd_run")
|
||||
checkout_spy = mocker.spy(base, "checkout")
|
||||
with dvc.lock, dvc.state:
|
||||
with dvc.lock:
|
||||
stage.run()
|
||||
|
||||
assert not run_spy.called
|
||||
|
||||
Reference in New Issue
Block a user