From 8f568ab197de9097dfc2d833a339f8da761c8c05 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 23 Feb 2021 20:51:34 +0200 Subject: [PATCH] state: use diskcache (#5505) * state: only load when needed * state: don't use separate load/dump * checkout: group link removal logic together * state: use diskcache.Cache * staging: use pbar and save to state * state: don't pass the whole repo to __init__ * staging: explicitly pass state object * fs: get rid of odb property * state: don't close the connection to cache * repo: close: close state --- dvc/api/__init__.py | 2 +- dvc/checkout.py | 4 +- dvc/command/git_hook.py | 11 +- dvc/config_schema.py | 4 +- dvc/dependency/repo.py | 11 +- dvc/fs/base.py | 4 - dvc/objects/__init__.py | 12 +- dvc/objects/db/base.py | 3 +- dvc/objects/stage.py | 95 +++--- dvc/output/base.py | 7 +- dvc/remote/base.py | 36 +-- dvc/repo/__init__.py | 15 +- dvc/repo/checkout.py | 30 +- dvc/repo/destroy.py | 1 + dvc/repo/diff.py | 7 +- dvc/repo/experiments/__init__.py | 2 - dvc/repo/experiments/executor/base.py | 2 +- dvc/repo/gc.py | 1 - dvc/stage/decorators.py | 4 - dvc/state.py | 404 +++----------------------- setup.py | 1 + tests/dir_helpers.py | 2 + tests/func/test_diff.py | 2 +- tests/func/test_init.py | 2 +- tests/func/test_remote.py | 6 +- tests/func/test_run_single_stage.py | 2 +- tests/func/test_stage.py | 2 +- tests/func/test_state.py | 75 ++--- tests/unit/fs/test_dvc.py | 18 +- tests/unit/fs/test_repo.py | 32 +- tests/unit/repo/test_repo.py | 25 +- tests/unit/stage/test_cache.py | 12 +- 32 files changed, 245 insertions(+), 589 deletions(-) diff --git a/dvc/api/__init__.py b/dvc/api/__init__.py index 98a163ac5..8471cf207 100644 --- a/dvc/api/__init__.py +++ b/dvc/api/__init__.py @@ -29,7 +29,7 @@ def get_url(path, repo=None, rev=None, remote=None): raise OutputNotFoundError(path, repo) cloud = metadata.repo.cloud - hash_info = get_hash(path_info, _repo.repo_fs, "md5") + hash_info = get_hash(path_info, _repo.repo_fs, "md5", _repo.odb) return cloud.get_url_for(remote, checksum=hash_info.value) diff --git a/dvc/checkout.py b/dvc/checkout.py index 45779710d..04659d5a6 100644 --- a/dvc/checkout.py +++ b/dvc/checkout.py @@ -30,7 +30,7 @@ def _changed(path_info, fs, obj, cache): return True try: - actual = get_hash(path_info, fs, obj.hash_info.name) + actual = get_hash(path_info, fs, obj.hash_info.name, cache) except FileNotFoundError: logger.debug("'%s' doesn't exist.", path_info) return True @@ -56,7 +56,7 @@ def _remove(path_info, fs, cache, force=False): fs.remove(path_info) return - current = get_hash(path_info, fs, fs.PARAM_CHECKSUM) + current = get_hash(path_info, fs, fs.PARAM_CHECKSUM, cache) try: obj = load(cache, current) check(cache, obj) diff --git a/dvc/command/git_hook.py b/dvc/command/git_hook.py index 0b2cc4ad9..e936ce6f3 100644 --- a/dvc/command/git_hook.py +++ b/dvc/command/git_hook.py @@ -69,14 +69,13 @@ class CmdMergeDriver(CmdHookBase): dvc = Repo() try: - with dvc.state: - ancestor = Dvcfile(dvc, self.args.ancestor, verify=False) - our = Dvcfile(dvc, self.args.our, verify=False) - their = Dvcfile(dvc, self.args.their, verify=False) + ancestor = Dvcfile(dvc, self.args.ancestor, verify=False) + our = Dvcfile(dvc, self.args.our, verify=False) + their = Dvcfile(dvc, self.args.their, verify=False) - our.merge(ancestor, their) + our.merge(ancestor, their) - return 0 + return 0 finally: dvc.close() diff --git a/dvc/config_schema.py b/dvc/config_schema.py index df84458fe..1bd509fdb 100644 --- a/dvc/config_schema.py +++ b/dvc/config_schema.py @@ -213,8 +213,8 @@ SCHEMA = { ) }, "state": { - "row_limit": All(Coerce(int), Range(1)), - "row_cleanup_quota": All(Coerce(int), Range(0, 100)), + "row_limit": All(Coerce(int), Range(1)), # obsoleted + "row_cleanup_quota": All(Coerce(int), Range(0, 100)), # obsoleted }, # section for experimental features "feature": { diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index e7878382a..9778a75ff 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -47,13 +47,16 @@ class RepoDependency(LocalDependency): return external_repo(d["url"], rev=rev, **kwargs) def _get_hash(self, locked=True): - from dvc.objects.stage import get_hash + from dvc.objects.stage import stage with self._make_repo(locked=locked) as repo: path_info = PathInfo(repo.root_dir) / self.def_path - return get_hash( - path_info, repo.repo_fs, "md5", follow_subrepos=False - ) + return stage( + self.repo.odb.local, + path_info, + repo.repo_fs, + follow_subrepos=False, + ).hash_info def workspace_status(self): current = self._get_hash(locked=True) diff --git a/dvc/fs/base.py b/dvc/fs/base.py index d0203fae3..9ffbe69a4 100644 --- a/dvc/fs/base.py +++ b/dvc/fs/base.py @@ -130,10 +130,6 @@ class BaseFileSystem: f"dependencies: {missing}. {hint}" ) - @property - def odb(self): - return getattr(self.repo.odb, self.scheme) - def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs): if hasattr(self, "_generate_download_url"): # pylint:disable=no-member diff --git a/dvc/objects/__init__.py b/dvc/objects/__init__.py index d2585f344..02689064b 100644 --- a/dvc/objects/__init__.py +++ b/dvc/objects/__init__.py @@ -36,7 +36,7 @@ class HashFile: return bool(self.hash_info) def check(self, odb): - actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM) + actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM, odb) logger.trace( "cache '%s' expected '%s' actual '%s'", @@ -64,7 +64,9 @@ class File(HashFile): @classmethod def stage(cls, odb, path_info, fs, **kwargs): - hash_info = get_hash(path_info, fs, odb.fs.PARAM_CHECKSUM, **kwargs) + hash_info = get_hash( + path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs + ) raw = odb.get(hash_info) obj = cls(raw.path_info, raw.fs, hash_info) obj.src = HashFile(path_info, fs, hash_info) @@ -114,7 +116,7 @@ class Tree(HashFile): with fs.open(path_info, "rb") as fobj: odb.fs.upload_fobj(fobj, tmp_info) - hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM) + hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM, odb) hash_info.value += odb.fs.CHECKSUM_DIR_SUFFIX hash_info.dir_info = dir_info hash_info.nfiles = dir_info.nfiles @@ -125,7 +127,9 @@ class Tree(HashFile): @classmethod def stage(cls, odb, path_info, fs, **kwargs): - hash_info = get_hash(path_info, fs, odb.fs.PARAM_CHECKSUM, **kwargs) + hash_info = get_hash( + path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs + ) hi = cls.save_dir_info(odb, hash_info.dir_info, hash_info) hi.size = hash_info.size raw = odb.get(hi) diff --git a/dvc/objects/db/base.py b/dvc/objects/db/base.py index 6bb51f879..cbe5b04ca 100644 --- a/dvc/objects/db/base.py +++ b/dvc/objects/db/base.py @@ -58,8 +58,7 @@ class ObjectDB: with fs.open(path_info, mode="rb") as fobj: self.fs.upload_fobj(fobj, cache_info) self.protect(cache_info) - with self.fs.repo.state: - self.fs.repo.state.save(cache_info, self.fs, hash_info) + self.fs.repo.state.save(cache_info, self.fs, hash_info) callback = kwargs.get("download_callback") if callback: diff --git a/dvc/objects/stage.py b/dvc/objects/stage.py index 978c2d020..b339b6558 100644 --- a/dvc/objects/stage.py +++ b/dvc/objects/stage.py @@ -27,22 +27,26 @@ def get_file_hash(path_info, fs, name): raise NotImplementedError -def _calculate_hashes(file_infos, fs, name): - def _get_file_hash(path_info): - return get_file_hash(path_info, fs, name) +def _calculate_hashes(path_info, fs, name, state, **kwargs): + def _get_file_hash(file_info): + hash_info = state.get( # pylint: disable=assignment-from-none + file_info, fs, + ) + if not hash_info: + hash_info = get_file_hash(file_info, fs, name) + state.save(file_info, fs, hash_info) + return file_info, hash_info with Tqdm( - total=len(file_infos), - unit="md5", - desc="Computing file/dir hashes (only done once)", + unit="md5", desc="Computing file/dir hashes (only done once)", ) as pbar: worker = pbar.wrap_fn(_get_file_hash) with ThreadPoolExecutor(max_workers=fs.hash_jobs) as executor: - hash_infos = executor.map(worker, file_infos) - return dict(zip(file_infos, hash_infos)) + pairs = executor.map(worker, fs.walk_files(path_info, **kwargs)) + return dict(pairs) -def _iter_hashes(path_info, fs, name, **kwargs): +def _iter_hashes(path_info, fs, name, state, **kwargs): if name in fs.DETAIL_FIELDS: for details in fs.ls(path_info, recursive=True, detail=True): file_info = path_info.replace(path=details["name"]) @@ -53,25 +57,14 @@ def _iter_hashes(path_info, fs, name, **kwargs): return None - file_infos = [] - for file_info in fs.walk_files(path_info, **kwargs): - hash_info = fs.repo.state.get( # pylint: disable=assignment-from-none - file_info, fs, - ) - if not hash_info: - file_infos.append(file_info) - continue - assert hash_info.name == name - yield file_info, hash_info - - yield from _calculate_hashes(file_infos, fs, name).items() + yield from _calculate_hashes(path_info, fs, name, state, **kwargs).items() -def _collect_dir(path_info, fs, name, **kwargs): +def _collect_dir(path_info, fs, name, state, **kwargs): from dvc.dir_info import DirInfo dir_info = DirInfo() - for fi, hi in _iter_hashes(path_info, fs, name, **kwargs): + for fi, hi in _iter_hashes(path_info, fs, name, state, **kwargs): if DvcIgnore.DVCIGNORE_FILE == fi.name: raise DvcIgnoreInCollectedDirError(fi.parent) @@ -88,17 +81,17 @@ def _collect_dir(path_info, fs, name, **kwargs): return dir_info -def get_dir_hash(path_info, fs, name, **kwargs): +def get_dir_hash(path_info, fs, name, state, **kwargs): from . import Tree - dir_info = _collect_dir(path_info, fs, name, **kwargs) + dir_info = _collect_dir(path_info, fs, name, state, **kwargs) hash_info = Tree.save_dir_info(fs.repo.odb.local, dir_info) hash_info.size = dir_info.size hash_info.dir_info = dir_info return hash_info -def get_hash(path_info, fs, name, **kwargs): +def get_hash(path_info, fs, name, odb, **kwargs): assert path_info and ( isinstance(path_info, str) or path_info.scheme == fs.scheme ) @@ -108,36 +101,36 @@ def get_hash(path_info, fs, name, **kwargs): errno.ENOENT, os.strerror(errno.ENOENT), path_info ) - with fs.repo.state: - # pylint: disable=assignment-from-none - hash_info = fs.repo.state.get(path_info, fs) + state = odb.repo.state + # pylint: disable=assignment-from-none + hash_info = state.get(path_info, fs) - # If we have dir hash in state db, but dir cache file is lost, - # then we need to recollect the dir via .get_dir_hash() call below, - # see https://github.com/iterative/dvc/issues/2219 for context - if ( - hash_info - and hash_info.isdir - and not fs.odb.fs.exists(fs.odb.hash_to_path_info(hash_info.value)) - ): - hash_info = None + # If we have dir hash in state db, but dir cache file is lost, + # then we need to recollect the dir via .get_dir_hash() call below, + # see https://github.com/iterative/dvc/issues/2219 for context + if ( + hash_info + and hash_info.isdir + and not odb.fs.exists(odb.hash_to_path_info(hash_info.value)) + ): + hash_info = None - if hash_info: - if hash_info.isdir: - from . import Tree + if hash_info: + if hash_info.isdir: + from . import Tree - # NOTE: loading the fs will restore hash_info.dir_info - Tree.load(fs.odb, hash_info) - assert hash_info.name == name - return hash_info + # NOTE: loading the fs will restore hash_info.dir_info + Tree.load(odb, hash_info) + assert hash_info.name == name + return hash_info - if fs.isdir(path_info): - hash_info = get_dir_hash(path_info, fs, name, **kwargs) - else: - hash_info = get_file_hash(path_info, fs, name) + if fs.isdir(path_info): + hash_info = get_dir_hash(path_info, fs, name, state, **kwargs) + else: + hash_info = get_file_hash(path_info, fs, name) - if hash_info and fs.exists(path_info): - fs.repo.state.save(path_info, fs, hash_info) + if hash_info and fs.exists(path_info): + state.save(path_info, fs, hash_info) return hash_info diff --git a/dvc/output/base.py b/dvc/output/base.py index cdc812afb..c74543639 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -193,7 +193,12 @@ class BaseOutput: def get_hash(self): if not self.use_cache: - return get_hash(self.path_info, self.fs, self.fs.PARAM_CHECKSUM) + return get_hash( + self.path_info, + self.fs, + self.fs.PARAM_CHECKSUM, + self.repo.odb.local, + ) return ostage(self.odb, self.path_info, self.fs).hash_info @property diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 6cf4ac0d8..e1d8f14bd 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -472,13 +472,12 @@ class Remote: ) if self.fs.scheme == "local": - with self.fs.repo.state: - for checksum in named_cache.scheme_keys("local"): - cache_file = self.odb.hash_to_path_info(checksum) - if self.fs.exists(cache_file): - hash_info = HashInfo(self.fs.PARAM_CHECKSUM, checksum) - self.fs.repo.state.save(cache_file, self.fs, hash_info) - self.odb.protect(cache_file) + for checksum in named_cache.scheme_keys("local"): + cache_file = self.odb.hash_to_path_info(checksum) + if self.fs.exists(cache_file): + hash_info = HashInfo(self.fs.PARAM_CHECKSUM, checksum) + self.fs.repo.state.save(cache_file, self.fs, hash_info) + self.odb.protect(cache_file) return ret @@ -493,19 +492,16 @@ class Remote: ) if not self.odb.verify: - with cache.fs.repo.state: - for checksum in named_cache.scheme_keys("local"): - cache_file = cache.hash_to_path_info(checksum) - if cache.fs.exists(cache_file): - # We can safely save here, as existing corrupted files - # will be removed upon status, while files corrupted - # during download will not be moved from tmp_file - # (see `BaseFileSystem.download()`) - hash_info = HashInfo(cache.fs.PARAM_CHECKSUM, checksum) - cache.fs.repo.state.save( - cache_file, cache.fs, hash_info - ) - cache.protect(cache_file) + for checksum in named_cache.scheme_keys("local"): + cache_file = cache.hash_to_path_info(checksum) + if cache.fs.exists(cache_file): + # We can safely save here, as existing corrupted files + # will be removed upon status, while files corrupted + # during download will not be moved from tmp_file + # (see `BaseFileSystem.download()`) + hash_info = HashInfo(cache.fs.PARAM_CHECKSUM, checksum) + cache.fs.repo.state.save(cache_file, cache.fs, hash_info) + cache.protect(cache_file) return ret diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 076b44dd5..92818a23f 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -33,7 +33,7 @@ def lock_repo(repo: "Repo"): if depth > 0: yield else: - with repo.lock, repo.state: + with repo.lock: repo._reset() yield # Graph cache is no longer valid after we release the repo.lock @@ -186,7 +186,7 @@ class Repo: # NOTE: storing state and link_state in the repository itself to # avoid any possible state corruption in 'shared cache dir' # scenario. - self.state = State(self) + self.state = State(self.root_dir, self.tmp_dir) self.stage_cache = StageCache(self) self._ignore() @@ -477,11 +477,10 @@ class Repo: fs = RepoFileSystem(self, subrepos=True) path = PathInfo(self.root_dir) / path try: - with self.state: - with fs.open( - path, mode=mode, encoding=encoding, remote=remote, - ) as fobj: - yield fobj + with fs.open( + path, mode=mode, encoding=encoding, remote=remote, + ) as fobj: + yield fobj except FileNotFoundError as exc: raise FileMissingError(path) from exc except IsADirectoryError as exc: @@ -489,8 +488,10 @@ class Repo: def close(self): self.scm.close() + self.state.close() def _reset(self): + self.state.close() self.scm._reset() # pylint: disable=protected-access self.__dict__.pop("outs_trie", None) self.__dict__.pop("outs_graph", None) diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index e506d7f14..e53906745 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -19,16 +19,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def _get_unused_links(repo): - used = [ - out.fspath - for stage in repo.stages - for out in stage.outs - if out.scheme == "local" - ] - return repo.state.get_unused_links(used, repo.fs) - - def _fspath_dir(path): if not os.path.exists(str(path)): return str(path) @@ -37,6 +27,20 @@ def _fspath_dir(path): return os.path.join(path, "") if os.path.isdir(path) else path +def _remove_unused_links(repo): + used = [ + out.fspath + for stage in repo.stages + for out in stage.outs + if out.scheme == "local" + ] + + unused = repo.state.get_unused_links(used, repo.fs) + ret = [_fspath_dir(u) for u in unused] + repo.state.remove_links(unused, repo.fs) + return ret + + def get_all_files_numbers(pairs): return sum( stage.get_all_files_number(filter_info) for stage, filter_info in pairs @@ -83,7 +87,6 @@ def checkout( **kwargs, ): - unused = [] stats = { "added": [], "deleted": [], @@ -92,10 +95,7 @@ def checkout( } if not targets: targets = [None] - unused = _get_unused_links(self) - - stats["deleted"] = [_fspath_dir(u) for u in unused] - self.state.remove_links(unused, self.fs) + stats["deleted"] = _remove_unused_links(self) if isinstance(targets, str): targets = [targets] diff --git a/dvc/repo/destroy.py b/dvc/repo/destroy.py index 5c8548800..ee5bbdc6b 100644 --- a/dvc/repo/destroy.py +++ b/dvc/repo/destroy.py @@ -14,4 +14,5 @@ def _destroy_stages(repo): # which will cause issues on Windows, as `.dvc/lock` will be busy. def destroy(repo): _destroy_stages(repo) + repo.close() remove(repo.dvc_dir) diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index 7dd968072..d75520ce1 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -2,7 +2,8 @@ import logging import os from dvc.exceptions import PathMissingError -from dvc.objects.stage import get_file_hash, get_hash +from dvc.objects.stage import get_file_hash +from dvc.objects.stage import stage as ostage from dvc.repo import locked logger = logging.getLogger(__name__) @@ -128,7 +129,9 @@ def _output_paths(repo, repo_fs, targets): def _to_checksum(output): if on_working_fs: - return get_hash(output.path_info, repo.odb.local.fs, "md5").value + return ostage( + repo.odb.local, output.path_info, repo.odb.local.fs, + ).hash_info.value return output.hash_info.value for stage in repo.stages: diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index 19b0f51da..c99ed3eac 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -51,14 +51,12 @@ def scm_locked(f): def unlocked_repo(f): @wraps(f) def wrapper(exp, *args, **kwargs): - exp.repo.state.dump() exp.repo.lock.unlock() exp.repo._reset() # pylint: disable=protected-access try: ret = f(exp, *args, **kwargs) finally: exp.repo.lock.lock() - exp.repo.state.load() return ret return wrapper diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 1010b4832..467c46363 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -337,7 +337,7 @@ class BaseExecutor(ABC): logger.exception("unexpected error") raise finally: - dvc.scm.close() + dvc.close() if old_cwd: os.chdir(old_cwd) diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index 14dee1387..896638fa7 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -55,7 +55,6 @@ def gc( with ExitStack() as stack: for repo in all_repos: stack.enter_context(repo.lock) - stack.enter_context(repo.state) used = NamedCache() for repo in all_repos + [self]: diff --git a/dvc/stage/decorators.py b/dvc/stage/decorators.py index 7b5f98588..668855ad8 100644 --- a/dvc/stage/decorators.py +++ b/dvc/stage/decorators.py @@ -39,14 +39,12 @@ def rwlocked(call, read=None, write=None): def unlocked_repo(f): @wraps(f) def wrapper(stage, *args, **kwargs): - stage.repo.state.dump() stage.repo.lock.unlock() stage.repo._reset() # pylint: disable=protected-access try: ret = f(stage, *args, **kwargs) finally: stage.repo.lock.lock() - stage.repo.state.load() return ret return wrapper @@ -56,11 +54,9 @@ def relock_repo(f): @wraps(f) def wrapper(stage, *args, **kwargs): stage.repo.lock.lock() - stage.repo.state.load() try: ret = f(stage, *args, **kwargs) finally: - stage.repo.state.dump() stage.repo.lock.unlock() stage.repo._reset() # pylint: disable=protected-access return ret diff --git a/dvc/state.py b/dvc/state.py index 0ba70cd48..3d7b4a307 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -9,7 +9,7 @@ from urllib.parse import urlencode, urlunparse from dvc.exceptions import DvcException from dvc.fs.local import LocalFileSystem from dvc.hash_info import HashInfo -from dvc.utils import current_timestamp, relpath, to_chunks +from dvc.utils import relpath from dvc.utils.fs import get_inode, get_mtime_and_size, remove SQLITE_MAX_VARIABLES_NUMBER = 999 @@ -32,8 +32,9 @@ class StateVersionTooNewError(DvcException): class StateBase(ABC): - def __init__(self): - self.count = 0 + @abstractmethod + def close(self): + pass @abstractmethod def save(self, path_info, fs, hash_info): @@ -47,27 +48,11 @@ class StateBase(ABC): def save_link(self, path_info, fs): pass - @abstractmethod - def load(self): - pass - - @abstractmethod - def dump(self): - pass - - def __enter__(self): - if self.count <= 0: - self.load() - self.count += 1 - - def __exit__(self, typ, value, tbck): - self.count -= 1 - if self.count > 0: - return - self.dump() - class StateNoop(StateBase): + def close(self): + pass + def save(self, path_info, fs, hash_info): pass @@ -77,307 +62,27 @@ class StateNoop(StateBase): def save_link(self, path_info, fs): pass - def load(self): - pass - - def dump(self): - pass - class State(StateBase): # pylint: disable=too-many-instance-attributes - """Class for the state database. - - Args: - repo (dvc.repo.Repo): repo instance that this state belongs to. - config (configobj.ConfigObj): config for the state. - - Raises: - StateVersionTooNewError: thrown when dvc version is older than the - state database version. - """ - - VERSION = 3 - STATE_FILE = "state" - STATE_TABLE = "state" - STATE_TABLE_LAYOUT = ( - "inode INTEGER PRIMARY KEY, " - "mtime TEXT NOT NULL, " - "size TEXT NOT NULL, " - "md5 TEXT NOT NULL, " - "timestamp TEXT NOT NULL" - ) - - STATE_INFO_TABLE = "state_info" - STATE_INFO_TABLE_LAYOUT = "count INTEGER" - STATE_INFO_ROW = 1 - - LINK_STATE_TABLE = "link_state" - LINK_STATE_TABLE_LAYOUT = ( - "path TEXT PRIMARY KEY, " - "inode INTEGER NOT NULL, " - "mtime TEXT NOT NULL" - ) - - STATE_ROW_LIMIT = 100000000 - STATE_ROW_CLEANUP_QUOTA = 50 - - MAX_INT = 2 ** 63 - 1 - MAX_UINT = 2 ** 64 - 2 - - def __init__(self, repo): + def __init__(self, root_dir=None, tmp_dir=None): + from diskcache import Cache super().__init__() - self.repo = repo - self.root_dir = repo.root_dir + self.tmp_dir = tmp_dir + self.root_dir = root_dir self.fs = LocalFileSystem(None, {"url": self.root_dir}) - state_config = repo.config.get("state", {}) - self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT) - self.row_cleanup_quota = state_config.get( - "row_cleanup_quota", self.STATE_ROW_CLEANUP_QUOTA - ) - - if not repo.tmp_dir: - self.state_file = None + if not tmp_dir: return - self.state_file = os.path.join(repo.tmp_dir, self.STATE_FILE) + config = {"eviction_policy": "least-recently-used"} + self.links = Cache(directory=os.path.join(tmp_dir, "links"), **config) + self.md5s = Cache(directory=os.path.join(tmp_dir, "md5s"), **config) - self.database = None - self.cursor = None - self.inserts = 0 - - def _execute(self, cmd, parameters=()): - logger.trace(cmd) - return self.cursor.execute(cmd, parameters) - - def _fetchall(self): - ret = self.cursor.fetchall() - logger.trace("fetched: %s", ret) - return ret - - def _to_sqlite(self, num): - assert num >= 0 - assert num < self.MAX_UINT - # NOTE: sqlite stores unit as signed ints, so maximum uint is 2^63-1 - # see http://jakegoulding.com/blog/2011/02/06/sqlite-64-bit-integers/ - if num > self.MAX_INT: - ret = -(num - self.MAX_INT) - else: - ret = num - assert self._from_sqlite(ret) == num - return ret - - def _from_sqlite(self, num): - assert abs(num) <= self.MAX_INT - if num < 0: - return abs(num) + self.MAX_INT - assert num < self.MAX_UINT - assert num >= 0 - return num - - def _prepare_db(self, empty=False): - from dvc import __version__ - - if not empty: - cmd = "PRAGMA user_version;" - self._execute(cmd) - ret = self._fetchall() - assert len(ret) == 1 - assert len(ret[0]) == 1 - assert isinstance(ret[0][0], int) - version = ret[0][0] - - if version > self.VERSION: - raise StateVersionTooNewError( - __version__, self.VERSION, version - ) - elif version < self.VERSION: - logger.warning( - "State file version '%d' is too old. " - "Reformatting to the current version '%d'.", - version, - self.VERSION, - ) - cmd = "DROP TABLE IF EXISTS {};" - self._execute(cmd.format(self.STATE_TABLE)) - self._execute(cmd.format(self.STATE_INFO_TABLE)) - self._execute(cmd.format(self.LINK_STATE_TABLE)) - - # Check that the state file is indeed a database - cmd = "CREATE TABLE IF NOT EXISTS {} ({})" - self._execute(cmd.format(self.STATE_TABLE, self.STATE_TABLE_LAYOUT)) - self._execute( - cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE_LAYOUT) - ) - self._execute( - cmd.format(self.LINK_STATE_TABLE, self.LINK_STATE_TABLE_LAYOUT) - ) - - cmd = ( - "INSERT OR IGNORE INTO {} (count) SELECT 0 " - "WHERE NOT EXISTS (SELECT * FROM {})" - ) - self._execute(cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE)) - - cmd = "PRAGMA user_version = {};" - self._execute(cmd.format(self.VERSION)) - - def load(self): - """Loads state database.""" - retries = 1 - while True: - assert self.database is None - assert self.cursor is None - assert self.inserts == 0 - empty = not os.path.exists(self.state_file) - # NOTE: we use nolock option because fcntl() lock sqlite uses - # doesn't work on some older NFS/CIFS filesystems. - # This opens a possibility of data corruption by concurrent writes, - # which is prevented by repo lock. - self.database = _connect_sqlite(self.state_file, {"nolock": 1}) - self.cursor = self.database.cursor() - - from sqlite3 import DatabaseError - - # Try loading once to check that the file is indeed a database - # and reformat it if it is not. - try: - self._prepare_db(empty=empty) - return - except DatabaseError: - self.cursor.close() - self.database.close() - self.database = None - self.cursor = None - self.inserts = 0 - if retries > 0: - os.unlink(self.state_file) - retries -= 1 - else: - raise - - def _vacuum(self): - # NOTE: see https://bugs.python.org/issue28518 - self.database.isolation_level = None - self._execute("VACUUM") - self.database.isolation_level = "" - - def dump(self): - """Saves state database.""" - assert self.database is not None - - cmd = "SELECT count from {} WHERE rowid=?".format( - self.STATE_INFO_TABLE - ) - self._execute(cmd, (self.STATE_INFO_ROW,)) - ret = self._fetchall() - assert len(ret) == 1 - assert len(ret[0]) == 1 - count = self._from_sqlite(ret[0][0]) + self.inserts - - if count > self.row_limit: - msg = "cleaning up state, this might take a while." - logger.warning(msg) - - delete = count - self.row_limit - delete += int(self.row_limit * (self.row_cleanup_quota / 100.0)) - cmd = ( - "DELETE FROM {} WHERE timestamp IN (" - "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});" - ) - self._execute( - cmd.format(self.STATE_TABLE, self.STATE_TABLE, delete) - ) - - self._vacuum() - - cmd = "SELECT COUNT(*) FROM {}" - - self._execute(cmd.format(self.STATE_TABLE)) - ret = self._fetchall() - assert len(ret) == 1 - assert len(ret[0]) == 1 - count = ret[0][0] - - cmd = "UPDATE {} SET count = ? WHERE rowid = ?".format( - self.STATE_INFO_TABLE - ) - self._execute(cmd, (self._to_sqlite(count), self.STATE_INFO_ROW)) - - self.database.commit() - self.cursor.close() - self.database.close() - self.database = None - self.cursor = None - self.inserts = 0 - - @staticmethod - def _file_metadata_changed(actual_mtime, mtime, actual_size, size): - return actual_mtime != mtime or actual_size != size - - def _update_state_record_timestamp_for_inode(self, actual_inode): - cmd = "UPDATE {} SET timestamp = ? WHERE inode = ?".format( - self.STATE_TABLE - ) - self._execute( - cmd, (current_timestamp(), self._to_sqlite(actual_inode)) - ) - - def _update_state_for_path_changed( - self, actual_inode, actual_mtime, actual_size, checksum - ): - cmd = ( - "UPDATE {} SET " - "mtime = ?, size = ?, " - "md5 = ?, timestamp = ? " - "WHERE inode = ?" - ).format(self.STATE_TABLE) - self._execute( - cmd, - ( - actual_mtime, - actual_size, - checksum, - current_timestamp(), - self._to_sqlite(actual_inode), - ), - ) - - def _insert_new_state_record( - self, actual_inode, actual_mtime, actual_size, checksum - ): - assert checksum is not None - - cmd = ( - "INSERT INTO {}(inode, mtime, size, md5, timestamp) " - "VALUES (?, ?, ?, ?, ?)" - ).format(self.STATE_TABLE) - self._execute( - cmd, - ( - self._to_sqlite(actual_inode), - actual_mtime, - actual_size, - checksum, - current_timestamp(), - ), - ) - self.inserts += 1 - - def get_state_record_for_inode(self, inode): - cmd = ( - "SELECT mtime, size, md5, timestamp from {} WHERE " - "inode=?".format(self.STATE_TABLE) - ) - self._execute(cmd, (self._to_sqlite(inode),)) - results = self._fetchall() - if results: - # uniqueness constrain on inode - assert len(results) == 1 - return results[0] - return None + def close(self): + self.md5s.close() + self.links.close() def save(self, path_info, fs, hash_info): """Save hash for the specified path info. @@ -395,20 +100,15 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes assert isinstance(hash_info, HashInfo) assert os.path.exists(path_info) - actual_mtime, actual_size = get_mtime_and_size(path_info, self.fs) - actual_inode = get_inode(path_info) + mtime, size = get_mtime_and_size(path_info, self.fs) + inode = get_inode(path_info) - existing_record = self.get_state_record_for_inode(actual_inode) - if not existing_record: - self._insert_new_state_record( - actual_inode, actual_mtime, actual_size, hash_info.value - ) - return - - self._update_state_for_path_changed( - actual_inode, actual_mtime, actual_size, hash_info.value + logger.debug( + "state save (%s, %s, %s) %s", inode, mtime, size, hash_info.value ) + self.md5s[inode] = (mtime, size, hash_info.value) + def get(self, path_info, fs): """Gets the hash for the specified path info. Hash will be retrieved from the state database if available. @@ -432,19 +132,15 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes if not os.path.exists(path): return None - actual_mtime, actual_size = get_mtime_and_size(path, self.fs) - actual_inode = get_inode(path) + mtime, size = get_mtime_and_size(path, self.fs) + inode = get_inode(path) - existing_record = self.get_state_record_for_inode(actual_inode) - if not existing_record: + value = self.md5s.get(inode) + + if not value or value[0] != mtime or value[1] != size: return None - mtime, size, value, _ = existing_record - if self._file_metadata_changed(actual_mtime, mtime, actual_size, size): - return None - - self._update_state_record_timestamp_for_inode(actual_inode) - return HashInfo("md5", value, size=int(actual_size)) + return HashInfo("md5", value[2], size=int(size)) def save_link(self, path_info, fs): """Adds the specified path to the list of links created by dvc. This @@ -465,10 +161,8 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes inode = get_inode(path_info) relative_path = relpath(path_info, self.root_dir) - cmd = "REPLACE INTO {}(path, inode, mtime) " "VALUES (?, ?, ?)".format( - self.LINK_STATE_TABLE - ) - self._execute(cmd, (relative_path, self._to_sqlite(inode), mtime)) + with self.links as ref: + ref[relative_path] = (inode, mtime) def get_unused_links(self, used, fs): """Removes all saved links except the ones that are used. @@ -481,21 +175,19 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes unused = [] - self._execute(f"SELECT * FROM {self.LINK_STATE_TABLE}") - for row in self.cursor: - relative_path, inode, mtime = row - inode = self._from_sqlite(inode) - path = os.path.join(self.root_dir, relative_path) + with self.links as ref: + for relative_path in ref: + path = os.path.join(self.root_dir, relative_path) - if path in used or not self.fs.exists(path): - continue + if path in used or not self.fs.exists(path): + continue - actual_inode = get_inode(path) - actual_mtime, _ = get_mtime_and_size(path, self.fs) + inode = get_inode(path) + mtime, _ = get_mtime_and_size(path, self.fs) - if (inode, mtime) == (actual_inode, actual_mtime): - logger.debug("Removing '%s' as unused link.", path) - unused.append(relative_path) + if ref[relative_path] == (inode, mtime): + logger.debug("Removing '%s' as unused link.", path) + unused.append(relative_path) return unused @@ -504,15 +196,11 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes return for path in unused: - remove(path) + remove(os.path.join(self.root_dir, path)) - for chunk_unused in to_chunks( - unused, chunk_size=SQLITE_MAX_VARIABLES_NUMBER - ): - cmd = "DELETE FROM {} WHERE path IN ({})".format( - self.LINK_STATE_TABLE, ",".join(["?"] * len(chunk_unused)) - ) - self._execute(cmd, tuple(chunk_unused)) + with self.links as ref: + for path in unused: + del ref[path] def _connect_sqlite(filename, options): diff --git a/setup.py b/setup.py index baf31f798..f0c21c0cb 100644 --- a/setup.py +++ b/setup.py @@ -89,6 +89,7 @@ install_requires = [ "typing_extensions>=3.7.4", "fsspec>=0.8.5", "dvclive>=0.0.1a0", + "diskcache>=5.2.1", ] diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 9220bddfc..9f5c17c07 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -114,6 +114,8 @@ class TmpDir(pathlib.Path): def close(self): if hasattr(self, "scm"): self.scm.close() + if hasattr(self, "dvc"): + self.dvc.close() def _require(self, name): if not hasattr(self, name): diff --git a/tests/func/test_diff.py b/tests/func/test_diff.py index 37cdbbb59..f3515fdf0 100644 --- a/tests/func/test_diff.py +++ b/tests/func/test_diff.py @@ -40,7 +40,6 @@ def test_no_cache_entry(tmp_dir, scm, dvc): tmp_dir.dvc_gen("file", "second") remove(tmp_dir / ".dvc" / "cache") - (tmp_dir / ".dvc" / "tmp" / "state").unlink() dir_checksum = "5fb6b29836c388e093ca0715c872fe2a.dir" @@ -131,6 +130,7 @@ def test_directories(tmp_dir, scm, dvc): tmp_dir.dvc_gen({"dir": {"2": "two"}}, commit="modify a file") (tmp_dir / "dir" / "2").unlink() + assert dvc.status() != {} # sanity check dvc.add("dir") scm.add(["dir.dvc"]) scm.commit("delete a file") diff --git a/tests/func/test_init.py b/tests/func/test_init.py index 79f7dc4ab..117301aa2 100644 --- a/tests/func/test_init.py +++ b/tests/func/test_init.py @@ -14,7 +14,7 @@ class TestInit(TestGit): self.assertTrue(os.path.isdir(DvcRepo.DVC_DIR)) def test_api(self): - DvcRepo.init() + DvcRepo.init().close() self._test_init() diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 3eddec5e1..90093d677 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -146,7 +146,7 @@ class TestRemoteShouldHandleUppercaseRemoteName(TestDvc): def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): - from dvc.objects.stage import get_hash + from dvc.objects.stage import stage tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}}) @@ -158,7 +158,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): with patch( "dvc.objects.stage._collect_dir", return_value=dir_info, ): - hash1 = get_hash(path_info, dvc.odb.local.fs, "md5") + hash1 = stage(dvc.odb.local, path_info, dvc.odb.local.fs).hash_info dir_info = DirInfo.from_list( [{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}] @@ -166,7 +166,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): with patch( "dvc.objects.stage._collect_dir", return_value=dir_info, ): - hash2 = get_hash(path_info, dvc.odb.local.fs, "md5") + hash2 = stage(dvc.odb.local, path_info, dvc.odb.local.fs).hash_info assert hash1 == hash2 diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py index fafba7543..8f03e1eb2 100644 --- a/tests/func/test_run_single_stage.py +++ b/tests/func/test_run_single_stage.py @@ -917,7 +917,7 @@ class TestShouldNotCheckoutUponCorruptedLocalHardlinkCache(TestDvc): patch_run = mock.patch("dvc.stage.run.cmd_run", wraps=cmd_run) - with self.dvc.lock, self.dvc.state: + with self.dvc.lock: with patch_checkout as mock_checkout: with patch_run as mock_run: stage.run() diff --git a/tests/func/test_stage.py b/tests/func/test_stage.py index d203b264b..47ba4d93f 100644 --- a/tests/func/test_stage.py +++ b/tests/func/test_stage.py @@ -107,7 +107,7 @@ class TestDefaultWorkingDirectory(TestDvc): d = load_yaml(stage.relpath) self.assertNotIn(Stage.PARAM_WDIR, d.keys()) - with self.dvc.lock, self.dvc.state: + with self.dvc.lock: stage = SingleStageFile(self.dvc, stage.relpath).stage self.assertFalse(stage.changed()) diff --git a/tests/func/test_state.py b/tests/func/test_state.py index 9aae8de0a..2a24a0ed5 100644 --- a/tests/func/test_state.py +++ b/tests/func/test_state.py @@ -1,7 +1,5 @@ import os -import mock - from dvc.hash_info import HashInfo from dvc.path_info import PathInfo from dvc.state import State @@ -14,21 +12,20 @@ def test_state(tmp_dir, dvc): path_info = PathInfo(path) hash_info = HashInfo("md5", file_md5(path, dvc.fs)) - state = State(dvc) + state = State(dvc.root_dir, dvc.tmp_dir) - with state: - state.save(path_info, dvc.fs, hash_info) - assert state.get(path_info, dvc.fs) == hash_info + state.save(path_info, dvc.fs, hash_info) + assert state.get(path_info, dvc.fs) == hash_info - path.unlink() - path.write_text("1") + path.unlink() + path.write_text("1") - assert state.get(path_info, dvc.fs) is None + assert state.get(path_info, dvc.fs) is None - hash_info = HashInfo("md5", file_md5(path, dvc.fs)) - state.save(path_info, dvc.fs, hash_info) + hash_info = HashInfo("md5", file_md5(path, dvc.fs)) + state.save(path_info, dvc.fs, hash_info) - assert state.get(path_info, dvc.fs) == hash_info + assert state.get(path_info, dvc.fs) == hash_info def test_state_overflow(tmp_dir, dvc): @@ -51,54 +48,26 @@ def mock_get_inode(inode): return get_inode_mocked -@mock.patch("dvc.state.get_inode", autospec=True) -def test_get_state_record_for_inode(get_inode_mock, tmp_dir, dvc): - tmp_dir.gen("foo", "foo content") - - state = State(dvc) - inode = state.MAX_INT + 2 - assert inode != state._to_sqlite(inode) - - foo = tmp_dir / "foo" - md5 = file_md5(foo, dvc.fs) - get_inode_mock.side_effect = mock_get_inode(inode) - - with state: - state.save(PathInfo(foo), dvc.fs, HashInfo("md5", md5)) - ret = state.get_state_record_for_inode(inode) - assert ret is not None - - def test_remove_links(tmp_dir, dvc): tmp_dir.dvc_gen({"foo": "foo_content", "bar": "bar_content"}) - with dvc.state: - cmd_count_links = "SELECT count(*) FROM {}".format( - State.LINK_STATE_TABLE - ) - result = dvc.state._execute(cmd_count_links).fetchone()[0] - assert result == 2 + assert len(dvc.state.links) == 2 - dvc.state.remove_links(["foo", "bar"], dvc.fs) + dvc.state.remove_links(["foo", "bar"], dvc.fs) - result = dvc.state._execute(cmd_count_links).fetchone()[0] - assert result == 0 + assert len(dvc.state.links) == 0 def test_get_unused_links(tmp_dir, dvc): tmp_dir.dvc_gen({"foo": "foo_content", "bar": "bar_content"}) - with dvc.state: - links = [os.path.join(dvc.root_dir, link) for link in ["foo", "bar"]] - assert set(dvc.state.get_unused_links([], dvc.fs)) == {"foo", "bar"} - assert set(dvc.state.get_unused_links(links[:1], dvc.fs)) == {"bar"} - assert set(dvc.state.get_unused_links(links, dvc.fs)) == set() - assert set( - dvc.state.get_unused_links( - ( - links[:1] - + [os.path.join(dvc.root_dir, "not-existing-file")] - ), - dvc.fs, - ) - ) == {"bar"} + links = [os.path.join(dvc.root_dir, link) for link in ["foo", "bar"]] + assert set(dvc.state.get_unused_links([], dvc.fs)) == {"foo", "bar"} + assert set(dvc.state.get_unused_links(links[:1], dvc.fs)) == {"bar"} + assert set(dvc.state.get_unused_links(links, dvc.fs)) == set() + assert set( + dvc.state.get_unused_links( + (links[:1] + [os.path.join(dvc.root_dir, "not-existing-file")]), + dvc.fs, + ) + ) == {"bar"} diff --git a/tests/unit/fs/test_dvc.py b/tests/unit/fs/test_dvc.py index 1a14cfb45..36b8bf29c 100644 --- a/tests/unit/fs/test_dvc.py +++ b/tests/unit/fs/test_dvc.py @@ -5,7 +5,7 @@ import pytest from dvc.fs.dvc import DvcFileSystem from dvc.hash_info import HashInfo -from dvc.objects.stage import get_hash +from dvc.objects.stage import stage from dvc.path_info import PathInfo @@ -229,13 +229,13 @@ def test_get_hash_granular(tmp_dir, dvc): fs = DvcFileSystem(dvc) subdir = PathInfo(tmp_dir) / "dir" / "subdir" assert fs.info(subdir).get("md5") is None - assert get_hash(subdir, fs, "md5") == HashInfo( + assert stage(dvc.odb.local, subdir, fs).hash_info == HashInfo( "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir", ) assert ( fs.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc" ) - assert get_hash(subdir / "data", fs, "md5") == HashInfo( + assert stage(dvc.odb.local, subdir / "data", fs).hash_info == HashInfo( "md5", "8d777f385d3dfec8815d20f7496026dc", ) @@ -247,9 +247,9 @@ def test_get_hash_dirty_file(tmp_dir, dvc): fs = DvcFileSystem(dvc) expected = "8c7dd922ad47494fc02c388e12c00eac" assert fs.info(PathInfo(tmp_dir) / "file").get("md5") == expected - assert get_hash(PathInfo(tmp_dir) / "file", fs, "md5") == HashInfo( - "md5", expected - ) + assert stage( + dvc.odb.local, PathInfo(tmp_dir) / "file", fs + ).hash_info == HashInfo("md5", expected) def test_get_hash_dirty_dir(tmp_dir, dvc): @@ -259,6 +259,6 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): fs = DvcFileSystem(dvc) expected = "5ea40360f5b4ec688df672a4db9c17d1.dir" assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") == expected - assert get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") == HashInfo( - "md5", expected - ) + assert stage( + dvc.odb.local, PathInfo(tmp_dir) / "dir", fs + ).hash_info == HashInfo("md5", expected) diff --git a/tests/unit/fs/test_repo.py b/tests/unit/fs/test_repo.py index a25e94b6f..64808b4fa 100644 --- a/tests/unit/fs/test_repo.py +++ b/tests/unit/fs/test_repo.py @@ -6,7 +6,7 @@ import pytest from dvc.fs.repo import RepoFileSystem from dvc.hash_info import HashInfo -from dvc.objects.stage import get_hash +from dvc.objects.stage import stage from dvc.path_info import PathInfo @@ -482,9 +482,9 @@ def test_get_hash_cached_file(tmp_dir, dvc, mocker): fs = RepoFileSystem(dvc) expected = "acbd18db4cc2f85cedef654fccc4a4d8" assert fs.info(PathInfo(tmp_dir) / "foo").get("md5") is None - assert get_hash(PathInfo(tmp_dir) / "foo", fs, "md5") == HashInfo( - "md5", expected, - ) + assert stage( + dvc.odb.local, PathInfo(tmp_dir) / "foo", fs + ).hash_info == HashInfo("md5", expected,) (tmp_dir / "foo").unlink() assert fs.info(PathInfo(tmp_dir) / "foo")["md5"] == expected @@ -496,15 +496,15 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker): fs = RepoFileSystem(dvc) expected = "8761c4e9acad696bee718615e23e22db.dir" assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") is None - assert get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") == HashInfo( - "md5", "8761c4e9acad696bee718615e23e22db.dir", - ) + assert stage( + dvc.odb.local, PathInfo(tmp_dir) / "dir", fs + ).hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir",) shutil.rmtree(tmp_dir / "dir") assert fs.info(PathInfo(tmp_dir) / "dir")["md5"] == expected - assert get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") == HashInfo( - "md5", "8761c4e9acad696bee718615e23e22db.dir", - ) + assert stage( + dvc.odb.local, PathInfo(tmp_dir) / "dir", fs + ).hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir",) def test_get_hash_cached_granular(tmp_dir, dvc, mocker): @@ -514,11 +514,11 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker): fs = RepoFileSystem(dvc) subdir = PathInfo(tmp_dir) / "dir" / "subdir" assert fs.info(subdir).get("md5") is None - assert get_hash(subdir, fs, "md5") == HashInfo( + assert stage(dvc.odb.local, subdir, fs).hash_info == HashInfo( "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir", ) assert fs.info(subdir / "data").get("md5") is None - assert get_hash(subdir / "data", fs, "md5") == HashInfo( + assert stage(dvc.odb.local, subdir / "data", fs).hash_info == HashInfo( "md5", "8d777f385d3dfec8815d20f7496026dc", ) (tmp_dir / "dir" / "subdir" / "data").unlink() @@ -540,7 +540,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc): tmp_dir.scm.commit("add dir") fs = RepoFileSystem(dvc) - actual = get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") + actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs).hash_info expected = HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir") assert actual == expected @@ -551,7 +551,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc): fs = RepoFileSystem(dvc) assert fs.info(PathInfo(tmp_dir) / "file").get("md5") is None - actual = get_hash(PathInfo(tmp_dir) / "file", fs, "md5") + actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs).hash_info expected = HashInfo("md5", "437b930db84b8079c2dd804a71936b5f") assert actual == expected @@ -560,7 +560,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc): fs.info(PathInfo(tmp_dir) / "file")["md5"] == "8c7dd922ad47494fc02c388e12c00eac" ) - actual = get_hash(PathInfo(tmp_dir) / "file", fs, "md5") + actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs).hash_info expected = HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac") assert actual == expected @@ -570,7 +570,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): (tmp_dir / "dir" / "baz").write_text("baz") fs = RepoFileSystem(dvc) - actual = get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") + actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs).hash_info expected = HashInfo("md5", "ba75a2162ca9c29acecb7957105a0bc2.dir") assert actual == expected assert actual.dir_info.nfiles == 3 diff --git a/tests/unit/repo/test_repo.py b/tests/unit/repo/test_repo.py index 345ceb164..4e797bc5f 100644 --- a/tests/unit/repo/test_repo.py +++ b/tests/unit/repo/test_repo.py @@ -5,7 +5,6 @@ import pytest from dvc.exceptions import OutputDuplicationError from dvc.repo import NotDvcRepoError, Repo, locked -from dvc.utils.fs import remove def test_is_dvc_internal(dvc): @@ -113,25 +112,28 @@ def test_skip_graph_checks(tmp_dir, dvc, mocker, run_copy): def test_branch_config(tmp_dir, scm): tmp_dir.scm_gen("foo", "foo", commit="init") + # sanity check + with pytest.raises(NotDvcRepoError): + Repo().close() + scm.checkout("branch", create_new=True) dvc = Repo.init() with dvc.config.edit() as conf: conf["remote"]["branch"] = {"url": "/some/path"} - scm.add([".dvc"]) + dvc.close() + + scm.add([os.path.join(".dvc", "config")]) scm.commit("init dvc") scm.checkout("master") - remove(".dvc") - - # sanity check with pytest.raises(NotDvcRepoError): - Repo() - - with pytest.raises(NotDvcRepoError): - Repo(scm=scm, rev="master") + Repo(scm=scm, rev="master").close() dvc = Repo(scm=scm, rev="branch") - assert dvc.config["remote"]["branch"]["url"] == "/some/path" + try: + assert dvc.config["remote"]["branch"]["url"] == "/some/path" + finally: + dvc.close() def test_dynamic_cache_initalization(tmp_dir, scm): @@ -139,5 +141,6 @@ def test_dynamic_cache_initalization(tmp_dir, scm): with dvc.config.edit() as conf: conf["cache"]["ssh"] = "foo" conf["remote"]["foo"] = {"url": "remote://bar/baz"} + dvc.close() - Repo(str(tmp_dir)) + Repo(str(tmp_dir)).close() diff --git a/tests/unit/stage/test_cache.py b/tests/unit/stage/test_cache.py index 52482d818..03112e31e 100644 --- a/tests/unit/stage/test_cache.py +++ b/tests/unit/stage/test_cache.py @@ -22,7 +22,7 @@ def test_stage_cache(tmp_dir, dvc, mocker): single_stage=True, ) - with dvc.lock, dvc.state: + with dvc.lock: stage.remove(remove_outs=True, force=True) assert not (tmp_dir / "out").exists() @@ -45,7 +45,7 @@ def test_stage_cache(tmp_dir, dvc, mocker): run_spy = mocker.patch("dvc.stage.run.cmd_run") checkout_spy = mocker.spy(base, "checkout") - with dvc.lock, dvc.state: + with dvc.lock: stage.run() assert not run_spy.called @@ -75,7 +75,7 @@ def test_stage_cache_params(tmp_dir, dvc, mocker): single_stage=True, ) - with dvc.lock, dvc.state: + with dvc.lock: stage.remove(remove_outs=True, force=True) assert not (tmp_dir / "out").exists() @@ -98,7 +98,7 @@ def test_stage_cache_params(tmp_dir, dvc, mocker): run_spy = mocker.patch("dvc.stage.run.cmd_run") checkout_spy = mocker.spy(base, "checkout") - with dvc.lock, dvc.state: + with dvc.lock: stage.run() assert not run_spy.called @@ -129,7 +129,7 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker): wdir="wdir", ) - with dvc.lock, dvc.state: + with dvc.lock: stage.remove(remove_outs=True, force=True) assert not (tmp_dir / "wdir" / "out").exists() @@ -152,7 +152,7 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker): run_spy = mocker.patch("dvc.stage.run.cmd_run") checkout_spy = mocker.spy(base, "checkout") - with dvc.lock, dvc.state: + with dvc.lock: stage.run() assert not run_spy.called