mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-07 15:28:28 +01:00
state: use granularly (#4676)
* state: add simple reentrance * state: pass Repo instead of LocalCache * state: use granularly Users could still load state preemptively if they want to bulk up the operations, but other than that tree/cache will just load it when they need it.
This commit is contained in:
+1
-2
@@ -34,8 +34,7 @@ def get_url(path, repo=None, rev=None, remote=None):
|
||||
raise OutputNotFoundError(path, repo)
|
||||
|
||||
cloud = metadata.repo.cloud
|
||||
with _repo.state:
|
||||
hash_info = _repo.repo_tree.get_hash(path_info)
|
||||
hash_info = _repo.repo_tree.get_hash(path_info)
|
||||
return cloud.get_url_for(remote, checksum=hash_info.value)
|
||||
|
||||
|
||||
|
||||
Vendored
+12
@@ -3,6 +3,7 @@ import logging
|
||||
from copy import copy
|
||||
from operator import itemgetter
|
||||
|
||||
from funcy import decorator
|
||||
from shortuuid import uuid
|
||||
|
||||
import dvc.prompt as prompt
|
||||
@@ -42,6 +43,13 @@ class DirCacheError(DvcException):
|
||||
)
|
||||
|
||||
|
||||
@decorator
|
||||
def use_state(call):
|
||||
tree = call._args[0].tree # pylint: disable=protected-access
|
||||
with tree.state:
|
||||
return call()
|
||||
|
||||
|
||||
class CloudCache:
|
||||
"""Cloud cache class."""
|
||||
|
||||
@@ -281,6 +289,7 @@ class CloudCache:
|
||||
|
||||
return hash_info, to_info
|
||||
|
||||
@use_state
|
||||
def save_dir_info(self, dir_info, hash_info=None):
|
||||
if (
|
||||
hash_info
|
||||
@@ -321,6 +330,7 @@ class CloudCache:
|
||||
cache_info = self.tree.hash_to_path_info(hi.value)
|
||||
self.tree.state.save(cache_info, hi.value)
|
||||
|
||||
@use_state
|
||||
def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
|
||||
if path_info.scheme != self.tree.scheme:
|
||||
raise RemoteActionNotImplemented(
|
||||
@@ -517,6 +527,7 @@ class CloudCache:
|
||||
|
||||
return bool(redundant_files)
|
||||
|
||||
@use_state
|
||||
def checkout(
|
||||
self,
|
||||
path_info,
|
||||
@@ -693,6 +704,7 @@ class CloudCache:
|
||||
merged = self._merge_dirs(ancestor, our, their)
|
||||
return self.save_dir_info(merged)
|
||||
|
||||
@use_state
|
||||
def get_hash(self, tree, path_info):
|
||||
hash_info = tree.get_hash(path_info)
|
||||
if not hash_info.isdir:
|
||||
|
||||
Vendored
+2
@@ -19,6 +19,7 @@ from .base import (
|
||||
STATUS_MISSING,
|
||||
STATUS_NEW,
|
||||
CloudCache,
|
||||
use_state,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -431,6 +432,7 @@ class LocalCache(CloudCache):
|
||||
download=False,
|
||||
)
|
||||
|
||||
@use_state
|
||||
@index_locked
|
||||
def pull(self, named_cache, remote, jobs=None, show_checksums=False):
|
||||
ret = self._process(
|
||||
|
||||
@@ -144,7 +144,7 @@ class Repo:
|
||||
# NOTE: storing state and link_state in the repository itself to
|
||||
# avoid any possible state corruption in 'shared cache dir'
|
||||
# scenario.
|
||||
self.state = State(self.cache.local)
|
||||
self.state = State(self)
|
||||
self.stage_cache = StageCache(self)
|
||||
|
||||
try:
|
||||
|
||||
+52
-13
@@ -4,6 +4,7 @@ import logging
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from abc import ABC, abstractmethod
|
||||
from urllib.parse import urlencode, urlunparse
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
@@ -29,8 +30,51 @@ class StateVersionTooNewError(DvcException):
|
||||
)
|
||||
|
||||
|
||||
class StateNoop:
|
||||
files = []
|
||||
class StateBase(ABC):
|
||||
def __init__(self):
|
||||
self.count = 0
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def files(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def save(self, path_info, checksum):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get(self, path_info):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def save_link(self, path_info):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def load(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def dump(self):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
if self.count <= 0:
|
||||
self.load()
|
||||
self.count += 1
|
||||
|
||||
def __exit__(self, typ, value, tbck):
|
||||
self.count -= 1
|
||||
if self.count > 0:
|
||||
return
|
||||
self.dump()
|
||||
|
||||
|
||||
class StateNoop(StateBase):
|
||||
@property
|
||||
def files(self):
|
||||
return []
|
||||
|
||||
def save(self, path_info, checksum):
|
||||
pass
|
||||
@@ -41,14 +85,14 @@ class StateNoop:
|
||||
def save_link(self, path_info):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
def load(self):
|
||||
pass
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
def dump(self):
|
||||
pass
|
||||
|
||||
|
||||
class State: # pylint: disable=too-many-instance-attributes
|
||||
class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
"""Class for the state database.
|
||||
|
||||
Args:
|
||||
@@ -88,10 +132,11 @@ class State: # pylint: disable=too-many-instance-attributes
|
||||
MAX_INT = 2 ** 63 - 1
|
||||
MAX_UINT = 2 ** 64 - 2
|
||||
|
||||
def __init__(self, local_cache):
|
||||
def __init__(self, repo):
|
||||
from dvc.tree.local import LocalTree
|
||||
|
||||
repo = local_cache.repo
|
||||
super().__init__()
|
||||
|
||||
self.repo = repo
|
||||
self.root_dir = repo.root_dir
|
||||
self.tree = LocalTree(None, {"url": self.root_dir})
|
||||
@@ -122,12 +167,6 @@ class State: # pylint: disable=too-many-instance-attributes
|
||||
def files(self):
|
||||
return self.temp_files + [self.state_file]
|
||||
|
||||
def __enter__(self):
|
||||
self.load()
|
||||
|
||||
def __exit__(self, typ, value, tbck):
|
||||
self.dump()
|
||||
|
||||
def _execute(self, cmd, parameters=()):
|
||||
logger.trace(cmd)
|
||||
return self.cursor.execute(cmd, parameters)
|
||||
|
||||
+10
-1
@@ -5,7 +5,7 @@ from functools import partial
|
||||
from multiprocessing import cpu_count
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import cached_property
|
||||
from funcy import cached_property, decorator
|
||||
|
||||
from dvc.exceptions import DvcException, DvcIgnoreInCollectedDirError
|
||||
from dvc.hash_info import HashInfo
|
||||
@@ -38,6 +38,13 @@ class RemoteMissingDepsError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
@decorator
|
||||
def use_state(call):
|
||||
tree = call._args[0] # pylint: disable=protected-access
|
||||
with tree.state:
|
||||
return call()
|
||||
|
||||
|
||||
class BaseTree:
|
||||
scheme = "base"
|
||||
REQUIRES = {}
|
||||
@@ -230,6 +237,7 @@ class BaseTree:
|
||||
return False
|
||||
return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
|
||||
|
||||
@use_state
|
||||
def get_hash(self, path_info, **kwargs):
|
||||
assert path_info and (
|
||||
isinstance(path_info, str) or path_info.scheme == self.scheme
|
||||
@@ -326,6 +334,7 @@ class BaseTree:
|
||||
for fi in file_infos
|
||||
]
|
||||
|
||||
@use_state
|
||||
def get_dir_hash(self, path_info, **kwargs):
|
||||
dir_info = self._collect_dir(path_info, **kwargs)
|
||||
return self.repo.cache.local.save_dir_info(dir_info)
|
||||
|
||||
@@ -24,19 +24,16 @@ def test_commit_force(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"file": "text1", "file2": "text2"}})
|
||||
(stage,) = dvc.add("dir", no_commit=True)
|
||||
|
||||
with dvc.state:
|
||||
assert stage.outs[0].changed_cache()
|
||||
assert stage.outs[0].changed_cache()
|
||||
|
||||
tmp_dir.gen("dir/file", "file content modified")
|
||||
|
||||
with dvc.state:
|
||||
assert stage.outs[0].changed_cache()
|
||||
assert stage.outs[0].changed_cache()
|
||||
|
||||
with pytest.raises(StageCommitError):
|
||||
dvc.commit(stage.path)
|
||||
|
||||
with dvc.state:
|
||||
assert stage.outs[0].changed_cache()
|
||||
assert stage.outs[0].changed_cache()
|
||||
|
||||
dvc.commit(stage.path, force=True)
|
||||
assert dvc.status([stage.path]) == {}
|
||||
@@ -53,14 +50,12 @@ def test_commit_with_deps(tmp_dir, dvc, run_copy, run_kw):
|
||||
assert stage is not None
|
||||
assert len(stage.outs) == 1
|
||||
|
||||
with dvc.state:
|
||||
assert foo_stage.outs[0].changed_cache()
|
||||
assert stage.outs[0].changed_cache()
|
||||
assert foo_stage.outs[0].changed_cache()
|
||||
assert stage.outs[0].changed_cache()
|
||||
|
||||
dvc.commit(stage.path, with_deps=True)
|
||||
with dvc.state:
|
||||
assert not foo_stage.outs[0].changed_cache()
|
||||
assert not stage.outs[0].changed_cache()
|
||||
assert not foo_stage.outs[0].changed_cache()
|
||||
assert not stage.outs[0].changed_cache()
|
||||
|
||||
|
||||
def test_commit_changed_md5(tmp_dir, dvc):
|
||||
@@ -125,5 +120,4 @@ def test_imported_entries_unchanged(tmp_dir, dvc, erepo_dir):
|
||||
|
||||
stage = dvc.imp(fspath(erepo_dir), "file")
|
||||
|
||||
with dvc.state:
|
||||
assert stage.changed_entries() == ([], [], None)
|
||||
assert stage.changed_entries() == ([], [], None)
|
||||
|
||||
@@ -55,76 +55,75 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
|
||||
md5_dir = out_dir.hash_info.value
|
||||
info_dir = NamedCache.make(out_dir.scheme, md5_dir, name_dir)
|
||||
|
||||
with dvc.state:
|
||||
# Check status
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_NEW}}
|
||||
assert status == expected
|
||||
# Check status
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_NEW}}
|
||||
assert status == expected
|
||||
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
|
||||
assert status_dir == expected
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
|
||||
assert status_dir == expected
|
||||
|
||||
# Move cache and check status
|
||||
# See issue https://github.com/iterative/dvc/issues/4383 for details
|
||||
backup_dir = dvc.cache.local.cache_dir + ".backup"
|
||||
move(dvc.cache.local.cache_dir, backup_dir)
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_MISSING}}
|
||||
assert status == expected
|
||||
# Move cache and check status
|
||||
# See issue https://github.com/iterative/dvc/issues/4383 for details
|
||||
backup_dir = dvc.cache.local.cache_dir + ".backup"
|
||||
move(dvc.cache.local.cache_dir, backup_dir)
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_MISSING}}
|
||||
assert status == expected
|
||||
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_MISSING}}
|
||||
assert status_dir == expected
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_MISSING}}
|
||||
assert status_dir == expected
|
||||
|
||||
# Restore original cache:
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
move(backup_dir, dvc.cache.local.cache_dir)
|
||||
# Restore original cache:
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
move(backup_dir, dvc.cache.local.cache_dir)
|
||||
|
||||
# Push and check status
|
||||
dvc.cloud.push(info)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
# Push and check status
|
||||
dvc.cloud.push(info)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
|
||||
dvc.cloud.push(info_dir)
|
||||
assert os.path.isfile(cache_dir)
|
||||
dvc.cloud.push(info_dir)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
assert status == expected
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
assert status == expected
|
||||
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
assert status_dir == expected
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
assert status_dir == expected
|
||||
|
||||
# Remove and check status
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
# Remove and check status
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_DELETED}}
|
||||
assert status == expected
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_DELETED}}
|
||||
assert status == expected
|
||||
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}}
|
||||
assert status_dir == expected
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}}
|
||||
assert status_dir == expected
|
||||
|
||||
# Pull and check status
|
||||
dvc.cloud.pull(info)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
with open(cache) as fd:
|
||||
assert fd.read() == "foo"
|
||||
# Pull and check status
|
||||
dvc.cloud.pull(info)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
with open(cache) as fd:
|
||||
assert fd.read() == "foo"
|
||||
|
||||
dvc.cloud.pull(info_dir)
|
||||
assert os.path.isfile(cache_dir)
|
||||
dvc.cloud.pull(info_dir)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
assert status == expected
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
assert status == expected
|
||||
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
assert status_dir == expected
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
assert status_dir == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote", all_clouds, indirect=True)
|
||||
|
||||
+18
-19
@@ -149,26 +149,25 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}})
|
||||
|
||||
path_info = PathInfo("data")
|
||||
with dvc.state:
|
||||
with patch.object(
|
||||
BaseTree,
|
||||
"_collect_dir",
|
||||
return_value=[
|
||||
{"relpath": "1", "md5": "1"},
|
||||
{"relpath": "2", "md5": "2"},
|
||||
],
|
||||
):
|
||||
hash1 = dvc.cache.local.tree.get_hash(path_info)
|
||||
with patch.object(
|
||||
BaseTree,
|
||||
"_collect_dir",
|
||||
return_value=[
|
||||
{"relpath": "1", "md5": "1"},
|
||||
{"relpath": "2", "md5": "2"},
|
||||
],
|
||||
):
|
||||
hash1 = dvc.cache.local.tree.get_hash(path_info)
|
||||
|
||||
with patch.object(
|
||||
BaseTree,
|
||||
"_collect_dir",
|
||||
return_value=[
|
||||
{"md5": "1", "relpath": "1"},
|
||||
{"md5": "2", "relpath": "2"},
|
||||
],
|
||||
):
|
||||
hash2 = dvc.cache.local.tree.get_hash(path_info)
|
||||
with patch.object(
|
||||
BaseTree,
|
||||
"_collect_dir",
|
||||
return_value=[
|
||||
{"md5": "1", "relpath": "1"},
|
||||
{"md5": "2", "relpath": "2"},
|
||||
],
|
||||
):
|
||||
hash2 = dvc.cache.local.tree.get_hash(path_info)
|
||||
|
||||
assert hash1 == hash2
|
||||
|
||||
|
||||
@@ -46,8 +46,7 @@ def test_save(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
assert _recurse_count_files(run_cache_dir) == 1
|
||||
with dvc.state:
|
||||
assert dvc.stage_cache._load(stage)
|
||||
assert dvc.stage_cache._load(stage)
|
||||
|
||||
|
||||
def test_do_not_save_on_no_exec_and_dry(tmp_dir, dvc, run_copy):
|
||||
@@ -58,14 +57,12 @@ def test_do_not_save_on_no_exec_and_dry(tmp_dir, dvc, run_copy):
|
||||
stage = run_copy("foo", "bar", name="copy-foo-bar", no_exec=True)
|
||||
|
||||
assert _recurse_count_files(run_cache_dir) == 0
|
||||
with dvc.state:
|
||||
assert not dvc.stage_cache._load(stage)
|
||||
assert not dvc.stage_cache._load(stage)
|
||||
|
||||
(stage,) = dvc.reproduce("copy-foo-bar", dry=True)
|
||||
|
||||
assert _recurse_count_files(run_cache_dir) == 0
|
||||
with dvc.state:
|
||||
assert not dvc.stage_cache._load(stage)
|
||||
assert not dvc.stage_cache._load(stage)
|
||||
|
||||
|
||||
def test_uncached_outs_are_cached(tmp_dir, dvc, run_copy):
|
||||
@@ -76,8 +73,7 @@ def test_uncached_outs_are_cached(tmp_dir, dvc, run_copy):
|
||||
outs_no_cache=["bar"],
|
||||
name="copy-foo-bar",
|
||||
)
|
||||
with dvc.state:
|
||||
stage.outs[0].hash_info = stage.outs[0].get_hash()
|
||||
stage.outs[0].hash_info = stage.outs[0].get_hash()
|
||||
assert os.path.exists(relpath(stage.outs[0].cache_path))
|
||||
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ def test_state(tmp_dir, dvc):
|
||||
path_info = PathInfo(path)
|
||||
md5 = file_md5(path)[0]
|
||||
|
||||
state = State(dvc.cache.local)
|
||||
state = State(dvc)
|
||||
|
||||
with state:
|
||||
state.save(path_info, md5)
|
||||
@@ -57,7 +57,7 @@ def mock_get_inode(inode):
|
||||
def test_get_state_record_for_inode(get_inode_mock, tmp_dir, dvc):
|
||||
tmp_dir.gen("foo", "foo content")
|
||||
|
||||
state = State(dvc.cache.local)
|
||||
state = State(dvc)
|
||||
inode = state.MAX_INT + 2
|
||||
assert inode != state._to_sqlite(inode)
|
||||
|
||||
|
||||
@@ -194,9 +194,8 @@ def test_repotree_walk_fetch(tmp_dir, dvc, scm, local_remote):
|
||||
remove(tmp_dir / "dir")
|
||||
|
||||
tree = RepoTree(dvc, fetch=True)
|
||||
with dvc.state:
|
||||
for _, _, _ in tree.walk("dir"):
|
||||
pass
|
||||
for _, _, _ in tree.walk("dir"):
|
||||
pass
|
||||
|
||||
assert os.path.exists(out.cache_path)
|
||||
for entry in out.dir_cache:
|
||||
@@ -223,12 +222,10 @@ def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, local_cloud):
|
||||
for path in ("dir/bar", "dir/subdir/foo")
|
||||
]
|
||||
|
||||
with erepo_dir.dvc.state:
|
||||
cache = dvc.cache.local
|
||||
with cache.tree.state:
|
||||
path_info = PathInfo(erepo_dir / "dir")
|
||||
hash_info = cache.tree.get_hash(path_info)
|
||||
cache.save(path_info, tree, hash_info)
|
||||
cache = dvc.cache.local
|
||||
path_info = PathInfo(erepo_dir / "dir")
|
||||
hash_info = cache.tree.get_hash(path_info)
|
||||
cache.save(path_info, tree, hash_info)
|
||||
|
||||
for hash_ in expected:
|
||||
assert os.path.exists(cache.tree.hash_to_path_info(hash_))
|
||||
|
||||
@@ -208,5 +208,4 @@ def test_params_with_false_values(tmp_dir, dvc, param_value):
|
||||
|
||||
dep.fill_values(load_yaml(DEFAULT_PARAMS_FILE))
|
||||
|
||||
with dvc.state:
|
||||
assert dep.status() == {}
|
||||
assert dep.status() == {}
|
||||
|
||||
@@ -50,12 +50,11 @@ def test_used_cache(tmp_dir, dvc, path):
|
||||
),
|
||||
)
|
||||
|
||||
with dvc.state:
|
||||
used_cache = dvc.used_cache([path])
|
||||
assert (
|
||||
used_cache._items == expected._items
|
||||
and used_cache.external == expected.external
|
||||
)
|
||||
used_cache = dvc.used_cache([path])
|
||||
assert (
|
||||
used_cache._items == expected._items
|
||||
and used_cache.external == expected.external
|
||||
)
|
||||
|
||||
|
||||
def test_locked(mocker):
|
||||
|
||||
+14
-19
@@ -23,9 +23,8 @@ def test_open(tmp_dir, dvc):
|
||||
(tmp_dir / "foo").unlink()
|
||||
|
||||
tree = DvcTree(dvc)
|
||||
with dvc.state:
|
||||
with tree.open("foo", "r") as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
with tree.open("foo", "r") as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
|
||||
|
||||
def test_open_dirty_hash(tmp_dir, dvc):
|
||||
@@ -169,11 +168,10 @@ def test_walk_dir(tmp_dir, dvc, fetch, expected):
|
||||
|
||||
expected = [str(tmp_dir / path) for path in expected]
|
||||
|
||||
with dvc.state:
|
||||
actual = []
|
||||
for root, dirs, files in tree.walk("dir"):
|
||||
for entry in dirs + files:
|
||||
actual.append(os.path.join(root, entry))
|
||||
actual = []
|
||||
for root, dirs, files in tree.walk("dir"):
|
||||
for entry in dirs + files:
|
||||
actual.append(os.path.join(root, entry))
|
||||
|
||||
assert set(actual) == set(expected)
|
||||
assert len(actual) == len(expected)
|
||||
@@ -235,13 +233,12 @@ def test_get_hash_granular(tmp_dir, dvc):
|
||||
)
|
||||
tree = DvcTree(dvc, fetch=True)
|
||||
subdir = PathInfo(tmp_dir) / "dir" / "subdir"
|
||||
with dvc.state:
|
||||
assert tree.get_hash(subdir) == HashInfo(
|
||||
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir",
|
||||
)
|
||||
assert tree.get_hash(subdir / "data") == HashInfo(
|
||||
"md5", "8d777f385d3dfec8815d20f7496026dc",
|
||||
)
|
||||
assert tree.get_hash(subdir) == HashInfo(
|
||||
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir",
|
||||
)
|
||||
assert tree.get_hash(subdir / "data") == HashInfo(
|
||||
"md5", "8d777f385d3dfec8815d20f7496026dc",
|
||||
)
|
||||
|
||||
|
||||
def test_get_hash_dirty_file(tmp_dir, dvc):
|
||||
@@ -249,8 +246,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
|
||||
(tmp_dir / "file").write_text("something")
|
||||
|
||||
tree = DvcTree(dvc)
|
||||
with dvc.state:
|
||||
actual = tree.get_hash(PathInfo(tmp_dir) / "file")
|
||||
actual = tree.get_hash(PathInfo(tmp_dir) / "file")
|
||||
expected = HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac")
|
||||
assert actual == expected
|
||||
|
||||
@@ -260,8 +256,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
|
||||
(tmp_dir / "dir" / "baz").write_text("baz")
|
||||
|
||||
tree = DvcTree(dvc)
|
||||
with dvc.state:
|
||||
actual = tree.get_hash(PathInfo(tmp_dir) / "dir")
|
||||
actual = tree.get_hash(PathInfo(tmp_dir) / "dir")
|
||||
expected = HashInfo("md5", "5ea40360f5b4ec688df672a4db9c17d1.dir")
|
||||
|
||||
assert actual == expected
|
||||
|
||||
@@ -24,9 +24,8 @@ def test_open(tmp_dir, dvc):
|
||||
(tmp_dir / "foo").unlink()
|
||||
|
||||
tree = RepoTree(dvc)
|
||||
with dvc.state:
|
||||
with tree.open("foo", "r") as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
with tree.open("foo", "r") as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
|
||||
|
||||
def test_open_dirty_hash(tmp_dir, dvc):
|
||||
@@ -494,19 +493,17 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker):
|
||||
tree = RepoTree(dvc)
|
||||
get_file_hash_spy = mocker.spy(tree, "get_file_hash")
|
||||
dvc_tree_spy = mocker.spy(tree._dvctrees[dvc.root_dir], "get_dir_hash")
|
||||
with dvc.state:
|
||||
assert tree.get_hash(PathInfo(tmp_dir) / "dir") == HashInfo(
|
||||
"md5", "8761c4e9acad696bee718615e23e22db.dir",
|
||||
)
|
||||
assert tree.get_hash(PathInfo(tmp_dir) / "dir") == HashInfo(
|
||||
"md5", "8761c4e9acad696bee718615e23e22db.dir",
|
||||
)
|
||||
assert get_file_hash_spy.called
|
||||
assert not dvc_tree_spy.called
|
||||
get_file_hash_spy.reset_mock()
|
||||
|
||||
shutil.rmtree(tmp_dir / "dir")
|
||||
with dvc.state:
|
||||
assert tree.get_hash(PathInfo(tmp_dir) / "dir") == HashInfo(
|
||||
"md5", "8761c4e9acad696bee718615e23e22db.dir",
|
||||
)
|
||||
assert tree.get_hash(PathInfo(tmp_dir) / "dir") == HashInfo(
|
||||
"md5", "8761c4e9acad696bee718615e23e22db.dir",
|
||||
)
|
||||
assert not get_file_hash_spy.called
|
||||
assert dvc_tree_spy.called
|
||||
|
||||
@@ -518,13 +515,12 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
|
||||
tree = RepoTree(dvc, fetch=True)
|
||||
dvc_tree_spy = mocker.spy(tree._dvctrees[dvc.root_dir], "get_file_hash")
|
||||
subdir = PathInfo(tmp_dir) / "dir" / "subdir"
|
||||
with dvc.state:
|
||||
assert tree.get_hash(subdir) == HashInfo(
|
||||
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir",
|
||||
)
|
||||
assert tree.get_hash(subdir / "data") == HashInfo(
|
||||
"md5", "8d777f385d3dfec8815d20f7496026dc",
|
||||
)
|
||||
assert tree.get_hash(subdir) == HashInfo(
|
||||
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir",
|
||||
)
|
||||
assert tree.get_hash(subdir / "data") == HashInfo(
|
||||
"md5", "8d777f385d3dfec8815d20f7496026dc",
|
||||
)
|
||||
assert dvc_tree_spy.called
|
||||
|
||||
|
||||
@@ -541,8 +537,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc):
|
||||
tmp_dir.scm.commit("add dir")
|
||||
|
||||
tree = RepoTree(dvc)
|
||||
with dvc.state:
|
||||
actual = tree.get_hash(PathInfo(tmp_dir) / "dir")
|
||||
actual = tree.get_hash(PathInfo(tmp_dir) / "dir")
|
||||
expected = HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir")
|
||||
assert actual == expected
|
||||
|
||||
@@ -552,8 +547,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
|
||||
(tmp_dir / "file").write_text("something")
|
||||
|
||||
tree = RepoTree(dvc)
|
||||
with dvc.state:
|
||||
actual = tree.get_hash(PathInfo(tmp_dir) / "file")
|
||||
actual = tree.get_hash(PathInfo(tmp_dir) / "file")
|
||||
expected = HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac")
|
||||
assert actual == expected
|
||||
|
||||
@@ -563,8 +557,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
|
||||
(tmp_dir / "dir" / "baz").write_text("baz")
|
||||
|
||||
tree = RepoTree(dvc)
|
||||
with dvc.state:
|
||||
actual = tree.get_hash(PathInfo(tmp_dir) / "dir")
|
||||
actual = tree.get_hash(PathInfo(tmp_dir) / "dir")
|
||||
expected = HashInfo("md5", "ba75a2162ca9c29acecb7957105a0bc2.dir")
|
||||
assert actual == expected
|
||||
assert len(actual.dir_info) == 3
|
||||
|
||||
Reference in New Issue
Block a user