Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-01 23:08:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

repo: rename odb -> cache

Browse Source 
We used to have it named `cache` before, but these days it is clear that
you can have three odbs: just odb (e.g. in-memory), cache (typically local)
or remote (typically cloud). Fixing naming here before proceeding with
introducing `out.odb` and `out.cache`.

In preparation for migration to https://github.com/iterative/dvc-data/pull/304
This commit is contained in:
Ruslan Kuprieiev
2023-02-08 23:05:45 +02:00
parent 4176007493
commit 58295ca0ab
46 changed files with 218 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/odbmgr.py → dvc/cachemgr.py
+1 -1
View File
@@ -14,7 +14,7 @@ def _get_odb(repo, settings, fs=None):
return get_odb(fs, fs_path, state=repo.state, **config)
class ODBManager:
class CacheManager:
CACHE_DIR = "cache"
CLOUD_SCHEMES = [
Schemes.S3,
dvc/data_cloud.py
+6 -6
View File
@@ -147,12 +147,12 @@ class DataCloud:
"""
odb = odb or self.get_remote_odb(remote, "push")
return self.transfer(
self.repo.odb.local,
self.repo.cache.local,
odb,
objs,
jobs=jobs,
dest_index=get_index(odb),
cache_odb=self.repo.odb.local,
cache_odb=self.repo.cache.local,
validate_status=self._log_missing,
)
@@ -175,11 +175,11 @@ class DataCloud:
odb = odb or self.get_remote_odb(remote, "pull")
return self.transfer(
odb,
self.repo.odb.local,
self.repo.cache.local,
objs,
jobs=jobs,
src_index=get_index(odb),
cache_odb=self.repo.odb.local,
cache_odb=self.repo.cache.local,
verify=odb.verify,
validate_status=self._log_missing,
)
@@ -206,12 +206,12 @@ class DataCloud:
if not odb:
odb = self.get_remote_odb(remote, "status")
return compare_status(
self.repo.odb.local,
self.repo.cache.local,
odb,
objs,
jobs=jobs,
dest_index=get_index(odb),
cache_odb=self.repo.odb.local,
cache_odb=self.repo.cache.local,
)
def get_url_for(self, remote, checksum):
dvc/dependency/repo.py
+2 -2
View File
@@ -80,7 +80,7 @@ class RepoDependency(Dependency):
to.fs_path,
to.fs,
obj,
self.repo.odb.local,
self.repo.cache.local,
ignore=None,
state=self.repo.state,
prompt=confirm,
@@ -111,7 +111,7 @@ class RepoDependency(Dependency):
from dvc_data.hashfile.build import build
from dvc_data.hashfile.tree import Tree, TreeError
local_odb = self.repo.odb.local
local_odb = self.repo.cache.local
locked = kwargs.pop("locked", True)
with self._make_repo(locked=locked, cache_dir=local_odb.path) as repo:
used_obj_ids = defaultdict(set)
dvc/external_repo.py
+1 -1
View File
@@ -149,7 +149,7 @@ def _get_remote_config(url):
name = "auto-generated-upstream"
return {
"core": {"remote": name},
"remote": {name: {"url": repo.odb.local.path}},
"remote": {name: {"url": repo.cache.local.path}},
}
# Use original remote to make sure that we are using correct url,
dvc/info.py
+4 -4
View File
@@ -41,14 +41,14 @@ def get_dvc_info():
# can't auto-create it, as it might cause issues if the user
# later decides to enable shared cache mode with
# `dvc config cache.shared group`.
if os.path.exists(repo.odb.local.path):
if os.path.exists(repo.cache.local.path):
info.append(f"Cache types: {_get_linktype_support_info(repo)}")
fs_type = get_fs_type(repo.odb.local.path)
fs_type = get_fs_type(repo.cache.local.path)
info.append(f"Cache directory: {fs_type}")
else:
info.append("Cache types: " + error_link("no-dvc-cache"))
info.append(f"Caches: {_get_caches(repo.odb)}")
info.append(f"Caches: {_get_caches(repo.cache)}")
info.append(f"Remotes: {_get_remotes(repo.config)}")
root_directory = repo.root_dir
@@ -84,7 +84,7 @@ def _get_remotes(config):
def _get_linktype_support_info(repo):
odb = repo.odb.local
odb = repo.cache.local
links = generic.test_links(
["reflink", "hardlink", "symlink"],
dvc/output.py
+3 -3
View File
@@ -502,7 +502,7 @@ class Output:
@property
def odb(self):
odb_name = "repo" if self.is_in_repo else self.protocol
odb = getattr(self.repo.odb, odb_name)
odb = getattr(self.repo.cache, odb_name)
if self.use_cache and odb is None:
raise RemoteCacheRequiredError(self.fs.protocol, self.fs_path)
return odb
@@ -519,7 +519,7 @@ class Output:
if self.use_cache:
odb = self.odb
else:
odb = self.repo.odb.local
odb = self.repo.cache.local
_, meta, obj = build(
odb,
self.fs_path,
@@ -690,7 +690,7 @@ class Output:
)
else:
_, self.meta, self.obj = build(
self.repo.odb.local,
self.repo.cache.local,
self.fs_path,
self.fs,
self.hash_name,
dvc/repo/__init__.py
+7 -7
View File
@@ -162,11 +162,11 @@ class Repo:
repo_factory: Optional[Callable] = None,
scm: Optional[Union["Git", "NoSCM"]] = None,
):
from dvc.cachemgr import CacheManager
from dvc.config import Config
from dvc.data_cloud import DataCloud
from dvc.fs import GitFileSystem, LocalFileSystem, localfs
from dvc.lock import LockNoop, make_lock
from dvc.odbmgr import ODBManager
from dvc.repo.metrics import Metrics
from dvc.repo.params import Params
from dvc.repo.plots import Plots
@@ -205,12 +205,12 @@ class Repo:
self.stage: "StageLoad" = StageLoad(self)
self.lock: "LockBase"
self.odb: ODBManager
self.cache: CacheManager
self.state: "StateBase"
if isinstance(self.fs, GitFileSystem) or not self.dvc_dir:
self.lock = LockNoop()
self.state = StateNoop()
self.odb = ODBManager(self)
self.cache = CacheManager(self)
self.tmp_dir = None
else:
self.fs.makedirs(cast(str, self.tmp_dir), exist_ok=True)
@@ -228,7 +228,7 @@ class Repo:
self.lock = LockNoop()
self.state = StateNoop()
self.odb = ODBManager(self)
self.cache = CacheManager(self)
self.stage_cache = StageCache(self)
@@ -385,14 +385,14 @@ class Repo:
return init(root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir)
def unprotect(self, target):
return self.odb.repo.unprotect(target)
return self.cache.repo.unprotect(target)
def _ignore(self):
flist = [self.config.files["local"]]
if tmp_dir := self.tmp_dir:
flist.append(tmp_dir)
if path_isin(self.odb.repo.path, self.root_dir):
flist.append(self.odb.repo.path)
if path_isin(self.cache.repo.path, self.root_dir):
flist.append(self.cache.repo.path)
for file in flist:
self.scm_context.ignore(file)
dvc/repo/diff.py
+3 -3
View File
@@ -162,9 +162,9 @@ def _output_paths( # noqa: C901
obj: Optional["HashFile"]
if on_working_fs:
_, _, obj = build(
repo.odb.repo,
repo.cache.repo,
output.fs_path,
repo.odb.repo.fs,
repo.cache.repo.fs,
"md5",
dry_run=True,
ignore=output.dvcignore,
@@ -219,7 +219,7 @@ def _filter_missing(dvcfs: "DVCFileSystem", paths: Set[str]) -> Iterator[str]:
if (
entry
and info["type"] == "directory"
and not dvcfs.repo.odb.local.exists(entry.hash_info.value)
and not dvcfs.repo.cache.local.exists(entry.hash_info.value)
):
yield path
except FileNotFoundError:
dvc/repo/experiments/executor/local.py
+1 -1
View File
@@ -153,7 +153,7 @@ class TempDirExecutor(BaseLocalExecutor):
self, repo: "Repo", rev: str, run_cache: bool = True # noqa: ARG002
):
"""Initialize DVC cache."""
self._config(repo.odb.repo.path)
self._config(repo.cache.repo.path)
def cleanup(self, infofile: str):
super().cleanup(infofile)
dvc/repo/experiments/executor/ssh.py
+2 -2
View File
@@ -217,12 +217,12 @@ class SSHExecutor(BaseExecutor):
@contextmanager
def get_odb(self):
from dvc.odbmgr import ODBManager, get_odb
from dvc.cachemgr import CacheManager, get_odb
cache_path = posixpath.join(
self._repo_abspath,
self.dvc_dir,
ODBManager.CACHE_DIR,
CacheManager.CACHE_DIR,
)
with self.sshfs() as fs:
dvc/repo/gc.py
+1 -1
View File
@@ -85,7 +85,7 @@ def gc(
).values():
used_obj_ids.update(obj_ids)
for scheme, odb in self.odb.by_scheme():
for scheme, odb in self.cache.by_scheme():
if not odb:
continue
dvc/repo/imports.py
+1 -1
View File
@@ -107,7 +107,7 @@ def save_imports(
data_view = unfetched.data["repo"]
if len(data_view):
cache = repo.odb.local
cache = repo.cache.local
if not cache.fs.exists(cache.path):
os.makedirs(cache.path)
with TemporaryDirectory(dir=cache.path) as tmpdir:
dvc/stage/cache.py
+9 -9
View File
@@ -61,7 +61,7 @@ def _get_stage_hash(stage):
class StageCache:
def __init__(self, repo):
self.repo = repo
self.cache_dir = os.path.join(self.repo.odb.local.path, "runs")
self.cache_dir = os.path.join(self.repo.cache.local.path, "runs")
def _get_cache_dir(self, key):
return os.path.join(self.cache_dir, key[:2], key)
@@ -120,12 +120,12 @@ class StageCache:
@contextmanager
def _cache_type_copy(self):
cache_types = self.repo.odb.local.cache_types
self.repo.odb.local.cache_types = ["copy"]
cache_types = self.repo.cache.local.cache_types
self.repo.cache.local.cache_types = ["copy"]
try:
yield
finally:
self.repo.odb.local.cache_types = cache_types
self.repo.cache.local.cache_types = cache_types
def _uncached_outs(self, stage, cache):
# NOTE: using temporary stage to avoid accidentally modifying original
@@ -167,13 +167,13 @@ class StageCache:
COMPILED_LOCK_FILE_STAGE_SCHEMA(cache)
path = self._get_cache_path(cache_key, cache_value)
parent = self.repo.odb.local.fs.path.parent(path)
self.repo.odb.local.makedirs(parent)
parent = self.repo.cache.local.fs.path.parent(path)
self.repo.cache.local.makedirs(parent)
tmp = tempfile.NamedTemporaryFile(delete=False, dir=parent).name
assert os.path.exists(parent)
assert os.path.isdir(parent)
dump_yaml(tmp, cache)
self.repo.odb.local.move(tmp, path)
self.repo.cache.local.move(tmp, path)
def restore(self, stage, run_cache=True, pull=False, dry=False):
from .serialize import to_single_stage_lockfile
@@ -259,11 +259,11 @@ class StageCache:
def push(self, remote: Optional[str], odb: Optional["ObjectDB"] = None):
dest_odb = odb or self.repo.cloud.get_remote_odb(remote, "push --run-cache")
return self.transfer(self.repo.odb.local, dest_odb)
return self.transfer(self.repo.cache.local, dest_odb)
def pull(self, remote: Optional[str], odb: Optional["ObjectDB"] = None):
odb = odb or self.repo.cloud.get_remote_odb(remote, "fetch --run-cache")
return self.transfer(odb, self.repo.odb.local)
return self.transfer(odb, self.repo.cache.local)
def get_used_objs(self, used_run_cache, *args, **kwargs):
"""Return used cache for the specified run-cached stages."""
dvc/testing/api_tests.py
+2 -2
View File
@@ -24,7 +24,7 @@ class TestAPI:
dvc.push()
# Remove cache to force download
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
with api.open("foo") as fobj:
assert fobj.read() == "foo-text"
@@ -62,7 +62,7 @@ class TestAPI:
dvc.push()
if clear_cache:
remove(dvc.odb.repo.path)
remove(dvc.cache.repo.path)
if url := fs_kwargs.get("url"):
fs_kwargs["url"] = url.format(path=tmp_dir, posixpath=tmp_dir.as_posix())
dvc/testing/fixtures.py
+2 -2
View File
@@ -183,7 +183,7 @@ def local_remote(make_remote):
@pytest.fixture
def make_workspace(tmp_dir, dvc, make_cloud):
def _make_workspace(name, typ="local"):
from dvc.odbmgr import ODBManager
from dvc.cachemgr import CacheManager
cloud = make_cloud(typ) # pylint: disable=W0621
@@ -197,7 +197,7 @@ def make_workspace(tmp_dir, dvc, make_cloud):
with dvc.config.edit() as conf:
conf["cache"][scheme] = f"{name}-cache"
dvc.odb = ODBManager(dvc)
dvc.cache = CacheManager(dvc)
return cloud
dvc/testing/remote_tests.py
+13 -13
View File
@@ -48,8 +48,8 @@ class TestRemote:
# Move cache and check status
# See issue https://github.com/iterative/dvc/issues/4383 for details
backup_dir = dvc.odb.local.path + ".backup"
shutil.move(dvc.odb.local.path, backup_dir)
backup_dir = dvc.cache.local.path + ".backup"
shutil.move(dvc.cache.local.path, backup_dir)
status = dvc.cloud.status(foo_hashes)
_check_status(status, missing={foo_hash})
@@ -57,8 +57,8 @@ class TestRemote:
_check_status(status_dir, missing=dir_hashes)
# Restore original cache:
remove(dvc.odb.local.path)
shutil.move(backup_dir, dvc.odb.local.path)
remove(dvc.cache.local.path)
shutil.move(backup_dir, dvc.cache.local.path)
# Push and check status
dvc.cloud.push(foo_hashes)
@@ -75,7 +75,7 @@ class TestRemote:
_check_status(status_dir, ok=dir_hashes)
# Remove and check status
dvc.odb.local.clear()
dvc.cache.local.clear()
status = dvc.cloud.status(foo_hashes)
_check_status(status, deleted={foo_hash})
@@ -133,7 +133,7 @@ class TestRemote:
status = dvc.cloud.status(expected_hashes)
_check_status(status, ok=expected_hashes)
dvc.odb.local.clear()
dvc.cache.local.clear()
remove(tmp_dir / "foo")
remove(tmp_dir / "bar")
@@ -159,7 +159,7 @@ class TestRemote:
status = dvc.cloud.status(expected_hashes)
_check_status(status, ok=expected_hashes)
dvc.odb.local.clear()
dvc.cache.local.clear()
remove(tmp_dir / "foo")
remove(tmp_dir / "bar")
@@ -178,7 +178,7 @@ class TestRemoteVersionAware:
out = stage.outs[0]
assert out.meta.version_id
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
remove(tmp_dir / "foo")
dvc.pull()
@@ -205,7 +205,7 @@ class TestRemoteVersionAware:
assert file["version_id"]
assert file["remote"] == "upstream"
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
remove(tmp_dir / "data_dir")
dvc.pull()
@@ -225,7 +225,7 @@ class TestRemoteWorktree:
out = stage.outs[0]
assert out.meta.version_id
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
remove(tmp_dir / "foo")
dvc.pull()
@@ -252,7 +252,7 @@ class TestRemoteWorktree:
assert file["version_id"]
assert file["remote"] == "upstream"
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
remove(tmp_dir / "data_dir")
dvc.pull()
@@ -285,7 +285,7 @@ class TestRemoteWorktree:
tmp_dir.scm_add([tmp_dir / "data_dir.dvc"], commit="v2")
assert not (remote_worktree / "data_dir" / "data").exists()
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
remove(tmp_dir / "data_dir")
# pulling the original pushed version should still succeed
scm.checkout(v1)
@@ -335,7 +335,7 @@ class TestRemoteWorktree:
assert (tmp_dir / "data_dir" / "data").read_text() == "modified"
assert (tmp_dir / "data_dir" / "new_data").read_text() == "new data"
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
remove(tmp_dir / "foo")
remove(tmp_dir / "data_dir")
dvc.pull()
dvc/testing/workspace_tests.py
+6 -6
View File
@@ -27,7 +27,7 @@ class TestImport:
pytest.skip()
def test_import_dir(self, tmp_dir, dvc, workspace, stage_md5, dir_md5):
from dvc.odbmgr import ODBManager
from dvc.cachemgr import CacheManager
workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}})
@@ -35,7 +35,7 @@ class TestImport:
# to import dirs
with dvc.config.edit() as conf:
del conf["cache"]
dvc.odb = ODBManager(dvc)
dvc.cache = CacheManager(dvc)
assert not (tmp_dir / "dir").exists() # sanity check
dvc.imp_url("remote://workspace/dir")
@@ -93,7 +93,7 @@ class TestImportURLVersionAware:
assert (tmp_dir / "file").read_text() == "file"
assert dvc.status() == {}
dvc.odb.local.clear()
dvc.cache.local.clear()
remove(tmp_dir / "file")
dvc.pull()
assert (tmp_dir / "file").read_text() == "file"
@@ -106,7 +106,7 @@ class TestImportURLVersionAware:
assert (tmp_dir / "file").read_text() == "modified"
assert dvc.status() == {}
dvc.odb.local.clear()
dvc.cache.local.clear()
remove(tmp_dir / "file")
dvc.pull()
assert (tmp_dir / "file").read_text() == "modified"
@@ -119,7 +119,7 @@ class TestImportURLVersionAware:
assert (tmp_dir / "data_dir" / "subdir" / "file").read_text() == "file"
assert dvc.status() == {}
dvc.odb.local.clear()
dvc.cache.local.clear()
remove(tmp_dir / "data_dir")
dvc.pull()
assert (tmp_dir / "data_dir" / "subdir" / "file").read_text() == "file"
@@ -134,7 +134,7 @@ class TestImportURLVersionAware:
assert (tmp_dir / "data_dir" / "new_file").read_text() == "new"
assert dvc.status() == {}
dvc.odb.local.clear()
dvc.cache.local.clear()
remove(tmp_dir / "data_dir")
dvc.pull()
assert (tmp_dir / "data_dir" / "subdir" / "file").read_text() == "modified"
tests/func/api/test_data.py
+5 -5
View File
@@ -44,7 +44,7 @@ def test_open_external(tmp_dir, erepo_dir, cloud):
erepo_dir.dvc.push(all_branches=True)
# Remove cache to force download
remove(erepo_dir.dvc.odb.local.path)
remove(erepo_dir.dvc.cache.local.path)
# Using file url to force clone to tmp repo
repo_url = f"file://{erepo_dir.as_posix()}"
@@ -59,7 +59,7 @@ def test_open_granular(tmp_dir, dvc, remote):
dvc.push()
# Remove cache to force download
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
with api.open("dir/foo") as fd:
assert fd.read() == "foo-text"
@@ -69,7 +69,7 @@ def test_missing(tmp_dir, dvc, remote):
tmp_dir.dvc_gen("foo", "foo")
# Remove cache to make foo missing
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
api.read("foo")
@@ -129,7 +129,7 @@ def test_api_missing_local_cache_exists_on_remote(
dvc.push()
# Remove cache to make foo missing
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
remove(first(files))
repo_url = f"file://{tmp_dir.as_posix()}" if as_external else None
@@ -202,7 +202,7 @@ def test_open_from_remote(tmp_dir, erepo_dir, cloud, local_cloud):
erepo_dir.add_remote(config=local_cloud.config, default=True)
erepo_dir.dvc_gen({"dir": {"foo": "foo content"}}, commit="create file")
erepo_dir.dvc.push(remote="other")
remove(erepo_dir.dvc.odb.local.path)
remove(erepo_dir.dvc.cache.local.path)
with api.open(
os.path.join("dir", "foo"),
tests/func/data/db/test_index.py
+1 -1
View File
@@ -59,7 +59,7 @@ def test_clear_on_download_err(tmp_dir, dvc, index, mocker):
dvc.push()
for _, _, hi in out.obj:
remove(dvc.odb.local.get(hi.value).path)
remove(dvc.cache.local.get(hi.value).path)
remove(out.fs_path)
assert list(index.hashes())
tests/func/experiments/test_remote.py
+2 -2
View File
@@ -303,12 +303,12 @@ def test_push_pull_cache(
with open(path, encoding="utf-8") as f:
assert f.read() == str(x)
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
dvc.experiments.pull(remote, [ref_info.name], pull_cache=True)
for x in range(2, checkpoint_stage.iterations + 1):
hash_ = digest(str(x))
path = os.path.join(dvc.odb.local.path, hash_[:2], hash_[2:])
path = os.path.join(dvc.cache.local.path, hash_[:2], hash_[2:])
assert os.path.exists(path)
with open(path, encoding="utf-8") as f:
assert f.read() == str(x)
tests/func/metrics/test_show.py
+1 -1
View File
@@ -259,7 +259,7 @@ def test_metrics_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run):
# so as it works even for optimized cases
if clear_before_run:
remove(data_dir)
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
dvc._reset()
tests/func/plots/test_show.py
+1 -1
View File
@@ -161,7 +161,7 @@ def test_plots_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run):
# so as it works even for optimized cases
if clear_before_run:
remove(data_dir)
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
dvc._reset()
tests/func/test_add.py
+14 -14
View File
@@ -12,6 +12,7 @@ import pytest
import dvc as dvc_module
import dvc_data
from dvc.annotations import Annotation
from dvc.cachemgr import CacheManager
from dvc.cli import main
from dvc.config import ConfigError
from dvc.dvcfile import DVC_FILE_SUFFIX
@@ -23,7 +24,6 @@ from dvc.exceptions import (
RecursiveAddingWhileUsingFilename,
)
from dvc.fs import LocalFileSystem, system
from dvc.odbmgr import ODBManager
from dvc.output import (
OutputAlreadyTrackedError,
OutputDoesNotExistError,
@@ -110,7 +110,7 @@ def test_add_directory(tmp_dir, dvc):
hash_info = stage.outs[0].hash_info
obj = load(dvc.odb.local, hash_info)
obj = load(dvc.cache.local, hash_info)
for key, _, _ in obj:
for part in key:
assert "\\" not in part
@@ -407,12 +407,12 @@ def test_add_commit(tmp_dir, dvc):
ret = main(["add", "foo", "--no-commit"])
assert ret == 0
assert os.path.isfile("foo")
assert not os.path.exists(dvc.odb.local.path)
assert not os.path.exists(dvc.cache.local.path)
ret = main(["commit", "foo.dvc"])
assert ret == 0
assert os.path.isfile("foo")
assert dvc.odb.local.exists("acbd18db4cc2f85cedef654fccc4a4d8")
assert dvc.cache.local.exists("acbd18db4cc2f85cedef654fccc4a4d8")
def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
@@ -583,7 +583,7 @@ def test_windows_should_add_when_cache_on_different_drive(
tmp_dir, dvc, temporary_windows_drive
):
dvc.config["cache"]["dir"] = temporary_windows_drive
dvc.odb = ODBManager(dvc)
dvc.cache = CacheManager(dvc)
(stage,) = tmp_dir.dvc_gen({"file": "file"})
cache_path = stage.outs[0].cache_path
@@ -596,12 +596,12 @@ def test_windows_should_add_when_cache_on_different_drive(
def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
tmp_dir.gen("dir/data", "data")
dvc.odb.local.cache_types = ["symlink"]
dvc.cache.local.cache_types = ["symlink"]
dvc.add("dir")
tmp_dir.gen("dir/new_file", "new_file_content")
unprotect_spy = mocker.spy(dvc.odb.local, "unprotect")
unprotect_spy = mocker.spy(dvc.cache.local, "unprotect")
dvc.add("dir")
assert not unprotect_spy.mock.called
@@ -609,13 +609,13 @@ def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
def test_should_not_checkout_when_adding_cached_copy(tmp_dir, dvc, mocker):
dvc.odb.local.cache_types = ["copy"]
dvc.cache.local.cache_types = ["copy"]
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
shutil.copy("bar", "foo")
copy_spy = mocker.spy(dvc.odb.local.fs, "copy")
copy_spy = mocker.spy(dvc.cache.local.fs, "copy")
dvc.add("foo")
@@ -637,11 +637,11 @@ def test_should_relink_on_repeated_add(link, new_link, link_test_func, tmp_dir,
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
os.remove("foo")
getattr(dvc.odb.local.fs, link)(
getattr(dvc.cache.local.fs, link)(
(tmp_dir / "bar").fs_path, (tmp_dir / "foo").fs_path
)
dvc.odb.local.cache_types = [new_link]
dvc.cache.local.cache_types = [new_link]
dvc.add("foo")
@@ -650,7 +650,7 @@ def test_should_relink_on_repeated_add(link, new_link, link_test_func, tmp_dir,
@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
def test_should_protect_on_repeated_add(link, tmp_dir, dvc):
dvc.odb.local.cache_types = [link]
dvc.cache.local.cache_types = [link]
tmp_dir.dvc_gen({"foo": "foo"})
@@ -709,7 +709,7 @@ def test_not_raises_on_re_add(tmp_dir, dvc):
@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
def test_add_empty_files(tmp_dir, dvc, link):
file = "foo"
dvc.odb.local.cache_types = [link]
dvc.cache.local.cache_types = [link]
stages = tmp_dir.dvc_gen(file, "")
assert (tmp_dir / file).exists()
@@ -718,7 +718,7 @@ def test_add_empty_files(tmp_dir, dvc, link):
def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker):
dvc.odb.local.cache_types = ["hardlink"]
dvc.cache.local.cache_types = ["hardlink"]
tmp_dir.gen({"foo": "", "bar": "", "lorem": "lorem", "ipsum": "ipsum"})
m = mocker.spy(LocalFileSystem, "is_hardlink")
stages = dvc.add(["foo", "bar", "lorem", "ipsum"])
tests/func/test_checkout.py
+7 -7
View File
@@ -105,9 +105,9 @@ def test_checkout_corrupted_cache_dir(tmp_dir, dvc):
# NOTE: modifying cache file for one of the files inside the directory
# to check if dvc will detect that the cache is corrupted.
obj = load(dvc.odb.local, out.hash_info)
obj = load(dvc.cache.local, out.hash_info)
_, _, entry_oid = list(obj)[0]
cache = dvc.odb.local.oid_to_path(entry_oid.value)
cache = dvc.cache.local.oid_to_path(entry_oid.value)
os.chmod(cache, 0o644)
with open(cache, "w+", encoding="utf-8") as fobj:
@@ -161,7 +161,7 @@ class TestCheckoutCleanWorkingDir:
def test_checkout_selective_remove(tmp_dir, dvc):
# Use copy to test for changes in the inodes
dvc.odb.local.cache_types = ["copy"]
dvc.cache.local.cache_types = ["copy"]
tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}})
foo_inode = system.inode(os.path.join("data", "foo"))
@@ -357,7 +357,7 @@ def test_checkout_moved_cache_dir_with_symlinks(tmp_dir, dvc):
assert system.is_symlink(os.path.join("data", "file"))
old_data_link = os.path.realpath(os.path.join("data", "file"))
old_cache_dir = dvc.odb.local.path
old_cache_dir = dvc.cache.local.path
new_cache_dir = old_cache_dir + "_new"
os.rename(old_cache_dir, new_cache_dir)
@@ -395,7 +395,7 @@ def test_checkout_no_checksum(tmp_dir, dvc):
[("hardlink", system.is_hardlink), ("symlink", system.is_symlink)],
)
def test_checkout_relink(tmp_dir, dvc, link, link_test_func):
dvc.odb.local.cache_types = [link]
dvc.cache.local.cache_types = [link]
tmp_dir.dvc_gen({"dir": {"data": "text"}})
dvc.unprotect("dir/data")
@@ -408,7 +408,7 @@ def test_checkout_relink(tmp_dir, dvc, link, link_test_func):
@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
def test_checkout_relink_protected(tmp_dir, dvc, link):
dvc.odb.local.cache_types = [link]
dvc.cache.local.cache_types = [link]
tmp_dir.dvc_gen("foo", "foo")
dvc.unprotect("foo")
@@ -595,7 +595,7 @@ def test_checkout_with_relink_existing(tmp_dir, dvc, link):
(tmp_dir / "foo").unlink()
tmp_dir.dvc_gen("bar", "bar")
dvc.odb.local.cache_types = [link]
dvc.cache.local.cache_types = [link]
stats = dvc.checkout(relink=True)
assert stats == {**empty_checkout, "added": ["foo"]}
tests/func/test_data_cloud.py
+12 -12
View File
@@ -46,7 +46,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker):
_kwargs["jobs"] == jobs for (_args, _kwargs) in oids_exist.call_args_list
)
dvc.odb.local.clear()
dvc.cache.local.clear()
oids_exist.reset_mock()
assert main(["fetch", *args]) == 0
@@ -86,7 +86,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker):
assert main(["gc", "-cw", "-f", *args]) == 0
assert _list_oids_traverse.called
assert all(_kwargs["jobs"] == 2 for (_args, _kwargs) in oids_exist.call_args_list)
shutil.move(dvc.odb.local.path, dvc.odb.local.path + ".back")
shutil.move(dvc.cache.local.path, dvc.cache.local.path + ".back")
assert main(["fetch", *args]) == 0
@@ -155,7 +155,7 @@ def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
# purge cache
dvc.odb.local.clear()
dvc.cache.local.clear()
header = (
"Some of the cache files do not exist "
@@ -194,7 +194,7 @@ def test_verify_hashes(tmp_dir, scm, dvc, mocker, tmp_path_factory, local_remote
# remove artifacts and cache to trigger fetching
remove("file")
remove("dir")
dvc.odb.local.clear()
dvc.cache.local.clear()
hash_spy = mocker.spy(dvc_data.hashfile.hash, "file_md5")
@@ -202,7 +202,7 @@ def test_verify_hashes(tmp_dir, scm, dvc, mocker, tmp_path_factory, local_remote
assert hash_spy.call_count == 0
# Removing cache will invalidate existing state entries
dvc.odb.local.clear()
dvc.cache.local.clear()
dvc.config["remote"]["upstream"]["verify"] = True
@@ -224,9 +224,9 @@ def test_pull_git_imports(tmp_dir, dvc, scm, erepo):
assert dvc.pull()["fetched"] == 0
for item in ["foo", "new_dir", dvc.odb.local.path]:
for item in ["foo", "new_dir", dvc.cache.local.path]:
remove(item)
os.makedirs(dvc.odb.local.path, exist_ok=True)
os.makedirs(dvc.cache.local.path, exist_ok=True)
clean_repos()
assert dvc.pull(force=True)["fetched"] == 3
@@ -295,7 +295,7 @@ def test_pull_external_dvc_imports_mixed(tmp_dir, dvc, scm, erepo_dir, local_rem
def clean(outs, dvc=None):
if dvc:
dvc.odb.local.clear()
dvc.cache.local.clear()
for path in outs:
remove(path)
if dvc:
@@ -471,11 +471,11 @@ def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
dvc.pull("copy-foo-bar")
assert (tmp_dir / "bar").exists()
assert len(recurse_list_dir(dvc.odb.local.path)) == 2
assert len(recurse_list_dir(dvc.cache.local.path)) == 2
clean(["bar"], dvc)
dvc.fetch("copy-foo-bar")
assert len(recurse_list_dir(dvc.odb.local.path)) == 2
assert len(recurse_list_dir(dvc.cache.local.path)) == 2
def test_pull_partial(tmp_dir, dvc, local_remote):
@@ -521,7 +521,7 @@ def test_output_remote(tmp_dir, dvc, make_remote):
dvc.pull()
assert set(dvc.odb.local.all()) == {
assert set(dvc.cache.local.all()) == {
"37b51d194a7513e45b56f6524f2d51f2",
"acbd18db4cc2f85cedef654fccc4a4d8",
"f97c5d29941bfb1b2fdab0874906ab82",
@@ -554,7 +554,7 @@ def test_target_remote(tmp_dir, dvc, make_remote):
dvc.pull(remote="myremote")
assert set(dvc.odb.local.all()) == {
assert set(dvc.cache.local.all()) == {
"acbd18db4cc2f85cedef654fccc4a4d8",
"f97c5d29941bfb1b2fdab0874906ab82",
"6b18131dc289fd37006705affe961ef8.dir",
tests/func/test_data_status.py
+7 -7
View File
@@ -234,7 +234,7 @@ def test_untracked_newly_added_files(M, tmp_dir, dvc, scm):
def test_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
tmp_dir.dvc_gen("foobar", "foobar")
remove(dvc.odb.repo.path)
remove(dvc.cache.repo.path)
assert dvc.data_status(untracked_files="all") == {
**EMPTY_STATUS,
@@ -260,7 +260,7 @@ def test_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
def test_missing_cache_missing_workspace(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
tmp_dir.dvc_gen("foobar", "foobar")
for path in [dvc.odb.repo.path, "dir", "foobar"]:
for path in [dvc.cache.repo.path, "dir", "foobar"]:
remove(path)
assert dvc.data_status(untracked_files="all") == {
@@ -285,7 +285,7 @@ def test_missing_cache_missing_workspace(M, tmp_dir, dvc, scm):
def test_git_committed_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
tmp_dir.dvc_gen("foobar", "foobar", commit="add foobar")
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
assert dvc.data_status(untracked_files="all") == {
**EMPTY_STATUS,
@@ -308,7 +308,7 @@ def test_git_committed_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
def test_git_committed_missing_cache_missing_workspace(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
tmp_dir.dvc_gen("foobar", "foobar", commit="add foobar")
for path in [dvc.odb.repo.path, "dir", "foobar"]:
for path in [dvc.cache.repo.path, "dir", "foobar"]:
remove(path)
assert dvc.data_status(untracked_files="all") == {
@@ -330,7 +330,7 @@ def test_partial_missing_cache(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
# remove "foo" from cache
odb = dvc.odb.repo
odb = dvc.cache.repo
odb.fs.rm(odb.oid_to_path("acbd18db4cc2f85cedef654fccc4a4d8"))
assert dvc.data_status() == {
@@ -354,7 +354,7 @@ def test_missing_dir_object_from_head(M, tmp_dir, dvc, scm):
(stage,) = tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
remove("dir")
tmp_dir.dvc_gen({"dir": {"foobar": "foobar"}})
odb = dvc.odb.repo
odb = dvc.cache.repo
odb.fs.rm(odb.oid_to_path(stage.outs[0].hash_info.value))
assert dvc.data_status() == {
@@ -376,7 +376,7 @@ def test_missing_dir_object_from_index(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
remove("dir")
(stage,) = tmp_dir.dvc_gen({"dir": {"foobar": "foobar"}})
odb = dvc.odb.repo
odb = dvc.cache.repo
odb.fs.rm(odb.oid_to_path(stage.outs[0].hash_info.value))
assert dvc.data_status() == {
tests/func/test_diff.py
+2 -2
View File
@@ -250,10 +250,10 @@ def test_diff_no_cache(tmp_dir, scm, dvc):
tmp_dir.dvc_gen({"dir": {"file": "modified file content"}}, commit="second")
scm.tag("v2")
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
# invalidate_dir_info to force cache loading
dvc.odb.local._dir_info = {}
dvc.cache.local._dir_info = {}
diff = dvc.diff("v1", "v2")
assert diff["added"] == []
tests/func/test_external_repo.py
+4 -4
View File
@@ -115,7 +115,7 @@ def test_relative_remote(erepo_dir, tmp_dir):
erepo_dir.dvc.push()
(erepo_dir / "file").unlink()
remove(erepo_dir.dvc.odb.local.path)
remove(erepo_dir.dvc.cache.local.path)
url = os.fspath(erepo_dir)
@@ -195,12 +195,12 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
assert (tmp_dir / "out").read_text() == expected_files
# clear cache to test saving to cache
cache_dir = tmp_dir / repo.odb.local.path
cache_dir = tmp_dir / repo.cache.local.path
remove(cache_dir)
os.makedirs(cache_dir)
staging, _, obj = build(
repo.odb.local,
repo.cache.local,
"dir",
repo.dvcfs,
"md5",
@@ -208,7 +208,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
)
transfer(
staging,
repo.odb.local,
repo.cache.local,
{obj.hash_info},
shallow=False,
hardlink=True,
tests/func/test_gc.py
+20 -18
View File
@@ -20,7 +20,7 @@ def good_and_bad_cache(tmp_dir, dvc):
{"sub": {"data_sub": "data_sub", "data": "data", "тест": "проверка"}},
)
raw_dir_hash = stage.outs[0].hash_info.as_raw().value
odb = dvc.odb.local
odb = dvc.cache.local
bad_cache = {raw_dir_hash}
for i in ["123", "234", "345"]:
@@ -33,14 +33,14 @@ def good_and_bad_cache(tmp_dir, dvc):
def test_gc_api(dvc, good_and_bad_cache):
dvc.gc(workspace=True)
odb = dvc.odb.local
odb = dvc.cache.local
good_cache, bad_cache = good_and_bad_cache
assert set(odb.oids_exist([*good_cache, *bad_cache])) == good_cache
def test_gc_cli(dvc, good_and_bad_cache):
assert main(["gc", "-wf"]) == 0
odb = dvc.odb.local
odb = dvc.cache.local
good_cache, bad_cache = good_and_bad_cache
assert set(odb.oids_exist([*good_cache, *bad_cache])) == good_cache
@@ -59,7 +59,7 @@ def test_gc_branches_tags(tmp_dir, dvc, scm):
dvc.remove("file.dvc")
tmp_dir.dvc_gen("file", "master", commit="trash")
odb = dvc.odb.local
odb = dvc.cache.local
assert len(list(odb.all())) == 4
dvc.gc(all_tags=True, all_branches=True)
@@ -76,12 +76,12 @@ def test_gc_multiple_dvc_repos(tmp_dir, scm, dvc, erepo_dir):
tmp_dir.dvc_gen("only_in_first", "only in main repo")
tmp_dir.dvc_gen("in_both", "in both repos")
erepo_dir.dvc.odb.local.path = dvc.odb.local.path
erepo_dir.dvc.cache.local.path = dvc.cache.local.path
with erepo_dir.chdir():
erepo_dir.dvc_gen("in_both", "in both repos")
erepo_dir.dvc_gen("only_in_second", "only in additional repo")
odb = dvc.odb.local
odb = dvc.cache.local
assert len(list(odb.all())) == 3
dvc.gc(repos=[erepo_dir], workspace=True)
@@ -97,11 +97,11 @@ def test_all_commits(tmp_dir, scm, dvc):
tmp_dir.dvc_gen("testfile", "modified", commit="modified")
tmp_dir.dvc_gen("testfile", "workspace")
n = _count_files(dvc.odb.local.path)
n = _count_files(dvc.cache.local.path)
dvc.gc(all_commits=True)
# Only one uncommitted file should go away
assert _count_files(dvc.odb.local.path) == n - 1
assert _count_files(dvc.cache.local.path) == n - 1
def test_gc_no_dir_cache(tmp_dir, dvc):
@@ -113,9 +113,9 @@ def test_gc_no_dir_cache(tmp_dir, dvc):
with pytest.raises(CollectCacheError):
dvc.gc(workspace=True)
assert _count_files(dvc.odb.local.path) == 4
assert _count_files(dvc.cache.local.path) == 4
dvc.gc(force=True, workspace=True)
assert _count_files(dvc.odb.local.path) == 2
assert _count_files(dvc.cache.local.path) == 2
def _count_files(path):
@@ -240,14 +240,14 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
run_copy("foo", "foo2", name="copy")
shutil.rmtree(dvc.stage_cache.cache_dir)
assert _count_files(dvc.odb.local.path) == 1
assert _count_files(dvc.cache.local.path) == 1
dvc.gc(workspace=True, force=True)
assert _count_files(dvc.odb.local.path) == 1
assert _count_files(dvc.cache.local.path) == 1
# remove pipeline file and lockfile and check
load_file(dvc, PROJECT_FILE).remove(force=True)
dvc.gc(workspace=True, force=True)
assert _count_files(dvc.odb.local.path) == 0
assert _count_files(dvc.cache.local.path) == 0
def test_gc_external_output(tmp_dir, dvc, workspace):
@@ -328,13 +328,15 @@ def test_date(tmp_dir, scm, dvc):
dvc.gc(commit_date=datestamp)
assert _count_files(dvc.odb.local.path) == 1
assert dvc.odb.local.exists("9ae73c65f418e6f79ceb4f0e4a4b98d5") # "modified"
assert _count_files(dvc.cache.local.path) == 1
assert dvc.cache.local.exists("9ae73c65f418e6f79ceb4f0e4a4b98d5") # "modified"
tmp_dir.dvc_gen("testfile", "modified, again", commit="modify")
datestamp = (now.date() - datetime.timedelta(days=1)).isoformat()
dvc.gc(commit_date=datestamp)
assert _count_files(dvc.odb.local.path) == 2
assert dvc.odb.local.exists("9ae73c65f418e6f79ceb4f0e4a4b98d5")
assert dvc.odb.local.exists("3bcf3b1be3e794a97a5a6b93a005784c") # "modified, again"
assert _count_files(dvc.cache.local.path) == 2
assert dvc.cache.local.exists("9ae73c65f418e6f79ceb4f0e4a4b98d5")
assert dvc.cache.local.exists(
"3bcf3b1be3e794a97a5a6b93a005784c"
) # "modified, again"
tests/func/test_get.py
+2 -2
View File
@@ -3,9 +3,9 @@ import os
import pytest
from dvc.cachemgr import CacheManager
from dvc.cli import main
from dvc.fs import system
from dvc.odbmgr import ODBManager
from dvc.repo import Repo
from dvc.repo.get import GetDVCFileError
from dvc.testing.tmp_dir import make_subrepo
@@ -76,7 +76,7 @@ def test_cache_type_is_properly_overridden(tmp_dir, erepo_dir):
with erepo_dir.chdir():
with erepo_dir.dvc.config.edit() as conf:
conf["cache"]["type"] = "symlink"
erepo_dir.dvc.odb = ODBManager(erepo_dir.dvc)
erepo_dir.dvc.cache = CacheManager(erepo_dir.dvc)
erepo_dir.scm_add(
[erepo_dir.dvc.config.files["repo"]], "set cache type to symlinks"
)
tests/func/test_ignore.py
+1 -1
View File
@@ -477,7 +477,7 @@ def test_pull_ignore(tmp_dir, dvc, local_cloud):
foo_path.unlink()
assert not foo_path.exists()
dvc.odb.local.clear()
dvc.cache.local.clear()
dvc.pull()
assert foo_path.exists()
tests/func/test_import.py
+10 -10
View File
@@ -5,11 +5,11 @@ import pytest
from funcy import first
from dvc.annotations import Annotation
from dvc.cachemgr import CacheManager
from dvc.config import NoRemoteError
from dvc.dvcfile import load_file
from dvc.exceptions import DownloadError, PathMissingError
from dvc.fs import system
from dvc.odbmgr import ODBManager
from dvc.scm import Git
from dvc.stage.exceptions import StagePathNotFoundError
from dvc.testing.tmp_dir import make_subrepo
@@ -272,7 +272,7 @@ def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir):
with erepo_dir.chdir():
with erepo_dir.dvc.config.edit() as conf:
conf["cache"]["type"] = "symlink"
erepo_dir.dvc.odb = ODBManager(erepo_dir.dvc)
erepo_dir.dvc.cache = CacheManager(erepo_dir.dvc)
erepo_dir.scm_add(
[erepo_dir.dvc.config.files["repo"]],
"set source repo cache type to symlink",
@@ -294,7 +294,7 @@ def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir):
dvc.imp(os.fspath(erepo_dir), "dir", "dir_imported")
remove("dir_imported")
dvc.odb.local.clear()
dvc.cache.local.clear()
dvc.pull(["dir_imported.dvc"])
@@ -308,7 +308,7 @@ def test_pull_wildcard_imported_directory_stage(tmp_dir, dvc, erepo_dir):
dvc.imp(os.fspath(erepo_dir), "dir123", "dir_imported123")
remove("dir_imported123")
dvc.odb.local.clear()
dvc.cache.local.clear()
dvc.pull(["dir_imported*.dvc"], glob=True)
@@ -536,7 +536,7 @@ def test_pull_imported_stage_from_subrepos(tmp_dir, dvc, erepo_dir, is_dvc, file
dvc.imp(os.fspath(erepo_dir), path, out="out")
# clean everything
dvc.odb.local.clear()
dvc.cache.local.clear()
remove("out")
stats = dvc.pull(["out.dvc"])
@@ -606,7 +606,7 @@ def test_chained_import(tmp_dir, dvc, make_tmp_dir, erepo_dir, local_cloud):
with erepo_dir.chdir():
erepo_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="init")
erepo_dir.dvc.push()
remove(erepo_dir.dvc.odb.local.path)
remove(erepo_dir.dvc.cache.local.path)
remove(os.fspath(erepo_dir / "dir"))
erepo2 = make_tmp_dir("erepo2", scm=True, dvc=True)
@@ -614,7 +614,7 @@ def test_chained_import(tmp_dir, dvc, make_tmp_dir, erepo_dir, local_cloud):
erepo2.dvc.imp(os.fspath(erepo_dir), "dir")
erepo2.scm.add("dir.dvc")
erepo2.scm.commit("import")
remove(erepo2.dvc.odb.local.path)
remove(erepo2.dvc.cache.local.path)
remove(os.fspath(erepo2 / "dir"))
dvc.imp(os.fspath(erepo2), "dir", "dir_imported")
@@ -622,14 +622,14 @@ def test_chained_import(tmp_dir, dvc, make_tmp_dir, erepo_dir, local_cloud):
assert (dst / "foo").read_text() == "foo"
assert (dst / "bar").read_text() == "bar"
dvc.odb.local.clear()
dvc.cache.local.clear()
remove("dir_imported")
# pulled objects should come from the original upstream repo's remote,
# no cache or remote should be needed from the intermediate repo
dvc.pull(["dir_imported.dvc"])
assert not os.path.exists(erepo_dir.dvc.odb.local.path)
assert not os.path.exists(erepo2.dvc.odb.local.path)
assert not os.path.exists(erepo_dir.dvc.cache.local.path)
assert not os.path.exists(erepo2.dvc.cache.local.path)
assert (dst / "foo").read_text() == "foo"
assert (dst / "bar").read_text() == "bar"
tests/func/test_odb.py
+17 -17
View File
@@ -4,8 +4,8 @@ import stat
import configobj
import pytest
from dvc.cachemgr import CacheManager
from dvc.cli import main
from dvc.odbmgr import ODBManager
from dvc.utils import relpath
from dvc_data.hashfile.hash_info import HashInfo
from dvc_objects.errors import ObjectFormatError
@@ -15,12 +15,12 @@ def test_cache(tmp_dir, dvc):
cache1_md5 = "123"
cache2_md5 = "234"
cache1 = os.path.join(
dvc.odb.local.path,
dvc.cache.local.path,
cache1_md5[0:2],
cache1_md5[2:],
)
cache2 = os.path.join(
dvc.odb.local.path,
dvc.cache.local.path,
cache2_md5[0:2],
cache2_md5[2:],
)
@@ -29,7 +29,7 @@ def test_cache(tmp_dir, dvc):
assert os.path.exists(cache1)
assert os.path.exists(cache2)
odb = ODBManager(dvc)
odb = CacheManager(dvc)
md5_list = list(odb.local.all())
assert len(md5_list) == 2
@@ -46,16 +46,16 @@ def test_cache_load_bad_dir_cache(tmp_dir, dvc):
from dvc_data.hashfile import load
dir_hash = "123.dir"
fname = os.fspath(dvc.odb.local.oid_to_path(dir_hash))
fname = os.fspath(dvc.cache.local.oid_to_path(dir_hash))
tmp_dir.gen({fname: "<clearly>not,json"})
with pytest.raises(ObjectFormatError):
load(dvc.odb.local, HashInfo("md5", dir_hash))
load(dvc.cache.local, HashInfo("md5", dir_hash))
dir_hash = "234.dir"
fname = os.fspath(dvc.odb.local.oid_to_path(dir_hash))
fname = os.fspath(dvc.cache.local.oid_to_path(dir_hash))
tmp_dir.gen({fname: '{"a": "b"}'})
with pytest.raises(ObjectFormatError):
load(dvc.odb.local, HashInfo("md5", dir_hash))
load(dvc.cache.local, HashInfo("md5", dir_hash))
def test_external_cache_dir(tmp_dir, dvc, make_tmp_dir):
@@ -63,8 +63,8 @@ def test_external_cache_dir(tmp_dir, dvc, make_tmp_dir):
with dvc.config.edit() as conf:
conf["cache"]["dir"] = cache_dir.fs_path
assert not os.path.exists(dvc.odb.local.path)
dvc.odb = ODBManager(dvc)
assert not os.path.exists(dvc.cache.local.path)
dvc.cache = CacheManager(dvc)
tmp_dir.dvc_gen({"foo": "foo"})
@@ -87,9 +87,9 @@ def test_remote_cache_references(tmp_dir, dvc):
conf["remote"]["cache"] = {"url": "remote://storage/tmp"}
conf["cache"]["ssh"] = "cache"
dvc.odb = ODBManager(dvc)
dvc.cache = CacheManager(dvc)
assert dvc.odb.ssh.path == "/tmp"
assert dvc.cache.ssh.path == "/tmp"
def test_shared_cache_dir(tmp_dir):
@@ -126,7 +126,7 @@ def test_shared_cache_dir(tmp_dir):
def test_cache_link_type(tmp_dir, scm, dvc):
with dvc.config.edit() as conf:
conf["cache"]["type"] = "reflink,copy"
dvc.odb = ODBManager(dvc)
dvc.cache = CacheManager(dvc)
stages = tmp_dir.dvc_gen({"foo": "foo"})
assert len(stages) == 1
@@ -154,7 +154,7 @@ def test_cmd_cache_relative_path(tmp_dir, scm, dvc, make_tmp_dir):
assert ret == 0
dvc.config.load()
dvc.odb = ODBManager(dvc)
dvc.cache = CacheManager(dvc)
# NOTE: we are in the repo's root and config is in .dvc/, so
# dir path written to config should be just one level above.
@@ -170,7 +170,7 @@ def test_cmd_cache_relative_path(tmp_dir, scm, dvc, make_tmp_dir):
def test_default_cache_type(dvc):
assert dvc.odb.local.cache_types == ["reflink", "copy"]
assert dvc.cache.local.cache_types == ["reflink", "copy"]
@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
@@ -181,8 +181,8 @@ def test_shared_cache(tmp_dir, dvc, group):
if group:
with dvc.config.edit() as conf:
conf["cache"].update({"shared": "group"})
dvc.odb = ODBManager(dvc)
cache_dir = dvc.odb.local.path
dvc.cache = CacheManager(dvc)
cache_dir = dvc.cache.local.path
assert not os.path.exists(cache_dir)
tests/func/test_remote.py
+6 -6
View File
@@ -139,16 +139,16 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
tree = Tree.from_list([{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}])
tree.digest()
with patch("dvc_data.hashfile.build._build_tree", return_value=(None, tree)):
_, _, obj = build(dvc.odb.local, path, dvc.odb.local.fs, "md5")
_, _, obj = build(dvc.cache.local, path, dvc.cache.local.fs, "md5")
hash1 = obj.hash_info
# remove the raw dir obj to force building the tree on the next build call
dvc.odb.local.fs.remove(dvc.odb.local.oid_to_path(hash1.as_raw().value))
dvc.cache.local.fs.remove(dvc.cache.local.oid_to_path(hash1.as_raw().value))
tree = Tree.from_list([{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}])
tree.digest()
with patch("dvc_data.hashfile.build._build_tree", return_value=(None, tree)):
_, _, obj = build(dvc.odb.local, path, dvc.odb.local.fs, "md5")
_, _, obj = build(dvc.cache.local, path, dvc.cache.local.fs, "md5")
hash2 = obj.hash_info
assert hash1 == hash2
@@ -201,7 +201,7 @@ def test_partial_push_n_pull( # noqa: C901
# Push everything and delete local cache
dvc.push()
dvc.odb.local.clear()
dvc.cache.local.clear()
baz._collect_used_dir_cache()
@@ -263,7 +263,7 @@ def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
external_dir = tmp_path_factory.mktemp("external_dir")
file = external_dir / "file"
dvc.odb.local = None
dvc.cache.local = None
with pytest.raises(RemoteCacheRequiredError):
dvc.run(
cmd=f"echo content > {file}",
@@ -426,7 +426,7 @@ def test_push_incomplete_dir(tmp_dir, dvc, mocker, local_remote):
(stage,) = tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
remote_odb = dvc.cloud.get_remote_odb("upstream")
odb = dvc.odb.local
odb = dvc.cache.local
out = stage.outs[0]
file_objs = [entry_obj for _, _, entry_obj in out.obj]
tests/func/test_remove.py
+2 -2
View File
@@ -53,10 +53,10 @@ def test_remove_non_existent_file(tmp_dir, dvc):
def test_remove_broken_symlink(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
dvc.odb.local.cache_types = ["symlink"]
dvc.cache.local.cache_types = ["symlink"]
(stage,) = dvc.add("foo")
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
assert system.is_symlink("foo")
with pytest.raises(ObjectDBError):
tests/func/test_repo.py
+2 -2
View File
@@ -1,11 +1,11 @@
from dvc.cachemgr import CacheManager
from dvc.dvcfile import LOCK_FILE, PROJECT_FILE
from dvc.fs import system
from dvc.odbmgr import ODBManager
def test_destroy(tmp_dir, dvc, run_copy):
dvc.config["cache"]["type"] = ["symlink"]
dvc.odb = ODBManager(dvc)
dvc.cache = CacheManager(dvc)
tmp_dir.dvc_gen("file", "text")
tmp_dir.dvc_gen({"dir": {"file": "lorem", "subdir/file": "ipsum"}})
tests/func/test_repro.py
+2 -2
View File
@@ -837,11 +837,11 @@ def test_repro_no_commit(tmp_dir, dvc, copy_script, run_stage):
cmd="python copy.py foo file1",
name="run1",
)
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
ret = main(["repro", stage.addressing, "--no-commit"])
assert ret == 0
# run-cache should be skipped if `-no-commit`.
assert not os.path.isdir(dvc.odb.local.path)
assert not os.path.isdir(dvc.cache.local.path)
class TestReproAlreadyCached:
tests/func/test_run_single_stage.py
+3 -3
View File
@@ -682,12 +682,12 @@ def test_run_commit(dvc):
)
assert ret == 0
assert os.path.isfile(fname)
assert not os.path.exists(dvc.odb.local.path)
assert not os.path.exists(dvc.cache.local.path)
ret = main(["commit", fname + ".dvc"])
assert ret == 0
assert os.path.isfile(fname)
assert len(list(dvc.odb.local.all())) == 1
assert len(list(dvc.cache.local.all())) == 1
@pytest.mark.parametrize(
@@ -789,7 +789,7 @@ def test_should_not_checkout_upon_corrupted_local_hardlink_cache(
mocker, tmp_dir, dvc, copy_script
):
tmp_dir.gen("foo", "foo")
dvc.odb.local.cache_types = ["hardlink"]
dvc.cache.local.cache_types = ["hardlink"]
stage = dvc.run(
deps=["foo"],
tests/func/test_unprotect.py
+1 -1
View File
@@ -4,7 +4,7 @@ import os
def test_unprotect(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
dvc.odb.local.cache_types = ["hardlink"]
dvc.cache.local.cache_types = ["hardlink"]
dvc.add("foo")
cache = os.path.join(".dvc", "cache", "ac", "bd18db4cc2f85cedef654fccc4a4d8")
assert not os.access("foo", os.W_OK)
tests/unit/data/db/test_local.py
+5 -5
View File
@@ -32,7 +32,7 @@ def test_status_download_optimization(mocker, dvc):
@pytest.mark.parametrize("link_name", ["hardlink", "symlink"])
def test_is_protected(tmp_dir, dvc, link_name):
odb = dvc.odb.local
odb = dvc.cache.local
fs = odb.fs
link_method = getattr(fs, link_name)
@@ -67,7 +67,7 @@ def test_protect_ignore_errors(tmp_dir, dvc, mocker, err):
tmp_dir.gen("foo", "foo")
mock_chmod = mocker.patch("os.chmod", side_effect=OSError(err, "something"))
dvc.odb.local.protect("foo")
dvc.cache.local.protect("foo")
assert mock_chmod.called
@@ -76,7 +76,7 @@ def test_set_exec_ignore_errors(tmp_dir, dvc, mocker, err):
tmp_dir.gen("foo", "foo")
mock_chmod = mocker.patch("os.chmod", side_effect=OSError(err, "something"))
dvc.odb.local.set_exec("foo")
dvc.cache.local.set_exec("foo")
assert mock_chmod.called
@@ -88,7 +88,7 @@ def test_staging_file(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
fs = LocalFileSystem()
local_odb = dvc.odb.local
local_odb = dvc.cache.local
staging_odb, _, obj = build(local_odb, (tmp_dir / "foo").fs_path, fs, "md5")
assert not local_odb.exists(obj.hash_info.value)
@@ -113,7 +113,7 @@ def test_staging_dir(tmp_dir, dvc):
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
fs = LocalFileSystem()
local_odb = dvc.odb.local
local_odb = dvc.cache.local
staging_odb, _, obj = build(local_odb, (tmp_dir / "dir").fs_path, fs, "md5")
tests/unit/fs/test_data.py
+5 -5
View File
@@ -58,7 +58,7 @@ def test_open_dirty_hash(tmp_dir, dvc):
def test_open_no_remote(tmp_dir, dvc):
tmp_dir.dvc_gen("file", "file")
(tmp_dir / "file").unlink()
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
fs = DataFileSystem(index=dvc.index.data["repo"])
with pytest.raises(FileNotFoundError):
@@ -238,11 +238,11 @@ def test_get_hash_granular(tmp_dir, dvc):
fs = DataFileSystem(index=dvc.index.data["repo"])
subdir = "dir/subdir"
assert fs.info(subdir).get("md5") is None
_, _, obj = build(dvc.odb.local, subdir, fs, "md5", dry_run=True)
_, _, obj = build(dvc.cache.local, subdir, fs, "md5", dry_run=True)
assert obj.hash_info == HashInfo("md5", "af314506f1622d107e0ed3f14ec1a3b5.dir")
data = posixpath.join(subdir, "data")
assert fs.info(data)["md5"] == "8d777f385d3dfec8815d20f7496026dc"
_, _, obj = build(dvc.odb.local, data, fs, "md5", dry_run=True)
_, _, obj = build(dvc.cache.local, data, fs, "md5", dry_run=True)
assert obj.hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc")
@@ -253,7 +253,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
fs = DataFileSystem(index=dvc.index.data["repo"])
expected = "8c7dd922ad47494fc02c388e12c00eac"
assert fs.info("file").get("md5") == expected
_, _, obj = build(dvc.odb.local, "file", fs, "md5", dry_run=True)
_, _, obj = build(dvc.cache.local, "file", fs, "md5", dry_run=True)
assert obj.hash_info == HashInfo("md5", expected)
@@ -264,5 +264,5 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
fs = DataFileSystem(index=dvc.index.data["repo"])
expected = "5ea40360f5b4ec688df672a4db9c17d1.dir"
assert fs.info("dir").get("md5") == expected
_, _, obj = build(dvc.odb.local, "dir", fs, "md5", dry_run=True)
_, _, obj = build(dvc.cache.local, "dir", fs, "md5", dry_run=True)
assert obj.hash_info == HashInfo("md5", expected)
tests/unit/fs/test_dvc.py
+9 -9
View File
@@ -493,7 +493,7 @@ def test_get_hash_cached_file(tmp_dir, dvc, mocker):
fs = DVCFileSystem(repo=dvc)
expected = "acbd18db4cc2f85cedef654fccc4a4d8"
assert fs.info("foo").get("md5") is None
_, _, obj = build(dvc.odb.local, "foo", fs, "md5")
_, _, obj = build(dvc.cache.local, "foo", fs, "md5")
assert obj.hash_info == HashInfo("md5", expected)
(tmp_dir / "foo").unlink()
assert fs.info("foo")["md5"] == expected
@@ -504,12 +504,12 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker):
fs = DVCFileSystem(repo=dvc)
expected = "8761c4e9acad696bee718615e23e22db.dir"
assert fs.info("dir").get("md5") is None
_, _, obj = build(dvc.odb.local, "dir", fs, "md5")
_, _, obj = build(dvc.cache.local, "dir", fs, "md5")
assert obj.hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir")
shutil.rmtree(tmp_dir / "dir")
assert fs.info("dir")["md5"] == expected
_, _, obj = build(dvc.odb.local, "dir", fs, "md5")
_, _, obj = build(dvc.cache.local, "dir", fs, "md5")
assert obj.hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir")
@@ -518,10 +518,10 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
fs = DVCFileSystem(repo=dvc)
subdir = "dir/subdir"
assert fs.info(subdir).get("md5") is None
_, _, obj = build(dvc.odb.local, subdir, fs, "md5")
_, _, obj = build(dvc.cache.local, subdir, fs, "md5")
assert obj.hash_info == HashInfo("md5", "af314506f1622d107e0ed3f14ec1a3b5.dir")
assert fs.info(posixpath.join(subdir, "data")).get("md5") is None
_, _, obj = build(dvc.odb.local, posixpath.join(subdir, "data"), fs, "md5")
_, _, obj = build(dvc.cache.local, posixpath.join(subdir, "data"), fs, "md5")
assert obj.hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc")
(tmp_dir / "dir" / "subdir" / "data").unlink()
assert (
@@ -543,7 +543,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc):
tmp_dir.scm.commit("add dir")
fs = DVCFileSystem(repo=dvc)
_, _, obj = build(dvc.odb.local, "dir", fs, "md5")
_, _, obj = build(dvc.cache.local, "dir", fs, "md5")
assert obj.hash_info == HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir")
@@ -561,7 +561,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
# hash_file(file) should return workspace hash, not DVC cached hash
fs = DVCFileSystem(repo=dvc)
assert fs.info("file").get("md5") is None
staging, _, obj = build(dvc.odb.local, "file", fs, "md5")
staging, _, obj = build(dvc.cache.local, "file", fs, "md5")
assert obj.hash_info == something_hash_info
check(staging, obj)
@@ -573,7 +573,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
# tmp_dir/file can be built even though it is missing in workspace since
# repofs will use the DVC cached hash (and refer to the local cache object)
_, _, obj = build(dvc.odb.local, "file", fs, "md5")
_, _, obj = build(dvc.cache.local, "file", fs, "md5")
assert obj.hash_info == file_hash_info
@@ -582,7 +582,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
(tmp_dir / "dir" / "baz").write_text("baz")
fs = DVCFileSystem(repo=dvc)
_, meta, obj = build(dvc.odb.local, "dir", fs, "md5")
_, meta, obj = build(dvc.cache.local, "dir", fs, "md5")
assert obj.hash_info == HashInfo("md5", "ba75a2162ca9c29acecb7957105a0bc2.dir")
assert meta.nfiles == 3
tests/unit/output/test_output.py
+1 -1
View File
@@ -113,7 +113,7 @@ def test_remote_missing_dependency_on_dir_pull(tmp_dir, scm, dvc, mocker):
conf["core"] = {"remote": "s3"}
remove("dir")
remove(dvc.odb.local.path)
remove(dvc.cache.local.path)
mocker.patch(
"dvc.data_cloud.DataCloud.get_remote_odb",
tests/unit/stage/test_cache.py
+4 -4
View File
@@ -146,16 +146,16 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker):
def test_shared_stage_cache(tmp_dir, dvc, run_copy):
import stat
from dvc.odbmgr import ODBManager
from dvc.cachemgr import CacheManager
tmp_dir.gen("foo", "foo")
with dvc.config.edit() as config:
config["cache"]["shared"] = "group"
dvc.odb = ODBManager(dvc)
dvc.cache = CacheManager(dvc)
assert not os.path.exists(dvc.odb.local.path)
assert not os.path.exists(dvc.cache.local.path)
run_copy("foo", "bar", name="copy-foo-bar")
@@ -184,7 +184,7 @@ def test_shared_stage_cache(tmp_dir, dvc, run_copy):
dir_mode = 0o2775
file_mode = 0o664
assert _mode(dvc.odb.local.path) == dir_mode
assert _mode(dvc.cache.local.path) == dir_mode
assert _mode(dvc.stage_cache.cache_dir) == dir_mode
assert _mode(parent_cache_dir) == dir_mode
assert _mode(cache_dir) == dir_mode
tests/unit/test_external_repo.py
+6 -6
View File
@@ -55,7 +55,7 @@ def test_subrepo_is_constructed_properly(
subrepo = tmp_dir / "subrepo"
make_subrepo(subrepo, scm)
local_cache = subrepo.dvc.odb.local.path
local_cache = subrepo.dvc.cache.local.path
tmp_dir.scm_gen("bar", "bar", commit="add bar")
subrepo.dvc_gen("foo", "foo", commit="add foo")
@@ -72,16 +72,16 @@ def test_subrepo_is_constructed_properly(
assert repo.url == str(tmp_dir)
assert repo.config["cache"]["dir"] == str(cache_dir)
assert repo.odb.local.path == str(cache_dir)
assert subrepo.odb.local.path == str(cache_dir)
assert repo.cache.local.path == str(cache_dir)
assert subrepo.cache.local.path == str(cache_dir)
assert repo.config["cache"]["type"] == ["symlink"]
assert repo.odb.local.cache_types == ["symlink"]
assert subrepo.odb.local.cache_types == ["symlink"]
assert repo.cache.local.cache_types == ["symlink"]
assert subrepo.cache.local.cache_types == ["symlink"]
assert subrepo.config["remote"]["auto-generated-upstream"]["url"] == local_cache
if root_is_dvc:
main_cache = tmp_dir.dvc.odb.local.path
main_cache = tmp_dir.dvc.cache.local.path
assert repo.config["remote"]["auto-generated-upstream"]["url"] == str(
main_cache
)
tests/unit/test_info.py
+2 -2
View File
@@ -45,7 +45,7 @@ def find_supported_remotes(string):
def test_info_in_repo(scm_init, tmp_dir):
tmp_dir.init(scm=scm_init, dvc=True)
# Create `.dvc/cache`, that is needed to check supported link types.
os.mkdir(tmp_dir.dvc.odb.local.path)
os.mkdir(tmp_dir.dvc.cache.local.path)
dvc_info = get_dvc_info()
@@ -113,7 +113,7 @@ def test_remotes(tmp_dir, dvc, caplog):
def test_fs_info_in_repo(tmp_dir, dvc, caplog):
os.mkdir(dvc.odb.local.path)
os.mkdir(dvc.cache.local.path)
dvc_info = get_dvc_info()
assert re.search(r"Cache directory: .* on .*", dvc_info)