diff --git a/dvc/cache/gdrive.py b/dvc/cache/gdrive.py deleted file mode 100644 index 7cf1f5275..000000000 --- a/dvc/cache/gdrive.py +++ /dev/null @@ -1,5 +0,0 @@ -from .base import CloudCache - - -class GDriveCache(CloudCache): - DEFAULT_VERIFY = True diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 652555b1d..d94811080 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -64,7 +64,7 @@ class DataCloud: remote = self.get_remote(remote, "push") return remote.push( - self.repo.cache.local, + self.repo.odb.local, cache, jobs=jobs, show_checksums=show_checksums, @@ -86,7 +86,7 @@ class DataCloud: remote = self.get_remote(remote, "pull") return remote.pull( - self.repo.cache.local, + self.repo.odb.local, cache, jobs=jobs, show_checksums=show_checksums, @@ -133,7 +133,7 @@ class DataCloud: """ remote = self.get_remote(remote, "status") return remote.status( - self.repo.cache.local, + self.repo.odb.local, cache, jobs=jobs, show_checksums=show_checksums, @@ -142,4 +142,4 @@ class DataCloud: def get_url_for(self, remote, checksum): remote = self.get_remote(remote) - return str(remote.cache.hash_to_path_info(checksum)) + return str(remote.odb.hash_to_path_info(checksum)) diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 5c3e8b64a..e7878382a 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -78,22 +78,18 @@ class RepoDependency(LocalDependency): from dvc.objects import save from dvc.objects.stage import stage - cache = self.repo.cache.local + odb = self.repo.odb.local - with self._make_repo(cache_dir=cache.cache_dir) as repo: + with self._make_repo(cache_dir=odb.cache_dir) as repo: if self.def_repo.get(self.PARAM_REV_LOCK) is None: self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev() path_info = PathInfo(repo.root_dir) / self.def_path obj = stage( - cache, - path_info, - repo.repo_fs, - jobs=jobs, - follow_subrepos=False, + odb, path_info, repo.repo_fs, jobs=jobs, follow_subrepos=False, ) - save(cache, obj, jobs=jobs) + save(odb, obj, jobs=jobs) - checkout(to.path_info, to.fs, obj, cache) + checkout(to.path_info, to.fs, obj, odb) def update(self, rev=None): if rev: diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 3dd2fc96e..f701e15de 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -127,7 +127,7 @@ def _get_remote_config(url): name = "auto-generated-upstream" return { "core": {"remote": name}, - "remote": {name: {"url": repo.cache.local.cache_dir}}, + "remote": {name: {"url": repo.odb.local.cache_dir}}, } # Use original remote to make sure that we are using correct url, diff --git a/dvc/fs/base.py b/dvc/fs/base.py index ed47327ea..d0203fae3 100644 --- a/dvc/fs/base.py +++ b/dvc/fs/base.py @@ -131,8 +131,8 @@ class BaseFileSystem: ) @property - def cache(self): - return getattr(self.repo.cache, self.scheme) + def odb(self): + return getattr(self.repo.odb, self.scheme) def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs): if hasattr(self, "_generate_download_url"): diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index c63bb3d65..fd50b7faf 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -78,14 +78,14 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method checksum = self._get_granular_hash(path, out).value else: checksum = out.hash_info.value - remote_info = remote_obj.cache.hash_to_path_info(checksum) + remote_info = remote_obj.odb.hash_to_path_info(checksum) return remote_obj.fs.open( remote_info, mode=mode, encoding=encoding ) if out.is_dir_checksum: checksum = self._get_granular_hash(path, out).value - cache_path = out.cache.hash_to_path_info(checksum).url + cache_path = out.odb.hash_to_path_info(checksum).url else: cache_path = out.cache_path return open(cache_path, mode=mode, encoding=encoding) @@ -137,7 +137,7 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method from dvc.objects import Tree - hash_info = Tree.save_dir_info(out.cache, dir_cache) + hash_info = Tree.save_dir_info(out.odb, dir_cache) if hash_info != out.hash_info: raise FileNotFoundError diff --git a/dvc/info.py b/dvc/info.py index f0e52ecdc..335265f5d 100644 --- a/dvc/info.py +++ b/dvc/info.py @@ -37,16 +37,16 @@ def get_dvc_info(): # can't auto-create it, as it might cause issues if the user # later decides to enable shared cache mode with # `dvc config cache.shared group`. - if os.path.exists(repo.cache.local.cache_dir): + if os.path.exists(repo.odb.local.cache_dir): info.append( "Cache types: {}".format(_get_linktype_support_info(repo)) ) - fs_type = get_fs_type(repo.cache.local.cache_dir) + fs_type = get_fs_type(repo.odb.local.cache_dir) info.append(f"Cache directory: {fs_type}") else: info.append("Cache types: " + error_link("no-dvc-cache")) - info.append(f"Caches: {_get_caches(repo.cache)}") + info.append(f"Caches: {_get_caches(repo.odb)}") info.append(f"Remotes: {_get_remotes(repo.config)}") @@ -91,7 +91,7 @@ def _get_linktype_support_info(repo): } fname = "." + str(uuid.uuid4()) - src = os.path.join(repo.cache.local.cache_dir, fname) + src = os.path.join(repo.odb.local.cache_dir, fname) open(src, "w").close() dst = os.path.join(repo.root_dir, fname) diff --git a/dvc/cache/__init__.py b/dvc/objects/db/__init__.py similarity index 79% rename from dvc/cache/__init__.py rename to dvc/objects/db/__init__.py index 7238a2378..dd75d2fd6 100644 --- a/dvc/cache/__init__.py +++ b/dvc/objects/db/__init__.py @@ -1,44 +1,37 @@ -"""Manages cache of a DVC repo.""" from collections import defaultdict -from ..scheme import Schemes +from dvc.scheme import Schemes -def get_cloud_cache(fs): - from .base import CloudCache - from .gdrive import GDriveCache - from .local import LocalCache - from .ssh import SSHCache +def get_odb(fs): + from .base import ObjectDB + from .gdrive import GDriveObjectDB + from .local import LocalObjectDB + from .ssh import SSHObjectDB if fs.scheme == Schemes.LOCAL: - return LocalCache(fs) + return LocalObjectDB(fs) if fs.scheme == Schemes.SSH: - return SSHCache(fs) + return SSHObjectDB(fs) if fs.scheme == Schemes.GDRIVE: - return GDriveCache(fs) + return GDriveObjectDB(fs) - return CloudCache(fs) + return ObjectDB(fs) -def _get_cache(repo, settings): - from ..fs import get_cloud_fs +def _get_odb(repo, settings): + from dvc.fs import get_cloud_fs if not settings: return None fs = get_cloud_fs(repo, **settings) - return get_cloud_cache(fs) + return get_odb(fs) -class Cache: - """Class that manages cache locations of a DVC repo. - - Args: - repo (dvc.repo.Repo): repo instance that this cache belongs to. - """ - +class ODBManager: CACHE_DIR = "cache" CLOUD_SCHEMES = [ Schemes.S3, @@ -51,7 +44,7 @@ class Cache: def __init__(self, repo): self.repo = repo self.config = config = repo.config["cache"] - self._cache = {} + self._odb = {} local = config.get("local") @@ -60,33 +53,33 @@ class Cache: elif "dir" not in config: settings = None else: - from ..config_schema import LOCAL_COMMON + from dvc.config_schema import LOCAL_COMMON settings = {"url": config["dir"]} for opt in LOCAL_COMMON.keys(): if opt in config: settings[str(opt)] = config.get(opt) - self._cache[Schemes.LOCAL] = _get_cache(repo, settings) + self._odb[Schemes.LOCAL] = _get_odb(repo, settings) - def _initalize_cloud_cache(self, schemes): + def _init_odb(self, schemes): for scheme in schemes: remote = self.config.get(scheme) settings = {"name": remote} if remote else None - self._cache[scheme] = _get_cache(self.repo, settings) + self._odb[scheme] = _get_odb(self.repo, settings) def __getattr__(self, name): - if name not in self._cache and name in self.CLOUD_SCHEMES: - self._initalize_cloud_cache([name]) + if name not in self._odb and name in self.CLOUD_SCHEMES: + self._init_odb([name]) try: - return self._cache[name] + return self._odb[name] except KeyError as exc: raise AttributeError from exc def by_scheme(self): - self._initalize_cloud_cache(self.CLOUD_SCHEMES) - yield from self._cache.items() + self._init_odb(self.CLOUD_SCHEMES) + yield from self._odb.items() class NamedCacheItem: diff --git a/dvc/cache/base.py b/dvc/objects/db/base.py similarity index 99% rename from dvc/cache/base.py rename to dvc/objects/db/base.py index 7c6ee117b..6bb51f879 100644 --- a/dvc/cache/base.py +++ b/dvc/objects/db/base.py @@ -6,12 +6,12 @@ from typing import Optional from dvc.progress import Tqdm -from ..objects import HashFile, ObjectFormatError +from .. import HashFile, ObjectFormatError logger = logging.getLogger(__name__) -class CloudCache: +class ObjectDB: DEFAULT_VERIFY = False DEFAULT_CACHE_TYPES = ["copy"] diff --git a/dvc/objects/db/gdrive.py b/dvc/objects/db/gdrive.py new file mode 100644 index 000000000..05368a6b2 --- /dev/null +++ b/dvc/objects/db/gdrive.py @@ -0,0 +1,5 @@ +from .base import ObjectDB + + +class GDriveObjectDB(ObjectDB): + DEFAULT_VERIFY = True diff --git a/dvc/cache/local.py b/dvc/objects/db/local.py similarity index 97% rename from dvc/cache/local.py rename to dvc/objects/db/local.py index 4c60b602e..46a6c296e 100644 --- a/dvc/cache/local.py +++ b/dvc/objects/db/local.py @@ -9,15 +9,15 @@ from dvc.hash_info import HashInfo from dvc.objects import ObjectFormatError from dvc.path_info import PathInfo from dvc.progress import Tqdm +from dvc.utils import relpath +from dvc.utils.fs import copyfile, remove, umask, walk_files -from ..utils import relpath -from ..utils.fs import copyfile, remove, umask, walk_files -from .base import CloudCache +from .base import ObjectDB logger = logging.getLogger(__name__) -class LocalCache(CloudCache): +class LocalObjectDB(ObjectDB): DEFAULT_CACHE_TYPES = ["reflink", "copy"] CACHE_MODE = 0o444 UNPACKED_DIR_SUFFIX = ".unpacked" diff --git a/dvc/cache/ssh.py b/dvc/objects/db/ssh.py similarity index 98% rename from dvc/cache/ssh.py rename to dvc/objects/db/ssh.py index 2033df0c8..e2b0198c9 100644 --- a/dvc/cache/ssh.py +++ b/dvc/objects/db/ssh.py @@ -7,12 +7,12 @@ from concurrent.futures import ThreadPoolExecutor from dvc.progress import Tqdm from dvc.utils import to_chunks -from .base import CloudCache +from .base import ObjectDB logger = logging.getLogger(__name__) -class SSHCache(CloudCache): +class SSHObjectDB(ObjectDB): def batch_exists(self, path_infos, callback): def _exists(chunk_and_channel): chunk, channel = chunk_and_channel diff --git a/dvc/objects/stage.py b/dvc/objects/stage.py index 60cfdd112..978c2d020 100644 --- a/dvc/objects/stage.py +++ b/dvc/objects/stage.py @@ -92,7 +92,7 @@ def get_dir_hash(path_info, fs, name, **kwargs): from . import Tree dir_info = _collect_dir(path_info, fs, name, **kwargs) - hash_info = Tree.save_dir_info(fs.repo.cache.local, dir_info) + hash_info = Tree.save_dir_info(fs.repo.odb.local, dir_info) hash_info.size = dir_info.size hash_info.dir_info = dir_info return hash_info @@ -118,9 +118,7 @@ def get_hash(path_info, fs, name, **kwargs): if ( hash_info and hash_info.isdir - and not fs.cache.fs.exists( - fs.cache.hash_to_path_info(hash_info.value) - ) + and not fs.odb.fs.exists(fs.odb.hash_to_path_info(hash_info.value)) ): hash_info = None @@ -129,7 +127,7 @@ def get_hash(path_info, fs, name, **kwargs): from . import Tree # NOTE: loading the fs will restore hash_info.dir_info - Tree.load(fs.cache, hash_info) + Tree.load(fs.odb, hash_info) assert hash_info.name == name return hash_info diff --git a/dvc/output/base.py b/dvc/output/base.py index 30affa175..cdc812afb 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -8,7 +8,6 @@ from voluptuous import Any import dvc.objects as objects import dvc.prompt as prompt -from dvc.cache import NamedCache from dvc.checkout import checkout from dvc.exceptions import ( CheckoutError, @@ -18,6 +17,7 @@ from dvc.exceptions import ( RemoteCacheRequiredError, ) from dvc.hash_info import HashInfo +from dvc.objects.db import NamedCache from dvc.objects.stage import get_hash from dvc.objects.stage import stage as ostage @@ -144,7 +144,7 @@ class BaseOutput: self.desc = desc self.path_info = self._parse_path(fs, path) - if self.use_cache and self.cache is None: + if self.use_cache and self.odb is None: raise RemoteCacheRequiredError(self.path_info) self.obj = None @@ -180,8 +180,8 @@ class BaseOutput: return self.use_cache or self.stage.is_repo_import @property - def cache(self): - return getattr(self.repo.cache, self.scheme) + def odb(self): + return getattr(self.repo.odb, self.scheme) @property def dir_cache(self): @@ -189,12 +189,12 @@ class BaseOutput: @property def cache_path(self): - return self.cache.hash_to_path_info(self.hash_info.value).url + return self.odb.hash_to_path_info(self.hash_info.value).url def get_hash(self): if not self.use_cache: return get_hash(self.path_info, self.fs, self.fs.PARAM_CHECKSUM) - return ostage(self.cache, self.path_info, self.fs).hash_info + return ostage(self.odb, self.path_info, self.fs).hash_info @property def is_dir_checksum(self): @@ -216,7 +216,7 @@ class BaseOutput: return True try: - objects.check(self.cache, obj) + objects.check(self.odb, obj) return False except (FileNotFoundError, objects.ObjectFormatError): return True @@ -302,13 +302,13 @@ class BaseOutput: logger.debug("Output '%s' didn't change. Skipping saving.", self) return - self.obj = ostage(self.cache, self.path_info, self.fs) + self.obj = ostage(self.odb, self.path_info, self.fs) self.hash_info = self.obj.hash_info self.isexec = self.isfile() and self.fs.isexec(self.path_info) def set_exec(self): if self.isfile() and self.isexec: - self.cache.set_exec(self.path_info) + self.odb.set_exec(self.path_info) def commit(self, filter_info=None): if not self.exists: @@ -317,13 +317,13 @@ class BaseOutput: assert self.hash_info if self.use_cache: - obj = ostage(self.cache, filter_info or self.path_info, self.fs) - objects.save(self.cache, obj) + obj = ostage(self.odb, filter_info or self.path_info, self.fs) + objects.save(self.odb, obj) checkout( filter_info or self.path_info, self.fs, obj, - self.cache, + self.odb, relink=True, ) self.set_exec() @@ -379,7 +379,7 @@ class BaseOutput: obj = self.obj elif self.hash_info: try: - obj = objects.load(self.cache, self.hash_info) + obj = objects.load(self.odb, self.hash_info) except FileNotFoundError: return None else: @@ -387,7 +387,7 @@ class BaseOutput: if filter_info and filter_info != self.path_info: prefix = filter_info.relative_to(self.path_info).parts - obj = obj.filter(self.cache, prefix) + obj = obj.filter(self.odb, prefix) return obj @@ -419,7 +419,7 @@ class BaseOutput: filter_info or self.path_info, self.fs, obj, - self.cache, + self.odb, force=force, progress_callback=progress_callback, relink=relink, @@ -476,7 +476,7 @@ class BaseOutput: def unprotect(self): if self.exists: - self.cache.unprotect(self.path_info) + self.odb.unprotect(self.path_info) def get_dir_cache(self, **kwargs): @@ -484,7 +484,7 @@ class BaseOutput: raise DvcException("cannot get dir cache for file checksum") try: - objects.check(self.cache, self.cache.get(self.hash_info)) + objects.check(self.odb, self.odb.get(self.hash_info)) except (FileNotFoundError, objects.ObjectFormatError): self.repo.cloud.pull( NamedCache.make("local", self.hash_info.value, str(self)), @@ -493,7 +493,7 @@ class BaseOutput: ) try: - objects.load(self.cache, self.hash_info) + objects.load(self.odb, self.hash_info) assert self.hash_info.dir_info except (objects.ObjectFormatError, FileNotFoundError): self.hash_info.dir_info = None @@ -530,7 +530,7 @@ class BaseOutput: logger.debug(f"failed to pull cache for '{self}'") try: - objects.check(self.cache, self.cache.get(self.hash_info)) + objects.check(self.odb, self.odb.get(self.hash_info)) except (FileNotFoundError, objects.ObjectFormatError): msg = ( "Missing cache for directory '{}'. " @@ -661,5 +661,5 @@ class BaseOutput: self._check_can_merge(other) self.hash_info = objects.merge( - self.cache, ancestor_info, self.hash_info, other.hash_info + self.odb, ancestor_info, self.hash_info, other.hash_info ) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 487a99879..6cf4ac0d8 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -68,11 +68,11 @@ class Remote: INDEX_CLS = RemoteIndex def __init__(self, fs): - from dvc.cache import get_cloud_cache + from dvc.objects.db import get_odb self.fs = fs self.repo = fs.repo - self.cache = get_cloud_cache(self.fs) + self.odb = get_odb(self.fs) config = fs.config url = config.get("url") @@ -92,7 +92,7 @@ class Remote: @index_locked def gc(self, *args, **kwargs): - removed = self.cache.gc(*args, **kwargs) + removed = self.odb.gc(*args, **kwargs) if removed: self.index.clear() @@ -129,7 +129,7 @@ class Remote: if not hashes: return indexed_hashes - return indexed_hashes + self.cache.hashes_exist(list(hashes), **kwargs) + return indexed_hashes + self.odb.hashes_exist(list(hashes), **kwargs) def _status( self, @@ -227,7 +227,7 @@ class Remote: indexed_dir_exists = set() if indexed_dirs: indexed_dir_exists.update( - self.cache.list_hashes_exists(indexed_dirs) + self.odb.list_hashes_exists(indexed_dirs) ) missing_dirs = indexed_dirs.difference(indexed_dir_exists) if missing_dirs: @@ -239,7 +239,7 @@ class Remote: # Check if non-indexed (new) dir hashes exist on remote dir_exists = dir_md5s.intersection(indexed_dir_exists) - dir_exists.update(self.cache.list_hashes_exists(dir_md5s - dir_exists)) + dir_exists.update(self.odb.list_hashes_exists(dir_md5s - dir_exists)) # If .dir hash exists on the remote, assume directory contents # still exists on the remote @@ -278,7 +278,7 @@ class Remote: ): if info["status"] == status: cache.append(cache_obj.hash_to_path_info(md5)) - path_infos.append(self.cache.hash_to_path_info(md5)) + path_infos.append(self.odb.hash_to_path_info(md5)) names.append(info["name"]) hashes.append(md5) elif info["status"] == STATUS_MISSING: @@ -474,11 +474,11 @@ class Remote: if self.fs.scheme == "local": with self.fs.repo.state: for checksum in named_cache.scheme_keys("local"): - cache_file = self.cache.hash_to_path_info(checksum) + cache_file = self.odb.hash_to_path_info(checksum) if self.fs.exists(cache_file): hash_info = HashInfo(self.fs.PARAM_CHECKSUM, checksum) self.fs.repo.state.save(cache_file, self.fs, hash_info) - self.cache.protect(cache_file) + self.odb.protect(cache_file) return ret @@ -492,7 +492,7 @@ class Remote: download=True, ) - if not self.cache.verify: + if not self.odb.verify: with cache.fs.repo.state: for checksum in named_cache.scheme_keys("local"): cache_file = cache.hash_to_path_info(checksum) @@ -513,7 +513,7 @@ class Remote: from dvc.objects import transfer return transfer( - self.cache, + self.odb, from_fs, from_info, jobs=jobs, diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 0ca9303d6..076b44dd5 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -131,11 +131,11 @@ class Repo: url=None, repo_factory=None, ): - from dvc.cache import Cache from dvc.config import Config from dvc.data_cloud import DataCloud from dvc.fs.local import LocalFileSystem from dvc.lock import LockNoop, make_lock + from dvc.objects.db import ODBManager from dvc.repo.live import Live from dvc.repo.metrics import Metrics from dvc.repo.params import Params @@ -168,7 +168,7 @@ class Repo: # used by RepoFileSystem to determine if it should traverse subrepos self.subrepos = subrepos - self.cache = Cache(self) + self.odb = ODBManager(self) self.cloud = DataCloud(self) self.stage = StageLoad(self) @@ -307,7 +307,7 @@ class Repo: return Repo(root_dir) def unprotect(self, target): - return self.cache.local.unprotect(PathInfo(target)) + return self.odb.local.unprotect(PathInfo(target)) def _ignore(self): flist = [ @@ -315,8 +315,8 @@ class Repo: self.tmp_dir, ] - if path_isin(self.cache.local.cache_dir, self.root_dir): - flist += [self.cache.local.cache_dir] + if path_isin(self.odb.local.cache_dir, self.root_dir): + flist += [self.odb.local.cache_dir] self.scm.ignore_list(flist) @@ -368,7 +368,7 @@ class Repo: to items containing the output's `dumpd` names and the output's children (if the given output is a directory). """ - from dvc.cache import NamedCache + from dvc.objects.db import NamedCache cache = NamedCache() diff --git a/dvc/repo/add.py b/dvc/repo/add.py index fc0f05472..d86b46360 100644 --- a/dvc/repo/add.py +++ b/dvc/repo/add.py @@ -170,7 +170,7 @@ def _process_stages( from_fs = get_cloud_fs(repo, url=target) out.hash_info = transfer( - out.cache, from_fs, from_fs.path_info, jobs=kwargs.get("jobs"), + out.odb, from_fs, from_fs.path_info, jobs=kwargs.get("jobs"), ) out.checkout() diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index db7a3ed5c..7dd968072 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -128,7 +128,7 @@ def _output_paths(repo, repo_fs, targets): def _to_checksum(output): if on_working_fs: - return get_hash(output.path_info, repo.cache.local.fs, "md5").value + return get_hash(output.path_info, repo.odb.local.fs, "md5").value return output.hash_info.value for stage in repo.stages: diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index ffa6d7afb..49c7db0cb 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -593,7 +593,7 @@ class Experiments: name=item.name, branch=item.branch, tmp_dir=base_tmp_dir, - cache_dir=self.repo.cache.local.cache_dir, + cache_dir=self.repo.odb.local.cache_dir, ) executors[stash_rev] = executor diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index b68b8678f..6bfc73db6 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -93,10 +93,10 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs): def cb(result): results.append(result) - cache = self.cache.local + odb = self.odb.local try: with external_repo( - repo_url, repo_rev, cache_dir=cache.cache_dir + repo_url, repo_rev, cache_dir=odb.cache_dir ) as repo: root = PathInfo(repo.root_dir) for path in files: @@ -112,14 +112,14 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs): except (NoOutputOrStageError, NoRemoteError): pass obj = stage( - cache, + odb, path_info, repo.repo_fs, jobs=jobs, follow_subrepos=False, ) save( - cache, obj, jobs=jobs, download_callback=cb, + odb, obj, jobs=jobs, download_callback=cb, ) except CloneError: failed += 1 diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index 955cb3c52..14dee1387 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -1,7 +1,7 @@ import logging -from dvc.cache import NamedCache from dvc.exceptions import InvalidArgumentError +from dvc.objects.db import NamedCache from ..scheme import Schemes from . import locked @@ -72,11 +72,11 @@ def gc( ) ) - for scheme, cache in self.cache.by_scheme(): - if not cache: + for scheme, odb in self.odb.by_scheme(): + if not odb: continue - removed = cache.gc(set(used.scheme_keys(scheme)), jobs=jobs) + removed = odb.gc(set(used.scheme_keys(scheme)), jobs=jobs) if not removed: logger.info(f"No unused '{scheme}' cache to remove.") diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index a38f1293e..bbbed8beb 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -614,7 +614,7 @@ class Stage(params.StageParams): ) def get_used_cache(self, *args, **kwargs): - from dvc.cache import NamedCache + from dvc.objects.db import NamedCache cache = NamedCache() for out in self.filter_outs(kwargs.get("filter_info")): diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index 4e0b5a78b..18bc5d809 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -58,7 +58,7 @@ class StageCache: @cached_property def cache_dir(self): - return os.path.join(self.repo.cache.local.cache_dir, "runs") + return os.path.join(self.repo.odb.local.cache_dir, "runs") def _get_cache_dir(self, key): return os.path.join(self.cache_dir, key[:2], key) @@ -117,12 +117,12 @@ class StageCache: @contextmanager def _cache_type_copy(self): - cache_types = self.repo.cache.local.cache_types - self.repo.cache.local.cache_types = ["copy"] + cache_types = self.repo.odb.local.cache_types + self.repo.odb.local.cache_types = ["copy"] try: yield finally: - self.repo.cache.local.cache_types = cache_types + self.repo.odb.local.cache_types = cache_types def _uncached_outs(self, stage, cache): # NOTE: using temporary stage to avoid accidentally modifying original @@ -166,12 +166,12 @@ class StageCache: COMPILED_LOCK_FILE_STAGE_SCHEMA(cache) path = PathInfo(self._get_cache_path(cache_key, cache_value)) - self.repo.cache.local.makedirs(path.parent) + self.repo.odb.local.makedirs(path.parent) tmp = tempfile.NamedTemporaryFile(delete=False, dir=path.parent).name assert os.path.exists(path.parent) assert os.path.isdir(path.parent) dump_yaml(tmp, cache) - self.repo.cache.local.move(PathInfo(tmp), path) + self.repo.odb.local.move(PathInfo(tmp), path) def restore(self, stage, run_cache=True, pull=False): from .serialize import to_single_stage_lockfile @@ -233,7 +233,7 @@ class StageCache: remote = self.repo.cloud.get_remote(remote) return self._transfer( _log_exceptions(remote.fs.upload, "upload"), - self.repo.cache.local.fs, + self.repo.odb.local.fs, remote.fs, ) @@ -244,11 +244,11 @@ class StageCache: return self._transfer( _log_exceptions(remote.fs.download, "download"), remote.fs, - self.repo.cache.local.fs, + self.repo.odb.local.fs, ) def get_used_cache(self, used_run_cache, *args, **kwargs): - from dvc.cache import NamedCache + from dvc.objects.db import NamedCache cache = NamedCache() diff --git a/tests/func/api/test_api.py b/tests/func/api/test_api.py index 5a5555479..6050e1a39 100644 --- a/tests/func/api/test_api.py +++ b/tests/func/api/test_api.py @@ -64,7 +64,7 @@ def test_open(tmp_dir, dvc, remote): dvc.push() # Remove cache to force download - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) with api.open("foo") as fd: assert fd.read() == "foo-text" @@ -100,7 +100,7 @@ def test_open_external(tmp_dir, erepo_dir, cloud): erepo_dir.dvc.push(all_branches=True) # Remove cache to force download - remove(erepo_dir.dvc.cache.local.cache_dir) + remove(erepo_dir.dvc.odb.local.cache_dir) # Using file url to force clone to tmp repo repo_url = f"file://{erepo_dir}" @@ -117,7 +117,7 @@ def test_open_granular(tmp_dir, dvc, remote): dvc.push() # Remove cache to force download - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) with api.open("dir/foo") as fd: assert fd.read() == "foo-text" @@ -145,7 +145,7 @@ def test_missing(tmp_dir, dvc, remote): tmp_dir.dvc_gen("foo", "foo") # Remove cache to make foo missing - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) api.read("foo") @@ -199,7 +199,7 @@ def test_api_missing_local_cache_exists_on_remote( dvc.push() # Remove cache to make foo missing - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) remove(first(files)) repo_url = f"file://{tmp_dir}" if as_external else None diff --git a/tests/func/experiments/test_remote.py b/tests/func/experiments/test_remote.py index 98e591233..cd2d338f6 100644 --- a/tests/func/experiments/test_remote.py +++ b/tests/func/experiments/test_remote.py @@ -262,11 +262,11 @@ def test_push_pull_cache( assert os.path.exists(path) assert open(path).read() == str(x) - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) dvc.experiments.pull(remote, ref_info.name, pull_cache=True) for x in range(2, checkpoint_stage.iterations + 1): hash_ = digest(str(x)) - path = os.path.join(dvc.cache.local.cache_dir, hash_[:2], hash_[2:]) + path = os.path.join(dvc.odb.local.cache_dir, hash_[:2], hash_[2:]) assert os.path.exists(path) assert open(path).read() == str(x) diff --git a/tests/func/metrics/test_show.py b/tests/func/metrics/test_show.py index 64fcf156c..c9cd85c39 100644 --- a/tests/func/metrics/test_show.py +++ b/tests/func/metrics/test_show.py @@ -220,7 +220,7 @@ def test_metrics_show_overlap( # so as it works even for optimized cases if clear_before_run: remove(data_dir) - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) dvc._reset() diff --git a/tests/func/plots/test_show.py b/tests/func/plots/test_show.py index 876ab5512..5d45c6d32 100644 --- a/tests/func/plots/test_show.py +++ b/tests/func/plots/test_show.py @@ -754,7 +754,7 @@ def test_plots_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run): # so as it works even for optimized cases if clear_before_run: remove(data_dir) - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) dvc._reset() diff --git a/tests/func/remote/test_index.py b/tests/func/remote/test_index.py index ea931f8df..150452342 100644 --- a/tests/func/remote/test_index.py +++ b/tests/func/remote/test_index.py @@ -82,7 +82,7 @@ def test_clear_on_gc(tmp_dir, dvc, tmp_path_factory, remote, mocker): def test_clear_on_download_err(tmp_dir, dvc, tmp_path_factory, remote, mocker): tmp_dir.dvc_gen({"foo": "foo content"}) dvc.push() - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) mocked_clear = mocker.patch.object(remote.INDEX_CLS, "clear") mocker.patch.object(LocalFileSystem, "_download", side_effect=Exception) diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 3ef388f25..d94b18a9f 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -12,7 +12,6 @@ import pytest from mock import call, patch import dvc as dvc_module -from dvc.cache import Cache from dvc.dvcfile import DVC_FILE_SUFFIX from dvc.exceptions import ( DvcException, @@ -24,6 +23,7 @@ from dvc.exceptions import ( from dvc.fs.local import LocalFileSystem from dvc.hash_info import HashInfo from dvc.main import main +from dvc.objects.db import ODBManager from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError from dvc.repo import Repo as DvcRepo from dvc.stage import Stage @@ -106,7 +106,7 @@ def test_add_directory(tmp_dir, dvc): hash_info = stage.outs[0].hash_info - dir_info = load(dvc.cache.local, hash_info).hash_info.dir_info + dir_info = load(dvc.odb.local, hash_info).hash_info.dir_info for path, _ in dir_info.trie.items(): assert "\\" not in path @@ -511,12 +511,12 @@ class TestAddCommit(TestDvc): ret = main(["add", self.FOO, "--no-commit"]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.FOO)) - self.assertFalse(os.path.exists(self.dvc.cache.local.cache_dir)) + self.assertFalse(os.path.exists(self.dvc.odb.local.cache_dir)) ret = main(["commit", self.FOO + ".dvc"]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.FOO)) - self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 1) + self.assertEqual(len(os.listdir(self.dvc.odb.local.cache_dir)), 1) def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc): @@ -700,7 +700,7 @@ def test_windows_should_add_when_cache_on_different_drive( tmp_dir, dvc, temporary_windows_drive ): dvc.config["cache"]["dir"] = temporary_windows_drive - dvc.cache = Cache(dvc) + dvc.odb = ODBManager(dvc) (stage,) = tmp_dir.dvc_gen({"file": "file"}) cache_path = stage.outs[0].cache_path @@ -713,12 +713,12 @@ def test_windows_should_add_when_cache_on_different_drive( def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker): tmp_dir.gen("dir/data", "data") - dvc.cache.local.cache_types = ["symlink"] + dvc.odb.local.cache_types = ["symlink"] dvc.add("dir") tmp_dir.gen("dir/new_file", "new_file_content") - unprotect_spy = mocker.spy(dvc.cache.local, "unprotect") + unprotect_spy = mocker.spy(dvc.odb.local, "unprotect") dvc.add("dir") assert not unprotect_spy.mock.called @@ -726,13 +726,13 @@ def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker): def test_should_not_checkout_when_adding_cached_copy(tmp_dir, dvc, mocker): - dvc.cache.local.cache_types = ["copy"] + dvc.odb.local.cache_types = ["copy"] tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) shutil.copy("bar", "foo") - copy_spy = mocker.spy(dvc.cache.local.fs, "copy") + copy_spy = mocker.spy(dvc.odb.local.fs, "copy") dvc.add("foo") @@ -758,9 +758,9 @@ def test_should_relink_on_repeated_add( tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) os.remove("foo") - getattr(dvc.cache.local.fs, link)(PathInfo("bar"), PathInfo("foo")) + getattr(dvc.odb.local.fs, link)(PathInfo("bar"), PathInfo("foo")) - dvc.cache.local.cache_types = [new_link] + dvc.odb.local.cache_types = [new_link] dvc.add("foo") @@ -769,7 +769,7 @@ def test_should_relink_on_repeated_add( @pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"]) def test_should_protect_on_repeated_add(link, tmp_dir, dvc): - dvc.cache.local.cache_types = [link] + dvc.odb.local.cache_types = [link] tmp_dir.dvc_gen({"foo": "foo"}) @@ -827,7 +827,7 @@ def test_not_raises_on_re_add(tmp_dir, dvc): @pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"]) def test_add_empty_files(tmp_dir, dvc, link): file = "foo" - dvc.cache.local.cache_types = [link] + dvc.odb.local.cache_types = [link] stages = tmp_dir.dvc_gen(file, "") assert (tmp_dir / file).exists() @@ -836,7 +836,7 @@ def test_add_empty_files(tmp_dir, dvc, link): def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker): - dvc.cache.local.cache_types = ["hardlink"] + dvc.odb.local.cache_types = ["hardlink"] tmp_dir.gen({"foo": "", "bar": "", "lorem": "lorem", "ipsum": "ipsum"}) m = mocker.spy(LocalFileSystem, "is_hardlink") stages = dvc.add(["foo", "bar", "lorem", "ipsum"]) diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index d9d4ca544..98d604684 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -101,12 +101,10 @@ class TestCheckoutCorruptedCacheDir(TestDvc): # NOTE: modifying cache file for one of the files inside the directory # to check if dvc will detect that the cache is corrupted. _, entry_hash = next( - load( - self.dvc.cache.local, out.hash_info - ).hash_info.dir_info.items() + load(self.dvc.odb.local, out.hash_info).hash_info.dir_info.items() ) cache = os.fspath( - self.dvc.cache.local.hash_to_path_info(entry_hash.value) + self.dvc.odb.local.hash_to_path_info(entry_hash.value) ) os.chmod(cache, 0o644) @@ -470,7 +468,7 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc): self.assertTrue(System.is_symlink(self.DATA)) old_data_link = os.path.realpath(self.DATA) - old_cache_dir = self.dvc.cache.local.cache_dir + old_cache_dir = self.dvc.odb.local.cache_dir new_cache_dir = old_cache_dir + "_new" os.rename(old_cache_dir, new_cache_dir) @@ -514,7 +512,7 @@ def test_checkout_no_checksum(tmp_dir, dvc): [("hardlink", System.is_hardlink), ("symlink", System.is_symlink)], ) def test_checkout_relink(tmp_dir, dvc, link, link_test_func): - dvc.cache.local.cache_types = [link] + dvc.odb.local.cache_types = [link] tmp_dir.dvc_gen({"dir": {"data": "text"}}) dvc.unprotect("dir/data") @@ -527,7 +525,7 @@ def test_checkout_relink(tmp_dir, dvc, link, link_test_func): @pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"]) def test_checkout_relink_protected(tmp_dir, dvc, link): - dvc.cache.local.cache_types = [link] + dvc.odb.local.cache_types = [link] tmp_dir.dvc_gen("foo", "foo") dvc.unprotect("foo") @@ -718,7 +716,7 @@ def test_checkout_with_relink_existing(tmp_dir, dvc, link): (tmp_dir / "foo").unlink() tmp_dir.dvc_gen("bar", "bar") - dvc.cache.local.cache_types = [link] + dvc.odb.local.cache_types = [link] stats = dvc.checkout(relink=True) assert stats == {**empty_checkout, "added": ["foo"]} diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index f9e8e2fa9..92021c8ce 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -6,9 +6,9 @@ import pytest from flaky.flaky_decorator import flaky import dvc as dvc_module -from dvc.cache import NamedCache from dvc.external_repo import clean_repos from dvc.main import main +from dvc.objects.db import NamedCache from dvc.remote.base import ( STATUS_DELETED, STATUS_MISSING, @@ -76,8 +76,8 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument # Move cache and check status # See issue https://github.com/iterative/dvc/issues/4383 for details - backup_dir = dvc.cache.local.cache_dir + ".backup" - move(dvc.cache.local.cache_dir, backup_dir) + backup_dir = dvc.odb.local.cache_dir + ".backup" + move(dvc.odb.local.cache_dir, backup_dir) status = dvc.cloud.status(info, show_checksums=True) expected = {md5: {"name": md5, "status": STATUS_MISSING}} assert status == expected @@ -87,8 +87,8 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument assert status_dir == expected # Restore original cache: - remove(dvc.cache.local.cache_dir) - move(backup_dir, dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) + move(backup_dir, dvc.odb.local.cache_dir) # Push and check status dvc.cloud.push(info) @@ -107,7 +107,7 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument assert status_dir == expected # Remove and check status - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) status = dvc.cloud.status(info, show_checksums=True) expected = {md5: {"name": md5, "status": STATUS_DELETED}} @@ -162,7 +162,7 @@ def test_cloud_cli(tmp_dir, dvc, remote): assert os.path.isfile(cache) assert os.path.isfile(cache_dir) - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) assert main(["fetch"] + args) == 0 assert os.path.exists(cache) @@ -187,7 +187,7 @@ def test_cloud_cli(tmp_dir, dvc, remote): # NOTE: check if remote gc works correctly on directories assert main(["gc", "-cw", "-f"] + args) == 0 shutil.move( - dvc.cache.local.cache_dir, dvc.cache.local.cache_dir + ".back", + dvc.odb.local.cache_dir, dvc.odb.local.cache_dir + ".back", ) assert main(["fetch"] + args) == 0 @@ -247,7 +247,7 @@ def test_missing_cache(tmp_dir, dvc, local_remote, caplog): tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) # purge cache - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) header = ( "Some of the cache files do not exist " @@ -285,7 +285,7 @@ def test_verify_hashes( # remove artifacts and cache to trigger fetching remove("file") remove("dir") - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) hash_spy = mocker.spy(dvc_module.objects.stage, "get_file_hash") @@ -293,7 +293,7 @@ def test_verify_hashes( assert hash_spy.call_count == 0 # Removing cache will invalidate existing state entries - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) dvc.config["remote"]["upstream"]["verify"] = True @@ -315,9 +315,9 @@ def test_pull_git_imports(tmp_dir, dvc, scm, erepo): assert dvc.pull()["fetched"] == 0 - for item in ["foo", "new_dir", dvc.cache.local.cache_dir]: + for item in ["foo", "new_dir", dvc.odb.local.cache_dir]: remove(item) - os.makedirs(dvc.cache.local.cache_dir, exist_ok=True) + os.makedirs(dvc.odb.local.cache_dir, exist_ok=True) clean_repos() assert dvc.pull(force=True)["fetched"] == 2 @@ -355,12 +355,12 @@ def test_pull_external_dvc_imports(tmp_dir, dvc, scm, erepo_dir): def clean(outs, dvc=None): if dvc: - outs = outs + [dvc.cache.local.cache_dir] + outs = outs + [dvc.odb.local.cache_dir] for path in outs: print(path) remove(path) if dvc: - os.makedirs(dvc.cache.local.cache_dir, exist_ok=True) + os.makedirs(dvc.odb.local.cache_dir, exist_ok=True) clean_repos() @@ -532,8 +532,8 @@ def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote): dvc.pull("copy-foo-bar") assert (tmp_dir / "bar").exists() - assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1 + assert len(recurse_list_dir(dvc.odb.local.cache_dir)) == 1 clean(["bar"], dvc) dvc.fetch("copy-foo-bar") - assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1 + assert len(recurse_list_dir(dvc.odb.local.cache_dir)) == 1 diff --git a/tests/func/test_diff.py b/tests/func/test_diff.py index a6abb696d..37cdbbb59 100644 --- a/tests/func/test_diff.py +++ b/tests/func/test_diff.py @@ -201,10 +201,10 @@ def test_diff_no_cache(tmp_dir, scm, dvc): ) scm.tag("v2") - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) # invalidate_dir_info to force cache loading - dvc.cache.local._dir_info = {} + dvc.odb.local._dir_info = {} diff = dvc.diff("v1", "v2") assert diff["added"] == [] diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 115e2ce95..84b3e62a1 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -114,7 +114,7 @@ def test_relative_remote(erepo_dir, tmp_dir): erepo_dir.dvc.push() (erepo_dir / "file").unlink() - remove(erepo_dir.dvc.cache.local.cache_dir) + remove(erepo_dir.dvc.odb.local.cache_dir) url = os.fspath(erepo_dir) @@ -197,17 +197,17 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir): assert (tmp_dir / "out").read_text() == expected_files # clear cache to test saving to cache - cache_dir = tmp_dir / repo.cache.local.cache_dir + cache_dir = tmp_dir / repo.odb.local.cache_dir remove(cache_dir) makedirs(cache_dir) obj = stage( - repo.cache.local, + repo.odb.local, PathInfo(repo.root_dir) / "dir", repo.repo_fs, follow_subrepos=False, ) - save(repo.cache.local, obj) + save(repo.odb.local, obj) assert set(cache_dir.glob("*/*")) == { cache_dir / "e1" / "d9e8eae5374860ae025ec84cfd85c7.dir", cache_dir / "37" / "b51d194a7513e45b56f6524f2d51f2", diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index 95314f208..c8f6521ca 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -6,10 +6,10 @@ import configobj import pytest from git import Repo -from dvc.cache.local import LocalCache from dvc.exceptions import CollectCacheError from dvc.fs.local import LocalFileSystem from dvc.main import main +from dvc.objects.db.local import LocalObjectDB from dvc.repo import Repo as DvcRepo from dvc.utils.fs import remove from tests.basic_env import TestDir, TestDvcGit @@ -22,13 +22,13 @@ class TestGC(TestDvcGit): self.dvc.add(self.FOO) self.dvc.add(self.DATA_DIR) self.good_cache = [ - self.dvc.cache.local.hash_to_path_info(md5) - for md5 in self.dvc.cache.local.all() + self.dvc.odb.local.hash_to_path_info(md5) + for md5 in self.dvc.odb.local.all() ] self.bad_cache = [] for i in ["123", "234", "345"]: - path = os.path.join(self.dvc.cache.local.cache_dir, i[0:2], i[2:]) + path = os.path.join(self.dvc.odb.local.cache_dir, i[0:2], i[2:]) self.create(path, i) self.bad_cache.append(path) @@ -42,7 +42,7 @@ class TestGC(TestDvcGit): self._test_gc() def _test_gc(self): - self.assertTrue(os.path.isdir(self.dvc.cache.local.cache_dir)) + self.assertTrue(os.path.isdir(self.dvc.odb.local.cache_dir)) for c in self.bad_cache: self.assertFalse(os.path.exists(c)) @@ -187,11 +187,11 @@ def test_all_commits(tmp_dir, scm, dvc): tmp_dir.dvc_gen("testfile", "modified", commit="modified") tmp_dir.dvc_gen("testfile", "workspace") - n = _count_files(dvc.cache.local.cache_dir) + n = _count_files(dvc.odb.local.cache_dir) dvc.gc(all_commits=True) # Only one uncommitted file should go away - assert _count_files(dvc.cache.local.cache_dir) == n - 1 + assert _count_files(dvc.odb.local.cache_dir) == n - 1 def test_gc_no_dir_cache(tmp_dir, dvc): @@ -203,9 +203,9 @@ def test_gc_no_dir_cache(tmp_dir, dvc): with pytest.raises(CollectCacheError): dvc.gc(workspace=True) - assert _count_files(dvc.cache.local.cache_dir) == 4 + assert _count_files(dvc.odb.local.cache_dir) == 4 dvc.gc(force=True, workspace=True) - assert _count_files(dvc.cache.local.cache_dir) == 2 + assert _count_files(dvc.odb.local.cache_dir) == 2 def _count_files(path): @@ -218,7 +218,7 @@ def test_gc_no_unpacked_dir(tmp_dir, dvc): os.remove("dir.dvc") unpackeddir = ( - dir_stages[0].outs[0].cache_path + LocalCache.UNPACKED_DIR_SUFFIX + dir_stages[0].outs[0].cache_path + LocalObjectDB.UNPACKED_DIR_SUFFIX ) # older (pre 1.0) versions of dvc used to generate this dir @@ -341,14 +341,14 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy): run_copy("foo", "foo2", name="copy") shutil.rmtree(dvc.stage_cache.cache_dir) - assert _count_files(dvc.cache.local.cache_dir) == 1 + assert _count_files(dvc.odb.local.cache_dir) == 1 dvc.gc(workspace=True, force=True) - assert _count_files(dvc.cache.local.cache_dir) == 1 + assert _count_files(dvc.odb.local.cache_dir) == 1 # remove pipeline file and lockfile and check Dvcfile(dvc, PIPELINE_FILE).remove(force=True) dvc.gc(workspace=True, force=True) - assert _count_files(dvc.cache.local.cache_dir) == 0 + assert _count_files(dvc.odb.local.cache_dir) == 0 @pytest.mark.parametrize( diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 202096cd2..892408327 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -3,8 +3,8 @@ import os import pytest -from dvc.cache import Cache from dvc.main import main +from dvc.objects.db import ODBManager from dvc.repo import Repo from dvc.repo.get import GetDVCFileError from dvc.system import System @@ -81,7 +81,7 @@ def test_cache_type_is_properly_overridden(tmp_dir, erepo_dir): with erepo_dir.chdir(): with erepo_dir.dvc.config.edit() as conf: conf["cache"]["type"] = "symlink" - erepo_dir.dvc.cache = Cache(erepo_dir.dvc) + erepo_dir.dvc.odb = ODBManager(erepo_dir.dvc) erepo_dir.scm_add( [erepo_dir.dvc.config.files["repo"]], "set cache type to symlinks" ) diff --git a/tests/func/test_import.py b/tests/func/test_import.py index eebbc3e92..10696efb8 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -6,10 +6,10 @@ from funcy import first from mock import patch import dvc.data_cloud as cloud -from dvc.cache import Cache from dvc.config import NoRemoteError from dvc.dvcfile import Dvcfile from dvc.exceptions import DownloadError +from dvc.objects.db import ODBManager from dvc.stage.exceptions import StagePathNotFoundError from dvc.system import System from dvc.utils.fs import makedirs, remove @@ -218,7 +218,7 @@ def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir): with erepo_dir.chdir(): with erepo_dir.dvc.config.edit() as conf: conf["cache"]["type"] = "symlink" - erepo_dir.dvc.cache = Cache(erepo_dir.dvc) + erepo_dir.dvc.odb = ODBManager(erepo_dir.dvc) erepo_dir.scm_add( [erepo_dir.dvc.config.files["repo"]], "set source repo cache type to symlink", @@ -240,7 +240,7 @@ def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir): dvc.imp(os.fspath(erepo_dir), "dir", "dir_imported") remove("dir_imported") - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) dvc.pull(["dir_imported.dvc"]) @@ -256,7 +256,7 @@ def test_pull_wildcard_imported_directory_stage(tmp_dir, dvc, erepo_dir): dvc.imp(os.fspath(erepo_dir), "dir123", "dir_imported123") remove("dir_imported123") - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) dvc.pull(["dir_imported*.dvc"], glob=True) @@ -486,9 +486,9 @@ def test_pull_imported_stage_from_subrepos( dvc.imp(os.fspath(erepo_dir), path, out="out") # clean everything - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) remove("out") - makedirs(dvc.cache.local.cache_dir) + makedirs(dvc.odb.local.cache_dir) stats = dvc.pull(["out.dvc"]) diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py index 925675c6d..896b8a815 100644 --- a/tests/func/test_import_url.py +++ b/tests/func/test_import_url.py @@ -5,10 +5,10 @@ from uuid import uuid4 import pytest -from dvc.cache import Cache from dvc.dependency.base import DependencyDoesNotExistError from dvc.exceptions import InvalidArgumentError from dvc.main import main +from dvc.objects.db import ODBManager from dvc.stage import Stage from dvc.utils.fs import makedirs from tests.basic_env import TestDvc @@ -185,7 +185,7 @@ def test_import_url_dir(tmp_dir, dvc, workspace, stage_md5, dir_md5): # remove external cache to make sure that we don't need it to import dirs with dvc.config.edit() as conf: del conf["cache"] - dvc.cache = Cache(dvc) + dvc.odb = ODBManager(dvc) assert not (tmp_dir / "dir").exists() # sanity check dvc.imp_url("remote://workspace/dir") diff --git a/tests/func/test_cache.py b/tests/func/test_odb.py similarity index 89% rename from tests/func/test_cache.py rename to tests/func/test_odb.py index 62d178cc4..4b7fa2b56 100644 --- a/tests/func/test_cache.py +++ b/tests/func/test_odb.py @@ -5,10 +5,10 @@ import textwrap import configobj import pytest -from dvc.cache import Cache from dvc.hash_info import HashInfo from dvc.main import main from dvc.objects import ObjectFormatError +from dvc.objects.db import ODBManager from dvc.utils import relpath from tests.basic_env import TestDir, TestDvc @@ -19,12 +19,12 @@ class TestCache(TestDvc): self.cache1_md5 = "123" self.cache2_md5 = "234" self.cache1 = os.path.join( - self.dvc.cache.local.cache_dir, + self.dvc.odb.local.cache_dir, self.cache1_md5[0:2], self.cache1_md5[2:], ) self.cache2 = os.path.join( - self.dvc.cache.local.cache_dir, + self.dvc.odb.local.cache_dir, self.cache2_md5[0:2], self.cache2_md5[2:], ) @@ -32,13 +32,13 @@ class TestCache(TestDvc): self.create(self.cache2, "2") def test_all(self): - md5_list = list(Cache(self.dvc).local.all()) + md5_list = list(ODBManager(self.dvc).local.all()) self.assertEqual(len(md5_list), 2) self.assertIn(self.cache1_md5, md5_list) self.assertIn(self.cache2_md5, md5_list) def test_get(self): - cache = Cache(self.dvc).local.hash_to_path_info(self.cache1_md5) + cache = ODBManager(self.dvc).local.hash_to_path_info(self.cache1_md5) self.assertEqual(os.fspath(cache), self.cache1) @@ -51,16 +51,16 @@ class TestCacheLoadBadDirCache(TestDvc): from dvc.objects import load dir_hash = "123.dir" - fname = os.fspath(self.dvc.cache.local.hash_to_path_info(dir_hash)) + fname = os.fspath(self.dvc.odb.local.hash_to_path_info(dir_hash)) self.create(fname, "not,json") with pytest.raises(ObjectFormatError): - load(self.dvc.cache.local, HashInfo("md5", dir_hash)) + load(self.dvc.odb.local, HashInfo("md5", dir_hash)) dir_hash = "234.dir" - fname = os.fspath(self.dvc.cache.local.hash_to_path_info(dir_hash)) + fname = os.fspath(self.dvc.odb.local.hash_to_path_info(dir_hash)) self.create(fname, '{"a": "b"}') with pytest.raises(ObjectFormatError): - load(self.dvc.cache.local, HashInfo("md5", dir_hash)) + load(self.dvc.odb.local, HashInfo("md5", dir_hash)) class TestExternalCacheDir(TestDvc): @@ -70,7 +70,7 @@ class TestExternalCacheDir(TestDvc): ret = main(["config", "cache.dir", cache_dir]) self.assertEqual(ret, 0) - self.assertFalse(os.path.exists(self.dvc.cache.local.cache_dir)) + self.assertFalse(os.path.exists(self.dvc.odb.local.cache_dir)) ret = main(["add", self.FOO]) self.assertEqual(ret, 0) @@ -89,7 +89,7 @@ class TestExternalCacheDir(TestDvc): self.dvc.__init__() - assert self.dvc.cache.ssh.fs.path_info == ssh_url + "/tmp" + assert self.dvc.odb.ssh.fs.path_info == ssh_url + "/tmp" class TestSharedCacheDir(TestDir): @@ -183,7 +183,7 @@ class TestCmdCacheDir(TestDvc): def test_default_cache_type(dvc): - assert dvc.cache.local.cache_types == ["reflink", "copy"] + assert dvc.odb.local.cache_types == ["reflink", "copy"] @pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.") @@ -196,8 +196,8 @@ def test_shared_cache(tmp_dir, dvc, group): if group: with dvc.config.edit() as conf: conf["cache"].update({"shared": "group"}) - dvc.cache = Cache(dvc) - cache_dir = dvc.cache.local.cache_dir + dvc.odb = ODBManager(dvc) + cache_dir = dvc.odb.local.cache_dir assert not os.path.exists(cache_dir) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 0dc0db531..3eddec5e1 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -158,7 +158,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): with patch( "dvc.objects.stage._collect_dir", return_value=dir_info, ): - hash1 = get_hash(path_info, dvc.cache.local.fs, "md5") + hash1 = get_hash(path_info, dvc.odb.local.fs, "md5") dir_info = DirInfo.from_list( [{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}] @@ -166,7 +166,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): with patch( "dvc.objects.stage._collect_dir", return_value=dir_info, ): - hash2 = get_hash(path_info, dvc.cache.local.fs, "md5") + hash2 = get_hash(path_info, dvc.odb.local.fs, "md5") assert hash1 == hash2 @@ -191,18 +191,18 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote): remote = dvc.cloud.get_remote("upstream") assert not remote.fs.exists( - remote.cache.hash_to_path_info(foo.hash_info.value) + remote.odb.hash_to_path_info(foo.hash_info.value) ) assert remote.fs.exists( - remote.cache.hash_to_path_info(bar.hash_info.value) + remote.odb.hash_to_path_info(bar.hash_info.value) ) assert not remote.fs.exists( - remote.cache.hash_to_path_info(baz.hash_info.value) + remote.odb.hash_to_path_info(baz.hash_info.value) ) # Push everything and delete local cache dvc.push() - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) with patch.object(LocalFileSystem, "_download", side_effect=Exception): with pytest.raises(DownloadError) as download_error_info: @@ -256,7 +256,7 @@ def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory): external_dir = tmp_path_factory.mktemp("external_dir") (external_dir / "file").write_text("content") - dvc.cache.local = None + dvc.odb.local = None with pytest.raises(RemoteCacheRequiredError): dvc.run( cmd="echo hello world", @@ -276,8 +276,8 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote): # foo .dir file should be uploaded after bar remote = dvc.cloud.get_remote("upstream") - foo_path = remote.cache.hash_to_path_info(foo.hash_info.value) - bar_path = remote.cache.hash_to_path_info( + foo_path = remote.odb.hash_to_path_info(foo.hash_info.value) + bar_path = remote.odb.hash_to_path_info( foo.hash_info.dir_info.trie[("bar",)].value ) paths = [args[1] for args, _ in mocked_upload.call_args_list] @@ -417,7 +417,7 @@ def test_protect_local_remote(tmp_dir, dvc, local_remote): dvc.push() remote = dvc.cloud.get_remote("upstream") - remote_cache_file = remote.cache.hash_to_path_info( + remote_cache_file = remote.odb.hash_to_path_info( stage.outs[0].hash_info.value ) @@ -429,18 +429,18 @@ def test_push_incomplete_dir(tmp_dir, dvc, mocker, local_remote): (stage,) = tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}) remote = dvc.cloud.get_remote("upstream") - cache = dvc.cache.local + odb = dvc.odb.local dir_hash = stage.outs[0].hash_info.value used = stage.get_used_cache(remote=remote) # remove one of the cache files for directory - file_hashes = list(used.child_keys(cache.fs.scheme, dir_hash)) - remove(cache.hash_to_path_info(file_hashes[0])) + file_hashes = list(used.child_keys(odb.fs.scheme, dir_hash)) + remove(odb.hash_to_path_info(file_hashes[0])) dvc.push() - assert not remote.fs.exists(remote.cache.hash_to_path_info(dir_hash)) - assert not remote.fs.exists(remote.cache.hash_to_path_info(file_hashes[0])) - assert remote.fs.exists(remote.cache.hash_to_path_info(file_hashes[1])) + assert not remote.fs.exists(remote.odb.hash_to_path_info(dir_hash)) + assert not remote.fs.exists(remote.odb.hash_to_path_info(file_hashes[0])) + assert remote.fs.exists(remote.odb.hash_to_path_info(file_hashes[1])) def test_upload_exists(tmp_dir, dvc, local_remote): diff --git a/tests/func/test_remove.py b/tests/func/test_remove.py index f7f7536ba..5f084ed62 100644 --- a/tests/func/test_remove.py +++ b/tests/func/test_remove.py @@ -41,10 +41,10 @@ def test_remove_non_existent_file(tmp_dir, dvc): def test_remove_broken_symlink(tmp_dir, dvc): tmp_dir.gen("foo", "foo") - dvc.cache.local.cache_types = ["symlink"] + dvc.odb.local.cache_types = ["symlink"] (stage,) = dvc.add("foo") - remove(dvc.cache.local.cache_dir) + remove(dvc.odb.local.cache_dir) assert System.is_symlink("foo") with pytest.raises(DvcException): diff --git a/tests/func/test_repo.py b/tests/func/test_repo.py index ae5d35f5b..fd4239d66 100644 --- a/tests/func/test_repo.py +++ b/tests/func/test_repo.py @@ -1,11 +1,11 @@ -from dvc.cache import Cache from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK +from dvc.objects.db import ODBManager from dvc.system import System def test_destroy(tmp_dir, dvc, run_copy): dvc.config["cache"]["type"] = ["symlink"] - dvc.cache = Cache(dvc) + dvc.odb = ODBManager(dvc) tmp_dir.dvc_gen("file", "text") tmp_dir.dvc_gen({"dir": {"file": "lorem", "subdir/file": "ipsum"}}) diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index c8633c938..e04af2a4a 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -829,12 +829,12 @@ class TestReproAllPipelines(SingleStageRun, TestDvc): class TestReproNoCommit(TestRepro): def test(self): - remove(self.dvc.cache.local.cache_dir) + remove(self.dvc.odb.local.cache_dir) ret = main( ["repro", self._get_stage_target(self.stage), "--no-commit"] ) self.assertEqual(ret, 0) - self.assertEqual(os.listdir(self.dvc.cache.local.cache_dir), ["runs"]) + self.assertEqual(os.listdir(self.dvc.odb.local.cache_dir), ["runs"]) class TestReproAlreadyCached(TestRepro): diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py index bb5b942b5..fafba7543 100644 --- a/tests/func/test_run_single_stage.py +++ b/tests/func/test_run_single_stage.py @@ -724,12 +724,12 @@ class TestRunCommit(TestDvc): ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(fname)) - self.assertFalse(os.path.exists(self.dvc.cache.local.cache_dir)) + self.assertFalse(os.path.exists(self.dvc.odb.local.cache_dir)) ret = main(["commit", fname + ".dvc"]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(fname)) - self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 1) + self.assertEqual(len(os.listdir(self.dvc.odb.local.cache_dir)), 1) class TestRunPersist(TestDvc): diff --git a/tests/func/test_s3.py b/tests/func/test_s3.py index 17804b34a..c16b3a4e8 100644 --- a/tests/func/test_s3.py +++ b/tests/func/test_s3.py @@ -8,8 +8,8 @@ import moto.s3.models as s3model import pytest from moto import mock_s3 -from dvc.cache.base import CloudCache from dvc.fs.s3 import S3FileSystem +from dvc.objects.db.base import ObjectDB from tests.remotes import S3 # from https://github.com/spulec/moto/blob/v1.3.5/tests/test_s3/test_s3.py#L40 @@ -61,16 +61,16 @@ def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm): from dvc.checkout import _link fs = S3FileSystem(dvc, {"url": str(base_info.parent)}) - cache = CloudCache(fs) - s3 = cache.fs.s3.meta.client + odb = ObjectDB(fs) + s3 = odb.fs.s3.meta.client s3.create_bucket(Bucket=base_info.bucket) s3.put_object( Bucket=base_info.bucket, Key=(base_info / "from").path, Body="data" ) - _link(cache, base_info / "from", base_info / "to") + _link(odb, base_info / "from", base_info / "to") - assert cache.fs.exists(base_info / "from") - assert cache.fs.exists(base_info / "to") + assert odb.fs.exists(base_info / "from") + assert odb.fs.exists(base_info / "to") @mock_s3 diff --git a/tests/remotes/__init__.py b/tests/remotes/__init__.py index 320665d94..72fa6e1a6 100644 --- a/tests/remotes/__init__.py +++ b/tests/remotes/__init__.py @@ -103,7 +103,7 @@ def remote(tmp_dir, dvc, request): @pytest.fixture def workspace(tmp_dir, dvc, request): - from dvc.cache import Cache + from dvc.objects.db import ODBManager cloud = request.param @@ -119,6 +119,6 @@ def workspace(tmp_dir, dvc, request): with dvc.config.edit() as conf: conf["cache"][scheme] = "cache" - dvc.cache = Cache(dvc) + dvc.odb = ODBManager(dvc) return cloud diff --git a/tests/unit/cache/__init__.py b/tests/unit/objects/db/__init__.py similarity index 100% rename from tests/unit/cache/__init__.py rename to tests/unit/objects/db/__init__.py diff --git a/tests/unit/cache/test_local.py b/tests/unit/objects/db/test_local.py similarity index 73% rename from tests/unit/cache/test_local.py rename to tests/unit/objects/db/test_local.py index 62cb4b9e9..20badab1b 100644 --- a/tests/unit/cache/test_local.py +++ b/tests/unit/objects/db/test_local.py @@ -3,9 +3,9 @@ import os import pytest -from dvc.cache import NamedCache -from dvc.cache.local import LocalCache from dvc.fs.local import LocalFileSystem +from dvc.objects.db import NamedCache +from dvc.objects.db.local import LocalObjectDB from dvc.path_info import PathInfo from dvc.remote.index import RemoteIndexNoop @@ -15,29 +15,29 @@ def test_status_download_optimization(mocker, dvc): And the desired files to fetch are already on the local cache, Don't check the existence of the desired files on the remote cache """ - cache = LocalCache(LocalFileSystem(dvc, {})) + odb = LocalObjectDB(LocalFileSystem(dvc, {})) infos = NamedCache() infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo") infos.add("local", "37b51d194a7513e45b56f6524f2d51f2", "bar") local_exists = list(infos["local"]) - mocker.patch.object(cache, "hashes_exist", return_value=local_exists) + mocker.patch.object(odb, "hashes_exist", return_value=local_exists) other_remote = mocker.Mock() other_remote.url = "other_remote" other_remote.hashes_exist.return_value = [] other_remote.index = RemoteIndexNoop() - other_remote.status(cache, infos, download=True) + other_remote.status(odb, infos, download=True) assert other_remote.hashes_exist.call_count == 0 @pytest.mark.parametrize("link_name", ["hardlink", "symlink"]) def test_is_protected(tmp_dir, dvc, link_name): - cache = dvc.cache.local - fs = cache.fs + odb = dvc.odb.local + fs = odb.fs link_method = getattr(fs, link_name) (tmp_dir / "foo").write_text("foo") @@ -47,23 +47,23 @@ def test_is_protected(tmp_dir, dvc, link_name): link_method(foo, link) - assert not cache.is_protected(foo) - assert not cache.is_protected(link) + assert not odb.is_protected(foo) + assert not odb.is_protected(link) - cache.protect(foo) + odb.protect(foo) - assert cache.is_protected(foo) - assert cache.is_protected(link) + assert odb.is_protected(foo) + assert odb.is_protected(link) - cache.unprotect(link) + odb.unprotect(link) - assert not cache.is_protected(link) + assert not odb.is_protected(link) if os.name == "nt" and link_name == "hardlink": # NOTE: NTFS doesn't allow deleting read-only files, which forces us to # set write perms on the link, which propagates to the source. - assert not cache.is_protected(foo) + assert not odb.is_protected(foo) else: - assert cache.is_protected(foo) + assert odb.is_protected(foo) @pytest.mark.parametrize("err", [errno.EPERM, errno.EACCES, errno.EROFS]) @@ -73,7 +73,7 @@ def test_protect_ignore_errors(tmp_dir, dvc, mocker, err): mock_chmod = mocker.patch( "os.chmod", side_effect=OSError(err, "something") ) - dvc.cache.local.protect(PathInfo("foo")) + dvc.odb.local.protect(PathInfo("foo")) assert mock_chmod.called @@ -84,5 +84,5 @@ def test_set_exec_ignore_errors(tmp_dir, dvc, mocker, err): mock_chmod = mocker.patch( "os.chmod", side_effect=OSError(err, "something") ) - dvc.cache.local.set_exec(PathInfo("foo")) + dvc.odb.local.set_exec(PathInfo("foo")) assert mock_chmod.called diff --git a/tests/unit/output/test_output.py b/tests/unit/output/test_output.py index 09127e628..0dfa7f003 100644 --- a/tests/unit/output/test_output.py +++ b/tests/unit/output/test_output.py @@ -4,8 +4,8 @@ import pytest from funcy import first from voluptuous import MultipleInvalid, Schema -from dvc.cache import NamedCache from dvc.ignore import _no_match +from dvc.objects.db import NamedCache from dvc.output import CHECKSUM_SCHEMA, BaseOutput diff --git a/tests/unit/remote/test_base.py b/tests/unit/remote/test_base.py index d45dcd2a9..322ec5e83 100644 --- a/tests/unit/remote/test_base.py +++ b/tests/unit/remote/test_base.py @@ -3,8 +3,8 @@ import math import mock import pytest -from dvc.cache.base import CloudCache from dvc.fs.base import BaseFileSystem, RemoteCmdError +from dvc.objects.db.base import ObjectDB from dvc.path_info import PathInfo @@ -34,36 +34,32 @@ def test_cmd_error(dvc): BaseFileSystem(dvc, config).remove("file") -@mock.patch.object(CloudCache, "list_hashes_traverse") -@mock.patch.object(CloudCache, "list_hashes_exists") +@mock.patch.object(ObjectDB, "list_hashes_traverse") +@mock.patch.object(ObjectDB, "list_hashes_exists") def test_hashes_exist(object_exists, traverse, dvc): - cache = CloudCache(BaseFileSystem(dvc, {})) + odb = ObjectDB(BaseFileSystem(dvc, {})) # remote does not support traverse - cache.fs.CAN_TRAVERSE = False - with mock.patch.object( - cache, "list_hashes", return_value=list(range(256)) - ): + odb.fs.CAN_TRAVERSE = False + with mock.patch.object(odb, "list_hashes", return_value=list(range(256))): hashes = set(range(1000)) - cache.hashes_exist(hashes) + odb.hashes_exist(hashes) object_exists.assert_called_with(hashes, None, None) traverse.assert_not_called() - cache.fs.CAN_TRAVERSE = True + odb.fs.CAN_TRAVERSE = True # large remote, small local object_exists.reset_mock() traverse.reset_mock() - with mock.patch.object( - cache, "list_hashes", return_value=list(range(256)) - ): + with mock.patch.object(odb, "list_hashes", return_value=list(range(256))): hashes = list(range(1000)) - cache.hashes_exist(hashes) - # verify that _cache_paths_with_max() short circuits + odb.hashes_exist(hashes) + # verify that _odb_paths_with_max() short circuits # before returning all 256 remote hashes max_hashes = math.ceil( - cache._max_estimation_size(hashes) - / pow(16, cache.fs.TRAVERSE_PREFIX_LEN) + odb._max_estimation_size(hashes) + / pow(16, odb.fs.TRAVERSE_PREFIX_LEN) ) assert max_hashes < 256 object_exists.assert_called_with( @@ -74,15 +70,13 @@ def test_hashes_exist(object_exists, traverse, dvc): # large remote, large local object_exists.reset_mock() traverse.reset_mock() - cache.fs.JOBS = 16 - with mock.patch.object( - cache, "list_hashes", return_value=list(range(256)) - ): + odb.fs.JOBS = 16 + with mock.patch.object(odb, "list_hashes", return_value=list(range(256))): hashes = list(range(1000000)) - cache.hashes_exist(hashes) + odb.hashes_exist(hashes) object_exists.assert_not_called() traverse.assert_called_with( - 256 * pow(16, cache.fs.TRAVERSE_PREFIX_LEN), + 256 * pow(16, odb.fs.TRAVERSE_PREFIX_LEN), set(range(256)), None, None, @@ -90,18 +84,18 @@ def test_hashes_exist(object_exists, traverse, dvc): @mock.patch.object( - CloudCache, "list_hashes", return_value=[], + ObjectDB, "list_hashes", return_value=[], ) @mock.patch.object( - CloudCache, "_path_to_hash", side_effect=lambda x: x, + ObjectDB, "_path_to_hash", side_effect=lambda x: x, ) def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc): - cache = CloudCache(BaseFileSystem(dvc, {})) - cache.fs.path_info = PathInfo("foo") + odb = ObjectDB(BaseFileSystem(dvc, {})) + odb.fs.path_info = PathInfo("foo") # parallel traverse - size = 256 / cache.fs.JOBS * cache.fs.LIST_OBJECT_PAGE_SIZE - list(cache.list_hashes_traverse(size, {0})) + size = 256 / odb.fs.JOBS * odb.fs.LIST_OBJECT_PAGE_SIZE + list(odb.list_hashes_traverse(size, {0})) for i in range(1, 16): list_hashes.assert_any_call( prefix=f"{i:03x}", progress_callback=CallableOrNone @@ -114,38 +108,36 @@ def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc): # default traverse (small remote) size -= 1 list_hashes.reset_mock() - list(cache.list_hashes_traverse(size - 1, {0})) + list(odb.list_hashes_traverse(size - 1, {0})) list_hashes.assert_called_with( prefix=None, progress_callback=CallableOrNone ) def test_list_hashes(dvc): - cache = CloudCache(BaseFileSystem(dvc, {})) - cache.fs.path_info = PathInfo("foo") + odb = ObjectDB(BaseFileSystem(dvc, {})) + odb.fs.path_info = PathInfo("foo") with mock.patch.object( - cache, "_list_paths", return_value=["12/3456", "bar"] + odb, "_list_paths", return_value=["12/3456", "bar"] ): - hashes = list(cache.list_hashes()) + hashes = list(odb.list_hashes()) assert hashes == ["123456"] def test_list_paths(dvc): - cache = CloudCache(BaseFileSystem(dvc, {})) - cache.fs.path_info = PathInfo("foo") + odb = ObjectDB(BaseFileSystem(dvc, {})) + odb.fs.path_info = PathInfo("foo") - with mock.patch.object( - cache.fs, "walk_files", return_value=[] - ) as walk_mock: - for _ in cache._list_paths(): + with mock.patch.object(odb.fs, "walk_files", return_value=[]) as walk_mock: + for _ in odb._list_paths(): pass - walk_mock.assert_called_with(cache.fs.path_info, prefix=False) + walk_mock.assert_called_with(odb.fs.path_info, prefix=False) - for _ in cache._list_paths(prefix="000"): + for _ in odb._list_paths(prefix="000"): pass walk_mock.assert_called_with( - cache.fs.path_info / "00" / "0", prefix=True + odb.fs.path_info / "00" / "0", prefix=True ) diff --git a/tests/unit/repo/test_repo.py b/tests/unit/repo/test_repo.py index 64495e782..345ceb164 100644 --- a/tests/unit/repo/test_repo.py +++ b/tests/unit/repo/test_repo.py @@ -45,7 +45,7 @@ def test_find_outs_by_path_does_graph_checks(tmp_dir, dvc): [os.path.join("dir", "subdir", "file"), os.path.join("dir", "subdir")], ) def test_used_cache(tmp_dir, dvc, path): - from dvc.cache import NamedCache + from dvc.objects.db import NamedCache tmp_dir.dvc_gen({"dir": {"subdir": {"file": "file"}, "other": "other"}}) expected = NamedCache.make( diff --git a/tests/unit/stage/test_cache.py b/tests/unit/stage/test_cache.py index 4e26dd70e..52482d818 100644 --- a/tests/unit/stage/test_cache.py +++ b/tests/unit/stage/test_cache.py @@ -167,16 +167,16 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker): def test_shared_stage_cache(tmp_dir, dvc, run_copy): import stat - from dvc.cache import Cache + from dvc.objects.db import ODBManager tmp_dir.gen("foo", "foo") with dvc.config.edit() as config: config["cache"]["shared"] = "group" - dvc.cache = Cache(dvc) + dvc.odb = ODBManager(dvc) - assert not os.path.exists(dvc.cache.local.cache_dir) + assert not os.path.exists(dvc.odb.local.cache_dir) run_copy("foo", "bar", name="copy-foo-bar") @@ -205,7 +205,7 @@ def test_shared_stage_cache(tmp_dir, dvc, run_copy): dir_mode = 0o2775 file_mode = 0o664 - assert _mode(dvc.cache.local.cache_dir) == dir_mode + assert _mode(dvc.odb.local.cache_dir) == dir_mode assert _mode(dvc.stage_cache.cache_dir) == dir_mode assert _mode(parent_cache_dir) == dir_mode assert _mode(cache_dir) == dir_mode diff --git a/tests/unit/test_external_repo.py b/tests/unit/test_external_repo.py index 34e8d6256..841c6efd2 100644 --- a/tests/unit/test_external_repo.py +++ b/tests/unit/test_external_repo.py @@ -58,7 +58,7 @@ def test_subrepo_is_constructed_properly( subrepo = tmp_dir / "subrepo" make_subrepo(subrepo, scm) - local_cache = subrepo.dvc.cache.local.cache_dir + local_cache = subrepo.dvc.odb.local.cache_dir tmp_dir.scm_gen("bar", "bar", commit="add bar") subrepo.dvc_gen("foo", "foo", commit="add foo") @@ -75,19 +75,19 @@ def test_subrepo_is_constructed_properly( assert repo.url == str(tmp_dir) assert repo.config["cache"]["dir"] == str(cache_dir) - assert repo.cache.local.cache_dir == str(cache_dir) - assert subrepo.cache.local.cache_dir == str(cache_dir) + assert repo.odb.local.cache_dir == str(cache_dir) + assert subrepo.odb.local.cache_dir == str(cache_dir) assert repo.config["cache"]["type"] == ["symlink"] - assert repo.cache.local.cache_types == ["symlink"] - assert subrepo.cache.local.cache_types == ["symlink"] + assert repo.odb.local.cache_types == ["symlink"] + assert subrepo.odb.local.cache_types == ["symlink"] assert ( subrepo.config["remote"]["auto-generated-upstream"]["url"] == local_cache ) if root_is_dvc: - main_cache = tmp_dir.dvc.cache.local.cache_dir + main_cache = tmp_dir.dvc.odb.local.cache_dir assert repo.config["remote"]["auto-generated-upstream"][ "url" ] == str(main_cache) @@ -112,14 +112,14 @@ def test_fetch_external_repo_jobs(tmp_dir, scm, mocker, dvc, local_remote): spy = mocker.spy(repo.cloud, "pull") obj = stage( - dvc.cache.local, + dvc.odb.local, PathInfo(repo.root_dir) / "dir1", repo.repo_fs, follow_subrepos=False, jobs=3, ) save( - dvc.cache.local, obj, jobs=3, + dvc.odb.local, obj, jobs=3, ) run_jobs = tuple(spy.call_args_list[0])[1].get("jobs") diff --git a/tests/unit/test_info.py b/tests/unit/test_info.py index 64a702fd0..9989e69f1 100644 --- a/tests/unit/test_info.py +++ b/tests/unit/test_info.py @@ -17,7 +17,7 @@ PYTHON_VERSION_REGEX = r"Python \d\.\d+\.\d+\S*" def test_info_in_repo(scm_init, tmp_dir): tmp_dir.init(scm=scm_init, dvc=True) # Create `.dvc/cache`, that is needed to check supported link types. - os.mkdir(tmp_dir.dvc.cache.local.cache_dir) + os.mkdir(tmp_dir.dvc.odb.local.cache_dir) dvc_info = get_dvc_info() @@ -86,7 +86,7 @@ def test_remotes(tmp_dir, dvc, caplog): def test_fs_info_in_repo(tmp_dir, dvc, caplog): - os.mkdir(dvc.cache.local.cache_dir) + os.mkdir(dvc.odb.local.cache_dir) dvc_info = get_dvc_info() assert re.search(r"Cache directory: .* on .*", dvc_info)