Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

dvc: rename cache -> odb (#5481)

Browse Source
This commit is contained in:
Ruslan Kuprieiev
2021-02-16 22:49:25 +02:00
committed by GitHub
parent a814a49181
commit e0842ce01e
53 changed files with 308 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cache/gdrive.py
Vendored
-5
View File
@@ -1,5 +0,0 @@
from .base import CloudCache
class GDriveCache(CloudCache):
DEFAULT_VERIFY = True
dvc/data_cloud.py
+4 -4
View File
@@ -64,7 +64,7 @@ class DataCloud:
remote = self.get_remote(remote, "push")
return remote.push(
self.repo.cache.local,
self.repo.odb.local,
cache,
jobs=jobs,
show_checksums=show_checksums,
@@ -86,7 +86,7 @@ class DataCloud:
remote = self.get_remote(remote, "pull")
return remote.pull(
self.repo.cache.local,
self.repo.odb.local,
cache,
jobs=jobs,
show_checksums=show_checksums,
@@ -133,7 +133,7 @@ class DataCloud:
"""
remote = self.get_remote(remote, "status")
return remote.status(
self.repo.cache.local,
self.repo.odb.local,
cache,
jobs=jobs,
show_checksums=show_checksums,
@@ -142,4 +142,4 @@ class DataCloud:
def get_url_for(self, remote, checksum):
remote = self.get_remote(remote)
return str(remote.cache.hash_to_path_info(checksum))
return str(remote.odb.hash_to_path_info(checksum))
dvc/dependency/repo.py
+5 -9
View File
@@ -78,22 +78,18 @@ class RepoDependency(LocalDependency):
from dvc.objects import save
from dvc.objects.stage import stage
cache = self.repo.cache.local
odb = self.repo.odb.local
with self._make_repo(cache_dir=cache.cache_dir) as repo:
with self._make_repo(cache_dir=odb.cache_dir) as repo:
if self.def_repo.get(self.PARAM_REV_LOCK) is None:
self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
path_info = PathInfo(repo.root_dir) / self.def_path
obj = stage(
cache,
path_info,
repo.repo_fs,
jobs=jobs,
follow_subrepos=False,
odb, path_info, repo.repo_fs, jobs=jobs, follow_subrepos=False,
)
save(cache, obj, jobs=jobs)
save(odb, obj, jobs=jobs)
checkout(to.path_info, to.fs, obj, cache)
checkout(to.path_info, to.fs, obj, odb)
def update(self, rev=None):
if rev:
dvc/external_repo.py
+1 -1
View File
@@ -127,7 +127,7 @@ def _get_remote_config(url):
name = "auto-generated-upstream"
return {
"core": {"remote": name},
"remote": {name: {"url": repo.cache.local.cache_dir}},
"remote": {name: {"url": repo.odb.local.cache_dir}},
}
# Use original remote to make sure that we are using correct url,
dvc/fs/base.py
+2 -2
View File
@@ -131,8 +131,8 @@ class BaseFileSystem:
)
@property
def cache(self):
return getattr(self.repo.cache, self.scheme)
def odb(self):
return getattr(self.repo.odb, self.scheme)
def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
if hasattr(self, "_generate_download_url"):
dvc/fs/dvc.py
+3 -3
View File
@@ -78,14 +78,14 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
checksum = self._get_granular_hash(path, out).value
else:
checksum = out.hash_info.value
remote_info = remote_obj.cache.hash_to_path_info(checksum)
remote_info = remote_obj.odb.hash_to_path_info(checksum)
return remote_obj.fs.open(
remote_info, mode=mode, encoding=encoding
)
if out.is_dir_checksum:
checksum = self._get_granular_hash(path, out).value
cache_path = out.cache.hash_to_path_info(checksum).url
cache_path = out.odb.hash_to_path_info(checksum).url
else:
cache_path = out.cache_path
return open(cache_path, mode=mode, encoding=encoding)
@@ -137,7 +137,7 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
from dvc.objects import Tree
hash_info = Tree.save_dir_info(out.cache, dir_cache)
hash_info = Tree.save_dir_info(out.odb, dir_cache)
if hash_info != out.hash_info:
raise FileNotFoundError
dvc/info.py
+4 -4
View File
@@ -37,16 +37,16 @@ def get_dvc_info():
# can't auto-create it, as it might cause issues if the user
# later decides to enable shared cache mode with
# `dvc config cache.shared group`.
if os.path.exists(repo.cache.local.cache_dir):
if os.path.exists(repo.odb.local.cache_dir):
info.append(
"Cache types: {}".format(_get_linktype_support_info(repo))
)
fs_type = get_fs_type(repo.cache.local.cache_dir)
fs_type = get_fs_type(repo.odb.local.cache_dir)
info.append(f"Cache directory: {fs_type}")
else:
info.append("Cache types: " + error_link("no-dvc-cache"))
info.append(f"Caches: {_get_caches(repo.cache)}")
info.append(f"Caches: {_get_caches(repo.odb)}")
info.append(f"Remotes: {_get_remotes(repo.config)}")
@@ -91,7 +91,7 @@ def _get_linktype_support_info(repo):
}
fname = "." + str(uuid.uuid4())
src = os.path.join(repo.cache.local.cache_dir, fname)
src = os.path.join(repo.odb.local.cache_dir, fname)
open(src, "w").close()
dst = os.path.join(repo.root_dir, fname)
dvc/cache/__init__.py → dvc/objects/db/__init__.py
+24 -31
View File
@@ -1,44 +1,37 @@
"""Manages cache of a DVC repo."""
from collections import defaultdict
from ..scheme import Schemes
from dvc.scheme import Schemes
def get_cloud_cache(fs):
from .base import CloudCache
from .gdrive import GDriveCache
from .local import LocalCache
from .ssh import SSHCache
def get_odb(fs):
from .base import ObjectDB
from .gdrive import GDriveObjectDB
from .local import LocalObjectDB
from .ssh import SSHObjectDB
if fs.scheme == Schemes.LOCAL:
return LocalCache(fs)
return LocalObjectDB(fs)
if fs.scheme == Schemes.SSH:
return SSHCache(fs)
return SSHObjectDB(fs)
if fs.scheme == Schemes.GDRIVE:
return GDriveCache(fs)
return GDriveObjectDB(fs)
return CloudCache(fs)
return ObjectDB(fs)
def _get_cache(repo, settings):
from ..fs import get_cloud_fs
def _get_odb(repo, settings):
from dvc.fs import get_cloud_fs
if not settings:
return None
fs = get_cloud_fs(repo, **settings)
return get_cloud_cache(fs)
return get_odb(fs)
class Cache:
"""Class that manages cache locations of a DVC repo.
Args:
repo (dvc.repo.Repo): repo instance that this cache belongs to.
"""
class ODBManager:
CACHE_DIR = "cache"
CLOUD_SCHEMES = [
Schemes.S3,
@@ -51,7 +44,7 @@ class Cache:
def __init__(self, repo):
self.repo = repo
self.config = config = repo.config["cache"]
self._cache = {}
self._odb = {}
local = config.get("local")
@@ -60,33 +53,33 @@ class Cache:
elif "dir" not in config:
settings = None
else:
from ..config_schema import LOCAL_COMMON
from dvc.config_schema import LOCAL_COMMON
settings = {"url": config["dir"]}
for opt in LOCAL_COMMON.keys():
if opt in config:
settings[str(opt)] = config.get(opt)
self._cache[Schemes.LOCAL] = _get_cache(repo, settings)
self._odb[Schemes.LOCAL] = _get_odb(repo, settings)
def _initalize_cloud_cache(self, schemes):
def _init_odb(self, schemes):
for scheme in schemes:
remote = self.config.get(scheme)
settings = {"name": remote} if remote else None
self._cache[scheme] = _get_cache(self.repo, settings)
self._odb[scheme] = _get_odb(self.repo, settings)
def __getattr__(self, name):
if name not in self._cache and name in self.CLOUD_SCHEMES:
self._initalize_cloud_cache([name])
if name not in self._odb and name in self.CLOUD_SCHEMES:
self._init_odb([name])
try:
return self._cache[name]
return self._odb[name]
except KeyError as exc:
raise AttributeError from exc
def by_scheme(self):
self._initalize_cloud_cache(self.CLOUD_SCHEMES)
yield from self._cache.items()
self._init_odb(self.CLOUD_SCHEMES)
yield from self._odb.items()
class NamedCacheItem:
dvc/cache/base.py → dvc/objects/db/base.py
+2 -2
View File
@@ -6,12 +6,12 @@ from typing import Optional
from dvc.progress import Tqdm
from ..objects import HashFile, ObjectFormatError
from .. import HashFile, ObjectFormatError
logger = logging.getLogger(__name__)
class CloudCache:
class ObjectDB:
DEFAULT_VERIFY = False
DEFAULT_CACHE_TYPES = ["copy"]
dvc/objects/db/gdrive.py
+5
View File
@@ -0,0 +1,5 @@
from .base import ObjectDB
class GDriveObjectDB(ObjectDB):
DEFAULT_VERIFY = True
dvc/cache/local.py → dvc/objects/db/local.py
+4 -4
View File
@@ -9,15 +9,15 @@ from dvc.hash_info import HashInfo
from dvc.objects import ObjectFormatError
from dvc.path_info import PathInfo
from dvc.progress import Tqdm
from dvc.utils import relpath
from dvc.utils.fs import copyfile, remove, umask, walk_files
from ..utils import relpath
from ..utils.fs import copyfile, remove, umask, walk_files
from .base import CloudCache
from .base import ObjectDB
logger = logging.getLogger(__name__)
class LocalCache(CloudCache):
class LocalObjectDB(ObjectDB):
DEFAULT_CACHE_TYPES = ["reflink", "copy"]
CACHE_MODE = 0o444
UNPACKED_DIR_SUFFIX = ".unpacked"
dvc/cache/ssh.py → dvc/objects/db/ssh.py
+2 -2
View File
@@ -7,12 +7,12 @@ from concurrent.futures import ThreadPoolExecutor
from dvc.progress import Tqdm
from dvc.utils import to_chunks
from .base import CloudCache
from .base import ObjectDB
logger = logging.getLogger(__name__)
class SSHCache(CloudCache):
class SSHObjectDB(ObjectDB):
def batch_exists(self, path_infos, callback):
def _exists(chunk_and_channel):
chunk, channel = chunk_and_channel
dvc/objects/stage.py
+3 -5
View File
@@ -92,7 +92,7 @@ def get_dir_hash(path_info, fs, name, **kwargs):
from . import Tree
dir_info = _collect_dir(path_info, fs, name, **kwargs)
hash_info = Tree.save_dir_info(fs.repo.cache.local, dir_info)
hash_info = Tree.save_dir_info(fs.repo.odb.local, dir_info)
hash_info.size = dir_info.size
hash_info.dir_info = dir_info
return hash_info
@@ -118,9 +118,7 @@ def get_hash(path_info, fs, name, **kwargs):
if (
hash_info
and hash_info.isdir
and not fs.cache.fs.exists(
fs.cache.hash_to_path_info(hash_info.value)
)
and not fs.odb.fs.exists(fs.odb.hash_to_path_info(hash_info.value))
):
hash_info = None
@@ -129,7 +127,7 @@ def get_hash(path_info, fs, name, **kwargs):
from . import Tree
# NOTE: loading the fs will restore hash_info.dir_info
Tree.load(fs.cache, hash_info)
Tree.load(fs.odb, hash_info)
assert hash_info.name == name
return hash_info
dvc/output/base.py
+20 -20
View File
@@ -8,7 +8,6 @@ from voluptuous import Any
import dvc.objects as objects
import dvc.prompt as prompt
from dvc.cache import NamedCache
from dvc.checkout import checkout
from dvc.exceptions import (
CheckoutError,
@@ -18,6 +17,7 @@ from dvc.exceptions import (
RemoteCacheRequiredError,
)
from dvc.hash_info import HashInfo
from dvc.objects.db import NamedCache
from dvc.objects.stage import get_hash
from dvc.objects.stage import stage as ostage
@@ -144,7 +144,7 @@ class BaseOutput:
self.desc = desc
self.path_info = self._parse_path(fs, path)
if self.use_cache and self.cache is None:
if self.use_cache and self.odb is None:
raise RemoteCacheRequiredError(self.path_info)
self.obj = None
@@ -180,8 +180,8 @@ class BaseOutput:
return self.use_cache or self.stage.is_repo_import
@property
def cache(self):
return getattr(self.repo.cache, self.scheme)
def odb(self):
return getattr(self.repo.odb, self.scheme)
@property
def dir_cache(self):
@@ -189,12 +189,12 @@ class BaseOutput:
@property
def cache_path(self):
return self.cache.hash_to_path_info(self.hash_info.value).url
return self.odb.hash_to_path_info(self.hash_info.value).url
def get_hash(self):
if not self.use_cache:
return get_hash(self.path_info, self.fs, self.fs.PARAM_CHECKSUM)
return ostage(self.cache, self.path_info, self.fs).hash_info
return ostage(self.odb, self.path_info, self.fs).hash_info
@property
def is_dir_checksum(self):
@@ -216,7 +216,7 @@ class BaseOutput:
return True
try:
objects.check(self.cache, obj)
objects.check(self.odb, obj)
return False
except (FileNotFoundError, objects.ObjectFormatError):
return True
@@ -302,13 +302,13 @@ class BaseOutput:
logger.debug("Output '%s' didn't change. Skipping saving.", self)
return
self.obj = ostage(self.cache, self.path_info, self.fs)
self.obj = ostage(self.odb, self.path_info, self.fs)
self.hash_info = self.obj.hash_info
self.isexec = self.isfile() and self.fs.isexec(self.path_info)
def set_exec(self):
if self.isfile() and self.isexec:
self.cache.set_exec(self.path_info)
self.odb.set_exec(self.path_info)
def commit(self, filter_info=None):
if not self.exists:
@@ -317,13 +317,13 @@ class BaseOutput:
assert self.hash_info
if self.use_cache:
obj = ostage(self.cache, filter_info or self.path_info, self.fs)
objects.save(self.cache, obj)
obj = ostage(self.odb, filter_info or self.path_info, self.fs)
objects.save(self.odb, obj)
checkout(
filter_info or self.path_info,
self.fs,
obj,
self.cache,
self.odb,
relink=True,
)
self.set_exec()
@@ -379,7 +379,7 @@ class BaseOutput:
obj = self.obj
elif self.hash_info:
try:
obj = objects.load(self.cache, self.hash_info)
obj = objects.load(self.odb, self.hash_info)
except FileNotFoundError:
return None
else:
@@ -387,7 +387,7 @@ class BaseOutput:
if filter_info and filter_info != self.path_info:
prefix = filter_info.relative_to(self.path_info).parts
obj = obj.filter(self.cache, prefix)
obj = obj.filter(self.odb, prefix)
return obj
@@ -419,7 +419,7 @@ class BaseOutput:
filter_info or self.path_info,
self.fs,
obj,
self.cache,
self.odb,
force=force,
progress_callback=progress_callback,
relink=relink,
@@ -476,7 +476,7 @@ class BaseOutput:
def unprotect(self):
if self.exists:
self.cache.unprotect(self.path_info)
self.odb.unprotect(self.path_info)
def get_dir_cache(self, **kwargs):
@@ -484,7 +484,7 @@ class BaseOutput:
raise DvcException("cannot get dir cache for file checksum")
try:
objects.check(self.cache, self.cache.get(self.hash_info))
objects.check(self.odb, self.odb.get(self.hash_info))
except (FileNotFoundError, objects.ObjectFormatError):
self.repo.cloud.pull(
NamedCache.make("local", self.hash_info.value, str(self)),
@@ -493,7 +493,7 @@ class BaseOutput:
)
try:
objects.load(self.cache, self.hash_info)
objects.load(self.odb, self.hash_info)
assert self.hash_info.dir_info
except (objects.ObjectFormatError, FileNotFoundError):
self.hash_info.dir_info = None
@@ -530,7 +530,7 @@ class BaseOutput:
logger.debug(f"failed to pull cache for '{self}'")
try:
objects.check(self.cache, self.cache.get(self.hash_info))
objects.check(self.odb, self.odb.get(self.hash_info))
except (FileNotFoundError, objects.ObjectFormatError):
msg = (
"Missing cache for directory '{}'. "
@@ -661,5 +661,5 @@ class BaseOutput:
self._check_can_merge(other)
self.hash_info = objects.merge(
self.cache, ancestor_info, self.hash_info, other.hash_info
self.odb, ancestor_info, self.hash_info, other.hash_info
)
dvc/remote/base.py
+11 -11
View File
@@ -68,11 +68,11 @@ class Remote:
INDEX_CLS = RemoteIndex
def __init__(self, fs):
from dvc.cache import get_cloud_cache
from dvc.objects.db import get_odb
self.fs = fs
self.repo = fs.repo
self.cache = get_cloud_cache(self.fs)
self.odb = get_odb(self.fs)
config = fs.config
url = config.get("url")
@@ -92,7 +92,7 @@ class Remote:
@index_locked
def gc(self, *args, **kwargs):
removed = self.cache.gc(*args, **kwargs)
removed = self.odb.gc(*args, **kwargs)
if removed:
self.index.clear()
@@ -129,7 +129,7 @@ class Remote:
if not hashes:
return indexed_hashes
return indexed_hashes + self.cache.hashes_exist(list(hashes), **kwargs)
return indexed_hashes + self.odb.hashes_exist(list(hashes), **kwargs)
def _status(
self,
@@ -227,7 +227,7 @@ class Remote:
indexed_dir_exists = set()
if indexed_dirs:
indexed_dir_exists.update(
self.cache.list_hashes_exists(indexed_dirs)
self.odb.list_hashes_exists(indexed_dirs)
)
missing_dirs = indexed_dirs.difference(indexed_dir_exists)
if missing_dirs:
@@ -239,7 +239,7 @@ class Remote:
# Check if non-indexed (new) dir hashes exist on remote
dir_exists = dir_md5s.intersection(indexed_dir_exists)
dir_exists.update(self.cache.list_hashes_exists(dir_md5s - dir_exists))
dir_exists.update(self.odb.list_hashes_exists(dir_md5s - dir_exists))
# If .dir hash exists on the remote, assume directory contents
# still exists on the remote
@@ -278,7 +278,7 @@ class Remote:
):
if info["status"] == status:
cache.append(cache_obj.hash_to_path_info(md5))
path_infos.append(self.cache.hash_to_path_info(md5))
path_infos.append(self.odb.hash_to_path_info(md5))
names.append(info["name"])
hashes.append(md5)
elif info["status"] == STATUS_MISSING:
@@ -474,11 +474,11 @@ class Remote:
if self.fs.scheme == "local":
with self.fs.repo.state:
for checksum in named_cache.scheme_keys("local"):
cache_file = self.cache.hash_to_path_info(checksum)
cache_file = self.odb.hash_to_path_info(checksum)
if self.fs.exists(cache_file):
hash_info = HashInfo(self.fs.PARAM_CHECKSUM, checksum)
self.fs.repo.state.save(cache_file, self.fs, hash_info)
self.cache.protect(cache_file)
self.odb.protect(cache_file)
return ret
@@ -492,7 +492,7 @@ class Remote:
download=True,
)
if not self.cache.verify:
if not self.odb.verify:
with cache.fs.repo.state:
for checksum in named_cache.scheme_keys("local"):
cache_file = cache.hash_to_path_info(checksum)
@@ -513,7 +513,7 @@ class Remote:
from dvc.objects import transfer
return transfer(
self.cache,
self.odb,
from_fs,
from_info,
jobs=jobs,
dvc/repo/__init__.py
+6 -6
View File
@@ -131,11 +131,11 @@ class Repo:
url=None,
repo_factory=None,
):
from dvc.cache import Cache
from dvc.config import Config
from dvc.data_cloud import DataCloud
from dvc.fs.local import LocalFileSystem
from dvc.lock import LockNoop, make_lock
from dvc.objects.db import ODBManager
from dvc.repo.live import Live
from dvc.repo.metrics import Metrics
from dvc.repo.params import Params
@@ -168,7 +168,7 @@ class Repo:
# used by RepoFileSystem to determine if it should traverse subrepos
self.subrepos = subrepos
self.cache = Cache(self)
self.odb = ODBManager(self)
self.cloud = DataCloud(self)
self.stage = StageLoad(self)
@@ -307,7 +307,7 @@ class Repo:
return Repo(root_dir)
def unprotect(self, target):
return self.cache.local.unprotect(PathInfo(target))
return self.odb.local.unprotect(PathInfo(target))
def _ignore(self):
flist = [
@@ -315,8 +315,8 @@ class Repo:
self.tmp_dir,
]
if path_isin(self.cache.local.cache_dir, self.root_dir):
flist += [self.cache.local.cache_dir]
if path_isin(self.odb.local.cache_dir, self.root_dir):
flist += [self.odb.local.cache_dir]
self.scm.ignore_list(flist)
@@ -368,7 +368,7 @@ class Repo:
to items containing the output's `dumpd` names and the output's
children (if the given output is a directory).
"""
from dvc.cache import NamedCache
from dvc.objects.db import NamedCache
cache = NamedCache()
dvc/repo/add.py
+1 -1
View File
@@ -170,7 +170,7 @@ def _process_stages(
from_fs = get_cloud_fs(repo, url=target)
out.hash_info = transfer(
out.cache, from_fs, from_fs.path_info, jobs=kwargs.get("jobs"),
out.odb, from_fs, from_fs.path_info, jobs=kwargs.get("jobs"),
)
out.checkout()
dvc/repo/diff.py
+1 -1
View File
@@ -128,7 +128,7 @@ def _output_paths(repo, repo_fs, targets):
def _to_checksum(output):
if on_working_fs:
return get_hash(output.path_info, repo.cache.local.fs, "md5").value
return get_hash(output.path_info, repo.odb.local.fs, "md5").value
return output.hash_info.value
for stage in repo.stages:
dvc/repo/experiments/__init__.py
+1 -1
View File
@@ -593,7 +593,7 @@ class Experiments:
name=item.name,
branch=item.branch,
tmp_dir=base_tmp_dir,
cache_dir=self.repo.cache.local.cache_dir,
cache_dir=self.repo.odb.local.cache_dir,
)
executors[stash_rev] = executor
dvc/repo/fetch.py
+4 -4
View File
@@ -93,10 +93,10 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs):
def cb(result):
results.append(result)
cache = self.cache.local
odb = self.odb.local
try:
with external_repo(
repo_url, repo_rev, cache_dir=cache.cache_dir
repo_url, repo_rev, cache_dir=odb.cache_dir
) as repo:
root = PathInfo(repo.root_dir)
for path in files:
@@ -112,14 +112,14 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs):
except (NoOutputOrStageError, NoRemoteError):
pass
obj = stage(
cache,
odb,
path_info,
repo.repo_fs,
jobs=jobs,
follow_subrepos=False,
)
save(
cache, obj, jobs=jobs, download_callback=cb,
odb, obj, jobs=jobs, download_callback=cb,
)
except CloneError:
failed += 1
dvc/repo/gc.py
+4 -4
View File
@@ -1,7 +1,7 @@
import logging
from dvc.cache import NamedCache
from dvc.exceptions import InvalidArgumentError
from dvc.objects.db import NamedCache
from ..scheme import Schemes
from . import locked
@@ -72,11 +72,11 @@ def gc(
)
)
for scheme, cache in self.cache.by_scheme():
if not cache:
for scheme, odb in self.odb.by_scheme():
if not odb:
continue
removed = cache.gc(set(used.scheme_keys(scheme)), jobs=jobs)
removed = odb.gc(set(used.scheme_keys(scheme)), jobs=jobs)
if not removed:
logger.info(f"No unused '{scheme}' cache to remove.")
dvc/stage/__init__.py
+1 -1
View File
@@ -614,7 +614,7 @@ class Stage(params.StageParams):
)
def get_used_cache(self, *args, **kwargs):
from dvc.cache import NamedCache
from dvc.objects.db import NamedCache
cache = NamedCache()
for out in self.filter_outs(kwargs.get("filter_info")):
dvc/stage/cache.py
+9 -9
View File
@@ -58,7 +58,7 @@ class StageCache:
@cached_property
def cache_dir(self):
return os.path.join(self.repo.cache.local.cache_dir, "runs")
return os.path.join(self.repo.odb.local.cache_dir, "runs")
def _get_cache_dir(self, key):
return os.path.join(self.cache_dir, key[:2], key)
@@ -117,12 +117,12 @@ class StageCache:
@contextmanager
def _cache_type_copy(self):
cache_types = self.repo.cache.local.cache_types
self.repo.cache.local.cache_types = ["copy"]
cache_types = self.repo.odb.local.cache_types
self.repo.odb.local.cache_types = ["copy"]
try:
yield
finally:
self.repo.cache.local.cache_types = cache_types
self.repo.odb.local.cache_types = cache_types
def _uncached_outs(self, stage, cache):
# NOTE: using temporary stage to avoid accidentally modifying original
@@ -166,12 +166,12 @@ class StageCache:
COMPILED_LOCK_FILE_STAGE_SCHEMA(cache)
path = PathInfo(self._get_cache_path(cache_key, cache_value))
self.repo.cache.local.makedirs(path.parent)
self.repo.odb.local.makedirs(path.parent)
tmp = tempfile.NamedTemporaryFile(delete=False, dir=path.parent).name
assert os.path.exists(path.parent)
assert os.path.isdir(path.parent)
dump_yaml(tmp, cache)
self.repo.cache.local.move(PathInfo(tmp), path)
self.repo.odb.local.move(PathInfo(tmp), path)
def restore(self, stage, run_cache=True, pull=False):
from .serialize import to_single_stage_lockfile
@@ -233,7 +233,7 @@ class StageCache:
remote = self.repo.cloud.get_remote(remote)
return self._transfer(
_log_exceptions(remote.fs.upload, "upload"),
self.repo.cache.local.fs,
self.repo.odb.local.fs,
remote.fs,
)
@@ -244,11 +244,11 @@ class StageCache:
return self._transfer(
_log_exceptions(remote.fs.download, "download"),
remote.fs,
self.repo.cache.local.fs,
self.repo.odb.local.fs,
)
def get_used_cache(self, used_run_cache, *args, **kwargs):
from dvc.cache import NamedCache
from dvc.objects.db import NamedCache
cache = NamedCache()
tests/func/api/test_api.py
+5 -5
View File
@@ -64,7 +64,7 @@ def test_open(tmp_dir, dvc, remote):
dvc.push()
# Remove cache to force download
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
with api.open("foo") as fd:
assert fd.read() == "foo-text"
@@ -100,7 +100,7 @@ def test_open_external(tmp_dir, erepo_dir, cloud):
erepo_dir.dvc.push(all_branches=True)
# Remove cache to force download
remove(erepo_dir.dvc.cache.local.cache_dir)
remove(erepo_dir.dvc.odb.local.cache_dir)
# Using file url to force clone to tmp repo
repo_url = f"file://{erepo_dir}"
@@ -117,7 +117,7 @@ def test_open_granular(tmp_dir, dvc, remote):
dvc.push()
# Remove cache to force download
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
with api.open("dir/foo") as fd:
assert fd.read() == "foo-text"
@@ -145,7 +145,7 @@ def test_missing(tmp_dir, dvc, remote):
tmp_dir.dvc_gen("foo", "foo")
# Remove cache to make foo missing
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
api.read("foo")
@@ -199,7 +199,7 @@ def test_api_missing_local_cache_exists_on_remote(
dvc.push()
# Remove cache to make foo missing
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
remove(first(files))
repo_url = f"file://{tmp_dir}" if as_external else None
tests/func/experiments/test_remote.py
+2 -2
View File
@@ -262,11 +262,11 @@ def test_push_pull_cache(
assert os.path.exists(path)
assert open(path).read() == str(x)
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
dvc.experiments.pull(remote, ref_info.name, pull_cache=True)
for x in range(2, checkpoint_stage.iterations + 1):
hash_ = digest(str(x))
path = os.path.join(dvc.cache.local.cache_dir, hash_[:2], hash_[2:])
path = os.path.join(dvc.odb.local.cache_dir, hash_[:2], hash_[2:])
assert os.path.exists(path)
assert open(path).read() == str(x)
tests/func/metrics/test_show.py
+1 -1
View File
@@ -220,7 +220,7 @@ def test_metrics_show_overlap(
# so as it works even for optimized cases
if clear_before_run:
remove(data_dir)
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
dvc._reset()
tests/func/plots/test_show.py
+1 -1
View File
@@ -754,7 +754,7 @@ def test_plots_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run):
# so as it works even for optimized cases
if clear_before_run:
remove(data_dir)
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
dvc._reset()
tests/func/remote/test_index.py
+1 -1
View File
@@ -82,7 +82,7 @@ def test_clear_on_gc(tmp_dir, dvc, tmp_path_factory, remote, mocker):
def test_clear_on_download_err(tmp_dir, dvc, tmp_path_factory, remote, mocker):
tmp_dir.dvc_gen({"foo": "foo content"})
dvc.push()
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
mocked_clear = mocker.patch.object(remote.INDEX_CLS, "clear")
mocker.patch.object(LocalFileSystem, "_download", side_effect=Exception)
tests/func/test_add.py
+14 -14
View File
@@ -12,7 +12,6 @@ import pytest
from mock import call, patch
import dvc as dvc_module
from dvc.cache import Cache
from dvc.dvcfile import DVC_FILE_SUFFIX
from dvc.exceptions import (
DvcException,
@@ -24,6 +23,7 @@ from dvc.exceptions import (
from dvc.fs.local import LocalFileSystem
from dvc.hash_info import HashInfo
from dvc.main import main
from dvc.objects.db import ODBManager
from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError
from dvc.repo import Repo as DvcRepo
from dvc.stage import Stage
@@ -106,7 +106,7 @@ def test_add_directory(tmp_dir, dvc):
hash_info = stage.outs[0].hash_info
dir_info = load(dvc.cache.local, hash_info).hash_info.dir_info
dir_info = load(dvc.odb.local, hash_info).hash_info.dir_info
for path, _ in dir_info.trie.items():
assert "\\" not in path
@@ -511,12 +511,12 @@ class TestAddCommit(TestDvc):
ret = main(["add", self.FOO, "--no-commit"])
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.FOO))
self.assertFalse(os.path.exists(self.dvc.cache.local.cache_dir))
self.assertFalse(os.path.exists(self.dvc.odb.local.cache_dir))
ret = main(["commit", self.FOO + ".dvc"])
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.FOO))
self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 1)
self.assertEqual(len(os.listdir(self.dvc.odb.local.cache_dir)), 1)
def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
@@ -700,7 +700,7 @@ def test_windows_should_add_when_cache_on_different_drive(
tmp_dir, dvc, temporary_windows_drive
):
dvc.config["cache"]["dir"] = temporary_windows_drive
dvc.cache = Cache(dvc)
dvc.odb = ODBManager(dvc)
(stage,) = tmp_dir.dvc_gen({"file": "file"})
cache_path = stage.outs[0].cache_path
@@ -713,12 +713,12 @@ def test_windows_should_add_when_cache_on_different_drive(
def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
tmp_dir.gen("dir/data", "data")
dvc.cache.local.cache_types = ["symlink"]
dvc.odb.local.cache_types = ["symlink"]
dvc.add("dir")
tmp_dir.gen("dir/new_file", "new_file_content")
unprotect_spy = mocker.spy(dvc.cache.local, "unprotect")
unprotect_spy = mocker.spy(dvc.odb.local, "unprotect")
dvc.add("dir")
assert not unprotect_spy.mock.called
@@ -726,13 +726,13 @@ def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
def test_should_not_checkout_when_adding_cached_copy(tmp_dir, dvc, mocker):
dvc.cache.local.cache_types = ["copy"]
dvc.odb.local.cache_types = ["copy"]
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
shutil.copy("bar", "foo")
copy_spy = mocker.spy(dvc.cache.local.fs, "copy")
copy_spy = mocker.spy(dvc.odb.local.fs, "copy")
dvc.add("foo")
@@ -758,9 +758,9 @@ def test_should_relink_on_repeated_add(
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
os.remove("foo")
getattr(dvc.cache.local.fs, link)(PathInfo("bar"), PathInfo("foo"))
getattr(dvc.odb.local.fs, link)(PathInfo("bar"), PathInfo("foo"))
dvc.cache.local.cache_types = [new_link]
dvc.odb.local.cache_types = [new_link]
dvc.add("foo")
@@ -769,7 +769,7 @@ def test_should_relink_on_repeated_add(
@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
def test_should_protect_on_repeated_add(link, tmp_dir, dvc):
dvc.cache.local.cache_types = [link]
dvc.odb.local.cache_types = [link]
tmp_dir.dvc_gen({"foo": "foo"})
@@ -827,7 +827,7 @@ def test_not_raises_on_re_add(tmp_dir, dvc):
@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
def test_add_empty_files(tmp_dir, dvc, link):
file = "foo"
dvc.cache.local.cache_types = [link]
dvc.odb.local.cache_types = [link]
stages = tmp_dir.dvc_gen(file, "")
assert (tmp_dir / file).exists()
@@ -836,7 +836,7 @@ def test_add_empty_files(tmp_dir, dvc, link):
def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker):
dvc.cache.local.cache_types = ["hardlink"]
dvc.odb.local.cache_types = ["hardlink"]
tmp_dir.gen({"foo": "", "bar": "", "lorem": "lorem", "ipsum": "ipsum"})
m = mocker.spy(LocalFileSystem, "is_hardlink")
stages = dvc.add(["foo", "bar", "lorem", "ipsum"])
tests/func/test_checkout.py
+6 -8
View File
@@ -101,12 +101,10 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
# NOTE: modifying cache file for one of the files inside the directory
# to check if dvc will detect that the cache is corrupted.
_, entry_hash = next(
load(
self.dvc.cache.local, out.hash_info
).hash_info.dir_info.items()
load(self.dvc.odb.local, out.hash_info).hash_info.dir_info.items()
)
cache = os.fspath(
self.dvc.cache.local.hash_to_path_info(entry_hash.value)
self.dvc.odb.local.hash_to_path_info(entry_hash.value)
)
os.chmod(cache, 0o644)
@@ -470,7 +468,7 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
self.assertTrue(System.is_symlink(self.DATA))
old_data_link = os.path.realpath(self.DATA)
old_cache_dir = self.dvc.cache.local.cache_dir
old_cache_dir = self.dvc.odb.local.cache_dir
new_cache_dir = old_cache_dir + "_new"
os.rename(old_cache_dir, new_cache_dir)
@@ -514,7 +512,7 @@ def test_checkout_no_checksum(tmp_dir, dvc):
[("hardlink", System.is_hardlink), ("symlink", System.is_symlink)],
)
def test_checkout_relink(tmp_dir, dvc, link, link_test_func):
dvc.cache.local.cache_types = [link]
dvc.odb.local.cache_types = [link]
tmp_dir.dvc_gen({"dir": {"data": "text"}})
dvc.unprotect("dir/data")
@@ -527,7 +525,7 @@ def test_checkout_relink(tmp_dir, dvc, link, link_test_func):
@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
def test_checkout_relink_protected(tmp_dir, dvc, link):
dvc.cache.local.cache_types = [link]
dvc.odb.local.cache_types = [link]
tmp_dir.dvc_gen("foo", "foo")
dvc.unprotect("foo")
@@ -718,7 +716,7 @@ def test_checkout_with_relink_existing(tmp_dir, dvc, link):
(tmp_dir / "foo").unlink()
tmp_dir.dvc_gen("bar", "bar")
dvc.cache.local.cache_types = [link]
dvc.odb.local.cache_types = [link]
stats = dvc.checkout(relink=True)
assert stats == {**empty_checkout, "added": ["foo"]}
tests/func/test_data_cloud.py
+17 -17
View File
@@ -6,9 +6,9 @@ import pytest
from flaky.flaky_decorator import flaky
import dvc as dvc_module
from dvc.cache import NamedCache
from dvc.external_repo import clean_repos
from dvc.main import main
from dvc.objects.db import NamedCache
from dvc.remote.base import (
STATUS_DELETED,
STATUS_MISSING,
@@ -76,8 +76,8 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
# Move cache and check status
# See issue https://github.com/iterative/dvc/issues/4383 for details
backup_dir = dvc.cache.local.cache_dir + ".backup"
move(dvc.cache.local.cache_dir, backup_dir)
backup_dir = dvc.odb.local.cache_dir + ".backup"
move(dvc.odb.local.cache_dir, backup_dir)
status = dvc.cloud.status(info, show_checksums=True)
expected = {md5: {"name": md5, "status": STATUS_MISSING}}
assert status == expected
@@ -87,8 +87,8 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
assert status_dir == expected
# Restore original cache:
remove(dvc.cache.local.cache_dir)
move(backup_dir, dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
move(backup_dir, dvc.odb.local.cache_dir)
# Push and check status
dvc.cloud.push(info)
@@ -107,7 +107,7 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
assert status_dir == expected
# Remove and check status
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
status = dvc.cloud.status(info, show_checksums=True)
expected = {md5: {"name": md5, "status": STATUS_DELETED}}
@@ -162,7 +162,7 @@ def test_cloud_cli(tmp_dir, dvc, remote):
assert os.path.isfile(cache)
assert os.path.isfile(cache_dir)
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
assert main(["fetch"] + args) == 0
assert os.path.exists(cache)
@@ -187,7 +187,7 @@ def test_cloud_cli(tmp_dir, dvc, remote):
# NOTE: check if remote gc works correctly on directories
assert main(["gc", "-cw", "-f"] + args) == 0
shutil.move(
dvc.cache.local.cache_dir, dvc.cache.local.cache_dir + ".back",
dvc.odb.local.cache_dir, dvc.odb.local.cache_dir + ".back",
)
assert main(["fetch"] + args) == 0
@@ -247,7 +247,7 @@ def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
# purge cache
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
header = (
"Some of the cache files do not exist "
@@ -285,7 +285,7 @@ def test_verify_hashes(
# remove artifacts and cache to trigger fetching
remove("file")
remove("dir")
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
hash_spy = mocker.spy(dvc_module.objects.stage, "get_file_hash")
@@ -293,7 +293,7 @@ def test_verify_hashes(
assert hash_spy.call_count == 0
# Removing cache will invalidate existing state entries
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
dvc.config["remote"]["upstream"]["verify"] = True
@@ -315,9 +315,9 @@ def test_pull_git_imports(tmp_dir, dvc, scm, erepo):
assert dvc.pull()["fetched"] == 0
for item in ["foo", "new_dir", dvc.cache.local.cache_dir]:
for item in ["foo", "new_dir", dvc.odb.local.cache_dir]:
remove(item)
os.makedirs(dvc.cache.local.cache_dir, exist_ok=True)
os.makedirs(dvc.odb.local.cache_dir, exist_ok=True)
clean_repos()
assert dvc.pull(force=True)["fetched"] == 2
@@ -355,12 +355,12 @@ def test_pull_external_dvc_imports(tmp_dir, dvc, scm, erepo_dir):
def clean(outs, dvc=None):
if dvc:
outs = outs + [dvc.cache.local.cache_dir]
outs = outs + [dvc.odb.local.cache_dir]
for path in outs:
print(path)
remove(path)
if dvc:
os.makedirs(dvc.cache.local.cache_dir, exist_ok=True)
os.makedirs(dvc.odb.local.cache_dir, exist_ok=True)
clean_repos()
@@ -532,8 +532,8 @@ def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
dvc.pull("copy-foo-bar")
assert (tmp_dir / "bar").exists()
assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1
assert len(recurse_list_dir(dvc.odb.local.cache_dir)) == 1
clean(["bar"], dvc)
dvc.fetch("copy-foo-bar")
assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1
assert len(recurse_list_dir(dvc.odb.local.cache_dir)) == 1
tests/func/test_diff.py
+2 -2
View File
@@ -201,10 +201,10 @@ def test_diff_no_cache(tmp_dir, scm, dvc):
)
scm.tag("v2")
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
# invalidate_dir_info to force cache loading
dvc.cache.local._dir_info = {}
dvc.odb.local._dir_info = {}
diff = dvc.diff("v1", "v2")
assert diff["added"] == []
tests/func/test_external_repo.py
+4 -4
View File
@@ -114,7 +114,7 @@ def test_relative_remote(erepo_dir, tmp_dir):
erepo_dir.dvc.push()
(erepo_dir / "file").unlink()
remove(erepo_dir.dvc.cache.local.cache_dir)
remove(erepo_dir.dvc.odb.local.cache_dir)
url = os.fspath(erepo_dir)
@@ -197,17 +197,17 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
assert (tmp_dir / "out").read_text() == expected_files
# clear cache to test saving to cache
cache_dir = tmp_dir / repo.cache.local.cache_dir
cache_dir = tmp_dir / repo.odb.local.cache_dir
remove(cache_dir)
makedirs(cache_dir)
obj = stage(
repo.cache.local,
repo.odb.local,
PathInfo(repo.root_dir) / "dir",
repo.repo_fs,
follow_subrepos=False,
)
save(repo.cache.local, obj)
save(repo.odb.local, obj)
assert set(cache_dir.glob("*/*")) == {
cache_dir / "e1" / "d9e8eae5374860ae025ec84cfd85c7.dir",
cache_dir / "37" / "b51d194a7513e45b56f6524f2d51f2",
tests/func/test_gc.py
+13 -13
View File
@@ -6,10 +6,10 @@ import configobj
import pytest
from git import Repo
from dvc.cache.local import LocalCache
from dvc.exceptions import CollectCacheError
from dvc.fs.local import LocalFileSystem
from dvc.main import main
from dvc.objects.db.local import LocalObjectDB
from dvc.repo import Repo as DvcRepo
from dvc.utils.fs import remove
from tests.basic_env import TestDir, TestDvcGit
@@ -22,13 +22,13 @@ class TestGC(TestDvcGit):
self.dvc.add(self.FOO)
self.dvc.add(self.DATA_DIR)
self.good_cache = [
self.dvc.cache.local.hash_to_path_info(md5)
for md5 in self.dvc.cache.local.all()
self.dvc.odb.local.hash_to_path_info(md5)
for md5 in self.dvc.odb.local.all()
]
self.bad_cache = []
for i in ["123", "234", "345"]:
path = os.path.join(self.dvc.cache.local.cache_dir, i[0:2], i[2:])
path = os.path.join(self.dvc.odb.local.cache_dir, i[0:2], i[2:])
self.create(path, i)
self.bad_cache.append(path)
@@ -42,7 +42,7 @@ class TestGC(TestDvcGit):
self._test_gc()
def _test_gc(self):
self.assertTrue(os.path.isdir(self.dvc.cache.local.cache_dir))
self.assertTrue(os.path.isdir(self.dvc.odb.local.cache_dir))
for c in self.bad_cache:
self.assertFalse(os.path.exists(c))
@@ -187,11 +187,11 @@ def test_all_commits(tmp_dir, scm, dvc):
tmp_dir.dvc_gen("testfile", "modified", commit="modified")
tmp_dir.dvc_gen("testfile", "workspace")
n = _count_files(dvc.cache.local.cache_dir)
n = _count_files(dvc.odb.local.cache_dir)
dvc.gc(all_commits=True)
# Only one uncommitted file should go away
assert _count_files(dvc.cache.local.cache_dir) == n - 1
assert _count_files(dvc.odb.local.cache_dir) == n - 1
def test_gc_no_dir_cache(tmp_dir, dvc):
@@ -203,9 +203,9 @@ def test_gc_no_dir_cache(tmp_dir, dvc):
with pytest.raises(CollectCacheError):
dvc.gc(workspace=True)
assert _count_files(dvc.cache.local.cache_dir) == 4
assert _count_files(dvc.odb.local.cache_dir) == 4
dvc.gc(force=True, workspace=True)
assert _count_files(dvc.cache.local.cache_dir) == 2
assert _count_files(dvc.odb.local.cache_dir) == 2
def _count_files(path):
@@ -218,7 +218,7 @@ def test_gc_no_unpacked_dir(tmp_dir, dvc):
os.remove("dir.dvc")
unpackeddir = (
dir_stages[0].outs[0].cache_path + LocalCache.UNPACKED_DIR_SUFFIX
dir_stages[0].outs[0].cache_path + LocalObjectDB.UNPACKED_DIR_SUFFIX
)
# older (pre 1.0) versions of dvc used to generate this dir
@@ -341,14 +341,14 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
run_copy("foo", "foo2", name="copy")
shutil.rmtree(dvc.stage_cache.cache_dir)
assert _count_files(dvc.cache.local.cache_dir) == 1
assert _count_files(dvc.odb.local.cache_dir) == 1
dvc.gc(workspace=True, force=True)
assert _count_files(dvc.cache.local.cache_dir) == 1
assert _count_files(dvc.odb.local.cache_dir) == 1
# remove pipeline file and lockfile and check
Dvcfile(dvc, PIPELINE_FILE).remove(force=True)
dvc.gc(workspace=True, force=True)
assert _count_files(dvc.cache.local.cache_dir) == 0
assert _count_files(dvc.odb.local.cache_dir) == 0
@pytest.mark.parametrize(
tests/func/test_get.py
+2 -2
View File
@@ -3,8 +3,8 @@ import os
import pytest
from dvc.cache import Cache
from dvc.main import main
from dvc.objects.db import ODBManager
from dvc.repo import Repo
from dvc.repo.get import GetDVCFileError
from dvc.system import System
@@ -81,7 +81,7 @@ def test_cache_type_is_properly_overridden(tmp_dir, erepo_dir):
with erepo_dir.chdir():
with erepo_dir.dvc.config.edit() as conf:
conf["cache"]["type"] = "symlink"
erepo_dir.dvc.cache = Cache(erepo_dir.dvc)
erepo_dir.dvc.odb = ODBManager(erepo_dir.dvc)
erepo_dir.scm_add(
[erepo_dir.dvc.config.files["repo"]], "set cache type to symlinks"
)
tests/func/test_import.py
+6 -6
View File
@@ -6,10 +6,10 @@ from funcy import first
from mock import patch
import dvc.data_cloud as cloud
from dvc.cache import Cache
from dvc.config import NoRemoteError
from dvc.dvcfile import Dvcfile
from dvc.exceptions import DownloadError
from dvc.objects.db import ODBManager
from dvc.stage.exceptions import StagePathNotFoundError
from dvc.system import System
from dvc.utils.fs import makedirs, remove
@@ -218,7 +218,7 @@ def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir):
with erepo_dir.chdir():
with erepo_dir.dvc.config.edit() as conf:
conf["cache"]["type"] = "symlink"
erepo_dir.dvc.cache = Cache(erepo_dir.dvc)
erepo_dir.dvc.odb = ODBManager(erepo_dir.dvc)
erepo_dir.scm_add(
[erepo_dir.dvc.config.files["repo"]],
"set source repo cache type to symlink",
@@ -240,7 +240,7 @@ def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir):
dvc.imp(os.fspath(erepo_dir), "dir", "dir_imported")
remove("dir_imported")
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
dvc.pull(["dir_imported.dvc"])
@@ -256,7 +256,7 @@ def test_pull_wildcard_imported_directory_stage(tmp_dir, dvc, erepo_dir):
dvc.imp(os.fspath(erepo_dir), "dir123", "dir_imported123")
remove("dir_imported123")
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
dvc.pull(["dir_imported*.dvc"], glob=True)
@@ -486,9 +486,9 @@ def test_pull_imported_stage_from_subrepos(
dvc.imp(os.fspath(erepo_dir), path, out="out")
# clean everything
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
remove("out")
makedirs(dvc.cache.local.cache_dir)
makedirs(dvc.odb.local.cache_dir)
stats = dvc.pull(["out.dvc"])
tests/func/test_import_url.py
+2 -2
View File
@@ -5,10 +5,10 @@ from uuid import uuid4
import pytest
from dvc.cache import Cache
from dvc.dependency.base import DependencyDoesNotExistError
from dvc.exceptions import InvalidArgumentError
from dvc.main import main
from dvc.objects.db import ODBManager
from dvc.stage import Stage
from dvc.utils.fs import makedirs
from tests.basic_env import TestDvc
@@ -185,7 +185,7 @@ def test_import_url_dir(tmp_dir, dvc, workspace, stage_md5, dir_md5):
# remove external cache to make sure that we don't need it to import dirs
with dvc.config.edit() as conf:
del conf["cache"]
dvc.cache = Cache(dvc)
dvc.odb = ODBManager(dvc)
assert not (tmp_dir / "dir").exists() # sanity check
dvc.imp_url("remote://workspace/dir")
tests/func/test_cache.py → tests/func/test_odb.py
+14 -14
View File
@@ -5,10 +5,10 @@ import textwrap
import configobj
import pytest
from dvc.cache import Cache
from dvc.hash_info import HashInfo
from dvc.main import main
from dvc.objects import ObjectFormatError
from dvc.objects.db import ODBManager
from dvc.utils import relpath
from tests.basic_env import TestDir, TestDvc
@@ -19,12 +19,12 @@ class TestCache(TestDvc):
self.cache1_md5 = "123"
self.cache2_md5 = "234"
self.cache1 = os.path.join(
self.dvc.cache.local.cache_dir,
self.dvc.odb.local.cache_dir,
self.cache1_md5[0:2],
self.cache1_md5[2:],
)
self.cache2 = os.path.join(
self.dvc.cache.local.cache_dir,
self.dvc.odb.local.cache_dir,
self.cache2_md5[0:2],
self.cache2_md5[2:],
)
@@ -32,13 +32,13 @@ class TestCache(TestDvc):
self.create(self.cache2, "2")
def test_all(self):
md5_list = list(Cache(self.dvc).local.all())
md5_list = list(ODBManager(self.dvc).local.all())
self.assertEqual(len(md5_list), 2)
self.assertIn(self.cache1_md5, md5_list)
self.assertIn(self.cache2_md5, md5_list)
def test_get(self):
cache = Cache(self.dvc).local.hash_to_path_info(self.cache1_md5)
cache = ODBManager(self.dvc).local.hash_to_path_info(self.cache1_md5)
self.assertEqual(os.fspath(cache), self.cache1)
@@ -51,16 +51,16 @@ class TestCacheLoadBadDirCache(TestDvc):
from dvc.objects import load
dir_hash = "123.dir"
fname = os.fspath(self.dvc.cache.local.hash_to_path_info(dir_hash))
fname = os.fspath(self.dvc.odb.local.hash_to_path_info(dir_hash))
self.create(fname, "<clearly>not,json")
with pytest.raises(ObjectFormatError):
load(self.dvc.cache.local, HashInfo("md5", dir_hash))
load(self.dvc.odb.local, HashInfo("md5", dir_hash))
dir_hash = "234.dir"
fname = os.fspath(self.dvc.cache.local.hash_to_path_info(dir_hash))
fname = os.fspath(self.dvc.odb.local.hash_to_path_info(dir_hash))
self.create(fname, '{"a": "b"}')
with pytest.raises(ObjectFormatError):
load(self.dvc.cache.local, HashInfo("md5", dir_hash))
load(self.dvc.odb.local, HashInfo("md5", dir_hash))
class TestExternalCacheDir(TestDvc):
@@ -70,7 +70,7 @@ class TestExternalCacheDir(TestDvc):
ret = main(["config", "cache.dir", cache_dir])
self.assertEqual(ret, 0)
self.assertFalse(os.path.exists(self.dvc.cache.local.cache_dir))
self.assertFalse(os.path.exists(self.dvc.odb.local.cache_dir))
ret = main(["add", self.FOO])
self.assertEqual(ret, 0)
@@ -89,7 +89,7 @@ class TestExternalCacheDir(TestDvc):
self.dvc.__init__()
assert self.dvc.cache.ssh.fs.path_info == ssh_url + "/tmp"
assert self.dvc.odb.ssh.fs.path_info == ssh_url + "/tmp"
class TestSharedCacheDir(TestDir):
@@ -183,7 +183,7 @@ class TestCmdCacheDir(TestDvc):
def test_default_cache_type(dvc):
assert dvc.cache.local.cache_types == ["reflink", "copy"]
assert dvc.odb.local.cache_types == ["reflink", "copy"]
@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
@@ -196,8 +196,8 @@ def test_shared_cache(tmp_dir, dvc, group):
if group:
with dvc.config.edit() as conf:
conf["cache"].update({"shared": "group"})
dvc.cache = Cache(dvc)
cache_dir = dvc.cache.local.cache_dir
dvc.odb = ODBManager(dvc)
cache_dir = dvc.odb.local.cache_dir
assert not os.path.exists(cache_dir)
tests/func/test_remote.py
+16 -16
View File
@@ -158,7 +158,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
with patch(
"dvc.objects.stage._collect_dir", return_value=dir_info,
):
hash1 = get_hash(path_info, dvc.cache.local.fs, "md5")
hash1 = get_hash(path_info, dvc.odb.local.fs, "md5")
dir_info = DirInfo.from_list(
[{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}]
@@ -166,7 +166,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
with patch(
"dvc.objects.stage._collect_dir", return_value=dir_info,
):
hash2 = get_hash(path_info, dvc.cache.local.fs, "md5")
hash2 = get_hash(path_info, dvc.odb.local.fs, "md5")
assert hash1 == hash2
@@ -191,18 +191,18 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
remote = dvc.cloud.get_remote("upstream")
assert not remote.fs.exists(
remote.cache.hash_to_path_info(foo.hash_info.value)
remote.odb.hash_to_path_info(foo.hash_info.value)
)
assert remote.fs.exists(
remote.cache.hash_to_path_info(bar.hash_info.value)
remote.odb.hash_to_path_info(bar.hash_info.value)
)
assert not remote.fs.exists(
remote.cache.hash_to_path_info(baz.hash_info.value)
remote.odb.hash_to_path_info(baz.hash_info.value)
)
# Push everything and delete local cache
dvc.push()
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
with patch.object(LocalFileSystem, "_download", side_effect=Exception):
with pytest.raises(DownloadError) as download_error_info:
@@ -256,7 +256,7 @@ def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
external_dir = tmp_path_factory.mktemp("external_dir")
(external_dir / "file").write_text("content")
dvc.cache.local = None
dvc.odb.local = None
with pytest.raises(RemoteCacheRequiredError):
dvc.run(
cmd="echo hello world",
@@ -276,8 +276,8 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
# foo .dir file should be uploaded after bar
remote = dvc.cloud.get_remote("upstream")
foo_path = remote.cache.hash_to_path_info(foo.hash_info.value)
bar_path = remote.cache.hash_to_path_info(
foo_path = remote.odb.hash_to_path_info(foo.hash_info.value)
bar_path = remote.odb.hash_to_path_info(
foo.hash_info.dir_info.trie[("bar",)].value
)
paths = [args[1] for args, _ in mocked_upload.call_args_list]
@@ -417,7 +417,7 @@ def test_protect_local_remote(tmp_dir, dvc, local_remote):
dvc.push()
remote = dvc.cloud.get_remote("upstream")
remote_cache_file = remote.cache.hash_to_path_info(
remote_cache_file = remote.odb.hash_to_path_info(
stage.outs[0].hash_info.value
)
@@ -429,18 +429,18 @@ def test_push_incomplete_dir(tmp_dir, dvc, mocker, local_remote):
(stage,) = tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
remote = dvc.cloud.get_remote("upstream")
cache = dvc.cache.local
odb = dvc.odb.local
dir_hash = stage.outs[0].hash_info.value
used = stage.get_used_cache(remote=remote)
# remove one of the cache files for directory
file_hashes = list(used.child_keys(cache.fs.scheme, dir_hash))
remove(cache.hash_to_path_info(file_hashes[0]))
file_hashes = list(used.child_keys(odb.fs.scheme, dir_hash))
remove(odb.hash_to_path_info(file_hashes[0]))
dvc.push()
assert not remote.fs.exists(remote.cache.hash_to_path_info(dir_hash))
assert not remote.fs.exists(remote.cache.hash_to_path_info(file_hashes[0]))
assert remote.fs.exists(remote.cache.hash_to_path_info(file_hashes[1]))
assert not remote.fs.exists(remote.odb.hash_to_path_info(dir_hash))
assert not remote.fs.exists(remote.odb.hash_to_path_info(file_hashes[0]))
assert remote.fs.exists(remote.odb.hash_to_path_info(file_hashes[1]))
def test_upload_exists(tmp_dir, dvc, local_remote):
tests/func/test_remove.py
+2 -2
View File
@@ -41,10 +41,10 @@ def test_remove_non_existent_file(tmp_dir, dvc):
def test_remove_broken_symlink(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
dvc.cache.local.cache_types = ["symlink"]
dvc.odb.local.cache_types = ["symlink"]
(stage,) = dvc.add("foo")
remove(dvc.cache.local.cache_dir)
remove(dvc.odb.local.cache_dir)
assert System.is_symlink("foo")
with pytest.raises(DvcException):
tests/func/test_repo.py
+2 -2
View File
@@ -1,11 +1,11 @@
from dvc.cache import Cache
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
from dvc.objects.db import ODBManager
from dvc.system import System
def test_destroy(tmp_dir, dvc, run_copy):
dvc.config["cache"]["type"] = ["symlink"]
dvc.cache = Cache(dvc)
dvc.odb = ODBManager(dvc)
tmp_dir.dvc_gen("file", "text")
tmp_dir.dvc_gen({"dir": {"file": "lorem", "subdir/file": "ipsum"}})
tests/func/test_repro.py
+2 -2
View File
@@ -829,12 +829,12 @@ class TestReproAllPipelines(SingleStageRun, TestDvc):
class TestReproNoCommit(TestRepro):
def test(self):
remove(self.dvc.cache.local.cache_dir)
remove(self.dvc.odb.local.cache_dir)
ret = main(
["repro", self._get_stage_target(self.stage), "--no-commit"]
)
self.assertEqual(ret, 0)
self.assertEqual(os.listdir(self.dvc.cache.local.cache_dir), ["runs"])
self.assertEqual(os.listdir(self.dvc.odb.local.cache_dir), ["runs"])
class TestReproAlreadyCached(TestRepro):
tests/func/test_run_single_stage.py
+2 -2
View File
@@ -724,12 +724,12 @@ class TestRunCommit(TestDvc):
)
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(fname))
self.assertFalse(os.path.exists(self.dvc.cache.local.cache_dir))
self.assertFalse(os.path.exists(self.dvc.odb.local.cache_dir))
ret = main(["commit", fname + ".dvc"])
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(fname))
self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 1)
self.assertEqual(len(os.listdir(self.dvc.odb.local.cache_dir)), 1)
class TestRunPersist(TestDvc):
tests/func/test_s3.py
+6 -6
View File
@@ -8,8 +8,8 @@ import moto.s3.models as s3model
import pytest
from moto import mock_s3
from dvc.cache.base import CloudCache
from dvc.fs.s3 import S3FileSystem
from dvc.objects.db.base import ObjectDB
from tests.remotes import S3
# from https://github.com/spulec/moto/blob/v1.3.5/tests/test_s3/test_s3.py#L40
@@ -61,16 +61,16 @@ def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm):
from dvc.checkout import _link
fs = S3FileSystem(dvc, {"url": str(base_info.parent)})
cache = CloudCache(fs)
s3 = cache.fs.s3.meta.client
odb = ObjectDB(fs)
s3 = odb.fs.s3.meta.client
s3.create_bucket(Bucket=base_info.bucket)
s3.put_object(
Bucket=base_info.bucket, Key=(base_info / "from").path, Body="data"
)
_link(cache, base_info / "from", base_info / "to")
_link(odb, base_info / "from", base_info / "to")
assert cache.fs.exists(base_info / "from")
assert cache.fs.exists(base_info / "to")
assert odb.fs.exists(base_info / "from")
assert odb.fs.exists(base_info / "to")
@mock_s3
tests/remotes/__init__.py
+2 -2
View File
@@ -103,7 +103,7 @@ def remote(tmp_dir, dvc, request):
@pytest.fixture
def workspace(tmp_dir, dvc, request):
from dvc.cache import Cache
from dvc.objects.db import ODBManager
cloud = request.param
@@ -119,6 +119,6 @@ def workspace(tmp_dir, dvc, request):
with dvc.config.edit() as conf:
conf["cache"][scheme] = "cache"
dvc.cache = Cache(dvc)
dvc.odb = ODBManager(dvc)
return cloud
tests/unit/cache/__init__.py → tests/unit/objects/db/__init__.py
View File
tests/unit/cache/test_local.py → tests/unit/objects/db/test_local.py
+18 -18
View File
@@ -3,9 +3,9 @@ import os
import pytest
from dvc.cache import NamedCache
from dvc.cache.local import LocalCache
from dvc.fs.local import LocalFileSystem
from dvc.objects.db import NamedCache
from dvc.objects.db.local import LocalObjectDB
from dvc.path_info import PathInfo
from dvc.remote.index import RemoteIndexNoop
@@ -15,29 +15,29 @@ def test_status_download_optimization(mocker, dvc):
And the desired files to fetch are already on the local cache,
Don't check the existence of the desired files on the remote cache
"""
cache = LocalCache(LocalFileSystem(dvc, {}))
odb = LocalObjectDB(LocalFileSystem(dvc, {}))
infos = NamedCache()
infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo")
infos.add("local", "37b51d194a7513e45b56f6524f2d51f2", "bar")
local_exists = list(infos["local"])
mocker.patch.object(cache, "hashes_exist", return_value=local_exists)
mocker.patch.object(odb, "hashes_exist", return_value=local_exists)
other_remote = mocker.Mock()
other_remote.url = "other_remote"
other_remote.hashes_exist.return_value = []
other_remote.index = RemoteIndexNoop()
other_remote.status(cache, infos, download=True)
other_remote.status(odb, infos, download=True)
assert other_remote.hashes_exist.call_count == 0
@pytest.mark.parametrize("link_name", ["hardlink", "symlink"])
def test_is_protected(tmp_dir, dvc, link_name):
cache = dvc.cache.local
fs = cache.fs
odb = dvc.odb.local
fs = odb.fs
link_method = getattr(fs, link_name)
(tmp_dir / "foo").write_text("foo")
@@ -47,23 +47,23 @@ def test_is_protected(tmp_dir, dvc, link_name):
link_method(foo, link)
assert not cache.is_protected(foo)
assert not cache.is_protected(link)
assert not odb.is_protected(foo)
assert not odb.is_protected(link)
cache.protect(foo)
odb.protect(foo)
assert cache.is_protected(foo)
assert cache.is_protected(link)
assert odb.is_protected(foo)
assert odb.is_protected(link)
cache.unprotect(link)
odb.unprotect(link)
assert not cache.is_protected(link)
assert not odb.is_protected(link)
if os.name == "nt" and link_name == "hardlink":
# NOTE: NTFS doesn't allow deleting read-only files, which forces us to
# set write perms on the link, which propagates to the source.
assert not cache.is_protected(foo)
assert not odb.is_protected(foo)
else:
assert cache.is_protected(foo)
assert odb.is_protected(foo)
@pytest.mark.parametrize("err", [errno.EPERM, errno.EACCES, errno.EROFS])
@@ -73,7 +73,7 @@ def test_protect_ignore_errors(tmp_dir, dvc, mocker, err):
mock_chmod = mocker.patch(
"os.chmod", side_effect=OSError(err, "something")
)
dvc.cache.local.protect(PathInfo("foo"))
dvc.odb.local.protect(PathInfo("foo"))
assert mock_chmod.called
@@ -84,5 +84,5 @@ def test_set_exec_ignore_errors(tmp_dir, dvc, mocker, err):
mock_chmod = mocker.patch(
"os.chmod", side_effect=OSError(err, "something")
)
dvc.cache.local.set_exec(PathInfo("foo"))
dvc.odb.local.set_exec(PathInfo("foo"))
assert mock_chmod.called
tests/unit/output/test_output.py
+1 -1
View File
@@ -4,8 +4,8 @@ import pytest
from funcy import first
from voluptuous import MultipleInvalid, Schema
from dvc.cache import NamedCache
from dvc.ignore import _no_match
from dvc.objects.db import NamedCache
from dvc.output import CHECKSUM_SCHEMA, BaseOutput
tests/unit/remote/test_base.py
+35 -43
View File
@@ -3,8 +3,8 @@ import math
import mock
import pytest
from dvc.cache.base import CloudCache
from dvc.fs.base import BaseFileSystem, RemoteCmdError
from dvc.objects.db.base import ObjectDB
from dvc.path_info import PathInfo
@@ -34,36 +34,32 @@ def test_cmd_error(dvc):
BaseFileSystem(dvc, config).remove("file")
@mock.patch.object(CloudCache, "list_hashes_traverse")
@mock.patch.object(CloudCache, "list_hashes_exists")
@mock.patch.object(ObjectDB, "list_hashes_traverse")
@mock.patch.object(ObjectDB, "list_hashes_exists")
def test_hashes_exist(object_exists, traverse, dvc):
cache = CloudCache(BaseFileSystem(dvc, {}))
odb = ObjectDB(BaseFileSystem(dvc, {}))
# remote does not support traverse
cache.fs.CAN_TRAVERSE = False
with mock.patch.object(
cache, "list_hashes", return_value=list(range(256))
):
odb.fs.CAN_TRAVERSE = False
with mock.patch.object(odb, "list_hashes", return_value=list(range(256))):
hashes = set(range(1000))
cache.hashes_exist(hashes)
odb.hashes_exist(hashes)
object_exists.assert_called_with(hashes, None, None)
traverse.assert_not_called()
cache.fs.CAN_TRAVERSE = True
odb.fs.CAN_TRAVERSE = True
# large remote, small local
object_exists.reset_mock()
traverse.reset_mock()
with mock.patch.object(
cache, "list_hashes", return_value=list(range(256))
):
with mock.patch.object(odb, "list_hashes", return_value=list(range(256))):
hashes = list(range(1000))
cache.hashes_exist(hashes)
# verify that _cache_paths_with_max() short circuits
odb.hashes_exist(hashes)
# verify that _odb_paths_with_max() short circuits
# before returning all 256 remote hashes
max_hashes = math.ceil(
cache._max_estimation_size(hashes)
/ pow(16, cache.fs.TRAVERSE_PREFIX_LEN)
odb._max_estimation_size(hashes)
/ pow(16, odb.fs.TRAVERSE_PREFIX_LEN)
)
assert max_hashes < 256
object_exists.assert_called_with(
@@ -74,15 +70,13 @@ def test_hashes_exist(object_exists, traverse, dvc):
# large remote, large local
object_exists.reset_mock()
traverse.reset_mock()
cache.fs.JOBS = 16
with mock.patch.object(
cache, "list_hashes", return_value=list(range(256))
):
odb.fs.JOBS = 16
with mock.patch.object(odb, "list_hashes", return_value=list(range(256))):
hashes = list(range(1000000))
cache.hashes_exist(hashes)
odb.hashes_exist(hashes)
object_exists.assert_not_called()
traverse.assert_called_with(
256 * pow(16, cache.fs.TRAVERSE_PREFIX_LEN),
256 * pow(16, odb.fs.TRAVERSE_PREFIX_LEN),
set(range(256)),
None,
None,
@@ -90,18 +84,18 @@ def test_hashes_exist(object_exists, traverse, dvc):
@mock.patch.object(
CloudCache, "list_hashes", return_value=[],
ObjectDB, "list_hashes", return_value=[],
)
@mock.patch.object(
CloudCache, "_path_to_hash", side_effect=lambda x: x,
ObjectDB, "_path_to_hash", side_effect=lambda x: x,
)
def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc):
cache = CloudCache(BaseFileSystem(dvc, {}))
cache.fs.path_info = PathInfo("foo")
odb = ObjectDB(BaseFileSystem(dvc, {}))
odb.fs.path_info = PathInfo("foo")
# parallel traverse
size = 256 / cache.fs.JOBS * cache.fs.LIST_OBJECT_PAGE_SIZE
list(cache.list_hashes_traverse(size, {0}))
size = 256 / odb.fs.JOBS * odb.fs.LIST_OBJECT_PAGE_SIZE
list(odb.list_hashes_traverse(size, {0}))
for i in range(1, 16):
list_hashes.assert_any_call(
prefix=f"{i:03x}", progress_callback=CallableOrNone
@@ -114,38 +108,36 @@ def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc):
# default traverse (small remote)
size -= 1
list_hashes.reset_mock()
list(cache.list_hashes_traverse(size - 1, {0}))
list(odb.list_hashes_traverse(size - 1, {0}))
list_hashes.assert_called_with(
prefix=None, progress_callback=CallableOrNone
)
def test_list_hashes(dvc):
cache = CloudCache(BaseFileSystem(dvc, {}))
cache.fs.path_info = PathInfo("foo")
odb = ObjectDB(BaseFileSystem(dvc, {}))
odb.fs.path_info = PathInfo("foo")
with mock.patch.object(
cache, "_list_paths", return_value=["12/3456", "bar"]
odb, "_list_paths", return_value=["12/3456", "bar"]
):
hashes = list(cache.list_hashes())
hashes = list(odb.list_hashes())
assert hashes == ["123456"]
def test_list_paths(dvc):
cache = CloudCache(BaseFileSystem(dvc, {}))
cache.fs.path_info = PathInfo("foo")
odb = ObjectDB(BaseFileSystem(dvc, {}))
odb.fs.path_info = PathInfo("foo")
with mock.patch.object(
cache.fs, "walk_files", return_value=[]
) as walk_mock:
for _ in cache._list_paths():
with mock.patch.object(odb.fs, "walk_files", return_value=[]) as walk_mock:
for _ in odb._list_paths():
pass
walk_mock.assert_called_with(cache.fs.path_info, prefix=False)
walk_mock.assert_called_with(odb.fs.path_info, prefix=False)
for _ in cache._list_paths(prefix="000"):
for _ in odb._list_paths(prefix="000"):
pass
walk_mock.assert_called_with(
cache.fs.path_info / "00" / "0", prefix=True
odb.fs.path_info / "00" / "0", prefix=True
)
tests/unit/repo/test_repo.py
+1 -1
View File
@@ -45,7 +45,7 @@ def test_find_outs_by_path_does_graph_checks(tmp_dir, dvc):
[os.path.join("dir", "subdir", "file"), os.path.join("dir", "subdir")],
)
def test_used_cache(tmp_dir, dvc, path):
from dvc.cache import NamedCache
from dvc.objects.db import NamedCache
tmp_dir.dvc_gen({"dir": {"subdir": {"file": "file"}, "other": "other"}})
expected = NamedCache.make(
tests/unit/stage/test_cache.py
+4 -4
View File
@@ -167,16 +167,16 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker):
def test_shared_stage_cache(tmp_dir, dvc, run_copy):
import stat
from dvc.cache import Cache
from dvc.objects.db import ODBManager
tmp_dir.gen("foo", "foo")
with dvc.config.edit() as config:
config["cache"]["shared"] = "group"
dvc.cache = Cache(dvc)
dvc.odb = ODBManager(dvc)
assert not os.path.exists(dvc.cache.local.cache_dir)
assert not os.path.exists(dvc.odb.local.cache_dir)
run_copy("foo", "bar", name="copy-foo-bar")
@@ -205,7 +205,7 @@ def test_shared_stage_cache(tmp_dir, dvc, run_copy):
dir_mode = 0o2775
file_mode = 0o664
assert _mode(dvc.cache.local.cache_dir) == dir_mode
assert _mode(dvc.odb.local.cache_dir) == dir_mode
assert _mode(dvc.stage_cache.cache_dir) == dir_mode
assert _mode(parent_cache_dir) == dir_mode
assert _mode(cache_dir) == dir_mode
tests/unit/test_external_repo.py
+8 -8
View File
@@ -58,7 +58,7 @@ def test_subrepo_is_constructed_properly(
subrepo = tmp_dir / "subrepo"
make_subrepo(subrepo, scm)
local_cache = subrepo.dvc.cache.local.cache_dir
local_cache = subrepo.dvc.odb.local.cache_dir
tmp_dir.scm_gen("bar", "bar", commit="add bar")
subrepo.dvc_gen("foo", "foo", commit="add foo")
@@ -75,19 +75,19 @@ def test_subrepo_is_constructed_properly(
assert repo.url == str(tmp_dir)
assert repo.config["cache"]["dir"] == str(cache_dir)
assert repo.cache.local.cache_dir == str(cache_dir)
assert subrepo.cache.local.cache_dir == str(cache_dir)
assert repo.odb.local.cache_dir == str(cache_dir)
assert subrepo.odb.local.cache_dir == str(cache_dir)
assert repo.config["cache"]["type"] == ["symlink"]
assert repo.cache.local.cache_types == ["symlink"]
assert subrepo.cache.local.cache_types == ["symlink"]
assert repo.odb.local.cache_types == ["symlink"]
assert subrepo.odb.local.cache_types == ["symlink"]
assert (
subrepo.config["remote"]["auto-generated-upstream"]["url"]
== local_cache
)
if root_is_dvc:
main_cache = tmp_dir.dvc.cache.local.cache_dir
main_cache = tmp_dir.dvc.odb.local.cache_dir
assert repo.config["remote"]["auto-generated-upstream"][
"url"
] == str(main_cache)
@@ -112,14 +112,14 @@ def test_fetch_external_repo_jobs(tmp_dir, scm, mocker, dvc, local_remote):
spy = mocker.spy(repo.cloud, "pull")
obj = stage(
dvc.cache.local,
dvc.odb.local,
PathInfo(repo.root_dir) / "dir1",
repo.repo_fs,
follow_subrepos=False,
jobs=3,
)
save(
dvc.cache.local, obj, jobs=3,
dvc.odb.local, obj, jobs=3,
)
run_jobs = tuple(spy.call_args_list[0])[1].get("jobs")
tests/unit/test_info.py
+2 -2
View File
@@ -17,7 +17,7 @@ PYTHON_VERSION_REGEX = r"Python \d\.\d+\.\d+\S*"
def test_info_in_repo(scm_init, tmp_dir):
tmp_dir.init(scm=scm_init, dvc=True)
# Create `.dvc/cache`, that is needed to check supported link types.
os.mkdir(tmp_dir.dvc.cache.local.cache_dir)
os.mkdir(tmp_dir.dvc.odb.local.cache_dir)
dvc_info = get_dvc_info()
@@ -86,7 +86,7 @@ def test_remotes(tmp_dir, dvc, caplog):
def test_fs_info_in_repo(tmp_dir, dvc, caplog):
os.mkdir(dvc.cache.local.cache_dir)
os.mkdir(dvc.odb.local.cache_dir)
dvc_info = get_dvc_info()
assert re.search(r"Cache directory: .* on .*", dvc_info)