mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-07 15:28:28 +01:00
eb46c03c6c
* objects: make HashInfo & HashFile hashable * stage/output: collect objects (HashFiles) instead of NamedCache * remote/data_cloud: use objs instead of NamedCache * repo: use objects instead of NamedCache * rerun black after rebase * remove NamedCache usage from run-cache * drop NamedCache * update HashInfo.__hash__ * update unit tests * revert remote_name change * start cleaning up test failures * fix tree obj expansion * update missing cache logging * fix external dict handling * update naive odb.gc * update state save behavior for pull * update test_data_cloud * update test_remote * fix gc with multiple repos * objects.tree: support filtering w/preserved hash_info * fix stage cache restore * drop fs.path_info from HashFile.__hash__ * objects: add optional name attribute * revert missing cache logging name/suffix changes * revert name changes in remote tests * review fixes * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update dvc/remote/base.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: Ruslan Kuprieiev <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
81 lines
2.1 KiB
Python
81 lines
2.1 KiB
Python
from dvc.scheme import Schemes
|
|
|
|
|
|
def get_odb(fs, path_info, **config):
|
|
from .base import ObjectDB
|
|
from .gdrive import GDriveObjectDB
|
|
from .local import LocalObjectDB
|
|
from .ssh import SSHObjectDB
|
|
|
|
if fs.scheme == Schemes.LOCAL:
|
|
return LocalObjectDB(fs, path_info, **config)
|
|
|
|
if fs.scheme == Schemes.SSH:
|
|
return SSHObjectDB(fs, path_info, **config)
|
|
|
|
if fs.scheme == Schemes.GDRIVE:
|
|
return GDriveObjectDB(fs, path_info, **config)
|
|
|
|
return ObjectDB(fs, path_info, **config)
|
|
|
|
|
|
def _get_odb(repo, settings):
|
|
from dvc.fs import get_cloud_fs
|
|
|
|
if not settings:
|
|
return None
|
|
|
|
cls, config, path_info = get_cloud_fs(repo, **settings)
|
|
return get_odb(cls(**config), path_info, state=repo.state, **config)
|
|
|
|
|
|
class ODBManager:
|
|
CACHE_DIR = "cache"
|
|
CLOUD_SCHEMES = [
|
|
Schemes.S3,
|
|
Schemes.GS,
|
|
Schemes.SSH,
|
|
Schemes.HDFS,
|
|
Schemes.WEBHDFS,
|
|
]
|
|
|
|
def __init__(self, repo):
|
|
self.repo = repo
|
|
self.config = config = repo.config["cache"]
|
|
self._odb = {}
|
|
|
|
local = config.get("local")
|
|
|
|
if local:
|
|
settings = {"name": local}
|
|
elif "dir" not in config:
|
|
settings = None
|
|
else:
|
|
from dvc.config_schema import LOCAL_COMMON
|
|
|
|
settings = {"url": config["dir"]}
|
|
for opt in LOCAL_COMMON.keys():
|
|
if opt in config:
|
|
settings[str(opt)] = config.get(opt)
|
|
|
|
self._odb[Schemes.LOCAL] = _get_odb(repo, settings)
|
|
|
|
def _init_odb(self, schemes):
|
|
for scheme in schemes:
|
|
remote = self.config.get(scheme)
|
|
settings = {"name": remote} if remote else None
|
|
self._odb[scheme] = _get_odb(self.repo, settings)
|
|
|
|
def __getattr__(self, name):
|
|
if name not in self._odb and name in self.CLOUD_SCHEMES:
|
|
self._init_odb([name])
|
|
|
|
try:
|
|
return self._odb[name]
|
|
except KeyError as exc:
|
|
raise AttributeError from exc
|
|
|
|
def by_scheme(self):
|
|
self._init_odb(self.CLOUD_SCHEMES)
|
|
yield from self._odb.items()
|