mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-11 07:18:28 +01:00
Alexander Schepanovski
e2eb4ae0c3
The older code instantiates non-local cache remotes on Repo creation if any of those are configured. And remote instantiation loads their API libs.
106 lines
3.1 KiB
Python
106 lines
3.1 KiB
Python
"""Manages cache of a dvc repo."""
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
import os
|
|
from funcy import cached_property
|
|
|
|
from dvc.utils.compat import builtin_str
|
|
from dvc.config import Config
|
|
|
|
|
|
class CacheConfig(object):
|
|
def __init__(self, config):
|
|
self.config = config
|
|
|
|
def set_dir(self, dname, level=None):
|
|
from dvc.remote.config import RemoteConfig
|
|
|
|
configobj = self.config.get_configobj(level)
|
|
path = RemoteConfig.resolve_path(dname, configobj.filename)
|
|
self.config.set(
|
|
Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level
|
|
)
|
|
|
|
|
|
def _make_remote_property(name):
|
|
"""
|
|
The config file is stored in a way that allows you to have a
|
|
cache for each remote.
|
|
|
|
This is needed when specifying external outputs
|
|
(as they require you to have an external cache location).
|
|
|
|
Imagine a config file like the following:
|
|
|
|
['remote "dvc-storage"']
|
|
url = ssh://localhost/tmp
|
|
ask_password = true
|
|
|
|
[cache]
|
|
ssh = dvc-storage
|
|
|
|
This method creates a cached property, containing cache named `name`:
|
|
|
|
self.config == {'ssh': 'dvc-storage'}
|
|
self.ssh # a RemoteSSH instance
|
|
"""
|
|
|
|
def getter(self):
|
|
from dvc.remote import Remote
|
|
|
|
remote = self.config.get(name)
|
|
if not remote:
|
|
return None
|
|
|
|
return Remote(self.repo, name=remote)
|
|
|
|
getter.__name__ = builtin_str(name)
|
|
return cached_property(getter)
|
|
|
|
|
|
class Cache(object):
|
|
"""Class that manages cache locations of a dvc repo.
|
|
|
|
Args:
|
|
repo (dvc.repo.Repo): repo instance that this cache belongs to.
|
|
"""
|
|
|
|
CACHE_DIR = "cache"
|
|
|
|
def __init__(self, repo):
|
|
from dvc.remote import Remote
|
|
|
|
self.repo = repo
|
|
|
|
self.config = config = repo.config.config[Config.SECTION_CACHE]
|
|
local = config.get(Config.SECTION_CACHE_LOCAL)
|
|
|
|
if local:
|
|
name = Config.SECTION_REMOTE_FMT.format(local)
|
|
settings = repo.config.config[name]
|
|
else:
|
|
default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
|
|
cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
|
|
cache_type = config.get(Config.SECTION_CACHE_TYPE)
|
|
protected = config.get(Config.SECTION_CACHE_PROTECTED)
|
|
shared = config.get(Config.SECTION_CACHE_SHARED)
|
|
|
|
settings = {
|
|
Config.PRIVATE_CWD: config.get(
|
|
Config.PRIVATE_CWD, repo.dvc_dir
|
|
),
|
|
Config.SECTION_REMOTE_URL: cache_dir,
|
|
Config.SECTION_CACHE_TYPE: cache_type,
|
|
Config.SECTION_CACHE_PROTECTED: protected,
|
|
Config.SECTION_CACHE_SHARED: shared,
|
|
}
|
|
|
|
self.local = Remote(repo, **settings)
|
|
|
|
s3 = _make_remote_property(Config.SECTION_CACHE_S3)
|
|
gs = _make_remote_property(Config.SECTION_CACHE_GS)
|
|
ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
|
|
hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
|
|
azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
|