diff --git a/dvc/config_schema.py b/dvc/config_schema.py index 5f7d15926..fdb7b3386 100644 --- a/dvc/config_schema.py +++ b/dvc/config_schema.py @@ -273,12 +273,12 @@ SCHEMA = { ) }, "state": { - "dir": str, + "dir": str, # obsoleted "row_limit": All(Coerce(int), Range(1)), # obsoleted "row_cleanup_quota": All(Coerce(int), Range(0, 100)), # obsoleted }, "index": { - "dir": str, + "dir": str, # obsoleted }, "machine": { str: { diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index b5094ca6e..c9401fa8d 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -75,7 +75,7 @@ class DataCloud: config["version_aware"] = True fs = cls(**config) - config["tmp_dir"] = self.repo.index_db_dir + config["tmp_dir"] = self.repo.site_cache_dir if self.repo.data_index is not None: index = self.repo.data_index.view(("remote", name)) else: diff --git a/dvc/info.py b/dvc/info.py index b204dc1d7..733a6e22e 100644 --- a/dvc/info.py +++ b/dvc/info.py @@ -54,6 +54,7 @@ def get_dvc_info(): fs_root = get_fs_type(os.path.abspath(root_directory)) info.append(f"Workspace directory: {fs_root}") info.append(f"Repo: {_get_dvc_repo_info(repo)}") + info.append(f"Repo.site_cache_dir: {repo.site_cache_dir}") except NotDvcRepoError: pass except SCMError: diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 34fad33c7..b7b2587a5 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -134,38 +134,6 @@ class Repo: assert root_dir return root_dir, dvc_dir - def _get_database_dir(self, db_name: str) -> Optional[str]: - from dvc.fs import localfs - - # NOTE: by default, store SQLite-based remote indexes and state's - # `links` and `md5s` caches in the repository itself to avoid any - # possible state corruption in 'shared cache dir' scenario, but allow - # user to override this through config when, say, the repository is - # located on a mounted volume — see - # https://github.com/iterative/dvc/issues/4420 - base_db_dir = self.config.get(db_name, {}).get("dir", None) - if not base_db_dir: - return self.tmp_dir - - import hashlib - - if self.local_dvc_dir: - fs: "FileSystem" = localfs - local_root = fs.path.parent(self.local_dvc_dir) - else: - fs = self.fs - local_root = self.root_dir - root_dir_hash = hashlib.sha224(local_root.encode("utf-8")).hexdigest() - - db_dir = fs.path.join( - base_db_dir, - self.DVC_DIR, - f"{fs.path.name(local_root)}-{root_dir_hash[0:7]}", - ) - - fs.makedirs(db_dir, exist_ok=True) - return db_dir - def __init__( # noqa: PLR0915 self, root_dir: Optional[str] = None, @@ -241,8 +209,8 @@ class Repo: hardlink_lock=self.config["core"].get("hardlink_lock", False), friendly=True, ) - state_db_dir = self._get_database_dir("state") - self.state = State(self.root_dir, state_db_dir, self.dvcignore) + os.makedirs(self.site_cache_dir, exist_ok=True) + self.state = State(self.root_dir, self.site_cache_dir, self.dvcignore) else: self.lock = LockNoop() self.state = StateNoop() @@ -401,13 +369,9 @@ class Repo: from dvc_data.index import DataIndex if self._data_index is None: - if self.index_db_dir: - index_dir = os.path.join(self.index_db_dir, "index", "data") - os.makedirs(index_dir, exist_ok=True) - - self._data_index = DataIndex.open(os.path.join(index_dir, "db.db")) - else: - self._data_index = DataIndex() + index_dir = os.path.join(self.site_cache_dir, "index", "data") + os.makedirs(index_dir, exist_ok=True) + self._data_index = DataIndex.open(os.path.join(index_dir, "db.db")) return self._data_index @@ -590,8 +554,27 @@ class Repo: return DVCFileSystem(repo=self, subrepos=self.subrepos, **self._fs_conf) @cached_property - def index_db_dir(self): - return self._get_database_dir("index") + def site_cache_dir(self) -> str: + import hashlib + + import platformdirs + + from dvc.fs import GitFileSystem + + cache_dir = platformdirs.site_cache_dir("dvc", "iterative", opinion=True) + + if isinstance(self.fs, GitFileSystem): + relparts = () + if self.root_dir != "/": + # subrepo + relparts = self.fs.path.relparts(self.root_dir, "/") + root_dir = os.path.join(self.scm.root_dir, *relparts) + else: + root_dir = self.root_dir + + repo_token = hashlib.md5(os.fsencode(root_dir)).hexdigest() # noqa: S324 + + return os.path.join(cache_dir, "repo", repo_token) @contextmanager def open_by_relpath(self, path, remote=None, mode="r", encoding=None): diff --git a/dvc/repo/init.py b/dvc/repo/init.py index 8d6c7c78e..5a9ab093d 100644 --- a/dvc/repo/init.py +++ b/dvc/repo/init.py @@ -12,7 +12,7 @@ from dvc.utils.fs import remove logger = logging.getLogger(__name__) -def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): +def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): # noqa: C901 """ Creates an empty repo on the given directory -- basically a `.dvc` directory with subdirectories for configuration and cache. @@ -75,6 +75,11 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): proj = Repo(root_dir) + if os.path.isdir(proj.site_cache_dir): + proj.close() + remove(proj.site_cache_dir) + proj = Repo(root_dir) + with proj.scm_context(autostage=True) as context: files = [ config.files["repo"], diff --git a/pyproject.toml b/pyproject.toml index 094398607..8c1a98867 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "networkx>=2.5", "packaging>=19", "pathspec>=0.10.3", - "platformdirs<4,>=3", + "platformdirs<4,>=3.1.1", "psutil>=5.8", "pydot>=1.2.4", "pygtrie>=2.3.2", diff --git a/tests/func/test_state.py b/tests/func/test_state.py index b113597a7..65c5ee262 100644 --- a/tests/func/test_state.py +++ b/tests/func/test_state.py @@ -1,7 +1,5 @@ import os -import re -from dvc.repo import Repo from dvc_data.hashfile.hash import file_md5 from dvc_data.hashfile.hash_info import HashInfo from dvc_data.hashfile.state import State @@ -71,15 +69,3 @@ def test_get_unused_links(tmp_dir, dvc): dvc.fs, ) ) == {"bar"} - - -def test_state_dir_config(make_tmp_dir, dvc): - assert dvc.state.tmp_dir == dvc.tmp_dir - - index_dir = str(make_tmp_dir("tmp_index")) - repo = Repo(config={"state": {"dir": index_dir}}) - assert os.path.dirname(repo.state.tmp_dir) == os.path.join(index_dir, ".dvc") - assert re.match( - r"^test_state_dir_config0-([0-9a-f]+)$", - os.path.basename(repo.state.tmp_dir), - ) diff --git a/tests/unit/remote/test_remote.py b/tests/unit/remote/test_remote.py index 0847a779c..89a2b6c3e 100644 --- a/tests/unit/remote/test_remote.py +++ b/tests/unit/remote/test_remote.py @@ -1,11 +1,8 @@ -import os - import pytest from dvc_gs import GSFileSystem from dvc_s3 import S3FileSystem from dvc.fs import get_cloud_fs -from dvc_data.hashfile.db import get_index def test_remote_with_hash_jobs(dvc): @@ -57,18 +54,3 @@ def test_makedirs_not_create_for_top_level_path(fs_cls, dvc, mocker): fs.makedirs(url) assert not mocked_client.called - - -def test_remote_index_dir_config(make_tmp_dir, dvc): - index_dir = str(make_tmp_dir("tmp_index")) - with dvc.config.edit() as conf: - conf["index"]["dir"] = index_dir - conf["remote"]["s3"] = {"url": "s3://bucket/name"} - - dvc.root_dir = "/usr/local/test_repo" - dvc.dvc_dir = "/usr/local/test_repo/.dvc" - dvc.__dict__.pop("local_dvc_dir") - - assert os.path.dirname( - get_index(dvc.cloud.get_remote_odb(name="s3")).index_dir - ) == os.path.join(index_dir, ".dvc", "test_repo-a473718", "index")