install: pre-commit: install all deps by default

Fixes #9178

dvc/config_schema.py

@@ -273,12 +273,12 @@ SCHEMA = {
         )
     },
     "state": {
-        "dir": str,
+        "dir": str,  # obsoleted
         "row_limit": All(Coerce(int), Range(1)),  # obsoleted
         "row_cleanup_quota": All(Coerce(int), Range(0, 100)),  # obsoleted
     },
     "index": {
-        "dir": str,
+        "dir": str,  # obsoleted
     },
     "machine": {
         str: {

dvc/data_cloud.py

@@ -75,7 +75,7 @@ class DataCloud:
                     config["version_aware"] = True
 
             fs = cls(**config)
-            config["tmp_dir"] = self.repo.index_db_dir
+            config["tmp_dir"] = self.repo.site_cache_dir
             if self.repo.data_index is not None:
                 index = self.repo.data_index.view(("remote", name))
             else:

dvc/info.py

@@ -54,6 +54,7 @@ def get_dvc_info():
             fs_root = get_fs_type(os.path.abspath(root_directory))
             info.append(f"Workspace directory: {fs_root}")
             info.append(f"Repo: {_get_dvc_repo_info(repo)}")
+            info.append(f"Repo.site_cache_dir: {repo.site_cache_dir}")
     except NotDvcRepoError:
         pass
     except SCMError:

dvc/repo/__init__.py

@@ -134,38 +134,6 @@ class Repo:
         assert root_dir
         return root_dir, dvc_dir
 
-    def _get_database_dir(self, db_name: str) -> Optional[str]:
-        from dvc.fs import localfs
-
-        # NOTE: by default, store SQLite-based remote indexes and state's
-        # `links` and `md5s` caches in the repository itself to avoid any
-        # possible state corruption in 'shared cache dir' scenario, but allow
-        # user to override this through config when, say, the repository is
-        # located on a mounted volume — see
-        # https://github.com/iterative/dvc/issues/4420
-        base_db_dir = self.config.get(db_name, {}).get("dir", None)
-        if not base_db_dir:
-            return self.tmp_dir
-
-        import hashlib
-
-        if self.local_dvc_dir:
-            fs: "FileSystem" = localfs
-            local_root = fs.path.parent(self.local_dvc_dir)
-        else:
-            fs = self.fs
-            local_root = self.root_dir
-        root_dir_hash = hashlib.sha224(local_root.encode("utf-8")).hexdigest()
-
-        db_dir = fs.path.join(
-            base_db_dir,
-            self.DVC_DIR,
-            f"{fs.path.name(local_root)}-{root_dir_hash[0:7]}",
-        )
-
-        fs.makedirs(db_dir, exist_ok=True)
-        return db_dir
-
     def __init__(  # noqa: PLR0915
         self,
         root_dir: Optional[str] = None,
@@ -241,8 +209,8 @@ class Repo:
                     hardlink_lock=self.config["core"].get("hardlink_lock", False),
                     friendly=True,
                 )
-                state_db_dir = self._get_database_dir("state")
-                self.state = State(self.root_dir, state_db_dir, self.dvcignore)
+                os.makedirs(self.site_cache_dir, exist_ok=True)
+                self.state = State(self.root_dir, self.site_cache_dir, self.dvcignore)
             else:
                 self.lock = LockNoop()
                 self.state = StateNoop()
@@ -401,13 +369,9 @@ class Repo:
         from dvc_data.index import DataIndex
 
         if self._data_index is None:
-            if self.index_db_dir:
-                index_dir = os.path.join(self.index_db_dir, "index", "data")
-                os.makedirs(index_dir, exist_ok=True)
-
-                self._data_index = DataIndex.open(os.path.join(index_dir, "db.db"))
-            else:
-                self._data_index = DataIndex()
+            index_dir = os.path.join(self.site_cache_dir, "index", "data")
+            os.makedirs(index_dir, exist_ok=True)
+            self._data_index = DataIndex.open(os.path.join(index_dir, "db.db"))
 
         return self._data_index
 
@@ -590,8 +554,27 @@ class Repo:
         return DVCFileSystem(repo=self, subrepos=self.subrepos, **self._fs_conf)
 
     @cached_property
-    def index_db_dir(self):
-        return self._get_database_dir("index")
+    def site_cache_dir(self) -> str:
+        import hashlib
+
+        import platformdirs
+
+        from dvc.fs import GitFileSystem
+
+        cache_dir = platformdirs.site_cache_dir("dvc", "iterative", opinion=True)
+
+        if isinstance(self.fs, GitFileSystem):
+            relparts = ()
+            if self.root_dir != "/":
+                # subrepo
+                relparts = self.fs.path.relparts(self.root_dir, "/")
+            root_dir = os.path.join(self.scm.root_dir, *relparts)
+        else:
+            root_dir = self.root_dir
+
+        repo_token = hashlib.md5(os.fsencode(root_dir)).hexdigest()  # noqa: S324
+
+        return os.path.join(cache_dir, "repo", repo_token)
 
     @contextmanager
     def open_by_relpath(self, path, remote=None, mode="r", encoding=None):

dvc/repo/experiments/__init__.py

@@ -65,6 +65,11 @@ class Experiments:
 
     @property
     def scm(self):
+        from dvc.scm import SCMError
+
+        if self.repo.scm.no_commits:
+            raise SCMError("Empty Git repo. Add a commit to use experiments.")
+
         return self.repo.scm
 
     @cached_property

dvc/repo/experiments/queue/base.py

@@ -17,6 +17,8 @@ from typing import (
     Union,
 )
 
+from dvc_studio_client.env import STUDIO_REPO_URL
+from dvc_studio_client.post_live_metrics import get_studio_repo_url
 from funcy import retry
 
 from dvc.dependency import ParamsDependency
@@ -415,6 +417,10 @@ class BaseStashQueue(ABC):
                     run_env[DVC_EXP_NAME] = name
                     if resume_rev:
                         run_env[DVCLIVE_RESUME] = "1"
+                    studio_repo_url = get_studio_repo_url()
+                    if studio_repo_url is not None:
+                        run_env[STUDIO_REPO_URL] = studio_repo_url
+
                     self._pack_args(*args, run_env=run_env, **kwargs)
 
                     # save experiment as a stash commit

dvc/repo/experiments/utils.py

@@ -215,8 +215,6 @@ def exp_commits(
     refs = ref_infos if ref_infos else exp_refs(scm)
     for ref_info in refs:
         shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha))
-        if ref_info.baseline_sha:
-            shas.add(ref_info.baseline_sha)
     yield from shas
 
 

dvc/repo/init.py

@@ -12,7 +12,7 @@ from dvc.utils.fs import remove
 logger = logging.getLogger(__name__)
 
 
-def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
+def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):  # noqa: C901
     """
     Creates an empty repo on the given directory -- basically a
     `.dvc` directory with subdirectories for configuration and cache.
@@ -75,6 +75,11 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
 
     proj = Repo(root_dir)
 
+    if os.path.isdir(proj.site_cache_dir):
+        proj.close()
+        remove(proj.site_cache_dir)
+        proj = Repo(root_dir)
+
     with proj.scm_context(autostage=True) as context:
         files = [
             config.files["repo"],

dvc/repo/install.py

@@ -20,16 +20,19 @@ def pre_commit_install(scm: "Git") -> None:
             "hooks": [
                 {
                     "id": "dvc-pre-commit",
+                    "additional_dependencies": [".[all]"],
                     "language_version": "python3",
                     "stages": ["commit"],
                 },
                 {
                     "id": "dvc-pre-push",
+                    "additional_dependencies": [".[all]"],
                     "language_version": "python3",
                     "stages": ["push"],
                 },
                 {
                     "id": "dvc-post-checkout",
+                    "additional_dependencies": [".[all]"],
                     "language_version": "python3",
                     "stages": ["post-checkout"],
                     "always_run": True,

pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=0.42.3,<0.43",
+    "dvc-data>=0.43.0,<0.44",
     "dvc-http",
     "dvc-render>=0.1.2",
     "dvc-studio-client>=0.5.0,<1",
@@ -48,7 +48,7 @@ dependencies = [
     "networkx>=2.5",
     "packaging>=19",
     "pathspec>=0.10.3",
-    "platformdirs<4,>=3",
+    "platformdirs<4,>=3.1.1",
     "psutil>=5.8",
     "pydot>=1.2.4",
     "pygtrie>=2.3.2",
@@ -205,6 +205,7 @@ module = [
     "celery.*",
     "configobj.*",
     "dpath.*",
+    "distro",
     "dvc_http",
     "dvc_render.*",
     "dvc_ssh",
@@ -219,9 +220,12 @@ module = [
     "networkx.*",  # https://github.com/networkx/networkx/issues/3988
     "pygtrie.*",
     "pyinstrument",
+    "pyparsing",
     "pytest_docker.plugin",
     "ruamel.*",
     "ruamel.yaml.*",
+    "shortuuid",
+    "shtab",
     "tpi.*",
     "viztracer",
     "voluptuous",

tests/conftest.py

@@ -1,4 +1,3 @@
-import gc
 import json
 import os
 import sys
@@ -244,17 +243,3 @@ def run_copy_metrics(tmp_dir, copy_script):
         return stage
 
     return run
-
-
-@pytest.fixture(autouse=True)
-def gc_collect_on_dvc_close_on_win(mocker):
-    from dvc.repo import Repo
-
-    close = Repo.close
-
-    def wrapped(repo):
-        close(repo)
-        gc.collect()
-
-    if os.name == "nt":
-        mocker.patch("dvc.repo.Repo.close", wrapped)

tests/func/experiments/test_diff.py

@@ -1,16 +1,6 @@
 from funcy import first
 
 
-def test_diff_no_commits(tmp_dir):
-    from dvc.repo import Repo
-    from dvc.scm import Git
-
-    git = Git.init(tmp_dir.fs_path)
-    assert git.no_commits
-
-    assert Repo.init().experiments.diff() == {}
-
-
 def test_diff_empty(tmp_dir, scm, dvc, exp_stage):
     assert dvc.experiments.diff() == {"params": {}, "metrics": {}}
 

tests/func/experiments/test_experiments.py

@@ -13,7 +13,7 @@ from dvc.repo.experiments.exceptions import ExperimentExistsError
 from dvc.repo.experiments.queue.base import BaseStashQueue
 from dvc.repo.experiments.refs import CELERY_STASH
 from dvc.repo.experiments.utils import exp_refs_by_rev
-from dvc.scm import resolve_rev
+from dvc.scm import SCMError, resolve_rev
 from dvc.stage.exceptions import StageFileDoesNotExistError
 from dvc.utils.serialize import PythonFileCorruptedError
 from tests.scripts import COPY_SCRIPT
@@ -597,17 +597,21 @@ def test_experiments_workspace_not_log_exception(caplog, dvc, scm):
     assert not caplog.text
 
 
+@pytest.mark.vscode
 def test_run_env(tmp_dir, dvc, scm, mocker):
     dump_run_env = dedent(
         """\
         import os
+        from dvc_studio_client.env import STUDIO_REPO_URL
         from dvc.env import DVC_EXP_BASELINE_REV, DVC_EXP_NAME
-        with open("DVC_EXP_BASELINE_REV", "w") as f:
-            f.write(os.environ.get(DVC_EXP_BASELINE_REV, ""))
-        with open("DVC_EXP_NAME", "w") as f:
-            f.write(os.environ.get(DVC_EXP_NAME, ""))
+        for v in (DVC_EXP_BASELINE_REV, DVC_EXP_NAME, STUDIO_REPO_URL):
+            with open(v, "w") as f:
+                f.write(os.environ.get(v, ""))
         """
     )
+    mocker.patch(
+        "dvc.repo.experiments.queue.base.get_studio_repo_url", return_value="REPO_URL"
+    )
     (tmp_dir / "dump_run_env.py").write_text(dump_run_env)
     baseline = scm.get_rev()
     dvc.stage.add(
@@ -617,10 +621,12 @@ def test_run_env(tmp_dir, dvc, scm, mocker):
     dvc.experiments.run()
     assert (tmp_dir / "DVC_EXP_BASELINE_REV").read_text().strip() == baseline
     assert (tmp_dir / "DVC_EXP_NAME").read_text().strip() != ""
+    assert (tmp_dir / "STUDIO_REPO_URL").read_text().strip() == "REPO_URL"
 
     dvc.experiments.run(name="foo")
     assert (tmp_dir / "DVC_EXP_BASELINE_REV").read_text().strip() == baseline
     assert (tmp_dir / "DVC_EXP_NAME").read_text().strip() == "foo"
+    assert (tmp_dir / "STUDIO_REPO_URL").read_text().strip() == "REPO_URL"
 
 
 def test_experiment_unchanged(tmp_dir, scm, dvc, exp_stage):
@@ -640,3 +646,20 @@ def test_clean(tmp_dir, scm, dvc, mocker):
     clean = mocker.spy(dvc.experiments.celery_queue.celery, "clean")
     dvc.experiments.clean()
     clean.assert_called_once_with()
+
+
+def test_experiment_no_commit(tmp_dir):
+    from scmrepo.git import Git
+
+    from dvc.repo import Repo
+
+    Git.init(tmp_dir.fs_path).close()
+
+    repo = Repo.init()
+    assert repo.scm.no_commits
+
+    try:
+        with pytest.raises(SCMError):  # noqa: PT011
+            repo.experiments.ls()
+    finally:
+        repo.close()

tests/func/experiments/test_show.py

@@ -41,17 +41,6 @@ def make_executor_info(**kwargs):
     return ExecutorInfo(**kwargs)
 
 
-@pytest.mark.vscode
-def test_show_no_commits(tmp_dir):
-    from dvc.repo import Repo
-    from dvc.scm import Git
-
-    git = Git.init(tmp_dir.fs_path)
-    assert git.no_commits
-
-    assert Repo.init().experiments.show() == {}
-
-
 @pytest.mark.vscode
 def test_show_branch_and_tag_name(tmp_dir, scm, dvc, exp_stage):
     with tmp_dir.branch("new/branch", new=True):

tests/func/test_gc.py

@@ -276,8 +276,7 @@ def test_gc_all_experiments(tmp_dir, scm, dvc):
     (foo,) = tmp_dir.dvc_gen("foo", "foo", commit="foo")
     foo_hash = foo.outs[0].hash_info.value
 
-    (bar,) = tmp_dir.dvc_gen("foo", "bar", commit="bar")
-    bar_hash = bar.outs[0].hash_info.value
+    tmp_dir.dvc_gen("foo", "bar", commit="bar")
     baseline = scm.get_rev()
 
     (baz,) = tmp_dir.dvc_gen("foo", "baz", commit="baz")
@@ -288,12 +287,7 @@ def test_gc_all_experiments(tmp_dir, scm, dvc):
 
     dvc.gc(all_experiments=True, force=True)
 
-    # all_experiments will include the experiment commit (baz) plus baseline
-    # commit (bar)
     assert not (tmp_dir / ".dvc" / "cache" / foo_hash[:2] / foo_hash[2:]).exists()
-    assert (
-        tmp_dir / ".dvc" / "cache" / bar_hash[:2] / bar_hash[2:]
-    ).read_text() == "bar"
     assert (
         tmp_dir / ".dvc" / "cache" / baz_hash[:2] / baz_hash[2:]
     ).read_text() == "baz"

tests/func/test_state.py

@@ -1,7 +1,5 @@
 import os
-import re
 
-from dvc.repo import Repo
 from dvc_data.hashfile.hash import file_md5
 from dvc_data.hashfile.hash_info import HashInfo
 from dvc_data.hashfile.state import State
@@ -71,15 +69,3 @@ def test_get_unused_links(tmp_dir, dvc):
             dvc.fs,
         )
     ) == {"bar"}
-
-
-def test_state_dir_config(make_tmp_dir, dvc):
-    assert dvc.state.tmp_dir == dvc.tmp_dir
-
-    index_dir = str(make_tmp_dir("tmp_index"))
-    repo = Repo(config={"state": {"dir": index_dir}})
-    assert os.path.dirname(repo.state.tmp_dir) == os.path.join(index_dir, ".dvc")
-    assert re.match(
-        r"^test_state_dir_config0-([0-9a-f]+)$",
-        os.path.basename(repo.state.tmp_dir),
-    )

tests/unit/remote/test_remote.py

@@ -1,11 +1,8 @@
-import os
-
 import pytest
 from dvc_gs import GSFileSystem
 from dvc_s3 import S3FileSystem
 
 from dvc.fs import get_cloud_fs
-from dvc_data.hashfile.db import get_index
 
 
 def test_remote_with_hash_jobs(dvc):
@@ -57,18 +54,3 @@ def test_makedirs_not_create_for_top_level_path(fs_cls, dvc, mocker):
 
     fs.makedirs(url)
     assert not mocked_client.called
-
-
-def test_remote_index_dir_config(make_tmp_dir, dvc):
-    index_dir = str(make_tmp_dir("tmp_index"))
-    with dvc.config.edit() as conf:
-        conf["index"]["dir"] = index_dir
-        conf["remote"]["s3"] = {"url": "s3://bucket/name"}
-
-    dvc.root_dir = "/usr/local/test_repo"
-    dvc.dvc_dir = "/usr/local/test_repo/.dvc"
-    dvc.__dict__.pop("local_dvc_dir")
-
-    assert os.path.dirname(
-        get_index(dvc.cloud.get_remote_odb(name="s3")).index_dir
-    ) == os.path.join(index_dir, ".dvc", "test_repo-a473718", "index")

tests/unit/scm/test_scm.py

@@ -64,7 +64,6 @@ def test_iter_revs(
     assert gen == {
         rev_new: [rev_new],
         rev_old: [rev_old],
-        rev_root: [rev_root],
     }
 
     def _resolve_commit(rev):