repo: move index_dir to RemoteIndex (#4631)

dvc/remote/index.py

@@ -6,6 +6,7 @@ import threading
 from funcy import lchunks
 
 from dvc.state import _connect_sqlite
+from dvc.utils.fs import makedirs
 
 logger = logging.getLogger(__name__)
 
@@ -67,9 +68,12 @@ class RemoteIndex:
     VERSION = 1
     INDEX_TABLE = "remote_index"
     INDEX_TABLE_LAYOUT = "checksum TEXT PRIMARY KEY, dir INTEGER NOT NULL"
+    INDEX_DIR = "index"
 
     def __init__(self, repo, name, dir_suffix=".dir"):
-        self.path = os.path.join(repo.index_dir, f"{name}{self.INDEX_SUFFIX}")
+        self.path = os.path.join(
+            repo.tmp_dir, self.INDEX_DIR, f"{name}{self.INDEX_SUFFIX}"
+        )
 
         self.dir_suffix = dir_suffix
         self.database = None
@@ -150,6 +154,7 @@ class RemoteIndex:
             assert self.cursor is None
 
             empty = not os.path.isfile(self.path)
+            makedirs(os.path.dirname(self.path), exist_ok=True)
             self.database = _connect_sqlite(self.path, {"nolock": 1})
             self.cursor = self.database.cursor()
 

dvc/repo/__init__.py

@@ -132,8 +132,7 @@ class Repo:
             self.scm = SCM(self.root_dir, no_scm=no_scm)
 
         self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
-        self.index_dir = os.path.join(self.tmp_dir, "index")
-        makedirs(self.index_dir, exist_ok=True)
+        makedirs(self.tmp_dir, exist_ok=True)
 
         hardlink_lock = self.config["core"].get("hardlink_lock", False)
         self.lock = make_lock(

dvc/repo/experiments/executor.py

@@ -59,7 +59,15 @@ class ExperimentExecutor:
     # TODO: come up with better way to stash repro arguments
     @staticmethod
     def pack_repro_args(path, *args, tree=None, **kwargs):
-        open_func = tree.open if tree else open
+        dpath = os.path.dirname(path)
+        if tree:
+            open_func = tree.open
+            tree.makedirs(dpath)
+        else:
+            from dvc.utils.fs import makedirs
+
+            open_func = open
+            makedirs(dpath, exist_ok=True)
         data = {"args": args, "kwargs": kwargs}
         with open_func(path, "wb") as fobj:
             pickle.dump(data, fobj)

tests/unit/remote/test_index.py

@@ -16,7 +16,7 @@ def index(dvc):
 
 
 def test_init(dvc, index):
-    assert str(index.path) == os.path.join(dvc.index_dir, "foo.idx")
+    assert str(index.path) == os.path.join(dvc.tmp_dir, "index", "foo.idx")
 
 
 def test_is_dir_hash(dvc, index):