state: test: refactor existing state test to pytest

dvc/state.py

@@ -7,11 +7,12 @@ import sqlite3
 import logging
 
 from dvc.config import Config
-from dvc.utils import remove, current_timestamp, relpath
+from dvc.utils import remove, current_timestamp, relpath, chunk
 from dvc.exceptions import DvcException
 from dvc.utils.fs import get_mtime_and_size, get_inode
 from dvc.utils.compat import fspath_py35
 
+SQLITE_MAX_VARIABLES_NUMBER = 999
 
 logger = logging.getLogger(__name__)
 
@@ -449,7 +450,8 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
                 remove(path)
                 unused.append(relpath)
 
-        cmd = "DELETE FROM {} WHERE path IN ({})".format(
-            self.LINK_STATE_TABLE, ",".join(["?"] * len(unused))
-        )
-        self._execute(cmd, tuple(unused))
+        for chunk_unused in chunk(unused, SQLITE_MAX_VARIABLES_NUMBER):
+            cmd = "DELETE FROM {} WHERE path IN ({})".format(
+                self.LINK_STATE_TABLE, ",".join(["?"] * len(chunk_unused))
+            )
+            self._execute(cmd, tuple(chunk_unused))

dvc/utils/__init__.py

@@ -382,3 +382,8 @@ def relpath(path, start=os.curdir):
     ):
         return path
     return os.path.relpath(path, start)
+
+
+def chunk(iterable, chunk_size):
+    for i in range(0, len(iterable), chunk_size):
+        yield iterable[i : i + chunk_size]

tests/func/test_state.py

@@ -8,75 +8,70 @@ from dvc.utils import file_md5
 from dvc.main import main
 from dvc.path_info import PathInfo
 
-from tests.basic_env import TestDvc
 
+def test_state(dvc_repo, repo_dir):
+    path = os.path.join(dvc_repo.root_dir, repo_dir.FOO)
+    path_info = PathInfo(path)
+    md5 = file_md5(path)[0]
+
+    state = State(dvc_repo, dvc_repo.config.config)
+
+    with state:
+        state.save(path_info, md5)
+        entry_md5 = state.get(path_info)
+        assert entry_md5 == md5
+
+        os.unlink(path)
+        with open(path, "a") as fd:
+            fd.write("1")
+
+        entry_md5 = state.get(path_info)
+        assert entry_md5 is None
 
-class TestState(TestDvc):
-    def test_update(self):
-        path = os.path.join(self.dvc.root_dir, self.FOO)
-        path_info = PathInfo(path)
         md5 = file_md5(path)[0]
+        state.save(path_info, md5)
 
-        state = State(self.dvc, self.dvc.config.config)
-
-        with state:
-            state.save(path_info, md5)
-            entry_md5 = state.get(path_info)
-            self.assertEqual(entry_md5, md5)
-
-            os.unlink(path)
-            with open(path, "a") as fd:
-                fd.write("1")
-
-            entry_md5 = state.get(path_info)
-            self.assertTrue(entry_md5 is None)
-
-            md5 = file_md5(path)[0]
-            state.save(path_info, md5)
-
-            entry_md5 = state.get(path_info)
-            self.assertEqual(entry_md5, md5)
+        entry_md5 = state.get(path_info)
+        assert entry_md5 == md5
 
 
-class TestStateOverflow(TestDvc):
-    def test(self):
-        # NOTE: trying to add more entries than state can handle,
-        # to see if it will clean up and vacuum successfully
-        ret = main(["config", "state.row_limit", "10"])
-        self.assertEqual(ret, 0)
+def test_state_overflow(dvc_repo):
+    # NOTE: trying to add more entries than state can handle,
+    # to see if it will clean up and vacuum successfully
+    ret = main(["config", "state.row_limit", "10"])
+    assert ret == 0
 
-        dname = "dir"
-        os.mkdir(dname)
-        for i in range(20):
-            with open(os.path.join(dname, str(i)), "w+") as fobj:
-                fobj.write(str(i))
+    dname = "dir"
+    os.mkdir(dname)
+    for i in range(20):
+        with open(os.path.join(dname, str(i)), "w+") as fobj:
+            fobj.write(str(i))
 
-        ret = main(["add", "dir"])
-        self.assertEqual(ret, 0)
+    ret = main(["add", "dir"])
+    assert ret == 0
 
 
-class TestGetStateRecordForInode(TestDvc):
-    @staticmethod
-    def mock_get_inode(inode):
-        def get_inode_mocked(path):
-            return inode
+def mock_get_inode(inode):
+    def get_inode_mocked(path):
+        return inode
 
-        return get_inode_mocked
+    return get_inode_mocked
 
-    @mock.patch("dvc.state.get_inode", autospec=True)
-    def test_transforms_inode(self, get_inode_mock):
-        state = State(self.dvc, self.dvc.config.config)
-        inode = state.MAX_INT + 2
-        self.assertNotEqual(inode, state._to_sqlite(inode))
 
-        path = os.path.join(self.dvc.root_dir, self.FOO)
-        md5 = file_md5(path)[0]
-        get_inode_mock.side_effect = self.mock_get_inode(inode)
+@mock.patch("dvc.state.get_inode", autospec=True)
+def test_get_state_record_for_inode(get_inode_mock, dvc_repo, repo_dir):
+    state = State(dvc_repo, dvc_repo.config.config)
+    inode = state.MAX_INT + 2
+    assert inode != state._to_sqlite(inode)
 
-        with state:
-            state.save(PathInfo(path), md5)
-            ret = state.get_state_record_for_inode(inode)
-            self.assertIsNotNone(ret)
+    path = os.path.join(dvc_repo.root_dir, repo_dir.FOO)
+    md5 = file_md5(path)[0]
+    get_inode_mock.side_effect = mock_get_inode(inode)
+
+    with state:
+        state.save(PathInfo(path), md5)
+        ret = state.get_state_record_for_inode(inode)
+        assert ret is not None
 
 
 def test_remove_unused_links(repo_dir, dvc_repo):

tests/unit/utils/test_utils.py

@@ -0,0 +1,16 @@
+import pytest
+
+from dvc.utils import chunk
+
+
+@pytest.mark.parametrize(
+    "list_to_chunk, chunk_size, expected_chunks",
+    [
+        ([1, 2, 3, 4], 1, [[1], [2], [3], [4]]),
+        ([1, 2, 3, 4], 2, [[1, 2], [3, 4]]),
+        ([1, 2, 3, 4], 3, [[1, 2, 3], [4]]),
+    ],
+)
+def test_chunk(list_to_chunk, chunk_size, expected_chunks):
+    result = list(chunk(list_to_chunk, chunk_size))
+    assert result == expected_chunks