From 3d5c42e2fe2b6b2e80396b50d200993ce6268036 Mon Sep 17 00:00:00 2001 From: pared Date: Wed, 26 Jun 2019 12:41:06 +0200 Subject: [PATCH] state: test: refactor existing state test to pytest --- dvc/state.py | 12 ++-- dvc/utils/__init__.py | 5 ++ tests/func/test_state.py | 105 ++++++++++++++++----------------- tests/unit/utils/test_utils.py | 16 +++++ 4 files changed, 78 insertions(+), 60 deletions(-) create mode 100644 tests/unit/utils/test_utils.py diff --git a/dvc/state.py b/dvc/state.py index 15a4b0678..bc55d7194 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -7,11 +7,12 @@ import sqlite3 import logging from dvc.config import Config -from dvc.utils import remove, current_timestamp, relpath +from dvc.utils import remove, current_timestamp, relpath, chunk from dvc.exceptions import DvcException from dvc.utils.fs import get_mtime_and_size, get_inode from dvc.utils.compat import fspath_py35 +SQLITE_MAX_VARIABLES_NUMBER = 999 logger = logging.getLogger(__name__) @@ -449,7 +450,8 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes remove(path) unused.append(relpath) - cmd = "DELETE FROM {} WHERE path IN ({})".format( - self.LINK_STATE_TABLE, ",".join(["?"] * len(unused)) - ) - self._execute(cmd, tuple(unused)) + for chunk_unused in chunk(unused, SQLITE_MAX_VARIABLES_NUMBER): + cmd = "DELETE FROM {} WHERE path IN ({})".format( + self.LINK_STATE_TABLE, ",".join(["?"] * len(chunk_unused)) + ) + self._execute(cmd, tuple(chunk_unused)) diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 7e83d2485..8c76c36dd 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -382,3 +382,8 @@ def relpath(path, start=os.curdir): ): return path return os.path.relpath(path, start) + + +def chunk(iterable, chunk_size): + for i in range(0, len(iterable), chunk_size): + yield iterable[i : i + chunk_size] diff --git a/tests/func/test_state.py b/tests/func/test_state.py index e3d914eeb..5ea87fc1e 100644 --- a/tests/func/test_state.py +++ b/tests/func/test_state.py @@ -8,75 +8,70 @@ from dvc.utils import file_md5 from dvc.main import main from dvc.path_info import PathInfo -from tests.basic_env import TestDvc +def test_state(dvc_repo, repo_dir): + path = os.path.join(dvc_repo.root_dir, repo_dir.FOO) + path_info = PathInfo(path) + md5 = file_md5(path)[0] + + state = State(dvc_repo, dvc_repo.config.config) + + with state: + state.save(path_info, md5) + entry_md5 = state.get(path_info) + assert entry_md5 == md5 + + os.unlink(path) + with open(path, "a") as fd: + fd.write("1") + + entry_md5 = state.get(path_info) + assert entry_md5 is None -class TestState(TestDvc): - def test_update(self): - path = os.path.join(self.dvc.root_dir, self.FOO) - path_info = PathInfo(path) md5 = file_md5(path)[0] + state.save(path_info, md5) - state = State(self.dvc, self.dvc.config.config) - - with state: - state.save(path_info, md5) - entry_md5 = state.get(path_info) - self.assertEqual(entry_md5, md5) - - os.unlink(path) - with open(path, "a") as fd: - fd.write("1") - - entry_md5 = state.get(path_info) - self.assertTrue(entry_md5 is None) - - md5 = file_md5(path)[0] - state.save(path_info, md5) - - entry_md5 = state.get(path_info) - self.assertEqual(entry_md5, md5) + entry_md5 = state.get(path_info) + assert entry_md5 == md5 -class TestStateOverflow(TestDvc): - def test(self): - # NOTE: trying to add more entries than state can handle, - # to see if it will clean up and vacuum successfully - ret = main(["config", "state.row_limit", "10"]) - self.assertEqual(ret, 0) +def test_state_overflow(dvc_repo): + # NOTE: trying to add more entries than state can handle, + # to see if it will clean up and vacuum successfully + ret = main(["config", "state.row_limit", "10"]) + assert ret == 0 - dname = "dir" - os.mkdir(dname) - for i in range(20): - with open(os.path.join(dname, str(i)), "w+") as fobj: - fobj.write(str(i)) + dname = "dir" + os.mkdir(dname) + for i in range(20): + with open(os.path.join(dname, str(i)), "w+") as fobj: + fobj.write(str(i)) - ret = main(["add", "dir"]) - self.assertEqual(ret, 0) + ret = main(["add", "dir"]) + assert ret == 0 -class TestGetStateRecordForInode(TestDvc): - @staticmethod - def mock_get_inode(inode): - def get_inode_mocked(path): - return inode +def mock_get_inode(inode): + def get_inode_mocked(path): + return inode - return get_inode_mocked + return get_inode_mocked - @mock.patch("dvc.state.get_inode", autospec=True) - def test_transforms_inode(self, get_inode_mock): - state = State(self.dvc, self.dvc.config.config) - inode = state.MAX_INT + 2 - self.assertNotEqual(inode, state._to_sqlite(inode)) - path = os.path.join(self.dvc.root_dir, self.FOO) - md5 = file_md5(path)[0] - get_inode_mock.side_effect = self.mock_get_inode(inode) +@mock.patch("dvc.state.get_inode", autospec=True) +def test_get_state_record_for_inode(get_inode_mock, dvc_repo, repo_dir): + state = State(dvc_repo, dvc_repo.config.config) + inode = state.MAX_INT + 2 + assert inode != state._to_sqlite(inode) - with state: - state.save(PathInfo(path), md5) - ret = state.get_state_record_for_inode(inode) - self.assertIsNotNone(ret) + path = os.path.join(dvc_repo.root_dir, repo_dir.FOO) + md5 = file_md5(path)[0] + get_inode_mock.side_effect = mock_get_inode(inode) + + with state: + state.save(PathInfo(path), md5) + ret = state.get_state_record_for_inode(inode) + assert ret is not None def test_remove_unused_links(repo_dir, dvc_repo): diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py new file mode 100644 index 000000000..8e3b3de83 --- /dev/null +++ b/tests/unit/utils/test_utils.py @@ -0,0 +1,16 @@ +import pytest + +from dvc.utils import chunk + + +@pytest.mark.parametrize( + "list_to_chunk, chunk_size, expected_chunks", + [ + ([1, 2, 3, 4], 1, [[1], [2], [3], [4]]), + ([1, 2, 3, 4], 2, [[1, 2], [3, 4]]), + ([1, 2, 3, 4], 3, [[1, 2, 3], [4]]), + ], +) +def test_chunk(list_to_chunk, chunk_size, expected_chunks): + result = list(chunk(list_to_chunk, chunk_size)) + assert result == expected_chunks