diff --git a/dvc/remote/local.py b/dvc/remote/local.py index ae29921ab..3d0c9a3bf 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -21,6 +21,8 @@ from dvc.exceptions import DvcException from dvc.progress import progress from concurrent.futures import ThreadPoolExecutor +from dvc.utils.fs import get_mtime_and_size, get_inode + class RemoteLOCAL(RemoteBase): scheme = "local" @@ -122,13 +124,11 @@ class RemoteLOCAL(RemoteBase): if self.changed_cache_file(md5): return True - if not self.is_dir_cache(cache): - continue - - for entry in self.load_dir_cache(md5): - md5 = entry[self.PARAM_CHECKSUM] - cache = self.get(md5) - clist.append((cache, md5)) + if self.is_dir_cache(cache) and self._cache_metadata_changed(): + for entry in self.load_dir_cache(md5): + md5 = entry[self.PARAM_CHECKSUM] + cache = self.get(md5) + clist.append((cache, md5)) return False @@ -695,3 +695,15 @@ class RemoteLOCAL(RemoteBase): show_checksums=show_checksums, download=True, ) + + def _cache_metadata_changed(self): + mtime, size = get_mtime_and_size(self.cache_dir) + inode = get_inode(self.cache_dir) + + existing_record = self.state.get_state_record_for_inode(inode) + + if existing_record: + cached_mtime, cached_size, _, _ = existing_record + return not (mtime == cached_mtime and size == cached_size) + + return True diff --git a/dvc/state.py b/dvc/state.py index 64d377e62..742eb8551 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -2,18 +2,14 @@ from __future__ import unicode_literals -from dvc.utils.compat import str - import os -import time import sqlite3 -import nanotime import dvc.logger as logger from dvc.config import Config -from dvc.system import System -from dvc.utils import file_md5, remove +from dvc.utils import file_md5, remove, current_timestamp from dvc.exceptions import DvcException +from dvc.utils.fs import get_mtime_and_size, get_inode class StateVersionTooNewError(DvcException): @@ -289,6 +285,8 @@ class State(object): # pylint: disable=too-many-instance-attributes ) ) + self._update_cache_directory_state() + self.database.commit() self.cursor.close() self.database.close() @@ -296,30 +294,6 @@ class State(object): # pylint: disable=too-many-instance-attributes self.cursor = None self.inserts = 0 - @staticmethod - def _mtime_and_size(path): - size = os.path.getsize(path) - mtime = os.path.getmtime(path) - - if os.path.isdir(path): - for root, dirs, files in os.walk(str(path)): - for name in dirs + files: - entry = os.path.join(root, name) - stat = os.stat(entry) - size += stat.st_size - entry_mtime = stat.st_mtime - if entry_mtime > mtime: - mtime = entry_mtime - - # State of files handled by dvc is stored in db as TEXT. - # We cast results to string for later comparisons with stored values. - return str(int(nanotime.timestamp(mtime))), str(size) - - @staticmethod - def _inode(path): - logger.debug("Path {} inode {}".format(path, System.inode(path))) - return System.inode(path) - def _do_update(self, path, known_checksum=None): """ Make sure the stored info for the given path is up to date. @@ -327,25 +301,24 @@ class State(object): # pylint: disable=too-many-instance-attributes if not os.path.exists(path): return None, None - actual_mtime, actual_size = self._mtime_and_size(path) - actual_inode = self._inode(path) + actual_mtime, actual_size = get_mtime_and_size(path) + actual_inode = get_inode(path) - existing_records = self._get_state_records_for_inode(actual_inode) - should_insert_new_record = not existing_records + existing_record = self.get_state_record_for_inode(actual_inode) - if should_insert_new_record: - md5, info = self._insert_new_state_record( - path, actual_inode, actual_mtime, actual_size, known_checksum - ) - else: + if existing_record: md5, info = self._update_existing_state_record( path, actual_inode, actual_mtime, actual_size, - existing_records, + existing_record, known_checksum, ) + else: + md5, info = self._insert_new_state_record( + path, actual_inode, actual_mtime, actual_size, known_checksum + ) return md5, info @@ -355,13 +328,11 @@ class State(object): # pylint: disable=too-many-instance-attributes actual_inode, actual_mtime, actual_size, - existing_records, + existing_record, known_checksum=None, ): - md5, mtime, size = self._get_existing_record_data( - actual_inode, actual_mtime, actual_size, existing_records - ) + mtime, size, md5, _ = existing_record if _file_metadata_changed(actual_mtime, mtime, actual_size, size): md5, info = self._update_state_for_path_changed( path, actual_inode, actual_mtime, actual_size, known_checksum @@ -371,28 +342,12 @@ class State(object): # pylint: disable=too-many-instance-attributes self._update_state_record_timestamp_for_inode(actual_inode) return md5, info - def _get_existing_record_data( - self, actual_inode, actual_mtime, actual_size, existing_records - ): - assert len(existing_records) == 1 - assert len(existing_records[0]) == 5 - inode, mtime, size, md5, _ = existing_records[0] - inode = self._from_sqlite(inode) - assert inode == actual_inode - logger.debug( - "Inode '{}', mtime '{}', actual mtime '{}', size '{}', " - "actual size '{}'.".format( - inode, mtime, actual_mtime, size, actual_size - ) - ) - return md5, mtime, size - def _update_state_record_timestamp_for_inode(self, actual_inode): cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}' self._execute( cmd.format( self.STATE_TABLE, - int(nanotime.timestamp(time.time())), + current_timestamp(), self._to_sqlite(actual_inode), ) ) @@ -421,7 +376,7 @@ class State(object): # pylint: disable=too-many-instance-attributes actual_mtime, actual_size, md5, - int(nanotime.timestamp(time.time())), + current_timestamp(), self._to_sqlite(actual_inode), ) ) @@ -445,19 +400,22 @@ class State(object): # pylint: disable=too-many-instance-attributes actual_mtime, actual_size, md5, - int(nanotime.timestamp(time.time())), + current_timestamp(), ) ) self.inserts += 1 return md5, info - def _get_state_records_for_inode(self, actual_inode): - cmd = "SELECT * from {} WHERE inode={}".format( - self.STATE_TABLE, self._to_sqlite(actual_inode) - ) + def get_state_record_for_inode(self, inode): + cmd = "SELECT mtime, size, md5, timestamp from {} " "WHERE inode={}" + cmd = cmd.format(self.STATE_TABLE, inode) self._execute(cmd) - ret = self._fetchall() - return ret + results = self._fetchall() + if results: + # uniquness constrain on inode + assert len(results) == 1 + return results[0] + return None def update(self, path, known_checksum=None): """Gets the checksum for the specified path. Checksum will be @@ -499,8 +457,8 @@ class State(object): # pylint: disable=too-many-instance-attributes if not os.path.exists(path): return - mtime, _ = self._mtime_and_size(path) - inode = self._inode(path) + mtime, _ = get_mtime_and_size(path) + inode = get_inode(path) relpath = os.path.relpath(path, self.root_dir) cmd = ( @@ -531,8 +489,8 @@ class State(object): # pylint: disable=too-many-instance-attributes if not os.path.exists(path): continue - actual_inode = self._inode(path) - actual_mtime, _ = self._mtime_and_size(path) + actual_inode = get_inode(path) + actual_mtime, _ = get_mtime_and_size(path) if inode == actual_inode and mtime == actual_mtime: logger.debug("Removing '{}' as unused link.".format(path)) @@ -542,3 +500,20 @@ class State(object): # pylint: disable=too-many-instance-attributes for relpath in unused: cmd = 'DELETE FROM {} WHERE path = "{}"' self._execute(cmd.format(self.LINK_STATE_TABLE, relpath)) + + def _update_cache_directory_state(self): + cache_path = self.repo.cache.local.cache_dir + mtime, size = get_mtime_and_size(cache_path) + inode = get_inode(cache_path) + + cmd = ( + "INSERT OR REPLACE INTO {}(inode, size, mtime, timestamp, md5) " + 'VALUES ({}, "{}", "{}", "{}", "")'.format( + self.STATE_TABLE, + self._to_sqlite(inode), + size, + mtime, + current_timestamp(), + ) + ) + self._execute(cmd) diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 093e291db..d6789d8d9 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -10,6 +10,8 @@ import math import json import shutil import hashlib +import nanotime +import time LOCAL_CHUNK_SIZE = 1024 * 1024 LARGE_FILE_SIZE = 1024 * 1024 * 1024 @@ -219,3 +221,7 @@ def tmp_fname(fname): from uuid import uuid4 return fname + "." + str(uuid4()) + ".tmp" + + +def current_timestamp(): + return int(nanotime.timestamp(time.time())) diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py new file mode 100644 index 000000000..86f5e1712 --- /dev/null +++ b/dvc/utils/fs.py @@ -0,0 +1,33 @@ +from __future__ import unicode_literals + +import nanotime +import os + +import dvc.logger as logger +from dvc.system import System +from dvc.utils.compat import str + + +def get_inode(path): + inode = System.inode(path) + logger.debug("Path {} inode {}".format(path, inode)) + return inode + + +def get_mtime_and_size(path): + size = os.path.getsize(path) + mtime = os.path.getmtime(path) + + if os.path.isdir(path): + for root, dirs, files in os.walk(str(path)): + for name in dirs + files: + entry = os.path.join(root, name) + stat = os.stat(entry) + size += stat.st_size + entry_mtime = stat.st_mtime + if entry_mtime > mtime: + mtime = entry_mtime + + # State of files handled by dvc is stored in db as TEXT. + # We cast results to string for later comparisons with stored values. + return str(int(nanotime.timestamp(mtime))), str(size) diff --git a/tests/test_add.py b/tests/test_add.py index f2ad4b150..cc66ffb6c 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -276,3 +276,26 @@ class TestAddCommit(TestDvc): self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.FOO)) self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 1) + + +class TestShouldNotCheckCacheForDirIfCacheMetadataDidNotChange(TestDvc): + def test(self): + remote_local_loader_spy = spy( + dvc.remote.local.RemoteLOCAL.load_dir_cache + ) + with patch.object( + dvc.remote.local.RemoteLOCAL, + "load_dir_cache", + remote_local_loader_spy, + ): + + ret = main(["config", "cache.type", "copy"]) + self.assertEqual(ret, 0) + + ret = main(["add", self.DATA_DIR]) + self.assertEqual(ret, 0) + self.assertEqual(1, remote_local_loader_spy.mock.call_count) + + ret = main(["status", "{}.dvc".format(self.DATA_DIR)]) + self.assertEqual(ret, 0) + self.assertEqual(1, remote_local_loader_spy.mock.call_count) diff --git a/tests/unit/state.py b/tests/unit/fs.py similarity index 72% rename from tests/unit/state.py rename to tests/unit/fs.py index ab585ea69..cb3397129 100644 --- a/tests/unit/state.py +++ b/tests/unit/fs.py @@ -1,15 +1,14 @@ -from dvc.utils.compat import str - import os -from dvc.state import State -from tests.basic_env import TestDvc +from dvc.utils.compat import str +from dvc.utils.fs import get_mtime_and_size +from tests.basic_env import TestDir -class TestMtimeAndSize(TestDvc): +class TestMtimeAndSize(TestDir): def test(self): - file_time, file_size = State._mtime_and_size(self.DATA) - dir_time, dir_size = State._mtime_and_size(self.DATA_DIR) + file_time, file_size = get_mtime_and_size(self.DATA) + dir_time, dir_size = get_mtime_and_size(self.DATA_DIR) actual_file_size = os.path.getsize(self.DATA) actual_dir_size = (