diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 5cde2a3ea..66be70dd3 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -158,11 +158,6 @@ class RemoteLOCAL(RemoteBase): progress.update_target(title, processed, total) processed += 1 - # FIXME: we could've used md5 = state.update(path, dump=False) - # here, but it is around twice as slow(on ssd, don't know about - # hdd) for a directory with small files. What we could do here - # is introduce some kind of a limit for file size, after which - # we would actually register it in our state file. md5 = self.state.update(path, dump=False) dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5}) diff --git a/dvc/state.py b/dvc/state.py index 312a668f7..63882ddf3 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -43,6 +43,8 @@ class StateDuplicateError(DvcException): class State(object): STATE_FILE = 'state' STATE_LOCK_FILE = 'state.lock' + STATE_ENTRY_LIMIT = 10000 + STATE_SIZE_LIMIT = 7 * 1024 * 1024 def __init__(self, project): self.project = project @@ -116,6 +118,14 @@ class State(object): def inode(path): return str(System.inode(path)) + def _check_db_size(self): + if len(self._db.keys()) > self.STATE_ENTRY_LIMIT: + msg = "Entry limit size '{}' has been reached for '{}'. " + msg += "Dropping current state cache." + self.project.logger.debug(msg.format(self.STATE_ENTRY_LIMIT, + self.state_file)) + self._db = {} + def _do_update(self, path, dump=True): if not os.path.exists(path): return (None, None) @@ -128,15 +138,21 @@ class State(object): return (md5, None) md5, info = self._collect(path) - state = StateEntry(md5, mtime) - d = state.dumpd() - with self._lock: - with self._lock_file: - self._db[inode] = d + if os.path.getsize(path) > self.STATE_SIZE_LIMIT: + msg = "Saving info about '{}' to state file." + self.project.logger.debug(msg.format(path)) - if dump: - self._dump() + state = StateEntry(md5, mtime) + d = state.dumpd() + + with self._lock: + with self._lock_file: + self._check_db_size() + self._db[inode] = d + + if dump: + self._dump() return (md5, info)