Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

state: optimize state storage and computation

Browse Source 
Reduces 'dvc add' time for 30K files from 25sec to 9sec.

Fixes #546

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-03-13 20:18:03 +02:00
parent c9f4a4076c
commit b728d942c7
6 changed files with 40 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/output.py
+7 -5
View File
@@ -86,7 +86,7 @@ class Dependency(object):
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
raise CmdOutputIsNotFileOrDirError(self.rel_path)
self.md5 = self.project.state.update(self.path).md5
self.md5 = self.project.state.update(self.path)
def dumpd(self, cwd):
return {
@@ -157,8 +157,8 @@ class Output(Dependency):
return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list]
def _changed_cache(self, cache):
state = self.project.state.update(cache)
if state.md5 != self.project.cache.path_to_md5(cache):
md5 = self.project.state.update(cache)
if md5 != self.project.cache.path_to_md5(cache):
self.project.logger.debug('Corrupted cache file {}'.format(cache))
self.project._remove_cache_file(cache)
return True
@@ -303,8 +303,10 @@ class Output(Dependency):
path = os.path.join(root, fname)
relpath = os.path.relpath(path, self.path)
state = self.project.state.update(path)
dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: state.md5})
md5 = self.project.state.update(path, dump=False)
dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5})
self.project.state.dump()
return dir_info
dvc/state.py
+29 -31
View File
@@ -1,5 +1,5 @@
import os
import tinydb
import json
from checksumdir import dirhash
from dvc.system import System
@@ -11,30 +11,25 @@ from dvc.exceptions import DvcException
class StateEntry(object):
PARAM_MTIME = 'mtime'
PARAM_MD5 = 'md5'
PARAM_INODE = 'inode'
def __init__(self, md5, mtime, inode):
def __init__(self, md5, mtime):
self.mtime = mtime
self.md5 = md5
self.inode = inode
def update(self, md5, mtime, inode):
def update(self, md5, mtime):
self.mtime = mtime
self.md5 = md5
self.inode = inode
@staticmethod
def loadd(d):
mtime = d[StateEntry.PARAM_MTIME]
md5 = d[StateEntry.PARAM_MD5]
inode = d[StateEntry.PARAM_INODE]
return StateEntry(md5, mtime, inode)
return StateEntry(md5, mtime)
def dumpd(self):
return {
self.PARAM_MD5: self.md5,
self.PARAM_MTIME: self.mtime,
self.PARAM_INODE: self.inode
}
@@ -49,8 +44,7 @@ class State(object):
self.root_dir = root_dir
self.dvc_dir = dvc_dir
self.state_file = os.path.join(dvc_dir, self.STATE_FILE)
self._db = tinydb.TinyDB(self.state_file)
self._q = tinydb.Query()
self._db = {}
@staticmethod
def init(root_dir, dvc_dir):
@@ -63,38 +57,42 @@ class State(object):
return file_md5(path)[0]
def changed(self, path, md5):
state = self.update(path)
return state.md5 != md5
return self.update(path) != md5
def update(self, path):
def load(self):
with open(self.state_file, 'r') as fd:
self._db = json.load(fd)
def dump(self):
with open(self.state_file, 'w+') as fd:
json.dump(self._db, fd)
def update(self, path, dump=True):
mtime = os.path.getmtime(path)
inode = System.inode(path)
state = self._get(inode, mtime)
if state:
return state
md5 = self._get(inode, mtime)
if md5:
return md5
md5 = self.compute_md5(path)
state = StateEntry(md5, mtime, inode)
state = StateEntry(md5, mtime)
d = state.dumpd()
if self._db.contains(self._q.inode == inode):
self._db.update(d, self._q.inode == inode)
else:
self._db.insert(d)
self._db[inode] = d
return state
if dump:
self.dump()
return md5
def _get(self, inode, mtime):
d_list = self._db.search(self._q.inode == inode)
if not len(d_list):
d = self._db.get(inode, None)
if not d:
return None
if len(d_list) > 1:
raise StateDuplicateError()
state = StateEntry.loadd(d_list[0])
if mtime == state.mtime and inode == state.inode:
return state
state = StateEntry.loadd(d)
if mtime == state.mtime:
return state.md5
return None
requirements.txt
-1
View File
@@ -9,7 +9,6 @@ colorama>=0.3.9
configobj>=5.0.6
networkx>=1.11
pyyaml>=3.12
tinydb>=3.7.0
gitpython>=2.1.8
ntfsutils>=0.1.3
checksumdir>=1.1.4
setup.py
-1
View File
@@ -14,7 +14,6 @@ install_requires = [
"configobj>=5.0.6",
"networkx>=1.11",
"pyyaml>=3.12",
"tinydb>=3.7.0",
"gitpython>=2.1.8",
"ntfsutils>=0.1.3",
"checksumdir>=1.1.4",
tests/test_checkout.py
-4
View File
@@ -37,10 +37,6 @@ class TestCheckoutCorruptedCacheFile(TestRepro):
self.dvc.checkout()
#FIXME debug
import time
os.system('cat .dvc/state')
print('TIME {}'.format(time.time()))
self.assertFalse(os.path.isfile(self.FOO))
self.assertFalse(os.path.isfile(cache))
tests/test_state.py
+4 -11
View File
@@ -18,12 +18,8 @@ class TestState(TestDvc):
state = State(self.dvc.root_dir, self.dvc.dvc_dir)
state.update(path)
entry = state.get(path)
self.assertIsInstance(entry, StateEntry)
self.assertEqual(entry.md5, md5)
self.assertEqual(entry.mtime, mtime)
self.assertEqual(entry.inode, inode)
entry_md5 = state.update(path)
self.assertEqual(entry_md5, md5)
# Sleep some time to simulate realistic behavior.
# Some filesystems have a bad date resolution for
@@ -41,8 +37,5 @@ class TestState(TestDvc):
mtime = os.path.getmtime(path)
inode = System.inode(path)
entry = state.update(path)
self.assertIsInstance(entry, StateEntry)
self.assertEqual(entry.md5, md5)
self.assertEqual(entry.mtime, mtime)
self.assertEqual(entry.inode, inode)
entry_md5 = state.update(path)
self.assertEqual(entry_md5, md5)