mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
dvc: switch from yaml to json for dir cache file
yaml is extremely slow without CLoader, which caused our checkout() and similar functions to take way too long to process dir cache files. These files are not really meant to be read by the users, so we can sacrifice the readability if yaml for better performance of json module. E.g. checkout for 30K files used to take 30sec and after this patch only takes 6sec, which is still quite slow, but much more bearable. Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
+3
-3
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import stat
|
||||
import yaml
|
||||
import json
|
||||
from checksumdir import dirhash
|
||||
|
||||
from dvc.system import System
|
||||
@@ -249,7 +249,7 @@ class Output(Dependency):
|
||||
|
||||
try:
|
||||
with open(path, 'r') as fd:
|
||||
d = yaml.safe_load(fd)
|
||||
d = json.load(fd)
|
||||
except Exception as exc:
|
||||
msg = u'Failed to load dir cache \'{}\''
|
||||
Logger.error(msg.format(relpath), exc)
|
||||
@@ -349,7 +349,7 @@ class Output(Dependency):
|
||||
os.makedirs(dname)
|
||||
|
||||
with open(self.cache, 'w+') as fd:
|
||||
yaml.safe_dump(dir_info, fd, default_flow_style=False)
|
||||
json.dump(dir_info, fd)
|
||||
|
||||
def save(self):
|
||||
super(Output, self).save()
|
||||
|
||||
Reference in New Issue
Block a user