From 3b9259ff1762b2140e8e039cf579848c3750d620 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 9 Mar 2018 08:18:30 +0200 Subject: [PATCH] dvc: switch from yaml to json for dir cache file yaml is extremely slow without CLoader, which caused our checkout() and similar functions to take way too long to process dir cache files. These files are not really meant to be read by the users, so we can sacrifice the readability if yaml for better performance of json module. E.g. checkout for 30K files used to take 30sec and after this patch only takes 6sec, which is still quite slow, but much more bearable. Signed-off-by: Ruslan Kuprieiev --- dvc/output.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dvc/output.py b/dvc/output.py index 4ebdb6948..8b0d0e3db 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -1,6 +1,6 @@ import os import stat -import yaml +import json from checksumdir import dirhash from dvc.system import System @@ -249,7 +249,7 @@ class Output(Dependency): try: with open(path, 'r') as fd: - d = yaml.safe_load(fd) + d = json.load(fd) except Exception as exc: msg = u'Failed to load dir cache \'{}\'' Logger.error(msg.format(relpath), exc) @@ -349,7 +349,7 @@ class Output(Dependency): os.makedirs(dname) with open(self.cache, 'w+') as fd: - yaml.safe_dump(dir_info, fd, default_flow_style=False) + json.dump(dir_info, fd) def save(self): super(Output, self).save()