Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-10 07:18:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #826 from efiop/master

Browse Source 
checkout: optimise for large directories
This commit is contained in:
Ruslan Kuprieiev
2018-06-29 02:57:10 +03:00
committed by GitHub
parent a4c63335d1 15d7debb7e
commit 7ba47f7c5b
2 changed files with 14 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/remote/base.py
+4 -4
View File
@@ -99,13 +99,11 @@ class RemoteBase(object):
return posixpath.join(self.prefix, relpath).strip('/')
def collect(self, arg):
from dvc.remote.local import RemoteLOCAL
path, local = arg
ret = [path]
md5 = self.project.cache.local.path_to_md5(path)
if not RemoteLOCAL.is_dir_cache(path):
if not self.project.cache.local.is_dir_cache(path):
return ret
if not local and self.project.cache.local.changed(md5):
@@ -118,7 +116,9 @@ class RemoteBase(object):
if self.project.cache.local.changed(md5):
return ret
for relpath, md5 in RemoteLOCAL.get_dir_cache(path).items():
for entry in self.project.cache.local.load_dir_cache(path):
md5 = entry[self.project.cache.local.PARAM_MD5]
relpath = entry[self.project.cache.local.PARAM_RELPATH]
cache = self.project.cache.local.get(md5)
ret.append(cache)
dvc/remote/local.py
+10 -34
View File
@@ -17,6 +17,7 @@ class RemoteLOCAL(RemoteBase):
scheme = ''
REGEX = r'^(?P<path>(/+|.:\\+).*)$'
PARAM_MD5 = State.PARAM_MD5
PARAM_RELPATH = State.PARAM_RELPATH
CACHE_TYPES = ['reflink', 'hardlink', 'symlink', 'copy']
CACHE_TYPE_MAP = {
@@ -94,8 +95,6 @@ class RemoteLOCAL(RemoteBase):
for typ in types:
try:
msg = u'Checking out \'{}\' with cache \'{}\''
Logger.debug(msg.format(os.path.relpath(src), os.path.relpath(link)))
self.CACHE_TYPE_MAP[typ](src, link)
self.link_state.update(link, dump=dump)
return
@@ -107,46 +106,21 @@ class RemoteLOCAL(RemoteBase):
@staticmethod
def load_dir_cache(path):
if os.path.isabs(path):
relpath = os.path.relpath(path)
else:
relpath = path
try:
with open(path, 'r') as fd:
d = json.load(fd)
except Exception as exc:
msg = u'Failed to load dir cache \'{}\''
Logger.error(msg.format(relpath), exc)
Logger.error(msg.format(os.path.relpath(path)), exc)
return []
if not isinstance(d, list):
msg = u'Dir cache file format error \'{}\': skipping the file'
Logger.error(msg.format(relpath))
Logger.error(msg.format(os.path.relpath(path)))
return []
return d
@classmethod
def get_dir_cache(cls, path):
res = {}
d = cls.load_dir_cache(path)
for entry in d:
res[entry[State.PARAM_RELPATH]] = entry[State.PARAM_MD5]
return res
def dir_cache(self, cache):
res = {}
dir_cache = self.get_dir_cache(cache)
for relpath, md5 in dir_cache.items():
res[relpath] = self.get(md5)
return res
@staticmethod
def is_dir_cache(cache):
return cache.endswith(State.MD5_DIR_SUFFIX)
@@ -168,20 +142,22 @@ class RemoteLOCAL(RemoteBase):
Logger.debug(msg.format(os.path.relpath(path)))
remove(path)
msg = u'Checking out \'{}\' with cache \'{}\''
Logger.debug(msg.format(os.path.relpath(path), os.path.relpath(cache)))
if not self.is_dir_cache(cache):
self.link(cache, path, dump=True)
return
msg = u'Checking out directory \'{}\' with cache \'{}\''
Logger.debug(msg.format(os.path.relpath(path), os.path.relpath(cache)))
# Create dir separately so that dir is created
# even if there are no files in it
if not os.path.exists(path):
os.makedirs(path)
dir_cache = self.dir_cache(cache)
for relpath, c in dir_cache.items():
for entry in self.load_dir_cache(cache):
md5 = entry[self.PARAM_MD5]
relpath = entry[self.PARAM_RELPATH]
c = self.get(md5)
p = os.path.join(path, relpath)
self.link(c, p, dump=False)
self.link_state.dump()