From 8d6813a11244301bd89c1a38225b6e333abb8bfc Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 2 Jul 2018 18:54:33 +0300 Subject: [PATCH 1/6] checkout: use link_state properly Signed-off-by: Ruslan Kuprieiev --- dvc/project.py | 14 ++++++++++++-- dvc/remote/local.py | 10 +++++----- dvc/state.py | 16 ++++++++++------ 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/dvc/project.py b/dvc/project.py index 09106c2d0..c71a0b2b7 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -241,14 +241,24 @@ class Project(object): raise ReproductionError(stages[n].relpath, ex) return result + def _cleanup_unused_links(self, all_stages): + used = [] + for stage in all_stages: + for out in stage.outs: + used.append(out.path) + self.link_state.remove_unused(used) + def checkout(self, target=None): + all_stages = self.stages() + if target: if not Stage.is_stage_file(target): raise NotDvcFileError(target) stages = [Stage.load(self, target)] else: - self.link_state.remove_all() - stages = self.stages() + stages = all_stages + + self._cleanup_unused_links(all_stages) for stage in stages: stage.checkout() diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 7ddb6ab95..3aa7ae519 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -85,7 +85,7 @@ class RemoteLOCAL(RemoteBase): return False - def link(self, md5, path, dump=True): + def link(self, md5, path): cache = self.get(md5) if not cache or not os.path.exists(cache) or self.changed(md5): if cache: @@ -105,7 +105,6 @@ class RemoteLOCAL(RemoteBase): for typ in types: try: self.CACHE_TYPE_MAP[typ](cache, path) - self.link_state.update(path, dump=dump) return except Exception as exc: msg = 'Cache type \'{}\' is not supported'.format(typ) @@ -152,7 +151,8 @@ class RemoteLOCAL(RemoteBase): Logger.debug(msg.format(os.path.relpath(path), md5)) if not self.is_dir_cache(cache): - self.link(md5, path, dump=True) + self.link(md5, path) + self.link_state.update(path) return # Create dir separately so that dir is created @@ -164,8 +164,8 @@ class RemoteLOCAL(RemoteBase): md5 = entry[self.PARAM_MD5] relpath = entry[self.PARAM_RELPATH] p = os.path.join(path, relpath) - self.link(md5, p, dump=False) - self.link_state.dump() + self.link(md5, p) + self.link_state.update(path) def _move(self, inp, outp): # moving in two stages to make last the move atomic in diff --git a/dvc/state.py b/dvc/state.py index 4f172e2c7..22c5b8fd4 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -209,11 +209,15 @@ class LinkState(State): with self._lock_file: self.dump() - def _do_remove_all(self): - for p, s in self._db.items(): + def _do_remove_unused(self, used): + items = self._db.copy().items() + for p, s in items: path = os.path.join(self.root_dir, p) state = LinkStateEntry.loadd(s) + if path in used: + continue + if not os.path.exists(path): continue @@ -221,12 +225,12 @@ class LinkState(State): mtime = self.mtime(path) if inode == state.inode and mtime == state.mtime: + Logger.debug('Removing \'{}\' as unused link.'.format(path)) remove(path) + del self._db[p] - self._db = {} - - def remove_all(self): + def remove_unused(self, used): with self._lock: with self._lock_file: - self._do_remove_all() + self._do_remove_unused(used) self.dump() From ecf804588707bde379a077cf4d84a47e005aafb2 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 2 Jul 2018 19:04:40 +0300 Subject: [PATCH 2/6] state: store cache state on per-repo basis Signed-off-by: Ruslan Kuprieiev --- dvc/project.py | 5 ++++- dvc/remote/local.py | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dvc/project.py b/dvc/project.py index c71a0b2b7..5321f20f3 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -10,7 +10,7 @@ from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.stage import Stage from dvc.config import Config -from dvc.state import LinkState +from dvc.state import LinkState, State from dvc.lock import Lock from dvc.scm import SCM, Base from dvc.cache import Cache @@ -50,6 +50,9 @@ class Project(object): self.config = Config(self.dvc_dir) self.scm = SCM(self.root_dir) self.lock = Lock(self.dvc_dir) + # NOTE: storing state and link_state in the repository itself to avoid + # any possible state corruption in 'shared cache dir' scenario. + self.state = State(self.dvc_dir) self.link_state = LinkState(self.root_dir, self.dvc_dir) self.logger = Logger(self.config._config[Config.SECTION_CORE].get(Config.SECTION_CORE_LOGLEVEL, None)) self.cache = Cache(self) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 3aa7ae519..3cd9f8804 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -31,6 +31,7 @@ class RemoteLOCAL(RemoteBase): def __init__(self, project, config): self.project = project + self.state = self.project.state self.link_state = project.link_state storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None) self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath) @@ -46,8 +47,6 @@ class RemoteLOCAL(RemoteBase): if self.cache_dir != None and not os.path.exists(self.cache_dir): os.mkdir(self.cache_dir) - self.state = State(self.cache_dir) - @property def prefix(self): return self.cache_dir From d9b1308447e10feb23dd94c0b2826922ba04a1d9 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 2 Jul 2018 19:36:31 +0300 Subject: [PATCH 3/6] state: store collected dir info Signed-off-by: Ruslan Kuprieiev --- dvc/remote/local.py | 3 +-- dvc/state.py | 35 ++++++++++++++++++++--------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 3cd9f8804..2cc9edad0 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -187,10 +187,9 @@ class RemoteLOCAL(RemoteBase): def _save_dir(self, path_info): path = path_info['path'] - md5 = self.state.update(path) + md5, dir_info = self.state.update_info(path) cache = self.get(md5) dname = os.path.dirname(cache) - dir_info = self.state.collect_dir(path) for entry in dir_info: relpath = entry[State.PARAM_RELPATH] diff --git a/dvc/state.py b/dvc/state.py index 22c5b8fd4..35f6dded8 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -63,7 +63,7 @@ class State(object): def init(dvc_dir): return State(dvc_dir) - def collect_dir(self, dname): + def _collect_dir(self, dname): dir_info = [] for root, dirs, files in os.walk(dname): @@ -74,16 +74,15 @@ class State(object): md5 = self.update(path, dump=False) dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5}) - self.dump() + md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX - return dir_info + return (md5, dir_info) - def compute_md5(self, path): + def _collect(self, path): if os.path.isdir(path): - dir_info = self.collect_dir(path) - return dict_md5(dir_info) + self.MD5_DIR_SUFFIX + return collect_dir else: - return file_md5(path)[0] + return (file_md5(path)[0], None) def changed(self, path, md5): actual = self.update(path) @@ -120,19 +119,19 @@ class State(object): def inode(path): return str(System.inode(path)) - def update(self, path, dump=True): + def _do_update(self, path, dump=True): if not os.path.exists(path): return None mtime = self.mtime(path) inode = self.inode(path) - md5 = self._get(inode, mtime) - if md5: - return md5 + entry = self._get(inode, mtime) + if entry: + return entry - md5 = self.compute_md5(path) - state = StateEntry(md5, mtime) + md5, info = self._collect(path) + state = StateEntry(md5, mtime, info) d = state.dumpd() with self._lock: @@ -142,7 +141,13 @@ class State(object): if dump: self.dump() - return md5 + return (md5, info) + + def update(self, path, dump=True): + return self._do_update(self, path, dump=True)[0] + + def update_info(self, path, dump=True): + return self._do_update(self, path, dump=True) def _get(self, inode, mtime): with self._lock: @@ -154,7 +159,7 @@ class State(object): state = StateEntry.loadd(d) if mtime == state.mtime: - return state.md5 + return (state.md5, state.info) return None From d37828c4e3ded663f59d144ebecaea0ae3120e20 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 2 Jul 2018 20:41:35 +0300 Subject: [PATCH 4/6] state: save collected dir to cache Signed-off-by: Ruslan Kuprieiev --- dvc/project.py | 4 ++-- dvc/remote/local.py | 35 ++++++++++++++++++++--------------- dvc/state.py | 44 +++++++++++++++++++++++++------------------- tests/test_cache.py | 4 ++-- tests/test_state.py | 4 ++-- 5 files changed, 51 insertions(+), 40 deletions(-) diff --git a/dvc/project.py b/dvc/project.py index 5321f20f3..b45ca1aee 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -52,8 +52,8 @@ class Project(object): self.lock = Lock(self.dvc_dir) # NOTE: storing state and link_state in the repository itself to avoid # any possible state corruption in 'shared cache dir' scenario. - self.state = State(self.dvc_dir) - self.link_state = LinkState(self.root_dir, self.dvc_dir) + self.state = State(self) + self.link_state = LinkState(self) self.logger = Logger(self.config._config[Config.SECTION_CORE].get(Config.SECTION_CORE_LOGLEVEL, None)) self.cache = Cache(self) self.cloud = DataCloud(self, config=self.config._config) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 2cc9edad0..c1e643b9e 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -111,8 +111,11 @@ class RemoteLOCAL(RemoteBase): if typ == types[-1]: raise DvcException(msg, cause=exc) - @staticmethod - def load_dir_cache(path): + def load_dir_cache(self, md5): + path = self.get(md5) + + assert self.is_dir_cache(path) + try: with open(path, 'r') as fd: d = json.load(fd) @@ -128,6 +131,19 @@ class RemoteLOCAL(RemoteBase): return d + def dump_dir_cache(self, md5, dir_info): + path = self.get(md5) + dname = os.path.dirname(path) + + assert self.is_dir_cache(path) + assert isinstance(dir_info, list) + + if not os.path.isdir(dname): + os.makedirs(dname) + + with open(path, 'w+') as fd: + json.dump(dir_info, fd, sort_keys=True) + @staticmethod def is_dir_cache(cache): return cache.endswith(State.MD5_DIR_SUFFIX) @@ -159,7 +175,7 @@ class RemoteLOCAL(RemoteBase): if not os.path.exists(path): os.makedirs(path) - for entry in self.load_dir_cache(cache): + for entry in self.load_dir_cache(md5): md5 = entry[self.PARAM_MD5] relpath = entry[self.PARAM_RELPATH] p = os.path.join(path, relpath) @@ -188,8 +204,6 @@ class RemoteLOCAL(RemoteBase): def _save_dir(self, path_info): path = path_info['path'] md5, dir_info = self.state.update_info(path) - cache = self.get(md5) - dname = os.path.dirname(cache) for entry in dir_info: relpath = entry[State.PARAM_RELPATH] @@ -197,15 +211,6 @@ class RemoteLOCAL(RemoteBase): self._save_file({'scheme': 'local', 'path': p}) - if not os.path.isdir(dname): - os.makedirs(dname) - - Logger.debug(u'Saving directory \'{}\' to \'{}\''.format(os.path.relpath(path), - os.path.relpath(cache))) - - with open(cache, 'w+') as fd: - json.dump(dir_info, fd, sort_keys=True) - return {self.PARAM_MD5: md5} def save(self, path_info): @@ -316,7 +321,7 @@ class RemoteLOCAL(RemoteBase): continue if not os.path.exists(cache): missing.append(info) - collected.extend(self.load_dir_cache(cache)) + collected.extend(self.load_dir_cache(md5)) collected.extend(checksum_infos) return collected, missing diff --git a/dvc/state.py b/dvc/state.py index 35f6dded8..0d20a3f51 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -48,20 +48,21 @@ class State(object): PARAM_MD5 = 'md5' MD5_DIR_SUFFIX = '.dir' - def __init__(self, dvc_dir): - self.dvc_dir = dvc_dir + def __init__(self, project): + self.project = project + self.dvc_dir = project.dvc_dir self._lock = threading.Lock() - if dvc_dir: - self.state_file = os.path.join(dvc_dir, self.STATE_FILE) - self._lock_file = Lock(dvc_dir, self.STATE_LOCK_FILE) + if self.dvc_dir: + self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE) + self._lock_file = Lock(self.dvc_dir, self.STATE_LOCK_FILE) else: self.state_file = None self._lock_file = threading.Lock() self._db = self.load() @staticmethod - def init(dvc_dir): - return State(dvc_dir) + def init(project): + return State(project) def _collect_dir(self, dname): dir_info = [] @@ -75,12 +76,14 @@ class State(object): dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5}) md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX + if self.project.cache.local.changed(md5): + self.project.cache.local.dump_dir_cache(md5, dir_info) return (md5, dir_info) def _collect(self, path): if os.path.isdir(path): - return collect_dir + return self._collect_dir(path) else: return (file_md5(path)[0], None) @@ -121,17 +124,17 @@ class State(object): def _do_update(self, path, dump=True): if not os.path.exists(path): - return None + return (None, None) mtime = self.mtime(path) inode = self.inode(path) - entry = self._get(inode, mtime) - if entry: - return entry + md5 = self._get(inode, mtime) + if md5: + return (md5, None) md5, info = self._collect(path) - state = StateEntry(md5, mtime, info) + state = StateEntry(md5, mtime) d = state.dumpd() with self._lock: @@ -144,10 +147,13 @@ class State(object): return (md5, info) def update(self, path, dump=True): - return self._do_update(self, path, dump=True)[0] + return self._do_update(path, dump=dump)[0] def update_info(self, path, dump=True): - return self._do_update(self, path, dump=True) + md5, info = self._do_update(path, dump=dump) + if not info: + info = self.project.cache.local.load_dir_cache(md5) + return (md5, info) def _get(self, inode, mtime): with self._lock: @@ -159,7 +165,7 @@ class State(object): state = StateEntry.loadd(d) if mtime == state.mtime: - return (state.md5, state.info) + return state.md5 return None @@ -195,9 +201,9 @@ class LinkState(State): STATE_FILE = 'link.state' STATE_LOCK_FILE = STATE_FILE + '.lock' - def __init__(self, root_dir, dvc_dir): - super(LinkState, self).__init__(dvc_dir) - self.root_dir = root_dir + def __init__(self, project): + super(LinkState, self).__init__(project) + self.root_dir = project.root_dir def update(self, path, dump=True): if not os.path.exists(path): diff --git a/tests/test_cache.py b/tests/test_cache.py index 37963f2ce..d8627b5d4 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -41,11 +41,11 @@ class TestCacheLoadBadDirCache(TestDvc): self.assertEqual(len(ret), 0) def test(self): - fname = 'not-json' + fname = self.dvc.cache.local.get('123') self.create(fname, 'not,json') self._do_test(RemoteLOCAL.load_dir_cache(fname)) - fname = 'not-list' + fname = self.dvc.cache.local.get('234') self.create(fname, '{"a": "b"}') self._do_test(RemoteLOCAL.load_dir_cache(fname)) diff --git a/tests/test_state.py b/tests/test_state.py index 10ec64005..7ea8af24a 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -16,7 +16,7 @@ class TestState(TestDvc): mtime = os.path.getmtime(path) inode = System.inode(path) - state = State(self.dvc.dvc_dir) + state = State(self.dvc) entry_md5 = state.update(path) self.assertEqual(entry_md5, md5) @@ -41,6 +41,6 @@ class TestState(TestDvc): # Reload state db to make sure that it stays the same old_db = state._db - state = State(self.dvc.dvc_dir) + state = State(self.dvc) new_db = state._db self.assertEqual(old_db, new_db) From f471f7714bb235bc8cd682b3b4863685c57d02a4 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 2 Jul 2018 23:46:35 +0300 Subject: [PATCH 5/6] dvc: uptimize add/checkout/state Signed-off-by: Ruslan Kuprieiev --- dvc/output/local.py | 28 ++++++++++++++--------- dvc/remote/local.py | 56 +++++++++++++++++++++++++++++---------------- dvc/state.py | 7 +++++- tests/test_cache.py | 8 +++---- 4 files changed, 63 insertions(+), 36 deletions(-) diff --git a/dvc/output/local.py b/dvc/output/local.py index 3c2627071..53fc11aab 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -41,13 +41,10 @@ class OutputLOCAL(DependencyLOCAL): return ret def changed(self): - if super(OutputLOCAL, self).changed(): - return True + if not self.use_cache: + return super(OutputLOCAL, self).changed() - if self.use_cache and self.info != self.project.cache.local.save_info(self.path_info): - return True - - return False + return self.info != self.project.cache.local.save_info(self.path_info) def checkout(self): if not self.use_cache: @@ -61,15 +58,24 @@ class OutputLOCAL(DependencyLOCAL): self.project.cache.local.checkout(self.path_info, self.info) def save(self): - if not self.changed(): - msg = 'Output \'{}\' didn\'t change. Skipping saving.' + if not self.use_cache: + super(OutputLOCAL, self).save() + msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.' self.project.logger.debug(msg.format(self.rel_path)) return - super(OutputLOCAL, self).save() + if not os.path.exists(self.path): + raise self.DoesNotExistError(self.rel_path) - if not self.use_cache: - msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.' + if not os.path.isfile(self.path) and not os.path.isdir(self.path): + raise self.NotFileOrDirError(self.rel_path) + + if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \ + (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0): + self.project.logger.warn("File/directory '{}' is empty.".format(self.rel_path)) + + if not self.changed(): + msg = 'Output \'{}\' didn\'t change. Skipping saving.' self.project.logger.debug(msg.format(self.rel_path)) return diff --git a/dvc/remote/local.py b/dvc/remote/local.py index c1e643b9e..1922a4801 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -84,13 +84,8 @@ class RemoteLOCAL(RemoteBase): return False - def link(self, md5, path): - cache = self.get(md5) - if not cache or not os.path.exists(cache) or self.changed(md5): - if cache: - msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.' - Logger.warn(msg.format(md5, os.path.relpath(path))) - return + def link(self, cache, path): + assert os.path.isfile(cache) dname = os.path.dirname(path) if not os.path.exists(dname): @@ -141,8 +136,12 @@ class RemoteLOCAL(RemoteBase): if not os.path.isdir(dname): os.makedirs(dname) - with open(path, 'w+') as fd: + # NOTE: Writing first and renaming after that + # to make sure that the operation is atomic. + tmp = '{}.{}'.format(path, str(uuid.uuid4())) + with open(tmp, 'w+') as fd: json.dump(dir_info, fd, sort_keys=True) + move(tmp, path) @staticmethod def is_dir_cache(cache): @@ -157,6 +156,12 @@ class RemoteLOCAL(RemoteBase): Logger.warn('No cache info for \'{}\'. Skipping checkout.'.format(os.path.relpath(path))) return + if self.changed(md5): + msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.' + Logger.warn(msg.format(md5, os.path.relpath(path))) + remove(path) + return + if os.path.exists(path): msg = u'Data \'{}\' exists. Removing before checkout' Logger.debug(msg.format(os.path.relpath(path))) @@ -166,7 +171,7 @@ class RemoteLOCAL(RemoteBase): Logger.debug(msg.format(os.path.relpath(path), md5)) if not self.is_dir_cache(cache): - self.link(md5, path) + self.link(cache, path) self.link_state.update(path) return @@ -177,9 +182,10 @@ class RemoteLOCAL(RemoteBase): for entry in self.load_dir_cache(md5): md5 = entry[self.PARAM_MD5] + c = self.get(md5) relpath = entry[self.PARAM_RELPATH] p = os.path.join(path, relpath) - self.link(md5, p) + self.link(c, p) self.link_state.update(path) def _move(self, inp, outp): @@ -192,12 +198,17 @@ class RemoteLOCAL(RemoteBase): def _save_file(self, path_info): path = path_info['path'] md5 = self.state.update(path) + assert md5 != None + cache = self.get(md5) + if self.changed(md5): - Logger.debug(u'Saving \'{}\' to \'{}\''.format(os.path.relpath(path), - os.path.relpath(cache))) self._move(path, cache) - self.state.update(cache) + else: + remove(path) + + self.link(cache, path) + self.link_state.update(path) return {self.PARAM_MD5: md5} @@ -207,9 +218,18 @@ class RemoteLOCAL(RemoteBase): for entry in dir_info: relpath = entry[State.PARAM_RELPATH] + m = entry[State.PARAM_MD5] p = os.path.join(path, relpath) + c = self.get(m) - self._save_file({'scheme': 'local', 'path': p}) + if self.changed(m): + self._move(p, c) + else: + remove(p) + + self.link(c, p) + + self.link_state.update(path) return {self.PARAM_MD5: md5} @@ -220,13 +240,9 @@ class RemoteLOCAL(RemoteBase): path = path_info['path'] if os.path.isdir(path): - checksum_info = self._save_dir(path_info) + return self._save_dir(path_info) else: - checksum_info = self._save_file(path_info) - - self.checkout(path_info, checksum_info) - - return checksum_info + return self._save_file(path_info) def save_info(self, path_info): if path_info['scheme'] != 'local': diff --git a/dvc/state.py b/dvc/state.py index 0d20a3f51..33733b126 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -72,7 +72,12 @@ class State(object): path = os.path.join(root, fname) relpath = os.path.relpath(path, dname) - md5 = self.update(path, dump=False) + # FIXME: we could've used `md5 = self.update(path, dump=False)` here, + # but it is around twice as slow(on ssd, don't know about hdd) for a + # directory with small files. What we could do here is introduce some + # kind of a limit for file size, after which we would actually register + # it in our state file. + md5 = file_md5(path)[0] dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5}) md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX diff --git a/tests/test_cache.py b/tests/test_cache.py index d8627b5d4..d148c4d05 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -41,13 +41,13 @@ class TestCacheLoadBadDirCache(TestDvc): self.assertEqual(len(ret), 0) def test(self): - fname = self.dvc.cache.local.get('123') + fname = self.dvc.cache.local.get('123.dir') self.create(fname, 'not,json') - self._do_test(RemoteLOCAL.load_dir_cache(fname)) + self._do_test(self.dvc.cache.local.load_dir_cache(fname)) - fname = self.dvc.cache.local.get('234') + fname = self.dvc.cache.local.get('234.dir') self.create(fname, '{"a": "b"}') - self._do_test(RemoteLOCAL.load_dir_cache(fname)) + self._do_test(self.dvc.cache.local.load_dir_cache(fname)) class TestExternalCacheDir(TestDvc): From e6bcdd3979fbe1b9114087ace90390a42b8b8cb4 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 3 Jul 2018 00:03:55 +0300 Subject: [PATCH 6/6] link: exclude unsupported link types after trying once Signed-off-by: Ruslan Kuprieiev --- dvc/remote/local.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 1922a4801..987369084 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -91,20 +91,19 @@ class RemoteLOCAL(RemoteBase): if not os.path.exists(dname): os.makedirs(dname) - if self.cache_types != None: - types = self.cache_types - else: - types = self.CACHE_TYPES - - for typ in types: + i = 0 + N = len(self.cache_types) + while i < N: try: - self.CACHE_TYPE_MAP[typ](cache, path) + self.CACHE_TYPE_MAP[self.cache_types[i]](cache, path) return except Exception as exc: - msg = 'Cache type \'{}\' is not supported'.format(typ) + msg = 'Cache type \'{}\' is not supported'.format(self.cache_types[i]) Logger.debug(msg) - if typ == types[-1]: - raise DvcException(msg, cause=exc) + del self.cache_types[i] + i += 1 + + raise DvcException('No possible cache types left to try out.') def load_dir_cache(self, md5): path = self.get(md5)