diff --git a/dvc/project.py b/dvc/project.py index 3ffef039b..08aaa3be6 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -253,45 +253,44 @@ class Project(object): for stage in stages: stage.checkout() - def _used_cache(self, target=None): - cache_set = set() + def _used_cache(self, target=None, all_branches=False): + cache = [] - if target: - stages = [Stage.load(self, target)] - else: - stages = self.stages() + for branch in self.scm.brancher(all_branches=all_branches): + if target: + stages = [Stage.load(self, target)] + else: + stages = self.stages() - for stage in stages: - for out in stage.outs: - if out.path_info['scheme'] != 'local': - continue + for stage in stages: + for out in stage.outs: + if out.path_info['scheme'] != 'local': + continue - if not out.use_cache or not out.cache: - continue + if not out.use_cache or not out.info: + continue - cache_set |= set([out.cache]) - if self.cache.local.is_dir_cache(out.cache) and os.path.isfile(out.cache): - dir_cache = self.cache.local.dir_cache(out.cache) - cache_set |= set(dir_cache.values()) + cache.append(out.info) - return list(cache_set) + if self.cache.local.is_dir_cache(out.cache) and os.path.isfile(out.cache): + dir_cache = self.cache.local.load_dir_cache(out.cache) + cache += dir_cache + + return cache + + def _used_cache_files(self, target=None, all_branches=False): + infos = self._used_cache(target=target, all_branches=all_branches) + return [self.cache.local.get(info[self.cache.local.PARAM_MD5]) for info in infos] def gc(self, all_branches=False): - for branch in self.scm.brancher(all_branches=all_branches): - clist = self._used_cache() - for cache in self.cache.local.all(): - if cache in clist: - continue - os.unlink(cache) - self.logger.info(u'\'{}\' was removed'.format(self.to_dvc_path(cache))) + clist = self._used_cache(target=None, all_branches=all_branches) + self.cache.local.gc(clist) def push(self, target=None, jobs=1, remote=None, all_branches=False): - for branch in self.scm.brancher(all_branches=all_branches): - self.cloud.push(self._used_cache(target), jobs, remote=remote) + self.cloud.push(self._used_cache_files(target, all_branches), jobs, remote=remote) def fetch(self, target=None, jobs=1, remote=None, all_branches=False): - for branch in self.scm.brancher(all_branches=all_branches): - self.cloud.pull(self._used_cache(target), jobs, remote=remote) + self.cloud.pull(self._used_cache_files(target, all_branches), jobs, remote=remote) def pull(self, target=None, jobs=1, remote=None, all_branches=False): self.fetch(target, jobs, remote=remote, all_branches=all_branches) @@ -314,7 +313,7 @@ class Project(object): import dvc.remote.base as cloud status = {} - for target, ret in self.cloud.status(self._used_cache(target), jobs, remote=remote): + for target, ret in self.cloud.status(self._used_cache_files(target), jobs, remote=remote): if ret == cloud.STATUS_UNKNOWN or ret == cloud.STATUS_OK: continue diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 473e6691f..66721812e 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -58,7 +58,7 @@ class RemoteLOCAL(RemoteBase): for cache in os.listdir(subdir): path = os.path.join(subdir, cache) - clist.append(path) + clist.append(self.path_to_md5(path)) return clist @@ -317,3 +317,11 @@ class RemoteLOCAL(RemoteBase): os.rename(tmp_file, path) return path + + def gc(self, checksum_infos): + used_md5s = [info[self.PARAM_MD5] for info in checksum_infos] + + for md5 in self.all(): + if md5 in used_md5s: + continue + remove(self.get(md5)) diff --git a/tests/test_cache.py b/tests/test_cache.py index e5b92291f..37963f2ce 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -25,10 +25,10 @@ class TestCache(TestDvc): self.create(self.cache2, '2') def test_all(self): - flist = Cache(self.dvc).local.all() - self.assertEquals(len(flist), 2) - self.assertTrue(self.cache1 in flist) - self.assertTrue(self.cache2 in flist) + md5_list = Cache(self.dvc).local.all() + self.assertEquals(len(md5_list), 2) + self.assertTrue(self.cache1_md5 in md5_list) + self.assertTrue(self.cache2_md5 in md5_list) def test_get(self): cache = Cache(self.dvc).local.get(self.cache1_md5) diff --git a/tests/test_gc.py b/tests/test_gc.py index 6af35b1ba..b2276ea5e 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -16,7 +16,7 @@ class TestGC(TestDvc): self.dvc.add(self.FOO) self.dvc.add(self.DATA_DIR) - self.good_cache = self.dvc.cache.local.all() + self.good_cache = [self.dvc.cache.local.get(md5) for md5 in self.dvc.cache.local.all()] self.bad_cache = [] for i in ['123', '234', '345']: