gc: move gc logic to Remote

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-06-28 18:05:55 +03:00
parent 67634649bd
commit add9498e82
4 changed files with 42 additions and 35 deletions
+28 -29
View File
@@ -253,45 +253,44 @@ class Project(object):
for stage in stages:
stage.checkout()
def _used_cache(self, target=None):
cache_set = set()
def _used_cache(self, target=None, all_branches=False):
cache = []
if target:
stages = [Stage.load(self, target)]
else:
stages = self.stages()
for branch in self.scm.brancher(all_branches=all_branches):
if target:
stages = [Stage.load(self, target)]
else:
stages = self.stages()
for stage in stages:
for out in stage.outs:
if out.path_info['scheme'] != 'local':
continue
for stage in stages:
for out in stage.outs:
if out.path_info['scheme'] != 'local':
continue
if not out.use_cache or not out.cache:
continue
if not out.use_cache or not out.info:
continue
cache_set |= set([out.cache])
if self.cache.local.is_dir_cache(out.cache) and os.path.isfile(out.cache):
dir_cache = self.cache.local.dir_cache(out.cache)
cache_set |= set(dir_cache.values())
cache.append(out.info)
return list(cache_set)
if self.cache.local.is_dir_cache(out.cache) and os.path.isfile(out.cache):
dir_cache = self.cache.local.load_dir_cache(out.cache)
cache += dir_cache
return cache
def _used_cache_files(self, target=None, all_branches=False):
infos = self._used_cache(target=target, all_branches=all_branches)
return [self.cache.local.get(info[self.cache.local.PARAM_MD5]) for info in infos]
def gc(self, all_branches=False):
for branch in self.scm.brancher(all_branches=all_branches):
clist = self._used_cache()
for cache in self.cache.local.all():
if cache in clist:
continue
os.unlink(cache)
self.logger.info(u'\'{}\' was removed'.format(self.to_dvc_path(cache)))
clist = self._used_cache(target=None, all_branches=all_branches)
self.cache.local.gc(clist)
def push(self, target=None, jobs=1, remote=None, all_branches=False):
for branch in self.scm.brancher(all_branches=all_branches):
self.cloud.push(self._used_cache(target), jobs, remote=remote)
self.cloud.push(self._used_cache_files(target, all_branches), jobs, remote=remote)
def fetch(self, target=None, jobs=1, remote=None, all_branches=False):
for branch in self.scm.brancher(all_branches=all_branches):
self.cloud.pull(self._used_cache(target), jobs, remote=remote)
self.cloud.pull(self._used_cache_files(target, all_branches), jobs, remote=remote)
def pull(self, target=None, jobs=1, remote=None, all_branches=False):
self.fetch(target, jobs, remote=remote, all_branches=all_branches)
@@ -314,7 +313,7 @@ class Project(object):
import dvc.remote.base as cloud
status = {}
for target, ret in self.cloud.status(self._used_cache(target), jobs, remote=remote):
for target, ret in self.cloud.status(self._used_cache_files(target), jobs, remote=remote):
if ret == cloud.STATUS_UNKNOWN or ret == cloud.STATUS_OK:
continue
+9 -1
View File
@@ -58,7 +58,7 @@ class RemoteLOCAL(RemoteBase):
for cache in os.listdir(subdir):
path = os.path.join(subdir, cache)
clist.append(path)
clist.append(self.path_to_md5(path))
return clist
@@ -317,3 +317,11 @@ class RemoteLOCAL(RemoteBase):
os.rename(tmp_file, path)
return path
def gc(self, checksum_infos):
used_md5s = [info[self.PARAM_MD5] for info in checksum_infos]
for md5 in self.all():
if md5 in used_md5s:
continue
remove(self.get(md5))
+4 -4
View File
@@ -25,10 +25,10 @@ class TestCache(TestDvc):
self.create(self.cache2, '2')
def test_all(self):
flist = Cache(self.dvc).local.all()
self.assertEquals(len(flist), 2)
self.assertTrue(self.cache1 in flist)
self.assertTrue(self.cache2 in flist)
md5_list = Cache(self.dvc).local.all()
self.assertEquals(len(md5_list), 2)
self.assertTrue(self.cache1_md5 in md5_list)
self.assertTrue(self.cache2_md5 in md5_list)
def test_get(self):
cache = Cache(self.dvc).local.get(self.cache1_md5)
+1 -1
View File
@@ -16,7 +16,7 @@ class TestGC(TestDvc):
self.dvc.add(self.FOO)
self.dvc.add(self.DATA_DIR)
self.good_cache = self.dvc.cache.local.all()
self.good_cache = [self.dvc.cache.local.get(md5) for md5 in self.dvc.cache.local.all()]
self.bad_cache = []
for i in ['123', '234', '345']: