mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-16 23:09:55 +01:00
gc: move gc logic to Remote
Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
+28
-29
@@ -253,45 +253,44 @@ class Project(object):
|
||||
for stage in stages:
|
||||
stage.checkout()
|
||||
|
||||
def _used_cache(self, target=None):
|
||||
cache_set = set()
|
||||
def _used_cache(self, target=None, all_branches=False):
|
||||
cache = []
|
||||
|
||||
if target:
|
||||
stages = [Stage.load(self, target)]
|
||||
else:
|
||||
stages = self.stages()
|
||||
for branch in self.scm.brancher(all_branches=all_branches):
|
||||
if target:
|
||||
stages = [Stage.load(self, target)]
|
||||
else:
|
||||
stages = self.stages()
|
||||
|
||||
for stage in stages:
|
||||
for out in stage.outs:
|
||||
if out.path_info['scheme'] != 'local':
|
||||
continue
|
||||
for stage in stages:
|
||||
for out in stage.outs:
|
||||
if out.path_info['scheme'] != 'local':
|
||||
continue
|
||||
|
||||
if not out.use_cache or not out.cache:
|
||||
continue
|
||||
if not out.use_cache or not out.info:
|
||||
continue
|
||||
|
||||
cache_set |= set([out.cache])
|
||||
if self.cache.local.is_dir_cache(out.cache) and os.path.isfile(out.cache):
|
||||
dir_cache = self.cache.local.dir_cache(out.cache)
|
||||
cache_set |= set(dir_cache.values())
|
||||
cache.append(out.info)
|
||||
|
||||
return list(cache_set)
|
||||
if self.cache.local.is_dir_cache(out.cache) and os.path.isfile(out.cache):
|
||||
dir_cache = self.cache.local.load_dir_cache(out.cache)
|
||||
cache += dir_cache
|
||||
|
||||
return cache
|
||||
|
||||
def _used_cache_files(self, target=None, all_branches=False):
|
||||
infos = self._used_cache(target=target, all_branches=all_branches)
|
||||
return [self.cache.local.get(info[self.cache.local.PARAM_MD5]) for info in infos]
|
||||
|
||||
def gc(self, all_branches=False):
|
||||
for branch in self.scm.brancher(all_branches=all_branches):
|
||||
clist = self._used_cache()
|
||||
for cache in self.cache.local.all():
|
||||
if cache in clist:
|
||||
continue
|
||||
os.unlink(cache)
|
||||
self.logger.info(u'\'{}\' was removed'.format(self.to_dvc_path(cache)))
|
||||
clist = self._used_cache(target=None, all_branches=all_branches)
|
||||
self.cache.local.gc(clist)
|
||||
|
||||
def push(self, target=None, jobs=1, remote=None, all_branches=False):
|
||||
for branch in self.scm.brancher(all_branches=all_branches):
|
||||
self.cloud.push(self._used_cache(target), jobs, remote=remote)
|
||||
self.cloud.push(self._used_cache_files(target, all_branches), jobs, remote=remote)
|
||||
|
||||
def fetch(self, target=None, jobs=1, remote=None, all_branches=False):
|
||||
for branch in self.scm.brancher(all_branches=all_branches):
|
||||
self.cloud.pull(self._used_cache(target), jobs, remote=remote)
|
||||
self.cloud.pull(self._used_cache_files(target, all_branches), jobs, remote=remote)
|
||||
|
||||
def pull(self, target=None, jobs=1, remote=None, all_branches=False):
|
||||
self.fetch(target, jobs, remote=remote, all_branches=all_branches)
|
||||
@@ -314,7 +313,7 @@ class Project(object):
|
||||
import dvc.remote.base as cloud
|
||||
|
||||
status = {}
|
||||
for target, ret in self.cloud.status(self._used_cache(target), jobs, remote=remote):
|
||||
for target, ret in self.cloud.status(self._used_cache_files(target), jobs, remote=remote):
|
||||
if ret == cloud.STATUS_UNKNOWN or ret == cloud.STATUS_OK:
|
||||
continue
|
||||
|
||||
|
||||
+9
-1
@@ -58,7 +58,7 @@ class RemoteLOCAL(RemoteBase):
|
||||
|
||||
for cache in os.listdir(subdir):
|
||||
path = os.path.join(subdir, cache)
|
||||
clist.append(path)
|
||||
clist.append(self.path_to_md5(path))
|
||||
|
||||
return clist
|
||||
|
||||
@@ -317,3 +317,11 @@ class RemoteLOCAL(RemoteBase):
|
||||
os.rename(tmp_file, path)
|
||||
|
||||
return path
|
||||
|
||||
def gc(self, checksum_infos):
|
||||
used_md5s = [info[self.PARAM_MD5] for info in checksum_infos]
|
||||
|
||||
for md5 in self.all():
|
||||
if md5 in used_md5s:
|
||||
continue
|
||||
remove(self.get(md5))
|
||||
|
||||
+4
-4
@@ -25,10 +25,10 @@ class TestCache(TestDvc):
|
||||
self.create(self.cache2, '2')
|
||||
|
||||
def test_all(self):
|
||||
flist = Cache(self.dvc).local.all()
|
||||
self.assertEquals(len(flist), 2)
|
||||
self.assertTrue(self.cache1 in flist)
|
||||
self.assertTrue(self.cache2 in flist)
|
||||
md5_list = Cache(self.dvc).local.all()
|
||||
self.assertEquals(len(md5_list), 2)
|
||||
self.assertTrue(self.cache1_md5 in md5_list)
|
||||
self.assertTrue(self.cache2_md5 in md5_list)
|
||||
|
||||
def test_get(self):
|
||||
cache = Cache(self.dvc).local.get(self.cache1_md5)
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ class TestGC(TestDvc):
|
||||
|
||||
self.dvc.add(self.FOO)
|
||||
self.dvc.add(self.DATA_DIR)
|
||||
self.good_cache = self.dvc.cache.local.all()
|
||||
self.good_cache = [self.dvc.cache.local.get(md5) for md5 in self.dvc.cache.local.all()]
|
||||
|
||||
self.bad_cache = []
|
||||
for i in ['123', '234', '345']:
|
||||
|
||||
Reference in New Issue
Block a user