Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

dvc: properly collect used cache

Browse Source 
Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-03-03 04:25:46 +02:00
parent d80b7da3ef
commit f814269b4c
2 changed files with 21 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/project.py
+14 -6
View File
@@ -159,14 +159,22 @@ class Project(object):
for stage in self.stages():
stage.checkout()
def _used_cache(self):
def _used_cache(self, collect_dir=False):
clist = []
for stage in self.stages():
for out in stage.outs:
if not out.use_cache:
continue
if out.cache not in clist:
if not os.path.isdir(out.cache) and \
out.cache not in clist:
clist.append(out.cache)
if os.path.isdir(out.cache):
dir_cache = out.dir_cache()
clist.extend(dir_cache.values())
if collect_dir:
clist.extend([os.path.join(out.cache, f) for f in dir_cache.keys()])
else:
clist.append(out.cache)
return clist
def _remove_cache_file(self, cache):
@@ -188,7 +196,7 @@ class Project(object):
os.rmdir(cache)
def gc(self):
clist = self._used_cache()
clist = self._used_cache(collect_dir=False)
for cache in self.cache.all():
if cache in clist:
continue
@@ -196,14 +204,14 @@ class Project(object):
self.logger.info(u'\'{}\' was removed'.format(self.to_dvc_path(cache)))
def push(self, jobs=1):
self.cloud.push(self._used_cache(), jobs)
self.cloud.push(self._used_cache(collect_dir=True), jobs)
def pull(self, jobs=1):
self.cloud.pull(self._used_cache(), jobs)
self.cloud.pull(self._used_cache(collect_dir=False), jobs)
self.checkout()
def status(self, jobs=1):
return self.cloud.status(self._used_cache(), jobs)
return self.cloud.status(self._used_cache(collect_dir=True), jobs)
def graph(self):
G = nx.DiGraph()
tests/test_gc.py
+7 -2
View File
@@ -17,7 +17,8 @@ class TestGC(TestDvc):
def setUp(self):
super(TestGC, self).setUp()
stage = self.dvc.add(self.FOO)
self.dvc.add(self.FOO)
self.dvc.add(self.DATA_DIR)
self.good_cache = self.dvc.cache.all()
self.bad_cache = []
@@ -26,6 +27,11 @@ class TestGC(TestDvc):
self.create(path, i)
self.bad_cache.append(path)
path = os.path.join(self.dvc.cache.cache_dir, '45', '6', 'data')
os.mkdir(os.path.dirname(path))
self.create(path, 'md5: "123"')
self.bad_cache.append(path)
def test_api(self):
self.dvc.gc()
self._test_gc()
@@ -41,4 +47,3 @@ class TestGC(TestDvc):
for c in self.good_cache:
self.assertTrue(os.path.exists(c))
self.assertTrue(os.path.isfile(c))