mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-09 23:10:24 +01:00
Merge pull request #1303 from ynop/gc-multi-project
gc: support used cache collection for several repositories using the same remote storage
This commit is contained in:
Ruslan Kuprieiev
@@ -591,6 +591,13 @@ def parse_args(argv=None):
|
||||
type=int,
|
||||
default=None,
|
||||
help='Number of jobs to run simultaneously.')
|
||||
gc_parser.add_argument(
|
||||
'-p',
|
||||
'--projects',
|
||||
type=str,
|
||||
nargs='*',
|
||||
default=None,
|
||||
help='Collect garbage for all given projects.')
|
||||
gc_parser.set_defaults(func=CmdGC)
|
||||
|
||||
# Config
|
||||
|
||||
+16
-5
@@ -1,3 +1,5 @@
|
||||
import os
|
||||
|
||||
from dvc.command.base import CmdBase
|
||||
|
||||
|
||||
@@ -5,13 +7,21 @@ class CmdGC(CmdBase):
|
||||
def run(self):
|
||||
msg = 'This will remove all cache except the cache that is used in '
|
||||
if not self.args.all_branches and not self.args.all_tags:
|
||||
msg += 'the current git branch.'
|
||||
msg += 'the current git branch'
|
||||
elif self.args.all_branches and not self.args.all_tags:
|
||||
msg += 'all git branches.'
|
||||
msg += 'all git branches'
|
||||
elif not self.args.all_branches and self.args.all_tags:
|
||||
msg += 'all git tags.'
|
||||
msg += 'all git tags'
|
||||
else:
|
||||
msg += 'all git branches and all git tags.'
|
||||
msg += 'all git branches and all git tags'
|
||||
|
||||
if self.args.projects is not None and len(self.args.projects) > 0:
|
||||
msg += ' of the current and the following projects:'
|
||||
|
||||
for project_path in self.args.projects:
|
||||
msg += '\n - %s' % os.path.abspath(project_path)
|
||||
else:
|
||||
msg += ' of the current project.'
|
||||
|
||||
self.project.logger.warn(msg)
|
||||
|
||||
@@ -24,5 +34,6 @@ class CmdGC(CmdBase):
|
||||
cloud=self.args.cloud,
|
||||
remote=self.args.remote,
|
||||
force=self.args.force,
|
||||
jobs=self.args.jobs)
|
||||
jobs=self.args.jobs,
|
||||
projects=self.args.projects)
|
||||
return 0
|
||||
|
||||
+76
-9
@@ -1,3 +1,4 @@
|
||||
import collections
|
||||
import os
|
||||
|
||||
from dvc.exceptions import DvcException, MoveNotDataSourceError
|
||||
@@ -108,6 +109,19 @@ class Project(object):
|
||||
|
||||
return proj
|
||||
|
||||
@staticmethod
|
||||
def load_all(projects_paths):
|
||||
"""
|
||||
Instantiate all projects in the given list of paths.
|
||||
|
||||
Args:
|
||||
projects_paths: List of paths to projects.
|
||||
|
||||
Returns:
|
||||
List of Project instances in the same order of the given paths.
|
||||
"""
|
||||
return [Project(path) for path in projects_paths]
|
||||
|
||||
def destroy(self):
|
||||
import shutil
|
||||
|
||||
@@ -602,6 +616,45 @@ class Project(object):
|
||||
|
||||
return cache
|
||||
|
||||
@staticmethod
|
||||
def merge_cache_lists(clists):
|
||||
merged_cache = collections.defaultdict(list)
|
||||
|
||||
for cache_list in clists:
|
||||
for scheme, cache in cache_list.items():
|
||||
for item in cache:
|
||||
if item not in merged_cache[scheme]:
|
||||
merged_cache[scheme].append(item)
|
||||
|
||||
return merged_cache
|
||||
|
||||
@staticmethod
|
||||
def load_all_used_cache(projects,
|
||||
target=None,
|
||||
all_branches=False,
|
||||
active=True,
|
||||
with_deps=False,
|
||||
all_tags=False,
|
||||
remote=None,
|
||||
force=False,
|
||||
jobs=None):
|
||||
clists = []
|
||||
|
||||
for project in projects:
|
||||
with project.state:
|
||||
project_clist = project._used_cache(target=None,
|
||||
all_branches=all_branches,
|
||||
active=False,
|
||||
with_deps=with_deps,
|
||||
all_tags=all_tags,
|
||||
remote=remote,
|
||||
force=force,
|
||||
jobs=jobs)
|
||||
|
||||
clists.append(project_clist)
|
||||
|
||||
return clists
|
||||
|
||||
def _do_gc(self, typ, func, clist):
|
||||
removed = func(clist)
|
||||
if not removed:
|
||||
@@ -614,16 +667,30 @@ class Project(object):
|
||||
with_deps=False,
|
||||
all_tags=False,
|
||||
force=False,
|
||||
jobs=None):
|
||||
jobs=None,
|
||||
projects=None):
|
||||
|
||||
all_projects = [self]
|
||||
|
||||
if projects is not None and len(projects) > 0:
|
||||
all_projects.extend(Project.load_all(projects))
|
||||
|
||||
all_clists = Project.load_all_used_cache(all_projects,
|
||||
target=None,
|
||||
all_branches=all_branches,
|
||||
active=False,
|
||||
with_deps=with_deps,
|
||||
all_tags=all_tags,
|
||||
remote=remote,
|
||||
force=force,
|
||||
jobs=jobs)
|
||||
|
||||
if len(all_clists) > 1:
|
||||
clist = Project.merge_cache_lists(all_clists)
|
||||
else:
|
||||
clist = all_clists[0]
|
||||
|
||||
with self.state:
|
||||
clist = self._used_cache(target=None,
|
||||
all_branches=all_branches,
|
||||
active=False,
|
||||
with_deps=with_deps,
|
||||
all_tags=all_tags,
|
||||
remote=remote,
|
||||
force=force,
|
||||
jobs=jobs)
|
||||
self._do_gc('local', self.cache.local.gc, clist)
|
||||
|
||||
if self.cache.s3:
|
||||
|
||||
@@ -129,6 +129,20 @@ class TestGC(TestDvc):
|
||||
self.assertIsInstance(args.func(args), CmdGC)
|
||||
|
||||
|
||||
class TestGCMultipleProjects(TestDvc):
|
||||
def test(self):
|
||||
args = parse_args([
|
||||
'gc',
|
||||
'-p',
|
||||
'/tmp/asdf',
|
||||
'/tmp/xyz'
|
||||
])
|
||||
|
||||
self.assertIsInstance(args.func(args), CmdGC)
|
||||
|
||||
self.assertEqual(args.projects, ['/tmp/asdf', '/tmp/xyz'])
|
||||
|
||||
|
||||
class TestConfig(TestDvc):
|
||||
def test(self):
|
||||
name = 'param'
|
||||
|
||||
+79
-1
@@ -1,13 +1,18 @@
|
||||
import os
|
||||
import shutil
|
||||
import filecmp
|
||||
import configobj
|
||||
|
||||
from git import Repo
|
||||
|
||||
from dvc.main import main
|
||||
from dvc.utils import file_md5
|
||||
from dvc.stage import Stage
|
||||
from dvc.project import Project
|
||||
from dvc.command.gc import CmdGC
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.basic_env import TestDir
|
||||
|
||||
|
||||
class TestGC(TestDvc):
|
||||
@@ -16,7 +21,8 @@ class TestGC(TestDvc):
|
||||
|
||||
self.dvc.add(self.FOO)
|
||||
self.dvc.add(self.DATA_DIR)
|
||||
self.good_cache = [self.dvc.cache.local.get(md5) for md5 in self.dvc.cache.local.all()]
|
||||
self.good_cache = [self.dvc.cache.local.get(
|
||||
md5) for md5 in self.dvc.cache.local.all()]
|
||||
|
||||
self.bad_cache = []
|
||||
for i in ['123', '234', '345']:
|
||||
@@ -100,3 +106,75 @@ class TestGCBranchesTags(TestDvc):
|
||||
self.dvc.gc(all_tags=True, all_branches=False)
|
||||
|
||||
self._check_cache(1)
|
||||
|
||||
|
||||
class TestGCMultipleProjects(TestDvc):
|
||||
def _check_cache(self, num):
|
||||
total = 0
|
||||
for root, dirs, files in os.walk(os.path.join('.dvc', 'cache')):
|
||||
total += len(files)
|
||||
self.assertEqual(total, num)
|
||||
|
||||
def setUp(self):
|
||||
super(TestGCMultipleProjects, self).setUp()
|
||||
self.additional_path = TestDir.mkdtemp()
|
||||
self.additional_git = Repo.init(self.additional_path)
|
||||
self.additional_dvc = Project.init(self.additional_path)
|
||||
|
||||
cache_path = os.path.join(self._root_dir, '.dvc', 'cache')
|
||||
config_path = os.path.join(self.additional_path, '.dvc',
|
||||
'config.local')
|
||||
cfg = configobj.ConfigObj()
|
||||
cfg.filename = config_path
|
||||
cfg['cache'] = {
|
||||
'dir': cache_path
|
||||
}
|
||||
cfg.write()
|
||||
|
||||
self.additional_dvc = Project(self.additional_path)
|
||||
|
||||
def test(self):
|
||||
|
||||
# ADD FILE ONLY IN MAIN PROJECT
|
||||
fname = 'only_in_first'
|
||||
with open(fname, 'w+') as fobj:
|
||||
fobj.write('only in main project')
|
||||
|
||||
stages = self.dvc.add(fname)
|
||||
self.assertEqual(len(stages), 1)
|
||||
|
||||
# ADD FILE IN MAIN PROJECT THAT IS ALSO IN SECOND PROJECT
|
||||
fname = 'in_both'
|
||||
with open(fname, 'w+') as fobj:
|
||||
fobj.write('in both projects')
|
||||
|
||||
stages = self.dvc.add(fname)
|
||||
self.assertEqual(len(stages), 1)
|
||||
|
||||
cwd = os.getcwd()
|
||||
os.chdir(self.additional_path)
|
||||
# ADD FILE ONLY IN SECOND PROJECT
|
||||
fname = os.path.join(self.additional_path, 'only_in_second')
|
||||
with open(fname, 'w+') as fobj:
|
||||
fobj.write('only in additional project')
|
||||
|
||||
stages = self.additional_dvc.add(fname)
|
||||
self.assertEqual(len(stages), 1)
|
||||
|
||||
# ADD FILE IN SECOND PROJECT THAT IS ALSO IN MAIN PROJECT
|
||||
fname = os.path.join(self.additional_path, 'in_both')
|
||||
with open(fname, 'w+') as fobj:
|
||||
fobj.write('in both projects')
|
||||
|
||||
stages = self.additional_dvc.add(fname)
|
||||
self.assertEqual(len(stages), 1)
|
||||
|
||||
os.chdir(cwd)
|
||||
|
||||
self._check_cache(3)
|
||||
|
||||
self.dvc.gc(projects=[self.additional_path])
|
||||
self._check_cache(3)
|
||||
|
||||
self.dvc.gc()
|
||||
self._check_cache(2)
|
||||
|
||||
Reference in New Issue
Block a user