From 44264857c3994a67219a4a0c1b1fd67057abe3ca Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Fri, 11 Oct 2019 17:56:37 +0700 Subject: [PATCH] ui/perf: do not clone same repo multiple times (#2583) * ui/perf: do not clone same repo multiple times Closes #2490 External repos are still created in /tmp. They might be reused until main() end. If no perfect match is found then a repo with the same url is copied (not cloned again) and adjusted to fit a request: rev and cache_dir. NOTE: test_download_error_pulling_imported_stage() was modified to not rely on clone error. * test: fix import update test * test: fix external_repo test under Windows * dvc: fix non-master default branch for external repos * dvc: remove unused .unset_dir() method --- dvc/external_repo.py | 111 +++++++++++++++++++++---------- dvc/main.py | 6 ++ dvc/repo/__init__.py | 9 --- tests/func/test_external_repo.py | 28 ++++++++ tests/func/test_import.py | 7 +- tests/func/test_update.py | 5 ++ 6 files changed, 119 insertions(+), 47 deletions(-) create mode 100644 tests/func/test_external_repo.py diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 3b055c920..b57f5cd80 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -1,50 +1,91 @@ from __future__ import unicode_literals import os -import logging import tempfile +from distutils.dir_util import copy_tree from funcy import retry from contextlib import contextmanager -from dvc.config import Config -from dvc.cache import CacheConfig from dvc.utils import remove -logger = logging.getLogger(__name__) - - -def _clone(cache_dir=None, url=None, rev=None, rev_lock=None): - from dvc.repo import Repo - - _path = tempfile.mkdtemp("dvc-repo") - - repo = Repo.clone(url, _path, rev=(rev_lock or rev)) - - try: - if cache_dir: - cache_config = CacheConfig(repo.config) - cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL) - finally: - repo.close() - - return Repo(_path) - - -def _remove(repo): - repo.close() - - if os.name == "nt": - # git.exe may hang for a while not permitting to remove temp dir - os_retry = retry(5, errors=OSError, timeout=0.1) - os_retry(remove)(repo.root_dir) - else: - remove(repo.root_dir) +REPO_CACHE = {} @contextmanager -def external_repo(**kwargs): - repo = _clone(**kwargs) +def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None): + from dvc.repo import Repo + + path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir) + repo = Repo(path) yield repo - _remove(repo) + repo.close() + + +def _external_repo(url=None, rev=None, cache_dir=None): + from dvc.config import Config + from dvc.cache import CacheConfig + from dvc.repo import Repo + + key = (url, rev, cache_dir) + if key in REPO_CACHE: + return REPO_CACHE[key] + + new_path = tempfile.mkdtemp("dvc-erepo") + + # Copy and adjust existing clone + if (url, None, None) in REPO_CACHE: + old_path = REPO_CACHE[url, None, None] + + # This one unlike shutil.copytree() works with an existing dir + copy_tree(old_path, new_path) + else: + # Create a new clone + _clone_repo(url, new_path) + + # Save clean clone dir so that we will have access to a default branch + clean_clone_path = tempfile.mkdtemp("dvc-erepo") + copy_tree(new_path, clean_clone_path) + REPO_CACHE[url, None, None] = clean_clone_path + + # Adjust new clone/copy to fit rev and cache_dir + repo = Repo(new_path) + try: + if rev is not None: + repo.scm.checkout(rev) + + if cache_dir is not None: + cache_config = CacheConfig(repo.config) + cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL) + finally: + # Need to close/reopen repo to force config reread + repo.close() + + REPO_CACHE[key] = new_path + return new_path + + +def clean_repos(): + # Outside code should not see cache while we are removing + repo_paths = list(REPO_CACHE.values()) + REPO_CACHE.clear() + + for path in repo_paths: + _remove(path) + + +def _remove(path): + if os.name == "nt": + # git.exe may hang for a while not permitting to remove temp dir + os_retry = retry(5, errors=OSError, timeout=0.1) + os_retry(remove)(path) + else: + remove(path) + + +def _clone_repo(url, path): + from dvc.scm.git import Git + + git = Git.clone(url, path) + git.close() diff --git a/dvc/main.py b/dvc/main.py index 6cb11baef..f8b94b8d3 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -10,6 +10,7 @@ from dvc.config import ConfigError from dvc.analytics import Analytics from dvc.exceptions import NotDvcRepoError, DvcParserError from dvc.remote.pool import close_pools +from dvc.external_repo import clean_repos from dvc.utils.compat import is_py2 @@ -65,10 +66,15 @@ def main(argv=None): ret = 255 finally: logger.setLevel(outerLogLevel) + # Python 2 fails to close these clean occasionally and users see # weird error messages, so we do it manually close_pools() + # Remove cached repos in the end of the call, these are anonymous + # so won't be reused by any other subsequent run anyway. + clean_repos() + Analytics().send_cmd(cmd, args, ret) return ret diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 440e1e061..c43868620 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -504,15 +504,6 @@ class Repo(object): def close(self): self.scm.close() - @staticmethod - def clone(url, to_path, rev=None): - from dvc.scm.git import Git - - git = Git.clone(url, to_path, rev=rev) - git.close() - - return Repo(to_path) - @locked def checkout(self, *args, **kwargs): return self._checkout(*args, **kwargs) diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py new file mode 100644 index 000000000..da2ad7059 --- /dev/null +++ b/tests/func/test_external_repo.py @@ -0,0 +1,28 @@ +import os +from mock import patch + +from dvc.external_repo import external_repo +from dvc.scm.git import Git + + +def test_external_repo(erepo): + url = erepo.root_dir + # We will share cache dir, to fetch version file + cache_dir = erepo.dvc.cache.local.cache_dir + + with patch.object(Git, "clone", wraps=Git.clone) as mock: + with external_repo(url, cache_dir=cache_dir) as repo: + with repo.open(os.path.join(repo.root_dir, "version")) as fd: + assert fd.read() == "master" + + with external_repo(url, rev="branch", cache_dir=cache_dir) as repo: + with repo.open(os.path.join(repo.root_dir, "version")) as fd: + assert fd.read() == "branch" + + # Check cache_dir is unset + with external_repo(url) as repo: + assert repo.cache.local.cache_dir.startswith( + repo.root_dir + os.sep + ) + + assert mock.call_count == 1 diff --git a/tests/func/test_import.py b/tests/func/test_import.py index b5b99d596..6711b9cf2 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -1,8 +1,8 @@ import os import filecmp -import shutil import pytest +from mock import patch from tests.utils import trees_equal from dvc.stage import Stage @@ -73,9 +73,10 @@ def test_download_error_pulling_imported_stage(dvc_repo, erepo): dst_stage = Stage.load(dvc_repo, "foo_imported.dvc") dst_cache = dst_stage.outs[0].cache_path - shutil.rmtree(erepo.root_dir) os.remove(dst) os.remove(dst_cache) - with pytest.raises(DownloadError): + with patch( + "dvc.remote.RemoteLOCAL._download", side_effect=Exception + ), pytest.raises(DownloadError): dvc_repo.pull(["foo_imported.dvc"]) diff --git a/tests/func/test_update.py b/tests/func/test_update.py index 1425ec49b..cbd48a415 100644 --- a/tests/func/test_update.py +++ b/tests/func/test_update.py @@ -3,6 +3,7 @@ import shutil import filecmp from dvc.repo import Repo +from dvc.external_repo import clean_repos def test_update_import(dvc_repo, erepo): @@ -34,6 +35,10 @@ def test_update_import(dvc_repo, erepo): os.chdir(saved_dir) + # Caching in external repos doesn't see upstream updates within single + # cli call, so we need to clean the caches to see the changes. + clean_repos() + assert dvc_repo.status([stage.path]) == {} dvc_repo.update(stage.path) assert dvc_repo.status([stage.path]) == {}