ui/perf: do not clone same repo multiple times (#2583)

* ui/perf: do not clone same repo multiple times

Closes #2490

External repos are still created in /tmp. They might be reused until
main() end. If no perfect match is found then a repo with the same url
is copied (not cloned again) and adjusted to fit a request: rev and
cache_dir.

NOTE: test_download_error_pulling_imported_stage() was modified to not
rely on clone error.

* test: fix import update test

* test: fix external_repo test under Windows

* dvc: fix non-master default branch for external repos

* dvc: remove unused .unset_dir() method
This commit is contained in:
Alexander Schepanovski
2019-10-11 13:56:37 +03:00
committed by Ruslan Kuprieiev
parent 5cabb6d115
commit 44264857c3
6 changed files with 119 additions and 47 deletions
+76 -35
View File
@@ -1,50 +1,91 @@
from __future__ import unicode_literals
import os
import logging
import tempfile
from distutils.dir_util import copy_tree
from funcy import retry
from contextlib import contextmanager
from dvc.config import Config
from dvc.cache import CacheConfig
from dvc.utils import remove
logger = logging.getLogger(__name__)
def _clone(cache_dir=None, url=None, rev=None, rev_lock=None):
from dvc.repo import Repo
_path = tempfile.mkdtemp("dvc-repo")
repo = Repo.clone(url, _path, rev=(rev_lock or rev))
try:
if cache_dir:
cache_config = CacheConfig(repo.config)
cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
finally:
repo.close()
return Repo(_path)
def _remove(repo):
repo.close()
if os.name == "nt":
# git.exe may hang for a while not permitting to remove temp dir
os_retry = retry(5, errors=OSError, timeout=0.1)
os_retry(remove)(repo.root_dir)
else:
remove(repo.root_dir)
REPO_CACHE = {}
@contextmanager
def external_repo(**kwargs):
repo = _clone(**kwargs)
def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
from dvc.repo import Repo
path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
repo = Repo(path)
yield repo
_remove(repo)
repo.close()
def _external_repo(url=None, rev=None, cache_dir=None):
from dvc.config import Config
from dvc.cache import CacheConfig
from dvc.repo import Repo
key = (url, rev, cache_dir)
if key in REPO_CACHE:
return REPO_CACHE[key]
new_path = tempfile.mkdtemp("dvc-erepo")
# Copy and adjust existing clone
if (url, None, None) in REPO_CACHE:
old_path = REPO_CACHE[url, None, None]
# This one unlike shutil.copytree() works with an existing dir
copy_tree(old_path, new_path)
else:
# Create a new clone
_clone_repo(url, new_path)
# Save clean clone dir so that we will have access to a default branch
clean_clone_path = tempfile.mkdtemp("dvc-erepo")
copy_tree(new_path, clean_clone_path)
REPO_CACHE[url, None, None] = clean_clone_path
# Adjust new clone/copy to fit rev and cache_dir
repo = Repo(new_path)
try:
if rev is not None:
repo.scm.checkout(rev)
if cache_dir is not None:
cache_config = CacheConfig(repo.config)
cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
finally:
# Need to close/reopen repo to force config reread
repo.close()
REPO_CACHE[key] = new_path
return new_path
def clean_repos():
# Outside code should not see cache while we are removing
repo_paths = list(REPO_CACHE.values())
REPO_CACHE.clear()
for path in repo_paths:
_remove(path)
def _remove(path):
if os.name == "nt":
# git.exe may hang for a while not permitting to remove temp dir
os_retry = retry(5, errors=OSError, timeout=0.1)
os_retry(remove)(path)
else:
remove(path)
def _clone_repo(url, path):
from dvc.scm.git import Git
git = Git.clone(url, path)
git.close()
+6
View File
@@ -10,6 +10,7 @@ from dvc.config import ConfigError
from dvc.analytics import Analytics
from dvc.exceptions import NotDvcRepoError, DvcParserError
from dvc.remote.pool import close_pools
from dvc.external_repo import clean_repos
from dvc.utils.compat import is_py2
@@ -65,10 +66,15 @@ def main(argv=None):
ret = 255
finally:
logger.setLevel(outerLogLevel)
# Python 2 fails to close these clean occasionally and users see
# weird error messages, so we do it manually
close_pools()
# Remove cached repos in the end of the call, these are anonymous
# so won't be reused by any other subsequent run anyway.
clean_repos()
Analytics().send_cmd(cmd, args, ret)
return ret
-9
View File
@@ -504,15 +504,6 @@ class Repo(object):
def close(self):
self.scm.close()
@staticmethod
def clone(url, to_path, rev=None):
from dvc.scm.git import Git
git = Git.clone(url, to_path, rev=rev)
git.close()
return Repo(to_path)
@locked
def checkout(self, *args, **kwargs):
return self._checkout(*args, **kwargs)
+28
View File
@@ -0,0 +1,28 @@
import os
from mock import patch
from dvc.external_repo import external_repo
from dvc.scm.git import Git
def test_external_repo(erepo):
url = erepo.root_dir
# We will share cache dir, to fetch version file
cache_dir = erepo.dvc.cache.local.cache_dir
with patch.object(Git, "clone", wraps=Git.clone) as mock:
with external_repo(url, cache_dir=cache_dir) as repo:
with repo.open(os.path.join(repo.root_dir, "version")) as fd:
assert fd.read() == "master"
with external_repo(url, rev="branch", cache_dir=cache_dir) as repo:
with repo.open(os.path.join(repo.root_dir, "version")) as fd:
assert fd.read() == "branch"
# Check cache_dir is unset
with external_repo(url) as repo:
assert repo.cache.local.cache_dir.startswith(
repo.root_dir + os.sep
)
assert mock.call_count == 1
+4 -3
View File
@@ -1,8 +1,8 @@
import os
import filecmp
import shutil
import pytest
from mock import patch
from tests.utils import trees_equal
from dvc.stage import Stage
@@ -73,9 +73,10 @@ def test_download_error_pulling_imported_stage(dvc_repo, erepo):
dst_stage = Stage.load(dvc_repo, "foo_imported.dvc")
dst_cache = dst_stage.outs[0].cache_path
shutil.rmtree(erepo.root_dir)
os.remove(dst)
os.remove(dst_cache)
with pytest.raises(DownloadError):
with patch(
"dvc.remote.RemoteLOCAL._download", side_effect=Exception
), pytest.raises(DownloadError):
dvc_repo.pull(["foo_imported.dvc"])
+5
View File
@@ -3,6 +3,7 @@ import shutil
import filecmp
from dvc.repo import Repo
from dvc.external_repo import clean_repos
def test_update_import(dvc_repo, erepo):
@@ -34,6 +35,10 @@ def test_update_import(dvc_repo, erepo):
os.chdir(saved_dir)
# Caching in external repos doesn't see upstream updates within single
# cli call, so we need to clean the caches to see the changes.
clean_repos()
assert dvc_repo.status([stage.path]) == {}
dvc_repo.update(stage.path)
assert dvc_repo.status([stage.path]) == {}