mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 15:28:28 +01:00
ui/perf: do not clone same repo multiple times (#2583)
* ui/perf: do not clone same repo multiple times Closes #2490 External repos are still created in /tmp. They might be reused until main() end. If no perfect match is found then a repo with the same url is copied (not cloned again) and adjusted to fit a request: rev and cache_dir. NOTE: test_download_error_pulling_imported_stage() was modified to not rely on clone error. * test: fix import update test * test: fix external_repo test under Windows * dvc: fix non-master default branch for external repos * dvc: remove unused .unset_dir() method
This commit is contained in:
committed by
Ruslan Kuprieiev
parent
5cabb6d115
commit
44264857c3
+76
-35
@@ -1,50 +1,91 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import os
|
||||
import logging
|
||||
import tempfile
|
||||
from distutils.dir_util import copy_tree
|
||||
|
||||
from funcy import retry
|
||||
from contextlib import contextmanager
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.cache import CacheConfig
|
||||
from dvc.utils import remove
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _clone(cache_dir=None, url=None, rev=None, rev_lock=None):
|
||||
from dvc.repo import Repo
|
||||
|
||||
_path = tempfile.mkdtemp("dvc-repo")
|
||||
|
||||
repo = Repo.clone(url, _path, rev=(rev_lock or rev))
|
||||
|
||||
try:
|
||||
if cache_dir:
|
||||
cache_config = CacheConfig(repo.config)
|
||||
cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
|
||||
finally:
|
||||
repo.close()
|
||||
|
||||
return Repo(_path)
|
||||
|
||||
|
||||
def _remove(repo):
|
||||
repo.close()
|
||||
|
||||
if os.name == "nt":
|
||||
# git.exe may hang for a while not permitting to remove temp dir
|
||||
os_retry = retry(5, errors=OSError, timeout=0.1)
|
||||
os_retry(remove)(repo.root_dir)
|
||||
else:
|
||||
remove(repo.root_dir)
|
||||
REPO_CACHE = {}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def external_repo(**kwargs):
|
||||
repo = _clone(**kwargs)
|
||||
def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
|
||||
from dvc.repo import Repo
|
||||
|
||||
path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
|
||||
repo = Repo(path)
|
||||
yield repo
|
||||
_remove(repo)
|
||||
repo.close()
|
||||
|
||||
|
||||
def _external_repo(url=None, rev=None, cache_dir=None):
|
||||
from dvc.config import Config
|
||||
from dvc.cache import CacheConfig
|
||||
from dvc.repo import Repo
|
||||
|
||||
key = (url, rev, cache_dir)
|
||||
if key in REPO_CACHE:
|
||||
return REPO_CACHE[key]
|
||||
|
||||
new_path = tempfile.mkdtemp("dvc-erepo")
|
||||
|
||||
# Copy and adjust existing clone
|
||||
if (url, None, None) in REPO_CACHE:
|
||||
old_path = REPO_CACHE[url, None, None]
|
||||
|
||||
# This one unlike shutil.copytree() works with an existing dir
|
||||
copy_tree(old_path, new_path)
|
||||
else:
|
||||
# Create a new clone
|
||||
_clone_repo(url, new_path)
|
||||
|
||||
# Save clean clone dir so that we will have access to a default branch
|
||||
clean_clone_path = tempfile.mkdtemp("dvc-erepo")
|
||||
copy_tree(new_path, clean_clone_path)
|
||||
REPO_CACHE[url, None, None] = clean_clone_path
|
||||
|
||||
# Adjust new clone/copy to fit rev and cache_dir
|
||||
repo = Repo(new_path)
|
||||
try:
|
||||
if rev is not None:
|
||||
repo.scm.checkout(rev)
|
||||
|
||||
if cache_dir is not None:
|
||||
cache_config = CacheConfig(repo.config)
|
||||
cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
|
||||
finally:
|
||||
# Need to close/reopen repo to force config reread
|
||||
repo.close()
|
||||
|
||||
REPO_CACHE[key] = new_path
|
||||
return new_path
|
||||
|
||||
|
||||
def clean_repos():
|
||||
# Outside code should not see cache while we are removing
|
||||
repo_paths = list(REPO_CACHE.values())
|
||||
REPO_CACHE.clear()
|
||||
|
||||
for path in repo_paths:
|
||||
_remove(path)
|
||||
|
||||
|
||||
def _remove(path):
|
||||
if os.name == "nt":
|
||||
# git.exe may hang for a while not permitting to remove temp dir
|
||||
os_retry = retry(5, errors=OSError, timeout=0.1)
|
||||
os_retry(remove)(path)
|
||||
else:
|
||||
remove(path)
|
||||
|
||||
|
||||
def _clone_repo(url, path):
|
||||
from dvc.scm.git import Git
|
||||
|
||||
git = Git.clone(url, path)
|
||||
git.close()
|
||||
|
||||
@@ -10,6 +10,7 @@ from dvc.config import ConfigError
|
||||
from dvc.analytics import Analytics
|
||||
from dvc.exceptions import NotDvcRepoError, DvcParserError
|
||||
from dvc.remote.pool import close_pools
|
||||
from dvc.external_repo import clean_repos
|
||||
from dvc.utils.compat import is_py2
|
||||
|
||||
|
||||
@@ -65,10 +66,15 @@ def main(argv=None):
|
||||
ret = 255
|
||||
finally:
|
||||
logger.setLevel(outerLogLevel)
|
||||
|
||||
# Python 2 fails to close these clean occasionally and users see
|
||||
# weird error messages, so we do it manually
|
||||
close_pools()
|
||||
|
||||
# Remove cached repos in the end of the call, these are anonymous
|
||||
# so won't be reused by any other subsequent run anyway.
|
||||
clean_repos()
|
||||
|
||||
Analytics().send_cmd(cmd, args, ret)
|
||||
|
||||
return ret
|
||||
|
||||
@@ -504,15 +504,6 @@ class Repo(object):
|
||||
def close(self):
|
||||
self.scm.close()
|
||||
|
||||
@staticmethod
|
||||
def clone(url, to_path, rev=None):
|
||||
from dvc.scm.git import Git
|
||||
|
||||
git = Git.clone(url, to_path, rev=rev)
|
||||
git.close()
|
||||
|
||||
return Repo(to_path)
|
||||
|
||||
@locked
|
||||
def checkout(self, *args, **kwargs):
|
||||
return self._checkout(*args, **kwargs)
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import os
|
||||
from mock import patch
|
||||
|
||||
from dvc.external_repo import external_repo
|
||||
from dvc.scm.git import Git
|
||||
|
||||
|
||||
def test_external_repo(erepo):
|
||||
url = erepo.root_dir
|
||||
# We will share cache dir, to fetch version file
|
||||
cache_dir = erepo.dvc.cache.local.cache_dir
|
||||
|
||||
with patch.object(Git, "clone", wraps=Git.clone) as mock:
|
||||
with external_repo(url, cache_dir=cache_dir) as repo:
|
||||
with repo.open(os.path.join(repo.root_dir, "version")) as fd:
|
||||
assert fd.read() == "master"
|
||||
|
||||
with external_repo(url, rev="branch", cache_dir=cache_dir) as repo:
|
||||
with repo.open(os.path.join(repo.root_dir, "version")) as fd:
|
||||
assert fd.read() == "branch"
|
||||
|
||||
# Check cache_dir is unset
|
||||
with external_repo(url) as repo:
|
||||
assert repo.cache.local.cache_dir.startswith(
|
||||
repo.root_dir + os.sep
|
||||
)
|
||||
|
||||
assert mock.call_count == 1
|
||||
@@ -1,8 +1,8 @@
|
||||
import os
|
||||
import filecmp
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
from mock import patch
|
||||
from tests.utils import trees_equal
|
||||
|
||||
from dvc.stage import Stage
|
||||
@@ -73,9 +73,10 @@ def test_download_error_pulling_imported_stage(dvc_repo, erepo):
|
||||
dst_stage = Stage.load(dvc_repo, "foo_imported.dvc")
|
||||
dst_cache = dst_stage.outs[0].cache_path
|
||||
|
||||
shutil.rmtree(erepo.root_dir)
|
||||
os.remove(dst)
|
||||
os.remove(dst_cache)
|
||||
|
||||
with pytest.raises(DownloadError):
|
||||
with patch(
|
||||
"dvc.remote.RemoteLOCAL._download", side_effect=Exception
|
||||
), pytest.raises(DownloadError):
|
||||
dvc_repo.pull(["foo_imported.dvc"])
|
||||
|
||||
@@ -3,6 +3,7 @@ import shutil
|
||||
import filecmp
|
||||
|
||||
from dvc.repo import Repo
|
||||
from dvc.external_repo import clean_repos
|
||||
|
||||
|
||||
def test_update_import(dvc_repo, erepo):
|
||||
@@ -34,6 +35,10 @@ def test_update_import(dvc_repo, erepo):
|
||||
|
||||
os.chdir(saved_dir)
|
||||
|
||||
# Caching in external repos doesn't see upstream updates within single
|
||||
# cli call, so we need to clean the caches to see the changes.
|
||||
clean_repos()
|
||||
|
||||
assert dvc_repo.status([stage.path]) == {}
|
||||
dvc_repo.update(stage.path)
|
||||
assert dvc_repo.status([stage.path]) == {}
|
||||
|
||||
Reference in New Issue
Block a user