diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index b273e34e3..424269d60 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -53,35 +53,37 @@ class DependencyREPO(DependencyLOCAL): PathInfo(os.path.join(repo.root_dir, self.def_path)) ) - def _has_changed(self): - with self._make_repo() as repo, self._make_repo( - rev_lock=None - ) as updated_repo: - try: - current = repo.find_out_by_relpath(self.def_path).info - updated = updated_repo.find_out_by_relpath(self.def_path).info + # fileinfo is a dictionary containing "checksum" and "path" + def _get_fileinfo(self, updated=False): + rev_lock = None + if not updated: + rev_lock = self.def_repo.get(self.PARAM_REV_LOCK) - return current != updated - except OutputNotFoundError: - # Need to load the state before calculating checksums - repo.state.load() - updated_repo.state.load() + try: + with self._make_repo(rev_lock=rev_lock) as repo: + path = os.path.join(repo.root_dir, self.def_path) + try: + checksum = repo.find_out_by_relpath(self.def_path).info[ + "md5" + ] + except OutputNotFoundError: + repo.state.load() + checksum = repo.cache.local.get_checksum(PathInfo(path)) - return self._get_checksum_in_repo( - repo - ) != self._get_checksum_in_repo(updated_repo) + assert checksum - def _has_changed_non_dvc(self): - url = self.def_repo[self.PARAM_URL] - rev = self.def_repo.get(self.PARAM_REV) - rev_lock = self.def_repo.get(self.PARAM_REV_LOCK, rev) + return {"checksum": checksum, "path": path, "repo": repo} - current_repo = cached_clone(url, rev=rev_lock or rev) - updated_repo = cached_clone(url, rev=rev) - - current_path = os.path.join(current_repo, self.def_path) - updated_path = os.path.join(updated_repo, self.def_path) + except NotDvcRepoError: + repo_path = cached_clone( + self.def_repo[self.PARAM_URL], + rev=rev_lock or self.def_repo.get(self.PARAM_REV), + ) + path = os.path.join(repo_path, self.def_path) + return {"checksum": None, "path": path, "repo": None} + @staticmethod + def _paths_changed(current_path, updated_path): if not os.path.exists(current_path): raise FileNotFoundError(current_path) @@ -102,11 +104,29 @@ class DependencyREPO(DependencyLOCAL): return not filecmp.cmp(current_path, updated_path, shallow=False) + def _checkout_if_needed(self, fileinfo): + if os.path.exists(fileinfo["path"]): + return + if "repo" not in fileinfo: + return + + fileinfo["repo"].checkout([fileinfo["path"] + ".dvc"], recursive=True) + def status(self): - try: - has_changed = self._has_changed() - except NotDvcRepoError: - has_changed = self._has_changed_non_dvc() + current_fileinfo = self._get_fileinfo(updated=False) + updated_fileinfo = self._get_fileinfo(updated=True) + print("CURRENT", current_fileinfo, "UPDATED", updated_fileinfo) + + if current_fileinfo["checksum"] and updated_fileinfo["checksum"]: + has_changed = ( + current_fileinfo["checksum"] != updated_fileinfo["checksum"] + ) + else: + self._checkout_if_needed(current_fileinfo) + self._checkout_if_needed(updated_fileinfo) + has_changed = DependencyREPO._paths_changed( + current_fileinfo["path"], updated_fileinfo["path"] + ) if has_changed: return {str(self): "update available"} diff --git a/tests/func/test_status.py b/tests/func/test_status.py index b122a9e7a..1b688803a 100644 --- a/tests/func/test_status.py +++ b/tests/func/test_status.py @@ -3,6 +3,7 @@ import shutil from mock import patch +from dvc.repo import Repo from dvc.main import main from dvc.compat import fspath from tests.basic_env import TestDvc @@ -58,3 +59,36 @@ def test_status_non_dvc_repo_import(tmp_dir, dvc, erepo_dir): "file ({})".format(fspath(erepo_dir)): "update available" } } + + +def test_status_before_and_after_dvc_init(tmp_dir, dvc, erepo_dir): + with erepo_dir.chdir(): + erepo_dir.scm.repo.index.remove([".dvc"], r=True) + shutil.rmtree(".dvc") + erepo_dir.scm_gen("file", "first version") + erepo_dir.scm.add(["file"]) + erepo_dir.scm.commit("first version") + old_rev = erepo_dir.scm.get_rev() + + dvc.imp(fspath(erepo_dir), "file", "file") + + assert dvc.status(["file.dvc"]) == {} + + with erepo_dir.chdir(): + Repo.init() + erepo_dir.scm.repo.index.remove(["file"]) + os.remove("file") + erepo_dir.dvc_gen("file", "second version") + erepo_dir.scm.add([".dvc", "file.dvc"]) + erepo_dir.scm.commit("version with dvc") + new_rev = erepo_dir.scm.get_rev() + + assert old_rev != new_rev + + # Caching in external repos doesn't see upstream updates within single + # cli call, so we need to clean the caches to see the changes. + clean_repos() + + status = dvc.status(["file.dvc"]) + + print("STATUS", status) diff --git a/tests/func/test_update.py b/tests/func/test_update.py index dd467bc30..6f483fff0 100644 --- a/tests/func/test_update.py +++ b/tests/func/test_update.py @@ -1,5 +1,8 @@ import pytest +import os +import shutil +from dvc.repo import Repo from dvc.stage import Stage from dvc.compat import fspath from dvc.external_repo import clean_repos @@ -107,6 +110,48 @@ def test_update_import_after_remote_updates_to_dvc(tmp_dir, dvc, erepo_dir): } +def test_update_before_and_after_dvc_init(tmp_dir, dvc, erepo_dir): + with erepo_dir.chdir(): + erepo_dir.scm.repo.index.remove([".dvc"], r=True) + shutil.rmtree(".dvc") + erepo_dir.scm_gen("file", "first version") + erepo_dir.scm.add(["file"]) + erepo_dir.scm.commit("first version") + old_rev = erepo_dir.scm.get_rev() + + stage = dvc.imp(fspath(erepo_dir), "file", "file") + + with erepo_dir.chdir(): + Repo.init() + erepo_dir.scm.repo.index.remove(["file"]) + os.remove("file") + erepo_dir.dvc_gen("file", "second version") + erepo_dir.scm.add([".dvc", "file.dvc"]) + erepo_dir.scm.commit("version with dvc") + new_rev = erepo_dir.scm.get_rev() + + assert old_rev != new_rev + + # Caching in external repos doesn't see upstream updates within single + # cli call, so we need to clean the caches to see the changes. + clean_repos() + + assert dvc.status([stage.path]) == { + "file.dvc": [ + { + "changed deps": { + "file ({})".format(fspath(erepo_dir)): "update available" + } + } + ] + } + + dvc.update(stage.path) + + assert (tmp_dir / "file").read_text() == "second version" + assert dvc.status([stage.path]) == {} + + def test_update_import_url(tmp_dir, dvc, tmp_path_factory): import_src = tmp_path_factory.mktemp("import_url_source") src = import_src / "file"