diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 33d3f337f..e71fed463 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple from voluptuous import Required from dvc.prompt import confirm +from dvc_data.hashfile.meta import Meta from .base import Dependency @@ -32,6 +33,7 @@ class RepoDependency(Dependency): def __init__(self, def_repo, stage, *args, **kwargs): self.def_repo = def_repo self._objs: Dict[str, "HashFile"] = {} + self._meta: Dict[str, Meta] = {} super().__init__(stage, *args, **kwargs) def _parse_path(self, fs, fs_path): @@ -94,12 +96,12 @@ class RepoDependency(Dependency): def get_used_objs( self, **kwargs ) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]: - used, _ = self._get_used_and_obj(**kwargs) + used, _, _ = self._get_used_and_obj(**kwargs) return used def _get_used_and_obj( self, obj_only=False, **kwargs - ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "HashFile"]: + ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], Meta, "HashFile"]: from dvc.config import NoRemoteError from dvc.exceptions import NoOutputOrStageError, PathMissingError from dvc.utils import as_posix @@ -131,7 +133,7 @@ class RepoDependency(Dependency): pass try: - object_store, _, obj = build( + object_store, meta, obj = build( local_odb, as_posix(self.def_path), repo.dvcfs, @@ -145,10 +147,12 @@ class RepoDependency(Dependency): object_store.read_only = True self._objs[rev] = obj + self._meta[rev] = meta + used_obj_ids[object_store].add(obj.hash_info) if isinstance(obj, Tree): used_obj_ids[object_store].update(oid for _, _, oid in obj) - return used_obj_ids, obj + return used_obj_ids, meta, obj def _check_circular_import(self, odb, obj_ids): from dvc.exceptions import CircularImportError @@ -190,11 +194,21 @@ class RepoDependency(Dependency): rev = self._get_rev(locked=locked) if rev in self._objs: return self._objs[rev] - _, obj = self._get_used_and_obj( + _, _, obj = self._get_used_and_obj( obj_only=True, filter_info=filter_info, **kwargs ) return obj + def get_meta(self, filter_info=None, **kwargs): + locked = kwargs.get("locked", True) + rev = self._get_rev(locked=locked) + if rev in self._meta: + return self._meta[rev] + _, meta, _ = self._get_used_and_obj( + obj_only=True, filter_info=filter_info, **kwargs + ) + return meta + def _make_repo(self, locked=True, **kwargs): from dvc.external_repo import external_repo diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 8166ae75e..2bdcb061e 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -155,9 +155,15 @@ def _fetch_partial_imports(repo, targets, **kwargs): raise DataSourceChanged(f"{stage} ({dep})") logger.info("Importing '%s' -> '%s'", dep, out) - out.transfer( - dep.def_path, jobs=kwargs.get("jobs"), odb=repo.odb.local - ) + if stage.is_repo_import: + obj = dep.get_obj() + meta = dep.get_meta() + out.hash_info = obj.hash_info + out.meta = meta + else: + out.transfer( + dep.def_path, jobs=kwargs.get("jobs"), odb=repo.odb.local + ) except DataSourceChanged as exc: logger.warning(f"{exc}") failed += 1 diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index dedf44274..64a27cfca 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -683,7 +683,7 @@ class Stage(params.StageParams): self, *args, **kwargs ) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]: """Return set of object IDs used by this stage.""" - if self.is_partial_import: + if self.is_partial_import and not self.is_repo_import: return {} used_objs = defaultdict(set) diff --git a/dvc/stage/imports.py b/dvc/stage/imports.py index ac1f28b7e..313d0d332 100644 --- a/dvc/stage/imports.py +++ b/dvc/stage/imports.py @@ -56,7 +56,10 @@ def sync_import( stage.outs[0].checkout() else: stage.save_deps() - if not no_download: + if no_download: + if stage.is_repo_import: + stage.deps[0].update() + else: stage.deps[0].download( stage.outs[0], jobs=jobs, diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 94fc343f9..cedcf231c 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -217,6 +217,68 @@ def test_pull_imported_stage(tmp_dir, dvc, erepo_dir): assert os.path.isfile(dst_cache) +def test_import_no_download(tmp_dir, scm, dvc, erepo_dir): + with erepo_dir.chdir(): + erepo_dir.dvc_gen("foo", "foo content", commit="create foo") + + dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", no_download=True) + + assert not os.path.exists("foo_imported") + + dst_stage = Dvcfile(dvc, "foo_imported.dvc").stage + + assert dst_stage.deps[0].def_repo == { + "url": os.fspath(erepo_dir), + "rev_lock": erepo_dir.scm.get_rev(), + } + assert scm.is_ignored("foo_imported") + + +def test_pull_import_no_download(tmp_dir, scm, dvc, erepo_dir): + with erepo_dir.chdir(): + erepo_dir.scm_gen(os.path.join("foo", "bar"), b"bar", commit="add bar") + erepo_dir.dvc_gen( + os.path.join("foo", "baz"), b"baz contents", commit="add baz" + ) + size = ( + len(b"bar") + + len(b"baz contents") + + len((erepo_dir / "foo" / ".gitignore").read_bytes()) + ) + + dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", no_download=True) + + dvc.pull(["foo_imported.dvc"]) + assert os.path.exists("foo_imported") + + stage = Dvcfile(dvc, "foo_imported.dvc").stage + + assert ( + stage.outs[0].hash_info.value == "bdb8641831d8fcb03939637e09011c21.dir" + ) + + assert stage.outs[0].meta.size == size + assert stage.outs[0].meta.nfiles == 3 + assert stage.outs[0].meta.isdir + + +def test_pull_import_no_download_rev_lock( + tmp_dir, + dvc, + erepo_dir, +): + with erepo_dir.chdir(): + erepo_dir.dvc_gen("foo", "foo content", commit="add") + + dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", no_download=True) + + with erepo_dir.chdir(): + erepo_dir.dvc_gen("foo", "modified foo content", commit="modify foo") + + dvc.pull(["foo_imported.dvc"]) + assert (tmp_dir / "foo_imported").read_text() == "foo content" + + def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir): with erepo_dir.chdir(): with erepo_dir.dvc.config.edit() as conf: