import: fix rev lock and pull with --no-download

This commit is contained in:
Daniele Trifirò
2022-10-03 14:25:50 +02:00
committed by daniele
parent 5d69f57abf
commit 4594087786
5 changed files with 95 additions and 10 deletions
+19 -5
View File
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple
from voluptuous import Required
from dvc.prompt import confirm
from dvc_data.hashfile.meta import Meta
from .base import Dependency
@@ -32,6 +33,7 @@ class RepoDependency(Dependency):
def __init__(self, def_repo, stage, *args, **kwargs):
self.def_repo = def_repo
self._objs: Dict[str, "HashFile"] = {}
self._meta: Dict[str, Meta] = {}
super().__init__(stage, *args, **kwargs)
def _parse_path(self, fs, fs_path):
@@ -94,12 +96,12 @@ class RepoDependency(Dependency):
def get_used_objs(
self, **kwargs
) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
used, _ = self._get_used_and_obj(**kwargs)
used, _, _ = self._get_used_and_obj(**kwargs)
return used
def _get_used_and_obj(
self, obj_only=False, **kwargs
) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "HashFile"]:
) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], Meta, "HashFile"]:
from dvc.config import NoRemoteError
from dvc.exceptions import NoOutputOrStageError, PathMissingError
from dvc.utils import as_posix
@@ -131,7 +133,7 @@ class RepoDependency(Dependency):
pass
try:
object_store, _, obj = build(
object_store, meta, obj = build(
local_odb,
as_posix(self.def_path),
repo.dvcfs,
@@ -145,10 +147,12 @@ class RepoDependency(Dependency):
object_store.read_only = True
self._objs[rev] = obj
self._meta[rev] = meta
used_obj_ids[object_store].add(obj.hash_info)
if isinstance(obj, Tree):
used_obj_ids[object_store].update(oid for _, _, oid in obj)
return used_obj_ids, obj
return used_obj_ids, meta, obj
def _check_circular_import(self, odb, obj_ids):
from dvc.exceptions import CircularImportError
@@ -190,11 +194,21 @@ class RepoDependency(Dependency):
rev = self._get_rev(locked=locked)
if rev in self._objs:
return self._objs[rev]
_, obj = self._get_used_and_obj(
_, _, obj = self._get_used_and_obj(
obj_only=True, filter_info=filter_info, **kwargs
)
return obj
def get_meta(self, filter_info=None, **kwargs):
locked = kwargs.get("locked", True)
rev = self._get_rev(locked=locked)
if rev in self._meta:
return self._meta[rev]
_, meta, _ = self._get_used_and_obj(
obj_only=True, filter_info=filter_info, **kwargs
)
return meta
def _make_repo(self, locked=True, **kwargs):
from dvc.external_repo import external_repo
+9 -3
View File
@@ -155,9 +155,15 @@ def _fetch_partial_imports(repo, targets, **kwargs):
raise DataSourceChanged(f"{stage} ({dep})")
logger.info("Importing '%s' -> '%s'", dep, out)
out.transfer(
dep.def_path, jobs=kwargs.get("jobs"), odb=repo.odb.local
)
if stage.is_repo_import:
obj = dep.get_obj()
meta = dep.get_meta()
out.hash_info = obj.hash_info
out.meta = meta
else:
out.transfer(
dep.def_path, jobs=kwargs.get("jobs"), odb=repo.odb.local
)
except DataSourceChanged as exc:
logger.warning(f"{exc}")
failed += 1
+1 -1
View File
@@ -683,7 +683,7 @@ class Stage(params.StageParams):
self, *args, **kwargs
) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
"""Return set of object IDs used by this stage."""
if self.is_partial_import:
if self.is_partial_import and not self.is_repo_import:
return {}
used_objs = defaultdict(set)
+4 -1
View File
@@ -56,7 +56,10 @@ def sync_import(
stage.outs[0].checkout()
else:
stage.save_deps()
if not no_download:
if no_download:
if stage.is_repo_import:
stage.deps[0].update()
else:
stage.deps[0].download(
stage.outs[0],
jobs=jobs,
+62
View File
@@ -217,6 +217,68 @@ def test_pull_imported_stage(tmp_dir, dvc, erepo_dir):
assert os.path.isfile(dst_cache)
def test_import_no_download(tmp_dir, scm, dvc, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", no_download=True)
assert not os.path.exists("foo_imported")
dst_stage = Dvcfile(dvc, "foo_imported.dvc").stage
assert dst_stage.deps[0].def_repo == {
"url": os.fspath(erepo_dir),
"rev_lock": erepo_dir.scm.get_rev(),
}
assert scm.is_ignored("foo_imported")
def test_pull_import_no_download(tmp_dir, scm, dvc, erepo_dir):
with erepo_dir.chdir():
erepo_dir.scm_gen(os.path.join("foo", "bar"), b"bar", commit="add bar")
erepo_dir.dvc_gen(
os.path.join("foo", "baz"), b"baz contents", commit="add baz"
)
size = (
len(b"bar")
+ len(b"baz contents")
+ len((erepo_dir / "foo" / ".gitignore").read_bytes())
)
dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", no_download=True)
dvc.pull(["foo_imported.dvc"])
assert os.path.exists("foo_imported")
stage = Dvcfile(dvc, "foo_imported.dvc").stage
assert (
stage.outs[0].hash_info.value == "bdb8641831d8fcb03939637e09011c21.dir"
)
assert stage.outs[0].meta.size == size
assert stage.outs[0].meta.nfiles == 3
assert stage.outs[0].meta.isdir
def test_pull_import_no_download_rev_lock(
tmp_dir,
dvc,
erepo_dir,
):
with erepo_dir.chdir():
erepo_dir.dvc_gen("foo", "foo content", commit="add")
dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", no_download=True)
with erepo_dir.chdir():
erepo_dir.dvc_gen("foo", "modified foo content", commit="modify foo")
dvc.pull(["foo_imported.dvc"])
assert (tmp_dir / "foo_imported").read_text() == "foo content"
def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir):
with erepo_dir.chdir():
with erepo_dir.dvc.config.edit() as conf: