mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
import: fix rev lock and pull with --no-download
This commit is contained in:
+19
-5
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple
|
||||
from voluptuous import Required
|
||||
|
||||
from dvc.prompt import confirm
|
||||
from dvc_data.hashfile.meta import Meta
|
||||
|
||||
from .base import Dependency
|
||||
|
||||
@@ -32,6 +33,7 @@ class RepoDependency(Dependency):
|
||||
def __init__(self, def_repo, stage, *args, **kwargs):
|
||||
self.def_repo = def_repo
|
||||
self._objs: Dict[str, "HashFile"] = {}
|
||||
self._meta: Dict[str, Meta] = {}
|
||||
super().__init__(stage, *args, **kwargs)
|
||||
|
||||
def _parse_path(self, fs, fs_path):
|
||||
@@ -94,12 +96,12 @@ class RepoDependency(Dependency):
|
||||
def get_used_objs(
|
||||
self, **kwargs
|
||||
) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
|
||||
used, _ = self._get_used_and_obj(**kwargs)
|
||||
used, _, _ = self._get_used_and_obj(**kwargs)
|
||||
return used
|
||||
|
||||
def _get_used_and_obj(
|
||||
self, obj_only=False, **kwargs
|
||||
) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "HashFile"]:
|
||||
) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], Meta, "HashFile"]:
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.exceptions import NoOutputOrStageError, PathMissingError
|
||||
from dvc.utils import as_posix
|
||||
@@ -131,7 +133,7 @@ class RepoDependency(Dependency):
|
||||
pass
|
||||
|
||||
try:
|
||||
object_store, _, obj = build(
|
||||
object_store, meta, obj = build(
|
||||
local_odb,
|
||||
as_posix(self.def_path),
|
||||
repo.dvcfs,
|
||||
@@ -145,10 +147,12 @@ class RepoDependency(Dependency):
|
||||
object_store.read_only = True
|
||||
|
||||
self._objs[rev] = obj
|
||||
self._meta[rev] = meta
|
||||
|
||||
used_obj_ids[object_store].add(obj.hash_info)
|
||||
if isinstance(obj, Tree):
|
||||
used_obj_ids[object_store].update(oid for _, _, oid in obj)
|
||||
return used_obj_ids, obj
|
||||
return used_obj_ids, meta, obj
|
||||
|
||||
def _check_circular_import(self, odb, obj_ids):
|
||||
from dvc.exceptions import CircularImportError
|
||||
@@ -190,11 +194,21 @@ class RepoDependency(Dependency):
|
||||
rev = self._get_rev(locked=locked)
|
||||
if rev in self._objs:
|
||||
return self._objs[rev]
|
||||
_, obj = self._get_used_and_obj(
|
||||
_, _, obj = self._get_used_and_obj(
|
||||
obj_only=True, filter_info=filter_info, **kwargs
|
||||
)
|
||||
return obj
|
||||
|
||||
def get_meta(self, filter_info=None, **kwargs):
|
||||
locked = kwargs.get("locked", True)
|
||||
rev = self._get_rev(locked=locked)
|
||||
if rev in self._meta:
|
||||
return self._meta[rev]
|
||||
_, meta, _ = self._get_used_and_obj(
|
||||
obj_only=True, filter_info=filter_info, **kwargs
|
||||
)
|
||||
return meta
|
||||
|
||||
def _make_repo(self, locked=True, **kwargs):
|
||||
from dvc.external_repo import external_repo
|
||||
|
||||
|
||||
+9
-3
@@ -155,9 +155,15 @@ def _fetch_partial_imports(repo, targets, **kwargs):
|
||||
raise DataSourceChanged(f"{stage} ({dep})")
|
||||
|
||||
logger.info("Importing '%s' -> '%s'", dep, out)
|
||||
out.transfer(
|
||||
dep.def_path, jobs=kwargs.get("jobs"), odb=repo.odb.local
|
||||
)
|
||||
if stage.is_repo_import:
|
||||
obj = dep.get_obj()
|
||||
meta = dep.get_meta()
|
||||
out.hash_info = obj.hash_info
|
||||
out.meta = meta
|
||||
else:
|
||||
out.transfer(
|
||||
dep.def_path, jobs=kwargs.get("jobs"), odb=repo.odb.local
|
||||
)
|
||||
except DataSourceChanged as exc:
|
||||
logger.warning(f"{exc}")
|
||||
failed += 1
|
||||
|
||||
@@ -683,7 +683,7 @@ class Stage(params.StageParams):
|
||||
self, *args, **kwargs
|
||||
) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
|
||||
"""Return set of object IDs used by this stage."""
|
||||
if self.is_partial_import:
|
||||
if self.is_partial_import and not self.is_repo_import:
|
||||
return {}
|
||||
|
||||
used_objs = defaultdict(set)
|
||||
|
||||
@@ -56,7 +56,10 @@ def sync_import(
|
||||
stage.outs[0].checkout()
|
||||
else:
|
||||
stage.save_deps()
|
||||
if not no_download:
|
||||
if no_download:
|
||||
if stage.is_repo_import:
|
||||
stage.deps[0].update()
|
||||
else:
|
||||
stage.deps[0].download(
|
||||
stage.outs[0],
|
||||
jobs=jobs,
|
||||
|
||||
@@ -217,6 +217,68 @@ def test_pull_imported_stage(tmp_dir, dvc, erepo_dir):
|
||||
assert os.path.isfile(dst_cache)
|
||||
|
||||
|
||||
def test_import_no_download(tmp_dir, scm, dvc, erepo_dir):
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
|
||||
|
||||
dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", no_download=True)
|
||||
|
||||
assert not os.path.exists("foo_imported")
|
||||
|
||||
dst_stage = Dvcfile(dvc, "foo_imported.dvc").stage
|
||||
|
||||
assert dst_stage.deps[0].def_repo == {
|
||||
"url": os.fspath(erepo_dir),
|
||||
"rev_lock": erepo_dir.scm.get_rev(),
|
||||
}
|
||||
assert scm.is_ignored("foo_imported")
|
||||
|
||||
|
||||
def test_pull_import_no_download(tmp_dir, scm, dvc, erepo_dir):
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.scm_gen(os.path.join("foo", "bar"), b"bar", commit="add bar")
|
||||
erepo_dir.dvc_gen(
|
||||
os.path.join("foo", "baz"), b"baz contents", commit="add baz"
|
||||
)
|
||||
size = (
|
||||
len(b"bar")
|
||||
+ len(b"baz contents")
|
||||
+ len((erepo_dir / "foo" / ".gitignore").read_bytes())
|
||||
)
|
||||
|
||||
dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", no_download=True)
|
||||
|
||||
dvc.pull(["foo_imported.dvc"])
|
||||
assert os.path.exists("foo_imported")
|
||||
|
||||
stage = Dvcfile(dvc, "foo_imported.dvc").stage
|
||||
|
||||
assert (
|
||||
stage.outs[0].hash_info.value == "bdb8641831d8fcb03939637e09011c21.dir"
|
||||
)
|
||||
|
||||
assert stage.outs[0].meta.size == size
|
||||
assert stage.outs[0].meta.nfiles == 3
|
||||
assert stage.outs[0].meta.isdir
|
||||
|
||||
|
||||
def test_pull_import_no_download_rev_lock(
|
||||
tmp_dir,
|
||||
dvc,
|
||||
erepo_dir,
|
||||
):
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("foo", "foo content", commit="add")
|
||||
|
||||
dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", no_download=True)
|
||||
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("foo", "modified foo content", commit="modify foo")
|
||||
|
||||
dvc.pull(["foo_imported.dvc"])
|
||||
assert (tmp_dir / "foo_imported").read_text() == "foo content"
|
||||
|
||||
|
||||
def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir):
|
||||
with erepo_dir.chdir():
|
||||
with erepo_dir.dvc.config.edit() as conf:
|
||||
|
||||
Reference in New Issue
Block a user