import-url/update: add --no-download flag

add --no-download flag to dvc import-url/dvc update to only create/update .dvc
files without downloading the associated data.

Created .dvc files can be fetched using `dvc pull` or can be
updated using `dvc update --no-download`.
This commit is contained in:
Daniele Trifirò
2022-08-23 23:07:01 +03:00
committed by Ruslan Kuprieiev
parent 92d680ff94
commit 26008f1559
18 changed files with 348 additions and 26 deletions
+11 -2
View File
@@ -19,6 +19,7 @@ class CmdImport(CmdBase):
fname=self.args.file,
rev=self.args.rev,
no_exec=self.args.no_exec,
no_download=self.args.no_download,
desc=self.args.desc,
jobs=self.args.jobs,
)
@@ -69,11 +70,19 @@ def add_parser(subparsers, parent_parser):
help="Specify name of the .dvc file this command will generate.",
metavar="<filename>",
)
import_parser.add_argument(
no_download_exec_group = import_parser.add_mutually_exclusive_group()
no_download_exec_group.add_argument(
"--no-exec",
action="store_true",
default=False,
help="Only create .dvc file without actually downloading it.",
help="Only create .dvc file without actually importing target data.",
)
no_download_exec_group.add_argument(
"--no-download",
action="store_true",
default=False,
help="Create .dvc file including target data hash value(s)"
" but do not actually download the file(s).",
)
import_parser.add_argument(
"--desc",
+11 -2
View File
@@ -17,6 +17,7 @@ class CmdImportUrl(CmdBase):
out=self.args.out,
fname=self.args.file,
no_exec=self.args.no_exec,
no_download=self.args.no_download,
remote=self.args.remote,
to_remote=self.args.to_remote,
desc=self.args.desc,
@@ -66,11 +67,19 @@ def add_parser(subparsers, parent_parser):
help="Specify name of the .dvc file this command will generate.",
metavar="<filename>",
).complete = completion.DIR
import_parser.add_argument(
no_download_exec_group = import_parser.add_mutually_exclusive_group()
no_download_exec_group.add_argument(
"--no-exec",
action="store_true",
default=False,
help="Only create .dvc file without actually downloading it.",
help="Only create .dvc file without actually importing target data.",
)
no_download_exec_group.add_argument(
"--no-download",
action="store_true",
default=False,
help="Create .dvc file including target data hash value(s)"
" but do not actually download the file(s).",
)
import_parser.add_argument(
"--to-remote",
+8
View File
@@ -18,6 +18,7 @@ class CmdUpdate(CmdBase):
rev=self.args.rev,
recursive=self.args.recursive,
to_remote=self.args.to_remote,
no_download=self.args.no_download,
remote=self.args.remote,
jobs=self.args.jobs,
)
@@ -55,6 +56,13 @@ def add_parser(subparsers, parent_parser):
default=False,
help="Update all stages in the specified directory.",
)
update_parser.add_argument(
"--no-download",
action="store_true",
default=False,
help="Update .dvc file hash value(s)"
" but do not download the file(s).",
)
update_parser.add_argument(
"--to-remote",
action="store_true",
+5
View File
@@ -375,6 +375,11 @@ class Output:
return self.fs.path.relpath(self.fs_path, self.repo.root_dir)
def clear(self):
self.hash_info = HashInfo.from_dict({})
self.meta = Meta.from_dict({})
self.obj = None
@property
def protocol(self):
return self.fs.protocol
+40 -1
View File
@@ -3,7 +3,7 @@ import os
from collections import defaultdict
from contextlib import contextmanager
from functools import wraps
from typing import TYPE_CHECKING, Callable, Optional
from typing import TYPE_CHECKING, Callable, List, Optional
from funcy import cached_property
@@ -18,6 +18,7 @@ if TYPE_CHECKING:
from dvc.fs import FileSystem
from dvc.repo.scm_context import SCMContext
from dvc.scm import Base
from dvc.stage import Stage
logger = logging.getLogger(__name__)
@@ -444,6 +445,44 @@ class Repo:
return used
def partial_imports(
self,
targets=None,
all_branches=False,
all_tags=False,
all_commits=False,
all_experiments=False,
commit_date: Optional[str] = None,
recursive=False,
revs=None,
num=1,
) -> List["Stage"]:
"""Get the stages related to the given target and collect dependencies
which are missing outputs.
This is useful to retrieve files which have been imported to the repo
using --no-download.
Returns:
A list of partially imported stages
"""
from itertools import chain
partial_imports = chain.from_iterable(
self.index.partial_imports(targets, recursive=recursive)
for _ in self.brancher(
revs=revs,
all_branches=all_branches,
all_tags=all_tags,
all_commits=all_commits,
all_experiments=all_experiments,
commit_date=commit_date,
num=num,
)
)
return list(partial_imports)
@property
def stages(self): # obsolete, only for backward-compatibility
return self.index.stages
+34
View File
@@ -93,6 +93,18 @@ def fetch(
downloaded += d
failed += f
d, f = _fetch_partial_imports(
self,
targets,
all_branches=all_branches,
all_tags=all_tags,
all_commits=all_commits,
recursive=recursive,
revs=revs,
)
downloaded += d
failed += f
if failed:
raise DownloadError(failed)
@@ -107,3 +119,25 @@ def _fetch(repo, obj_ids, **kwargs):
except FileTransferError as exc:
failed += exc.amount
return downloaded, failed
def _fetch_partial_imports(repo, targets, **kwargs):
from dvc.stage.exceptions import DataSourceChanged
downloaded = 0
failed = 0
for stage in repo.partial_imports(targets, **kwargs):
try:
stage.run()
except DataSourceChanged as exc:
logger.warning(f"{exc}")
failed += 1
continue
if not any(
kwargs.get(kw, None)
for kw in ("all_branches", "all_tags", "all_commits", "revs")
):
stage.dump()
downloaded += 1
return downloaded, failed
+4 -3
View File
@@ -17,6 +17,7 @@ def imp_url(
fname=None,
erepo=None,
frozen=True,
no_download=False,
no_exec=False,
remote=None,
to_remote=False,
@@ -31,9 +32,9 @@ def imp_url(
self, out, always_local=to_remote and not out
)
if to_remote and no_exec:
if to_remote and (no_exec or no_download):
raise InvalidArgumentError(
"--no-exec can't be combined with --to-remote"
"--no-exec/--no-download cannot be combined with --to-remote"
)
if not to_remote and remote:
@@ -80,7 +81,7 @@ def imp_url(
stage.save_deps()
stage.md5 = stage.compute_md5()
else:
stage.run(jobs=jobs)
stage.run(jobs=jobs, no_download=no_download)
stage.frozen = frozen
+22
View File
@@ -246,6 +246,28 @@ class Index:
used[odb].update(objs)
return used
def partial_imports(
self,
targets: "TargetType" = None,
recursive: bool = False,
) -> List["Stage"]:
from itertools import chain
from dvc.utils.collections import ensure_list
collect_targets: Sequence[Optional[str]] = (None,)
if targets:
collect_targets = ensure_list(targets)
pairs = chain.from_iterable(
self.stage_collector.collect_granular(
target, recursive=recursive, with_deps=True
)
for target in collect_targets
)
return [stage for stage, _ in pairs if stage.is_partial_import]
# Following methods help us treat the collection as a set-like structure
# and provides faux-immutability.
# These methods do not preserve stages order.
+13 -1
View File
@@ -10,6 +10,7 @@ def update(
rev=None,
recursive=False,
to_remote=False,
no_download=False,
remote=None,
jobs=None,
):
@@ -21,6 +22,11 @@ def update(
if isinstance(targets, str):
targets = [targets]
if to_remote and no_download:
raise InvalidArgumentError(
"--to-remote can't be used with --no-download"
)
if not to_remote and remote:
raise InvalidArgumentError(
"--remote can't be used without --to-remote"
@@ -31,7 +37,13 @@ def update(
stages.update(self.stage.collect(target, recursive=recursive))
for stage in stages:
stage.update(rev, to_remote=to_remote, remote=remote, jobs=jobs)
stage.update(
rev,
to_remote=to_remote,
remote=remote,
no_download=no_download,
jobs=jobs,
)
dvcfile = Dvcfile(self, stage.path)
dvcfile.dump(stage)
+39 -7
View File
@@ -78,6 +78,7 @@ def create_stage(cls, repo, path, external=False, **kwargs):
wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
path = os.path.abspath(path)
check_dvcfile_path(repo, path)
check_stage_path(repo, wdir, is_wdir=kwargs.get("wdir"))
check_stage_path(repo, os.path.dirname(path))
@@ -242,6 +243,14 @@ class Stage(params.StageParams):
"""Whether the DVC file was created with `dvc import`."""
return not self.cmd and len(self.deps) == 1 and len(self.outs) == 1
@property
def is_partial_import(self) -> bool:
"""
Whether the DVC file was created using `dvc import --no-download`
or `dvc import-url --no-download`.
"""
return self.is_import and (not self.outs[0].hash_info)
@property
def is_repo_import(self):
if not self.is_import:
@@ -433,11 +442,23 @@ class Stage(params.StageParams):
return self
def update(self, rev=None, to_remote=False, remote=None, jobs=None):
def update(
self,
rev=None,
to_remote=False,
remote=None,
no_download=None,
jobs=None,
):
if not (self.is_repo_import or self.is_import):
raise StageUpdateError(self.relpath)
update_import(
self, rev=rev, to_remote=to_remote, remote=remote, jobs=jobs
self,
rev=rev,
to_remote=to_remote,
remote=remote,
no_download=no_download,
jobs=jobs,
)
def reload(self):
@@ -457,6 +478,7 @@ class Stage(params.StageParams):
def save(self, allow_missing=False):
self.save_deps(allow_missing=allow_missing)
self.save_outs(allow_missing=allow_missing)
self.md5 = self.compute_md5()
@@ -525,14 +547,17 @@ class Stage(params.StageParams):
no_commit=False,
force=False,
allow_missing=False,
no_download=False,
**kwargs,
):
if (self.cmd or self.is_import) and not self.frozen and not dry:
self.remove_outs(ignore_remove=False, force=False)
if not self.frozen and self.is_import:
if (not self.frozen and self.is_import) or self.is_partial_import:
jobs = kwargs.get("jobs", None)
self._sync_import(dry, force, jobs)
self._sync_import(
dry, force, jobs, no_download, check_changed=self.frozen
)
elif not self.frozen and self.cmd:
self._run_stage(dry, force, **kwargs)
else:
@@ -544,7 +569,7 @@ class Stage(params.StageParams):
self._check_missing_outputs()
if not dry:
if kwargs.get("checkpoint_func", None):
if kwargs.get("checkpoint_func", None) or no_download:
allow_missing = True
self.save(allow_missing=allow_missing)
if not no_commit:
@@ -555,8 +580,10 @@ class Stage(params.StageParams):
return run_stage(self, dry, force, **kwargs)
@rwlocked(read=["deps"], write=["outs"])
def _sync_import(self, dry, force, jobs):
sync_import(self, dry, force, jobs)
def _sync_import(
self, dry, force, jobs, no_download, check_changed: bool = False
):
sync_import(self, dry, force, jobs, no_download, check_changed)
@rwlocked(read=["outs"])
def _check_missing_outputs(self):
@@ -571,6 +598,8 @@ class Stage(params.StageParams):
@rwlocked(write=["outs"])
def checkout(self, allow_missing=False, **kwargs):
stats = defaultdict(list)
if self.is_partial_import:
return {}
for out in self.filter_outs(kwargs.get("filter_info")):
key, outs = self._checkout(
out,
@@ -658,6 +687,9 @@ class Stage(params.StageParams):
self, *args, **kwargs
) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
"""Return set of object IDs used by this stage."""
if self.is_partial_import:
return {}
used_objs = defaultdict(set)
for out in self.filter_outs(kwargs.get("filter_info")):
for odb, objs in out.get_used_objs(*args, **kwargs).items():
+5
View File
@@ -80,6 +80,11 @@ class MissingDataSource(DvcException):
super().__init__(msg)
class DataSourceChanged(DvcException):
def __init__(self, path: str):
super().__init__(f"data source changed: {path}")
class StageNotFound(DvcException, KeyError):
def __init__(self, file, name):
self.file = file.relpath
+29 -4
View File
@@ -17,20 +17,37 @@ def _update_import_on_remote(stage, remote, jobs):
stage.outs[0].transfer(url, odb=odb, jobs=jobs, update=True)
def update_import(stage, rev=None, to_remote=False, remote=None, jobs=None):
def update_import(
stage, rev=None, to_remote=False, remote=None, no_download=None, jobs=None
):
stage.deps[0].update(rev=rev)
outs = stage.outs
deps = stage.deps
frozen = stage.frozen
stage.frozen = False
if stage.outs:
stage.outs[0].clear()
try:
if to_remote:
_update_import_on_remote(stage, remote, jobs)
else:
stage.reproduce(jobs=jobs)
stage.reproduce(no_download=no_download, jobs=jobs)
finally:
if deps == stage.deps:
stage.outs = outs
stage.frozen = frozen
def sync_import(stage, dry=False, force=False, jobs=None):
def sync_import(
stage,
dry=False,
force=False,
jobs=None,
no_download=False,
check_changed=False,
):
"""Synchronize import's outs to the workspace."""
logger.info("Importing '%s' -> '%s'", stage.deps[0], stage.outs[0])
if dry:
@@ -39,5 +56,13 @@ def sync_import(stage, dry=False, force=False, jobs=None):
if not force and stage.already_cached():
stage.outs[0].checkout()
else:
if check_changed:
old_hash_info = stage.deps[0].hash_info
stage.save_deps()
stage.deps[0].download(stage.outs[0], jobs=jobs)
if check_changed and not old_hash_info == stage.deps[0].hash_info:
from dvc.stage.exceptions import DataSourceChanged
raise DataSourceChanged(f"{stage} ({stage.deps[0]})")
if not no_download:
stage.deps[0].download(stage.outs[0], jobs=jobs)
+14
View File
@@ -273,6 +273,20 @@ def test_pull_external_dvc_imports(tmp_dir, dvc, scm, erepo_dir):
assert (tmp_dir / "new_dir" / "bar").read_text() == "bar"
def test_pull_partial_import(tmp_dir, dvc, local_workspace):
local_workspace.gen("file", "file content")
dst = tmp_dir / "file"
stage = dvc.imp_url(
"remote://workspace/file", os.fspath(dst), no_download=True
)
result = dvc.pull("file")
assert result["fetched"] == 1
assert dst.exists()
assert stage.outs[0].get_hash().value == "d10b4c3ff123b26dc068d43a8bef2d23"
def test_pull_external_dvc_imports_mixed(
tmp_dir, dvc, scm, erepo_dir, local_remote
):
+22
View File
@@ -267,3 +267,25 @@ def test_import_url_to_remote_status(tmp_dir, dvc, local_cloud, local_remote):
status = dvc.status()
assert len(status) == 0
def test_import_url_no_download(tmp_dir, dvc, local_workspace):
local_workspace.gen("file", "file content")
dst = tmp_dir / "file"
stage = dvc.imp_url(
"remote://workspace/file", os.fspath(dst), no_download=True
)
assert stage.deps[0].hash_info.value == "d10b4c3ff123b26dc068d43a8bef2d23"
assert not dst.exists()
out = stage.outs[0]
assert out.hash_info.value is None
assert out.hash_info.name is None
assert out.meta.size is None
status = dvc.status()
assert status["file.dvc"] == [
{"changed outs": {"file": "deleted"}},
]
+27
View File
@@ -176,6 +176,33 @@ def test_update_import_url(tmp_dir, dvc, workspace):
assert dst.read_text() == "updated file content"
def test_update_import_url_no_download(tmp_dir, dvc, workspace):
workspace.gen("file", "file content")
dst = tmp_dir / "imported_file"
stage = dvc.imp_url(
"remote://workspace/file", os.fspath(dst), no_download=True
)
hash_info = stage.deps[0].hash_info
assert not dst.is_file()
assert stage.deps[0].hash_info.value == "d10b4c3ff123b26dc068d43a8bef2d23"
workspace.gen("file", "updated file content")
updated_stage = dvc.update([stage.path], no_download=True)[0]
assert not dst.exists()
updated_hash_info = updated_stage.deps[0].hash_info
assert updated_hash_info != hash_info
assert updated_hash_info.value == "6ffba511ce3aa40b8231d1b1f8c5fba5"
out = updated_stage.outs[0]
assert out.hash_info.value is None
assert out.hash_info.name is None
assert out.meta.size is None
def test_update_rev(tmp_dir, dvc, scm, git_dir):
with git_dir.chdir():
git_dir.scm_gen({"foo": "foo"}, commit="first")
+38
View File
@@ -34,6 +34,7 @@ def test_import(mocker):
fname="file",
rev="version",
no_exec=False,
no_download=False,
desc="description",
jobs=3,
)
@@ -69,6 +70,43 @@ def test_import_no_exec(mocker):
fname="file",
rev="version",
no_exec=True,
no_download=False,
desc="description",
jobs=None,
)
def test_import_no_download(mocker):
cli_args = parse_args(
[
"import",
"repo_url",
"src",
"--out",
"out",
"--file",
"file",
"--rev",
"version",
"--no-download",
"--desc",
"description",
]
)
cmd = cli_args.func(cli_args)
m = mocker.patch.object(cmd.repo, "imp", autospec=True)
assert cmd.run() == 0
m.assert_called_once_with(
"repo_url",
path="src",
out="out",
fname="file",
rev="version",
no_exec=False,
no_download=True,
desc="description",
jobs=None,
)
+24 -6
View File
@@ -1,5 +1,7 @@
import logging
import pytest
from dvc.cli import parse_args
from dvc.commands.imp_url import CmdImportUrl
from dvc.exceptions import DvcException
@@ -31,6 +33,7 @@ def test_import_url(mocker):
out="out",
fname="file",
no_exec=False,
no_download=False,
remote=None,
to_remote=False,
desc="description",
@@ -54,11 +57,19 @@ def test_failed_import_url(mocker, caplog):
assert expected_error in caplog.text
def test_import_url_no_exec(mocker):
@pytest.mark.parametrize(
"flag,expected",
[
("--no-exec", {"no_exec": True, "no_download": False}),
("--no-download", {"no_download": True, "no_exec": False}),
],
)
def test_import_url_no_exec_download_flags(mocker, flag, expected):
cli_args = parse_args(
[
"import-url",
"--no-exec",
flag,
"src",
"out",
"--file",
@@ -77,11 +88,11 @@ def test_import_url_no_exec(mocker):
"src",
out="out",
fname="file",
no_exec=True,
remote=None,
to_remote=False,
desc="description",
jobs=None,
**expected
)
@@ -110,6 +121,7 @@ def test_import_url_to_remote(mocker):
out="bar",
fname=None,
no_exec=False,
no_download=False,
remote="remote",
to_remote=True,
desc="description",
@@ -117,7 +129,8 @@ def test_import_url_to_remote(mocker):
)
def test_import_url_to_remote_invalid_combination(dvc, mocker, caplog):
@pytest.mark.parametrize("flag", ["--no-exec", "--no-download"])
def test_import_url_to_remote_invalid_combination(dvc, mocker, caplog, flag):
cli_args = parse_args(
[
"import-url",
@@ -126,7 +139,7 @@ def test_import_url_to_remote_invalid_combination(dvc, mocker, caplog):
"--to-remote",
"--remote",
"remote",
"--no-exec",
flag,
]
)
assert cli_args.func == CmdImportUrl
@@ -134,9 +147,14 @@ def test_import_url_to_remote_invalid_combination(dvc, mocker, caplog):
cmd = cli_args.func(cli_args)
with caplog.at_level(logging.ERROR, logger="dvc"):
assert cmd.run() == 1
expected_msg = "--no-exec can't be combined with --to-remote"
expected_msg = (
"--no-exec/--no-download cannot be combined with --to-remote"
)
assert expected_msg in caplog.text
def test_import_url_to_remote_flag(dvc, mocker, caplog):
cli_args = parse_args(
["import-url", "s3://bucket/foo", "bar", "--remote", "remote"]
)
+2
View File
@@ -26,6 +26,7 @@ def test_update(dvc, mocker):
rev="REV",
recursive=True,
to_remote=False,
no_download=False,
remote=None,
jobs=8,
)
@@ -56,6 +57,7 @@ def test_update_to_remote(dvc, mocker):
rev=None,
recursive=True,
to_remote=True,
no_download=False,
remote="remote",
jobs=5,
)