From 1f5f2f5c97b999eecf283e76eca499998e497a2e Mon Sep 17 00:00:00 2001 From: Samuel Knight Date: Fri, 22 Apr 2022 11:22:25 +0100 Subject: [PATCH] dvcfile: preserve 'remote' on add and commit Preserves 'remote' output on dvc add and dvc commit. Fixes #7356 --- dvc/output.py | 3 +++ dvc/repo/imp_url.py | 4 ++-- dvc/repo/stage.py | 9 +++++++-- dvc/stage/__init__.py | 8 +++++--- tests/func/test_add.py | 5 +++-- tests/func/test_commit.py | 31 +++++++++++++++++++++++++++++++ 6 files changed, 51 insertions(+), 9 deletions(-) diff --git a/dvc/output.py b/dvc/output.py index c064f10b9..b2ac7d7ec 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -670,6 +670,9 @@ class Output: if self.live: ret[self.PARAM_LIVE] = self.live + if self.remote: + ret[self.PARAM_REMOTE] = self.remote + return ret def verify_metric(self): diff --git a/dvc/repo/imp_url.py b/dvc/repo/imp_url.py index aa8ec83ba..35a684f66 100644 --- a/dvc/repo/imp_url.py +++ b/dvc/repo/imp_url.py @@ -24,7 +24,7 @@ def imp_url( jobs=None, ): from dvc.dvcfile import Dvcfile - from dvc.stage import Stage, create_stage, restore_meta + from dvc.stage import Stage, create_stage, restore_fields out = resolve_output(url, out) path, wdir, out = resolve_paths( @@ -58,7 +58,7 @@ def imp_url( outs=[out], erepo=erepo, ) - restore_meta(stage) + restore_fields(stage) if desc: stage.outs[0].desc = desc diff --git a/dvc/repo/stage.py b/dvc/repo/stage.py index 702cad896..ad19af396 100644 --- a/dvc/repo/stage.py +++ b/dvc/repo/stage.py @@ -168,7 +168,12 @@ class StageLoad: stage_data: Stage data to create from (see create_stage and loads_from for more information) """ - from dvc.stage import PipelineStage, Stage, create_stage, restore_meta + from dvc.stage import ( + PipelineStage, + Stage, + create_stage, + restore_fields, + ) from dvc.stage.exceptions import InvalidStageName from dvc.stage.utils import ( is_valid_name, @@ -201,7 +206,7 @@ class StageLoad: new_index = self.repo.index.add(stage) new_index.check_graph() - restore_meta(stage) + restore_fields(stage) return stage def from_target( diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 34eab5206..b027934c8 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -95,7 +95,7 @@ def create_stage(cls, repo, path, external=False, **kwargs): return stage -def restore_meta(stage): +def restore_fields(stage): from .exceptions import StageNotFound if not stage.dvcfile.exists(): @@ -113,10 +113,12 @@ def restore_meta(stage): stage.meta = old.meta stage.desc = old.desc - old_desc = {out.def_path: out.desc for out in old.outs} + old_fields = {out.def_path: (out.desc, out.remote) for out in old.outs} for out in stage.outs: - out.desc = old_desc.get(out.def_path, None) + out_fields = old_fields.get(out.def_path, None) + if out_fields: + out.desc, out.remote = out_fields class Stage(params.StageParams): diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 610710b48..289274b62 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -894,7 +894,7 @@ def test_add_with_cache_link_error(tmp_dir, dvc, mocker, capsys): } -def test_add_preserve_meta(tmp_dir, dvc): +def test_add_preserve_fields(tmp_dir, dvc): text = textwrap.dedent( """\ # top comment @@ -902,11 +902,11 @@ def test_add_preserve_meta(tmp_dir, dvc): outs: - path: foo # out comment desc: out desc + remote: testremote meta: some metadata """ ) tmp_dir.gen("foo.dvc", text) - tmp_dir.dvc_gen("foo", "foo") assert (tmp_dir / "foo.dvc").read_text() == textwrap.dedent( """\ @@ -915,6 +915,7 @@ def test_add_preserve_meta(tmp_dir, dvc): outs: - path: foo # out comment desc: out desc + remote: testremote md5: acbd18db4cc2f85cedef654fccc4a4d8 size: 3 meta: some metadata diff --git a/tests/func/test_commit.py b/tests/func/test_commit.py index afa7bec22..6284c0f43 100644 --- a/tests/func/test_commit.py +++ b/tests/func/test_commit.py @@ -1,4 +1,5 @@ import os +import textwrap import pytest @@ -39,6 +40,36 @@ def test_commit_force(tmp_dir, dvc): assert dvc.status([stage.path]) == {} +def test_commit_preserve_fields(tmp_dir, dvc): + text = textwrap.dedent( + """\ + # top comment + desc: top desc + outs: + - path: foo # out comment + desc: out desc + remote: testremote + meta: some metadata + """ + ) + tmp_dir.gen("foo.dvc", text) + tmp_dir.dvc_gen("foo", "foo", commit=False) + dvc.commit("foo") + assert (tmp_dir / "foo.dvc").read_text() == textwrap.dedent( + """\ + # top comment + desc: top desc + outs: + - path: foo # out comment + desc: out desc + remote: testremote + md5: acbd18db4cc2f85cedef654fccc4a4d8 + size: 3 + meta: some metadata + """ + ) + + @pytest.mark.parametrize("run_kw", [{"single_stage": True}, {"name": "copy"}]) def test_commit_with_deps(tmp_dir, dvc, run_copy, run_kw): tmp_dir.gen("foo", "foo")