push: fix cloud versioning

dvc/repo/fetch.py

@@ -1,4 +1,4 @@
-from typing import List, Tuple
+from typing import TYPE_CHECKING, List, Tuple
 
 from dvc.exceptions import DownloadError
 from dvc.log import logger
@@ -7,6 +7,10 @@ from dvc_data.index import DataIndex, FileStorage
 
 from . import locked
 
+if TYPE_CHECKING:
+    from dvc.output import Output
+    from dvc.stage import Stage
+
 logger = logger.getChild(__name__)
 
 
@@ -32,6 +36,7 @@ def _collect_indexes(  # noqa: PLR0913
     types=None,
     config=None,
     onerror=None,
+    push=False,
 ):
     indexes = {}
     collection_exc = None
@@ -42,6 +47,16 @@ def _collect_indexes(  # noqa: PLR0913
         core["remote"] = remote
         config["core"] = core
 
+    def stage_filter(stage: "Stage") -> bool:
+        if push and stage.is_repo_import:
+            return False
+        return True
+
+    def outs_filter(out: "Output") -> bool:
+        if push and not out.can_push:
+            return False
+        return True
+
     for rev in repo.brancher(
         revs=revs,
         all_branches=all_branches,
@@ -57,6 +72,8 @@ def _collect_indexes(  # noqa: PLR0913
                 recursive=recursive,
                 max_size=max_size,
                 types=types,
+                stage_filter=stage_filter,
+                outs_filter=outs_filter,
             )
 
             idx.data["repo"].onerror = _make_index_onerror(onerror, rev)

dvc/repo/push.py

@@ -87,6 +87,7 @@ def push(  # noqa: PLR0913
         recursive=recursive,
         all_commits=all_commits,
         revs=revs,
+        push=True,
     )
 
     cache_key = (
@@ -107,6 +108,7 @@ def push(  # noqa: PLR0913
             push=True,
         )
 
+    push_transferred, push_failed = 0, 0
     try:
         with ui.progress(
             desc="Pushing",
@@ -131,6 +133,11 @@ def push(  # noqa: PLR0913
         for fs_index in data:
             fs_index.close()
 
+        if push_transferred:
+            # NOTE: dropping cached index to force reloading from newly saved
+            # metadata from version-aware remotes
+            self.drop_data_index()
+
     transferred_count += push_transferred
     failed_count += push_failed
     if failed_count:

dvc/testing/remote_tests.py

@@ -172,6 +172,8 @@ class TestRemoteVersionAware:
         run_copy("foo", "foo_copy", name="copy")
 
         dvc.push()
+        assert (remote_version_aware / "foo").read_text() == "foo"
+        assert (remote_version_aware / "foo_copy").read_text() == "foo"
         foo_dvc = (tmp_dir / "foo.dvc").read_text()
         assert "version_id" in foo_dvc
         stage = stage.reload()
@@ -181,18 +183,24 @@ class TestRemoteVersionAware:
 
         remove(dvc.cache.local.path)
         remove(tmp_dir / "foo")
+        remove(tmp_dir / "foo_copy")
 
         dvc.pull()
         assert (tmp_dir / "foo").read_text() == "foo"
+        assert (tmp_dir / "foo_copy").read_text() == "foo"
         assert (tmp_dir / "foo.dvc").read_text() == foo_dvc
         assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
 
         dvc.push()
+        assert (remote_version_aware / "foo").read_text() == "foo"
+        assert (remote_version_aware / "foo_copy").read_text() == "foo"
         assert (tmp_dir / "foo.dvc").read_text() == foo_dvc
         assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
 
         dvc.reproduce()
         dvc.push()
+        assert (remote_version_aware / "foo").read_text() == "foo"
+        assert (remote_version_aware / "foo_copy").read_text() == "foo"
         assert (tmp_dir / "foo.dvc").read_text() == foo_dvc
         assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
 
@@ -241,6 +249,12 @@ class TestRemoteVersionAware:
         assert (tmp_dir / "data_dir.dvc").read_text() == data_dir_dvc
         assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
 
+        dvc.cache.local.clear()
+        remove(tmp_dir / "data_dir")
+        dvc.push()
+        assert (remote_version_aware / "data_dir").exists()
+        assert (remote_version_aware / "data_dir" / "data").exists()
+
 
 class TestRemoteWorktree:
     def test_file(self, tmp_dir, dvc, remote_worktree):

pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=2.20.0,<2.21.0",
+    "dvc-data>=2.21.2,<2.22.0",
     "dvc-http>=2.29.0",
     "dvc-render>=0.3.1,<1",
     "dvc-studio-client>=0.13.0,<1",

tests/func/api/test_data.py

@@ -41,7 +41,7 @@ def test_open_external(tmp_dir, erepo_dir, cloud):
             # NOTE: need file to be other size for Mac
             erepo_dir.dvc_gen("version", "branchver", commit="add version")
 
-    erepo_dir.dvc.push(all_branches=True)
+    assert erepo_dir.dvc.push(all_branches=True) == 2
 
     # Remove cache to force download
     remove(erepo_dir.dvc.cache.local.path)

tests/func/test_data_cloud.py

@@ -486,7 +486,7 @@ def test_pull_partial(tmp_dir, dvc, local_remote):
     clean(["foo"], dvc)
 
     stats = dvc.pull(os.path.join("foo", "bar"))
-    assert stats["fetched"] == 2
+    assert stats["fetched"] == 3
     assert (tmp_dir / "foo").read_text() == {"bar": {"baz": "baz"}}
 
 

tests/func/test_virtual_directory.py

@@ -182,7 +182,7 @@ def test_partial_checkout_and_update(M, tmp_dir, dvc, remote):
 
     assert dvc.pull("dir/subdir") == M.dict(
         added=[join("dir", "")],
-        fetched=2,
+        fetched=3,
     )
     assert (tmp_dir / "dir").read_text() == {"subdir": {"lorem": "lorem"}}