dvc dependency repo: Catch FileNotFoundError (#10501)

- When checkouting non-existent file the _checkout_file recursively
  calls os.stat on a symlinked file which destination doesn't exist.
  This causes a FileNotFoundError but it is not catched and it causes
  an unexpected error instead.

Fixes #10500.

.pre-commit-config.yaml

@@ -22,7 +22,7 @@ repos:
       - id: sort-simple-yaml
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.5.4'
+    rev: 'v0.5.6'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

README.rst

@@ -1,4 +1,6 @@
-|Banner|
+🚀 Check out our new product `DataChain <https://github.com/iterative/datachain>`_ (and give it a ⭐!) if you need to version and process a large number of files. Contact us at [email protected] to discuss commercial solutions and support for AI reproducibility and data management scenarios.
+
+--------------------------
 
 `Website <https://dvc.org>`_
 • `Docs <https://dvc.org/doc>`_

dvc/commands/data_sync.py

@@ -195,7 +195,7 @@ def add_parser(subparsers, _parent_parser):
     pull_parser.add_argument(
         "--run-cache",
         action=argparse.BooleanOptionalAction,
-        default=True,
+        default=False,
         help="Fetch run history for all stages.",
     )
     pull_parser.add_argument(
@@ -263,7 +263,7 @@ def add_parser(subparsers, _parent_parser):
     push_parser.add_argument(
         "--run-cache",
         action=argparse.BooleanOptionalAction,
-        default=True,
+        default=False,
         help="Push run history for all stages.",
     )
     push_parser.add_argument(
@@ -325,7 +325,7 @@ def add_parser(subparsers, _parent_parser):
     fetch_parser.add_argument(
         "--run-cache",
         action=argparse.BooleanOptionalAction,
-        default=True,
+        default=False,
         help="Fetch run history for all stages.",
     )
     fetch_parser.add_argument(

dvc/commands/experiments/pull.py

@@ -84,7 +84,7 @@ def add_parser(experiments_subparsers, parent_parser):
     experiments_pull_parser.add_argument(
         "--run-cache",
         action=argparse.BooleanOptionalAction,
-        default=True,
+        default=False,
         help="Pull run history for all stages.",
     )
     experiments_pull_parser.add_argument(

dvc/commands/experiments/push.py

@@ -122,7 +122,7 @@ def add_parser(experiments_subparsers, parent_parser):
     experiments_push_parser.add_argument(
         "--run-cache",
         action=argparse.BooleanOptionalAction,
-        default=True,
+        default=False,
         help="Push run history for all stages.",
     )
     experiments_push_parser.add_argument(

dvc/dependency/db.py

@@ -44,6 +44,10 @@ class AbstractDependency(Dependency):
         self.def_path = None  # type: ignore[assignment]
         self.info = info or {}
 
+    @property
+    def is_in_repo(self):
+        return False
+
 
 class DbDependency(AbstractDependency):
     PARAM_CONNECTION = "connection"
@@ -70,8 +74,8 @@ class DbDependency(AbstractDependency):
         return self.db_info.get(self.PARAM_QUERY) or self.db_info.get(self.PARAM_TABLE)
 
     def __repr__(self):
-        return "{}:{}".format(
-            self.__class__.__name__, "".join(f"{k}=={v}" for k, v in self.info.items())
+        return "{}: {}".format(
+            self.__class__.__name__, "".join(f"{k}={v}" for k, v in self.info.items())
         )
 
     def __str__(self):

dvc/dependency/repo.py

@@ -3,12 +3,14 @@ from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
 
 import voluptuous as vol
 
+from dvc.prompt import confirm
 from dvc.utils import as_posix
 
 from .base import Dependency
 
 if TYPE_CHECKING:
     from dvc.fs import DVCFileSystem
+    from dvc.output import Output
     from dvc.stage import Stage
 
 
@@ -91,6 +93,31 @@ class RepoDependency(Dependency):
             self.PARAM_REPO: self._dump_def_repo(self.def_repo),
         }
 
+    def download(self, to: "Output", jobs: Optional[int] = None):
+        from dvc_data.hashfile.build import build
+        from dvc_data.hashfile.checkout import CheckoutError, checkout
+
+        try:
+            repo = self._make_fs(locked=True).repo
+
+            _, _, obj = build(
+                repo.cache.local,
+                self.fs_path,
+                repo.dvcfs,
+                repo.cache.local.fs.PARAM_CHECKSUM,
+            )
+            checkout(
+                to.fs_path,
+                to.fs,
+                obj,
+                self.repo.cache.local,
+                ignore=None,
+                state=self.repo.state,
+                prompt=confirm,
+            )
+        except (CheckoutError, FileNotFoundError):
+            super().download(to=to, jobs=jobs)
+
     def update(self, rev: Optional[str] = None):
         if rev:
             self.def_repo[self.PARAM_REV] = rev

dvc/repo/experiments/serialize.py

@@ -50,7 +50,12 @@ class SerializableExp:
         Params, metrics, deps, outs are filled via repo fs/index, all other fields
         should be passed via kwargs.
         """
-        from dvc.dependency import ParamsDependency, RepoDependency
+        from dvc.dependency import (
+            DatasetDependency,
+            DbDependency,
+            ParamsDependency,
+            RepoDependency,
+        )
 
         rev = rev or repo.get_rev()
         assert rev
@@ -68,7 +73,10 @@ class SerializableExp:
                     nfiles=dep.meta.nfiles if dep.meta else None,
                 )
                 for dep in repo.index.deps
-                if not isinstance(dep, (ParamsDependency, RepoDependency))
+                if not isinstance(
+                    dep,
+                    (ParamsDependency, RepoDependency, DatasetDependency, DbDependency),
+                )
             },
             outs={
                 relpath(out.fs_path, repo.root_dir): ExpOut(

dvc/repo/imp_db.py

@@ -4,7 +4,6 @@ from funcy import compact
 
 from dvc.exceptions import OutputDuplicationError
 from dvc.repo.scm_context import scm_context
-from dvc.ui import ui
 from dvc.utils import resolve_output, resolve_paths
 
 if TYPE_CHECKING:
@@ -25,11 +24,6 @@ def imp_db(
     force: bool = False,
     connection: Optional[str] = None,
 ):
-    ui.warn("WARNING: import-db is an experimental feature.")
-    ui.warn(
-        "The functionality may change or break without notice, "
-        "which could lead to unexpected behavior."
-    )
     assert sql or table
     assert output_format in ("csv", "json")
 

dvc/stage/run.py

@@ -154,7 +154,11 @@ def cmd_run(stage, dry=False, run_env=None):
 
 
 def _pull_missing_deps(stage):
+    from dvc.dependency import DatasetDependency, DbDependency
+
     for dep in stage.deps:
+        if isinstance(dep, (DatasetDependency, DbDependency)):
+            continue
         if not dep.exists:
             stage.repo.pull(dep.def_path)
 

tests/func/test_import.py

@@ -6,6 +6,7 @@ from funcy import first
 
 from dvc.cachemgr import CacheManager
 from dvc.config import NoRemoteError
+from dvc.dependency import base
 from dvc.dvcfile import load_file
 from dvc.fs import system
 from dvc.scm import Git
@@ -722,3 +723,16 @@ def test_import_invalid_configs(tmp_dir, scm, dvc, erepo_dir):
             remote="myremote",
             remote_config={"key": "value"},
         )
+
+
+def test_reimport(tmp_dir, scm, dvc, erepo_dir, mocker):
+    with erepo_dir.chdir():
+        erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
+
+    spy = mocker.spy(base, "fs_download")
+    dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported")
+    assert spy.called
+
+    spy.reset_mock()
+    dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", force=True)
+    assert not spy.called