build: pyinstaller: use __pyinstaller (#9680)

Fixes #9462
Fixes #9512

dvc/__pyinstaller/__init__.py

@@ -0,0 +1,5 @@
+import os
+
+
+def get_hook_dirs():
+    return [os.path.dirname(__file__)]

dvc/__pyinstaller/hook-asyncssh.py

@@ -1 +1,2 @@
+# ruff: noqa: N999
 hiddenimports = ["win32timezone"]

dvc/__pyinstaller/hook-celery.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 # pylint: disable=import-error
 from PyInstaller.utils.hooks import collect_submodules, is_module_or_submodule
 

dvc/__pyinstaller/hook-dvc.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 from PyInstaller.utils.hooks import copy_metadata  # pylint:disable=import-error
 
 # needed for `dvc doctor` to show dep versions

dvc/__pyinstaller/hook-dvc.system.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 import os
 
 if os.name == "nt":

dvc/__pyinstaller/hook-dvc.tree.gs.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 from PyInstaller.utils.hooks import copy_metadata  # pylint:disable=import-error
 
 datas = copy_metadata("google-cloud-storage")

dvc/__pyinstaller/hook-dvc.utils.flatten.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 from PyInstaller.utils.hooks import copy_metadata  # pylint:disable=import-error
 
 datas = copy_metadata("flatten-dict")

dvc/__pyinstaller/hook-dvc_task.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 # pylint: disable=import-error
 from PyInstaller.utils.hooks import collect_submodules
 

dvc/__pyinstaller/hook-fsspec.py

@@ -1 +1,2 @@
+# ruff: noqa: N999
 hiddenimports = ["fsspec.implementations.memory"]

dvc/__pyinstaller/hook-google_compute_engine.logger.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 from PyInstaller.utils.hooks import copy_metadata  # pylint:disable=import-error
 
 datas = copy_metadata("google-compute-engine")

dvc/__pyinstaller/hook-pydrive2.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 from PyInstaller.utils.hooks import copy_metadata  # pylint:disable=import-error
 
 datas = copy_metadata("pydrive2")

dvc/dagascii.py

@@ -209,6 +209,30 @@ def draw(vertices, edges):
     Args:
         vertices (list): list of graph vertices.
         edges (list): list of graph edges.
+
+    Returns:
+        str: ASCII representation
+
+    Example:
+        >>> from dvc.dagascii import draw
+        >>> vertices = [1, 2, 3, 4]
+        >>> edges = [(1, 2), (2, 3), (2, 4), (1, 4)]
+        >>> print(draw(vertices, edges))
+        +---+     +---+
+        | 3 |     | 4 |
+        +---+    *+---+
+          *    **   *
+          *  **     *
+          * *       *
+        +---+       *
+        | 2 |      *
+        +---+     *
+             *    *
+              *  *
+               **
+             +---+
+             | 1 |
+             +---+
     """
     # pylint: disable=too-many-locals
     # NOTE: coordinates might me negative, so we need to shift

dvc/repo/experiments/executor/base.py

@@ -459,25 +459,16 @@ class BaseExecutor(ABC):
             should force overwrite any existing duplicates.
         """
         from dvc.repo.checkout import checkout as dvc_checkout
-        from dvc.repo.reproduce import reproduce as dvc_reproduce
-        from dvc.stage import PipelineStage
         from dvc.ui import ui
 
         auto_push = env2bool(DVC_EXP_AUTO_PUSH)
         git_remote = os.getenv(DVC_EXP_GIT_REMOTE, None)
 
-        unchanged = []
-
         if queue is not None:
             queue.put((rev, os.getpid()))
         if log_errors and log_level is not None:
             cls._set_log_level(log_level)
 
-        def filter_pipeline(stages):
-            unchanged.extend(
-                [stage for stage in stages if isinstance(stage, PipelineStage)]
-            )
-
         exp_hash: Optional[str] = None
         exp_ref: Optional["ExpRefInfo"] = None
         repro_force: bool = False
@@ -519,12 +510,7 @@ class BaseExecutor(ABC):
                     recursive=kwargs.get("recursive", False),
                 )
 
-            stages = dvc_reproduce(
-                dvc,
-                *args,
-                on_unchanged=filter_pipeline,
-                **kwargs,
-            )
+            stages = dvc.reproduce(*args, **kwargs)
             if paths := cls._get_top_level_paths(dvc):
                 logger.debug("Staging top-level files: %s", paths)
                 dvc.scm_context.add(paths)

dvc/repo/reproduce.py

@@ -122,7 +122,6 @@ def _reproduce_stages(  # noqa: C901
     stages,
     downstream=False,
     single_item=False,
-    on_unchanged=None,
     **kwargs,
 ):
     r"""Derive the evaluation of the given node for the given graph.
@@ -175,24 +174,20 @@ def _reproduce_stages(  # noqa: C901
         try:
             kwargs["upstream"] = result + unchanged
             ret = _reproduce_stage(stage, **kwargs)
-
-            if len(ret) == 0:
+            if not ret:
                 unchanged.extend([stage])
-            elif force_downstream:
+                continue
+
+            result.extend(ret)
+            if force_downstream:
                 # NOTE: we are walking our pipeline from the top to the
                 # bottom. If one stage is changed, it will be reproduced,
                 # which tells us that we should force reproducing all of
                 # the other stages down below, even if their direct
                 # dependencies didn't change.
                 kwargs["force"] = True
-
-            if ret:
-                result.extend(ret)
         except Exception as exc:  # noqa: BLE001
             raise ReproductionError(stage.addressing) from exc
-
-    if on_unchanged is not None:
-        on_unchanged(unchanged)
     return result
 
 
@@ -233,7 +228,3 @@ def _get_steps(graph, stages, downstream, single_item):
             steps.append(stage)
 
     return steps
-
-
-def _repro_callback(experiments_callback, unchanged, stages):
-    experiments_callback(unchanged, stages)

dvc/stage/cache.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List, Optional, Tuple
 from funcy import first
 
 from dvc import fs
-from dvc.exceptions import DvcException
+from dvc.exceptions import CollectCacheError, DvcException
 from dvc.utils import dict_sha256, relpath
 
 if TYPE_CHECKING:
@@ -179,7 +179,7 @@ class StageCache:
         dump_yaml(tmp, cache)
         self.repo.cache.legacy.move(tmp, path)
 
-    def restore(self, stage, run_cache=True, pull=False, dry=False):
+    def restore(self, stage, run_cache=True, pull=False, dry=False):  # noqa: C901
         from .serialize import to_single_stage_lockfile
 
         if not _can_hash(stage):
@@ -203,8 +203,11 @@ class StageCache:
         cached_stage = self._create_stage(cache, wdir=stage.wdir)
 
         if pull and not dry:
-            for objs in cached_stage.get_used_objs().values():
-                self.repo.cloud.pull(objs)
+            try:
+                for objs in cached_stage.get_used_objs().values():
+                    self.repo.cloud.pull(objs)
+            except CollectCacheError as exc:
+                raise RunCacheNotFoundError(stage) from exc
 
         if not cached_stage.outs_cached():
             raise RunCacheNotFoundError(stage)

pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=2.3.0,<2.4.0",
+    "dvc-data>=2.3.2,<2.4.0",
     "dvc-http>=2.29.0",
     "dvc-render>=0.3.1,<1",
     "dvc-studio-client>=0.9.2,<1",
@@ -82,6 +82,7 @@ lint = [
     "types-tabulate",
     "types-toml",
     "types-tqdm",
+    "types-pyinstaller",
 ]
 oss = ["dvc-oss==2.19"]
 s3 = ["dvc-s3==2.23.0"]
@@ -128,6 +129,10 @@ dvc = "dvc.api:DVCFileSystem"
 [project.entry-points."pytest11"]
 dvc-testing = "dvc.testing.plugin"
 
+[project.entry-points."pyinstaller40"]
+hook-dirs = "dvc.__pyinstaller:get_hook_dirs"
+tests = "dvc.__pyinstaller:get_PyInstaller_tests"
+
 [tool.setuptools]
 license-files = ["LICENSE"]
 

scripts/build-requirements.txt

@@ -1,3 +0,0 @@
-# NOTE: using strict version due to
-# https://github.com/iterative/dvc/issues/7949
-PyInstaller==5.9.0

scripts/pyinstaller/.gitignore

@@ -1,4 +0,0 @@
-/build
-/dist
-/dvc.spec
-/dvc

scripts/pyinstaller/build.py

@@ -1,47 +0,0 @@
-import os
-import pathlib
-from subprocess import STDOUT, check_call, check_output
-
-path = pathlib.Path(__file__).parent.absolute()
-hooks = path / "hooks"
-dvc = path.parent.parent / "dvc"
-entry = dvc / "__main__.py"
-
-check_call(
-    [
-        "pyinstaller",
-        "--additional-hooks-dir",
-        os.fspath(hooks),
-        "--name",
-        "dvc",
-        "-y",
-        os.fspath(entry),
-    ],
-    cwd=path,
-    stderr=STDOUT,
-)
-
-out = check_output(
-    [
-        path / "dist" / "dvc" / "dvc",
-        "doctor",
-    ],
-    stderr=STDOUT,
-).decode()
-
-remotes = [
-    "s3",
-    "oss",
-    "gdrive",
-    "gs",
-    "hdfs",
-    "http",
-    "webhdfs",
-    "azure",
-    "ssh",
-    "webdav",
-]
-
-print(out)
-for remote in remotes:
-    assert f"\t{remote}" in out, f"Missing support for {remote}"