dvcfs: handle NotADirectoryError from datafs.ls (#9746)

* dvc_fs: ignore NotADirectoryError in ls

* deps: bump dvc-data to >= 2.6.0

* add test

---------

Co-authored-by: Ruslan Kuprieiev <[email protected]>

.github/workflows/build.yaml

@@ -1,4 +1,4 @@
-name: Build packages
+name: Build package
 on:
   push:
     branches:
@@ -7,39 +7,35 @@ on:
     types: [released, prereleased]
   workflow_dispatch:
 
-permissions:  # added using https://github.com/step-security/secure-workflows
+permissions:
   contents: read
+  id-token: write
 
 jobs:
   pip:
-    runs-on: ubuntu-20.04
-    permissions:
-      id-token: write
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
 
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4
+    - uses: actions/setup-python@v4
       with:
         python-version: 3.8
 
-    - name: Install
-      run: pip install --upgrade pip wheel
-
-    - name: Force version for Test PyPI uploads
+    - run: python -m pip install --upgrade pip setuptools_scm build twine
+    - name: Force version for Test PyPI upload
       if: ${{ !startsWith(github.ref, 'refs/tags') }}
-      run: |
-        pip install setuptools_scm
-        echo version=$(python -m setuptools_scm | awk -F+ '{print $1}' | tail -1) >> $GITHUB_ENV
+      run: echo version=$(python -m setuptools_scm | awk -F+ '{print $1}' | tail -1) >> $GITHUB_ENV
 
-    - name: Build packages
-      run: ./scripts/build_package.sh
+    - run: |
+        echo 'PKG = "pip"'>dvc/_build.py
+        python -m build
       env:
         SETUPTOOLS_SCM_PRETEND_VERSION: ${{ env.version }}
+    - run: twine check --strict dist/*
 
-    - name: Publish packages to PyPI
+    - name: Publish to PyPI
       if: github.event_name == 'release'
       uses: pypa/gh-action-pypi-publish@release/v1
 
@@ -47,5 +43,5 @@ jobs:
       if: ${{ github.event_name == 'release' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
-        repository_url: https://test.pypi.org/legacy/
-        skip_existing: true
+        repository-url: https://test.pypi.org/legacy/
+        skip-existing: true

.pre-commit-config.yaml

@@ -29,25 +29,25 @@ repos:
       - id: beautysh
         args: ["-i", "2"]
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
+    rev: v3.9.0
     hooks:
       - id: pyupgrade
         args:
           - --py38-plus
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 23.7.0
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/isort
     rev: 5.12.0
     hooks:
       - id: isort
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.261'
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: 'v0.0.278'
     hooks:
       - id: ruff
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.4
+    rev: v2.2.5
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]

README.rst

@@ -280,8 +280,8 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://pypi.org/project/dvc
    :alt: PyPI Downloads
 
-.. |Packages| image:: https://img.shields.io/github/v/release/iterative/dvc?label=deb|pkg|rpm|exe&logo=GitHub
-   :target: https://github.com/iterative/dvc/releases/latest
+.. |Packages| image:: https://img.shields.io/badge/deb|pkg|rpm|exe-blue
+   :target: https://dvc.org/doc/install
    :alt: deb|pkg|rpm|exe
 
 .. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.3677553-blue.svg

dvc/__pyinstaller/hook-dvc.system.py

@@ -1,5 +0,0 @@
-# ruff: noqa: N999
-import os
-
-if os.name == "nt":
-    hiddenimports = ["win32file"]

dvc/__pyinstaller/hook-dvc.tree.gs.py

@@ -1,10 +0,0 @@
-# ruff: noqa: N999
-from PyInstaller.utils.hooks import copy_metadata  # pylint:disable=import-error
-
-datas = copy_metadata("google-cloud-storage")
-
-# NOTE: including "requests" metadata because of google-resumable-media
-# bug. See [1] for more info.
-#
-# [1] https://github.com/googleapis/google-resumable-media-python/issues/59
-datas += copy_metadata("requests")

dvc/__pyinstaller/hook-google_compute_engine.logger.py

@@ -1,5 +0,0 @@
-# ruff: noqa: N999
-from PyInstaller.utils.hooks import copy_metadata  # pylint:disable=import-error
-
-datas = copy_metadata("google-compute-engine")
-hiddenimports = ["google_cloud_engine"]

dvc/_debug.py

@@ -198,33 +198,84 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs):
 def add_debugging_flags(parser):
     from argparse import SUPPRESS
 
-    parser.add_argument("--cprofile", action="store_true", default=False, help=SUPPRESS)
-    parser.add_argument("--yappi", action="store_true", default=False, help=SUPPRESS)
+    # For detailed info see:
+    # https://github.com/iterative/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC
+    args, _ = parser.parse_known_args()
+    verbose = args.verbose
+
+    def debug_help(msg):
+        if verbose:
+            return msg
+        return SUPPRESS
+
+    parser = parser.add_argument_group("debug options")
+
+    parser.add_argument(
+        "--cprofile",
+        action="store_true",
+        default=False,
+        help=debug_help("Generate cprofile data for tools like snakeviz / tuna"),
+    )
+    parser.add_argument(
+        "--cprofile-dump", help=debug_help("Location to dump cprofile file")
+    )
+    parser.add_argument(
+        "--yappi",
+        action="store_true",
+        default=False,
+        help=debug_help(
+            "Generate a callgrind file for use with tools like "
+            "kcachegrind / qcachegrind"
+        ),
+    )
     parser.add_argument(
         "--yappi-separate-threads",
         action="store_true",
         default=False,
-        help=SUPPRESS,
+        help=debug_help("Generate one callgrind file per thread"),
     )
     parser.add_argument(
-        "--viztracer", action="store_true", default=False, help=SUPPRESS
-    )
-    parser.add_argument("--viztracer-depth", type=int, help=SUPPRESS)
-    parser.add_argument(
-        "--viztracer-async", action="store_true", default=False, help=SUPPRESS
-    )
-    parser.add_argument("--cprofile-dump", help=SUPPRESS)
-    parser.add_argument("--pdb", action="store_true", default=False, help=SUPPRESS)
-    parser.add_argument(
-        "--instrument", action="store_true", default=False, help=SUPPRESS
+        "--viztracer",
+        action="store_true",
+        default=False,
+        help=debug_help("Generate a viztracer file for use with vizviewer"),
     )
     parser.add_argument(
-        "--instrument-open", action="store_true", default=False, help=SUPPRESS
+        "--viztracer-depth",
+        type=int,
+        help=debug_help("Set viztracer maximum stack depth"),
+    )
+    parser.add_argument(
+        "--viztracer-async",
+        action="store_true",
+        default=False,
+        help=debug_help("Treat async tasks as threads"),
+    )
+    parser.add_argument(
+        "--pdb",
+        action="store_true",
+        default=False,
+        help=debug_help("Drop into the pdb/ipdb debugger on any exception"),
+    )
+    parser.add_argument(
+        "--instrument",
+        action="store_true",
+        default=False,
+        help=debug_help("Use pyinstrument CLI profiler"),
+    )
+    parser.add_argument(
+        "--instrument-open",
+        action="store_true",
+        default=False,
+        help=debug_help("Use pyinstrument web profiler"),
     )
     parser.add_argument(
         "--show-stack",
         "--ss",
         action="store_true",
         default=False,
-        help=SUPPRESS,
+        help=debug_help(
+            r"Use Ctrl+T on macOS or Ctrl+\ on Linux to print the stack "
+            "frame currently executing. Unavailable on Windows."
+        ),
     )

dvc/api/data.py

@@ -36,21 +36,22 @@ def get_url(path, repo=None, rev=None, remote=None):
     NOTE: This function does not check for the actual existence of the file or
     directory in the remote storage.
     """
-    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
-        with _wrap_exceptions(_repo, path):
-            fs_path = _repo.dvcfs.from_os_path(path)
+    from dvc.config import NoRemoteError
+    from dvc_data.index import StorageKeyError
 
-            with reraise(FileNotFoundError, PathMissingError(path, repo)):
-                info = _repo.dvcfs.info(fs_path)
-
-            dvc_info = info.get("dvc_info")
-            if not dvc_info:
-                raise OutputNotFoundError(path, repo)
-
-            dvc_repo = info["repo"]  # pylint: disable=unsubscriptable-object
-            md5 = dvc_info["md5"]
-
-            return dvc_repo.cloud.get_url_for(remote, checksum=md5)
+    repo_kwargs: Dict[str, Any] = {}
+    if remote:
+        repo_kwargs["config"] = {"core": {"remote": remote}}
+    with Repo.open(
+        repo, rev=rev, subrepos=True, uninitialized=True, **repo_kwargs
+    ) as _repo:
+        index, entry = _repo.get_data_index_entry(path)
+        with reraise(
+            (StorageKeyError, ValueError),
+            NoRemoteError(f"no remote specified in {_repo}"),
+        ):
+            remote_fs, remote_path = index.storage_map.get_remote(entry)
+            return remote_fs.unstrip_protocol(remote_path)
 
 
 class _OpenContextManager(GCM):

dvc/cli/__init__.py

@@ -192,7 +192,7 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
 
             from dvc import PKG, __version__
 
-            pyv = " ".join([python_implementation(), python_version()])
+            pyv = f"{python_implementation()} {python_version()}"
             pkg = f" ({PKG})" if PKG else ""
             logger.debug("v%s%s, %s on %s", __version__, pkg, pyv, platform())
             logger.debug("command: %s", " ".join(argv or sys.argv))

dvc/cli/parser.py

@@ -137,7 +137,6 @@ def get_parent_parser():
     from dvc._debug import add_debugging_flags
 
     parent_parser = argparse.ArgumentParser(add_help=False)
-    add_debugging_flags(parent_parser)
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
         "-q", "--quiet", action="count", default=0, help="Be quiet."
@@ -145,6 +144,7 @@ def get_parent_parser():
     log_level_group.add_argument(
         "-v", "--verbose", action="count", default=0, help="Be verbose."
     )
+    add_debugging_flags(parent_parser)
 
     return parent_parser
 

dvc/cli/utils.py

@@ -16,9 +16,7 @@ def append_doc_link(help_message, path):
     if not path:
         return help_message
     doc_base = "https://man.dvc.org/"
-    return "{message}\nDocumentation: {link}".format(
-        message=help_message, link=format_link(doc_base + path)
-    )
+    return f"{help_message}\nDocumentation: {format_link(doc_base + path)}"
 
 
 def hide_subparsers_from_help(subparsers):

dvc/commands/data.py

@@ -90,9 +90,7 @@ class CmdDataStatus(CmdBase):
                     ui.write(f"  ({hint})")
 
             if isinstance(stage_status, dict):
-                items = [
-                    ": ".join([state, file]) for file, state in stage_status.items()
-                ]
+                items = [f"{state}: {file}" for file, state in stage_status.items()]
             else:
                 items = stage_status
 

dvc/commands/data_sync.py

@@ -91,6 +91,8 @@ class CmdDataFetch(CmdDataBase):
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
+                max_size=self.args.max_size,
+                types=self.args.types,
             )
             self.log_summary({"fetched": processed_files_count})
         except DvcException:
@@ -322,6 +324,22 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch run history for all stages.",
     )
+    fetch_parser.add_argument(
+        "--max-size",
+        type=int,
+        help="Fetch data files/directories that are each below specified size (bytes).",
+    )
+    fetch_parser.add_argument(
+        "--type",
+        dest="types",
+        action="append",
+        default=[],
+        help=(
+            "Only fetch data files/directories that are of a particular "
+            "type (metrics, plots)."
+        ),
+        choices=["metrics", "plots"],
+    )
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status

dvc/commands/destroy.py

@@ -21,7 +21,7 @@ class CmdDestroy(CmdBase):
             )
 
             if not self.args.force and not ui.confirm(statement):
-                raise DvcException(
+                raise DvcException(  # noqa: TRY301
                     "cannot destroy without a confirmation from the user."
                     " Use `-f` to force."
                 )

dvc/commands/remote.py

@@ -127,9 +127,7 @@ class CmdRemoteRename(CmdRemote):
         all_config = self.config.load_config_to_level(None)
         if self.args.new in all_config.get("remote", {}):
             raise ConfigError(
-                "Rename failed. Remote name '{}' already exists.".format(
-                    {self.args.new}
-                )
+                f"Rename failed. Remote name {self.args.new!r} already exists."
             )
 
         with self.config.edit(self.args.level) as conf:

dvc/commands/repro.py

@@ -33,6 +33,7 @@ class CmdRepro(CmdBase):
             "force_downstream": self.args.force_downstream,
             "pull": self.args.pull,
             "allow_missing": self.args.allow_missing,
+            "on_error": self.args.on_error,
         }
 
     @property
@@ -137,6 +138,26 @@ and then the stage name name.
             "Only print the commands that would be executed without actually executing."
         ),
     )
+    repro_parser.add_argument(
+        "-k",
+        "--keep-going",
+        action="store_const",
+        default="fail",
+        const="keep-going",
+        dest="on_error",
+        help=(
+            "Continue executing, skipping stages having dependencies "
+            "on the failed stages"
+        ),
+    )
+    repro_parser.add_argument(
+        "--ignore-errors",
+        action="store_const",
+        default="fail",
+        const="ignore",
+        dest="on_error",
+        help="Ignore errors from stages.",
+    )
 
 
 def add_parser(subparsers, parent_parser):

dvc/commands/stage.py

@@ -195,12 +195,6 @@ def _add_common_args(parser):
         help="Declare output file or directory (do not put into DVC cache).",
         metavar="<filename>",
     ).complete = completion.FILE
-    parser.add_argument(
-        "--external",
-        action="store_true",
-        default=False,
-        help="Allow outputs that are outside of the DVC repository.",
-    )
     parser.add_argument(
         "--outs-persist",
         action="append",

dvc/config_schema.py

@@ -75,7 +75,7 @@ def ByUrl(mapping):  # noqa: N802
 
 
 class RelPath(str):
-    pass
+    __slots__ = ()
 
 
 class FeatureSchema(Schema):

dvc/daemon.py

@@ -28,7 +28,9 @@ def _popen(cmd, **kwargs) -> Popen:
             os.path.abspath(os.path.dirname(__file__)), "__main__.py"
         )
         prefix += [main_entrypoint]
-    return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)  # nosec B603
+    return Popen(
+        prefix + cmd, close_fds=True, shell=False, **kwargs  # nosec B603  # noqa: S603
+    )
 
 
 def _spawn_windows(cmd, env):

dvc/dvcfile.py

@@ -97,9 +97,7 @@ class FileMixin:
         self.verify = verify
 
     def __repr__(self):
-        return "{}: {}".format(
-            self.__class__.__name__, relpath(self.path, self.repo.root_dir)
-        )
+        return f"{self.__class__.__name__}: {relpath(self.path, self.repo.root_dir)}"
 
     def __hash__(self):
         return hash(self.path)

dvc/exceptions.py

@@ -63,9 +63,7 @@ class OutputNotFoundError(DvcException):
         self.output = output
         self.repo = repo
         super().__init__(
-            "Unable to find DVC file with output '{path}'".format(
-                path=relpath(self.output)
-            )
+            f"Unable to find DVC file with output {relpath(self.output)!r}"
         )
 
 
@@ -80,11 +78,7 @@ class StagePathAsOutputError(DvcException):
 
     def __init__(self, stage, output):
         assert isinstance(output, str)
-        super().__init__(
-            "{stage} is within an output '{output}' of another stage".format(
-                stage=stage, output=output
-            )
-        )
+        super().__init__(f"{stage} is within an output {output!r} of another stage")
 
 
 class CircularDependencyError(DvcException):
@@ -156,9 +150,7 @@ class CyclicGraphError(DvcException):
 class ConfirmRemoveError(DvcException):
     def __init__(self, path):
         super().__init__(
-            "unable to remove '{}' without a confirmation. Use `-f` to force.".format(
-                path
-            )
+            f"unable to remove {path!r} without a confirmation. Use `-f` to force."
         )
 
 
@@ -167,9 +159,7 @@ class InitError(DvcException):
 
 
 class ReproductionError(DvcException):
-    def __init__(self, name):
-        self.name = name
-        super().__init__(f"failed to reproduce '{name}'")
+    pass
 
 
 class BadMetricError(DvcException):

dvc/fs/dvc.py

@@ -5,7 +5,7 @@ import ntpath
 import os
 import posixpath
 import threading
-from contextlib import suppress
+from contextlib import ExitStack, suppress
 from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union
 
 from fsspec.spec import AbstractFileSystem
@@ -54,6 +54,8 @@ def _merge_info(repo, key, fs_info, dvc_info):
         ret["size"] = dvc_info["size"]
         if not fs_info and "md5" in dvc_info:
             ret["md5"] = dvc_info["md5"]
+        if not fs_info and "md5-dos2unix" in dvc_info:
+            ret["md5-dos2unix"] = dvc_info["md5-dos2unix"]
 
     if fs_info:
         ret["type"] = fs_info["type"]
@@ -121,11 +123,13 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         from pygtrie import Trie
 
         super().__init__()
+        self._repo_stack = ExitStack()
         if repo is None:
             repo = self._make_repo(url=url, rev=rev, subrepos=subrepos, **repo_kwargs)
             assert repo is not None
             # pylint: disable=protected-access
             repo_factory = repo._fs_conf["repo_factory"]
+            self._repo_stack.enter_context(repo)
 
         if not repo_factory:
             from dvc.repo import Repo
@@ -214,6 +218,7 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
                     scm=self.repo.scm,
                     repo_factory=self.repo_factory,
                 )
+                self._repo_stack.enter_context(repo)
                 self._datafss[key] = DataFileSystem(index=repo.index.data["repo"])
             self._subrepos_trie[key] = repo
 
@@ -283,7 +288,7 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         dvc_infos = {}
         if dvc_fs:
             dvc_path = _get_dvc_path(dvc_fs, subkey)
-            with suppress(FileNotFoundError):
+            with suppress(FileNotFoundError, NotADirectoryError):
                 for info in dvc_fs.ls(dvc_path, detail=True):
                     dvc_infos[dvc_fs.path.name(info["name"])] = info
             dvc_exists = bool(dvc_infos) or dvc_fs.exists(dvc_path)
@@ -394,6 +399,9 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         dvc_path = _get_dvc_path(dvc_fs, subkey)
         return dvc_fs.get_file(dvc_path, lpath, **kwargs)
 
+    def close(self):
+        self._repo_stack.close()
+
 
 class DVCFileSystem(FileSystem):
     protocol = "local"
@@ -404,7 +412,7 @@ class DVCFileSystem(FileSystem):
 
     @functools.cached_property
     # pylint: disable-next=invalid-overridden-method
-    def fs(self) -> "DVCFileSystem":
+    def fs(self) -> "_DVCFileSystem":
         return _DVCFileSystem(**self.fs_args)
 
     def isdvc(self, path, **kwargs) -> bool:
@@ -427,3 +435,7 @@ class DVCFileSystem(FileSystem):
             path = os.path.relpath(path, self.repo.root_dir)
 
         return as_posix(path)
+
+    def close(self):
+        if "fs" in self.__dict__:
+            self.fs.close()

dvc/logger.py

@@ -56,7 +56,7 @@ def add_logging_level(level_name, level_num, method_name=None):
 
 class LoggingException(Exception):
     def __init__(self, record):
-        msg = f"failed to log {str(record)}"
+        msg = f"failed to log {record!s}"
         super().__init__(msg)
 
 

dvc/output.py

@@ -471,9 +471,7 @@ class Output:
         return fs.path.abspath(fs.path.normpath(fs_path))
 
     def __repr__(self):
-        return "{class_name}: '{def_path}'".format(
-            class_name=type(self).__name__, def_path=self.def_path
-        )
+        return f"{type(self).__name__}: {self.def_path!r}"
 
     def __str__(self):
         if self.fs.protocol != "local":
@@ -548,12 +546,24 @@ class Output:
         _, hash_info = self._get_hash_meta()
         return hash_info
 
+    def _build(
+        self, *args, no_progress_bar=False, **kwargs
+    ) -> Tuple["HashFileDB", "Meta", "HashFile"]:
+        from dvc.ui import ui
+
+        with ui.progress(
+            unit="file",
+            desc=f"Collecting files and computing hashes in {self}",
+            disable=no_progress_bar,
+        ) as pb:
+            return build(*args, callback=pb.as_callback(), **kwargs)
+
     def _get_hash_meta(self):
         if self.use_cache:
             odb = self.cache
         else:
             odb = self.local_cache
-        _, meta, obj = build(
+        _, meta, obj = self._build(
             odb,
             self.fs_path,
             self.fs,
@@ -681,7 +691,7 @@ class Output:
     def save(self) -> None:
         if self.use_cache and not self.is_in_repo:
             raise DvcException(
-                f"Saving cached external output {str(self)} is not supported "
+                f"Saving cached external output {self!s} is not supported "
                 "since DVC 3.0. See "
                 f"{format_link('https://dvc.org/doc/user-guide/upgrade')} "
                 "for more info."
@@ -703,7 +713,7 @@ class Output:
 
         self._update_legacy_hash_name()
         if self.use_cache:
-            _, self.meta, self.obj = build(
+            _, self.meta, self.obj = self._build(
                 self.cache,
                 self.fs_path,
                 self.fs,
@@ -711,7 +721,7 @@ class Output:
                 ignore=self.dvcignore,
             )
         else:
-            _, self.meta, self.obj = build(
+            _, self.meta, self.obj = self._build(
                 self.local_cache,
                 self.fs_path,
                 self.fs,
@@ -763,7 +773,7 @@ class Output:
             if granular:
                 obj = self._commit_granular_dir(filter_info, hardlink)
             else:
-                staging, _, obj = build(
+                staging, _, obj = self._build(
                     self.cache,
                     filter_info or self.fs_path,
                     self.fs,
@@ -794,7 +804,7 @@ class Output:
 
     def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]:
         prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
-        staging, _, obj = build(
+        staging, _, obj = self._build(
             self.cache,
             self.fs_path,
             self.fs,
@@ -1021,7 +1031,7 @@ class Output:
 
         upload = not (update and from_fs.isdir(from_info))
         jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
-        staging, self.meta, obj = build(
+        staging, self.meta, obj = self._build(
             odb,
             from_info,
             from_fs,
@@ -1357,7 +1367,7 @@ class Output:
         new: "HashFile"
         try:
             assert self.hash_name
-            staging, meta, obj = build(
+            staging, meta, obj = self._build(
                 cache,
                 path,
                 self.fs,
@@ -1402,7 +1412,9 @@ class Output:
         otransfer(staging, self.cache, {obj.hash_info}, hardlink=relink, shallow=False)
 
         if relink:
-            with CheckoutCallback(desc="Checking out {path}", unit="files") as callback:
+            with CheckoutCallback(
+                desc=f"Checking out {path}", unit="files"
+            ) as callback:
                 self._checkout(
                     path,
                     self.fs,

dvc/parsing/__init__.py

@@ -135,6 +135,7 @@ def make_definition(
 class DataResolver:
     def __init__(self, repo: "Repo", wdir: str, d: dict):
         self.fs = fs = repo.fs
+        self.parsing_config = repo.config.get("parsing", {})
 
         if os.path.isabs(wdir):
             wdir = fs.path.relpath(wdir)
@@ -301,7 +302,10 @@ class EntryDefinition:
     ) -> "DictStrAny":
         try:
             return context.resolve(
-                value, skip_interpolation_checks=skip_checks, key=key
+                value,
+                skip_interpolation_checks=skip_checks,
+                key=key,
+                config=self.resolver.parsing_config,
             )
         except (ParseError, KeyNotInContext) as exc:
             format_and_raise(exc, f"'{self.where}.{self.name}.{key}'", self.relpath)

dvc/parsing/context.py

@@ -495,7 +495,12 @@ class Context(CtxDict):
                 self.data.pop(key, None)
 
     def resolve(
-        self, src, unwrap=True, skip_interpolation_checks=False, key=None
+        self,
+        src,
+        unwrap=True,
+        skip_interpolation_checks=False,
+        key=None,
+        config=None,
     ) -> Any:
         """Recursively resolves interpolation and returns resolved data.
 
@@ -511,10 +516,15 @@ class Context(CtxDict):
         {'lst': [1, 2, 3]}
         """
         func = recurse(self.resolve_str)
-        return func(src, unwrap, skip_interpolation_checks, key)
+        return func(src, unwrap, skip_interpolation_checks, key, config)
 
     def resolve_str(
-        self, src: str, unwrap=True, skip_interpolation_checks=False, key=None
+        self,
+        src: str,
+        unwrap=True,
+        skip_interpolation_checks=False,
+        key=None,
+        config=None,
     ) -> str:
         """Resolves interpolated string to it's original value,
         or in case of multiple interpolations, a combined string.
@@ -535,7 +545,12 @@ class Context(CtxDict):
             return value
         # but not "${num} days"
         return str_interpolate(
-            src, matches, self, skip_checks=skip_interpolation_checks, key=key
+            src,
+            matches,
+            self,
+            skip_checks=skip_interpolation_checks,
+            key=key,
+            config=config,
         )
 
 

dvc/parsing/interpolate.py

@@ -82,20 +82,18 @@ def escape_str(value):
 
 
 @singledispatch
-def to_str(obj) -> str:
+def to_str(obj, config=None) -> str:  # noqa: ARG001, pylint: disable=unused-argument
     return str(obj)
 
 
 @to_str.register(bool)
-def _(obj: bool):
+def _(obj: bool, config=None):  # noqa: ARG001, pylint: disable=unused-argument
     return "true" if obj else "false"
 
 
 @to_str.register(dict)
-def _(obj: dict):  # noqa: C901
-    from dvc.config import Config
-
-    config = Config.from_cwd().get("parsing", {})
+def _(obj: dict, config=None):  # noqa: C901
+    config = config or {}
 
     result = ""
     for k, v in flatten(obj).items():
@@ -211,6 +209,7 @@ def str_interpolate(
     context: "Context",
     skip_checks: bool = False,
     key=None,
+    config=None,
 ):
     index, buf = 0, ""
     for match in matches:
@@ -218,7 +217,7 @@ def str_interpolate(
         expr = get_expression(match, skip_checks=skip_checks)
         value = context.select(expr, unwrap=True)
         validate_value(value, key)
-        buf += template[index:start] + to_str(value)
+        buf += template[index:start] + to_str(value, config=config)
         index = end
     buf += template[index:]
     # regex already backtracks and avoids any `${` starting with

dvc/render/match.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from collections import defaultdict
 from typing import TYPE_CHECKING, DefaultDict, Dict, List, NamedTuple, Optional
@@ -17,6 +18,7 @@ if TYPE_CHECKING:
 
 
 dpath.options.ALLOW_EMPTY_STRING_KEYS = True
+logger = logging.getLogger(__name__)
 
 
 def _squash_plots_properties(data: List) -> Dict:
@@ -113,6 +115,7 @@ def match_defs_renderers(  # noqa: C901, PLR0912
             try:
                 dps, rev_props = converter.flat_datapoints(rev)
             except Exception as e:  # noqa: BLE001, pylint: disable=broad-except
+                logger.warning("In %r, %s", rev, str(e).lower())
                 def_errors[rev] = e
                 continue
 

dvc/repo/__init__.py

@@ -29,7 +29,7 @@ if TYPE_CHECKING:
     from dvc.stage import Stage
     from dvc.types import DictStrAny
     from dvc_data.hashfile.state import StateBase
-    from dvc_data.index import DataIndex
+    from dvc_data.index import DataIndex, DataIndexEntry
 
     from .experiments import Experiments
     from .index import Index
@@ -285,7 +285,7 @@ class Repo:
             new.check_graph()
 
     @staticmethod
-    def open(url, *args, **kwargs):  # noqa: A003
+    def open(url: Optional[str], *args, **kwargs) -> "Repo":  # noqa: A003
         from .open_repo import open_repo
 
         return open_repo(url, *args, **kwargs)
@@ -375,6 +375,28 @@ class Repo:
         self.data_index.commit()
         self._reset()
 
+    def get_data_index_entry(
+        self,
+        path: str,
+        workspace: str = "repo",
+    ) -> Tuple["DataIndex", "DataIndexEntry"]:
+        if self.subrepos:
+            fs_path = self.dvcfs.from_os_path(path)
+            fs = self.dvcfs.fs
+            # pylint: disable-next=protected-access
+            key = fs._get_key_from_relative(fs_path)
+            # pylint: disable-next=protected-access
+            subrepo, _, key = fs._get_subrepo_info(key)
+            index = subrepo.index.data[workspace]
+        else:
+            index = self.index.data[workspace]
+            key = self.fs.path.relparts(path, self.root_dir)
+
+        try:
+            return index, index[key]
+        except KeyError as exc:
+            raise OutputNotFoundError(path, self) from exc
+
     def __repr__(self):
         return f"{self.__class__.__name__}: '{self.root_dir}'"
 
@@ -460,7 +482,7 @@ class Repo:
         used_run_cache=None,
         revs=None,
         num=1,
-        push: bool = False,
+        push: bool = False,  # noqa: F811
     ):
         """Get the stages related to the given target and collect
         the `info` of its outputs.
@@ -557,6 +579,7 @@ class Repo:
 
         from dvc.dirs import site_cache_dir
         from dvc.fs import GitFileSystem
+        from dvc.version import version_tuple
 
         cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir()
 
@@ -578,7 +601,13 @@ class Repo:
             os.umask(umask)
 
         md5 = hashlib.md5(  # noqa: S324  # nosec B324, B303
-            str((root_dir, getpass.getuser())).encode()
+            str(
+                (
+                    root_dir,
+                    getpass.getuser(),
+                    version_tuple[0],
+                )
+            ).encode()
         )
         repo_token = md5.hexdigest()
         return os.path.join(repos_dir, repo_token)
@@ -586,12 +615,16 @@ class Repo:
     def close(self):
         self.scm.close()
         self.state.close()
+        if "dvcfs" in self.__dict__:
+            self.dvcfs.close()
         if self._data_index is not None:
             self._data_index.close()
 
     def _reset(self):
         self.scm._reset()  # pylint: disable=protected-access
         self.state.close()
+        if "dvcfs" in self.__dict__:
+            self.dvcfs.close()
         self.__dict__.pop("index", None)
         self.__dict__.pop("dvcignore", None)
         self.__dict__.pop("dvcfs", None)

dvc/repo/experiments/executor/base.py

@@ -44,7 +44,7 @@ if TYPE_CHECKING:
     from dvc.repo import Repo
     from dvc.repo.experiments.stash import ExpStashEntry
     from dvc.scm import Git
-    from dvc.stage import PipelineStage
+    from dvc.stage import PipelineStage, Stage
 
 logger = logging.getLogger(__name__)
 
@@ -255,6 +255,15 @@ class BaseExecutor(ABC):
             **kwargs,
         )
 
+    @classmethod
+    def _get_stage_files(cls, stages: List["Stage"]) -> List[str]:
+        from dvc.stage.utils import _get_stage_files
+
+        ret: List[str] = []
+        for stage in stages:
+            ret.extend(_get_stage_files(stage))
+        return ret
+
     @classmethod
     def _get_top_level_paths(cls, repo: "Repo") -> List["str"]:
         return list(
@@ -511,6 +520,9 @@ class BaseExecutor(ABC):
                 )
 
             stages = dvc.reproduce(*args, **kwargs)
+            if paths := cls._get_stage_files(stages):
+                logger.debug("Staging stage-related files: %s", paths)
+                dvc.scm_context.add(paths)
             if paths := cls._get_top_level_paths(dvc):
                 logger.debug("Staging top-level files: %s", paths)
                 dvc.scm_context.add(paths)

dvc/repo/experiments/queue/tempdir.py

@@ -115,7 +115,9 @@ class TempDirQueue(WorkspaceQueue):
                 message=message,
             )
             if not exec_result.exp_hash:
-                raise DvcException(f"Failed to reproduce experiment '{rev[:7]}'")
+                raise DvcException(  # noqa: TRY301
+                    f"Failed to reproduce experiment '{rev[:7]}'"
+                )
             if exec_result.ref_info:
                 results[rev].update(
                     self.collect_executor(self.repo.experiments, executor, exec_result)

dvc/repo/experiments/queue/workspace.py

@@ -121,7 +121,9 @@ class WorkspaceQueue(BaseStashQueue):
                 message=kwargs.get("message"),
             )
             if not exec_result.exp_hash:
-                raise DvcException(f"Failed to reproduce experiment '{rev[:7]}'")
+                raise DvcException(  # noqa: TRY301
+                    f"Failed to reproduce experiment '{rev[:7]}'"
+                )
             if exec_result.ref_info:
                 results[rev].update(
                     self.collect_executor(self.repo.experiments, executor, exec_result)

dvc/repo/experiments/utils.py

@@ -234,7 +234,7 @@ def fix_exp_head(scm: Union["Git", "NoSCM"], ref: Optional[str]) -> Optional[str
     if ref:
         name, tail = Git.split_ref_pattern(ref)
         if name == "HEAD" and scm.get_ref(EXEC_BASELINE):
-            return "".join((EXEC_BASELINE, tail))
+            return f"{EXEC_BASELINE}{tail}"
     return ref
 
 

dvc/repo/fetch.py

@@ -1,12 +1,61 @@
 import logging
+from typing import List, Tuple
 
 from dvc.exceptions import DownloadError
+from dvc_data.index import DataIndex, FileStorage
 
 from . import locked
 
 logger = logging.getLogger(__name__)
 
 
+def _collect_indexes(  # noqa: PLR0913
+    repo,
+    targets=None,
+    remote=None,
+    all_branches=False,
+    with_deps=False,
+    all_tags=False,
+    recursive=False,
+    all_commits=False,
+    revs=None,
+    max_size=None,
+    types=None,
+):
+    indexes = {}
+    collection_exc = None
+    for rev in repo.brancher(
+        revs=revs,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        all_commits=all_commits,
+    ):
+        saved_remote = repo.config["core"].get("remote")
+        try:
+            if remote:
+                repo.config["core"]["remote"] = remote
+
+            idx = repo.index.targets_view(
+                targets,
+                with_deps=with_deps,
+                recursive=recursive,
+                max_size=max_size,
+                types=types,
+            )
+            indexes[idx.data_tree.hash_info.value] = idx.data["repo"]
+        except Exception as exc:  # pylint: disable=broad-except
+            collection_exc = exc
+            logger.exception("failed to collect '%s'", rev or "workspace")
+        finally:
+            if remote:
+                repo.config["core"]["remote"] = saved_remote
+
+    if not indexes and collection_exc:
+        raise collection_exc
+
+    return indexes
+
+
 @locked
 def fetch(  # noqa: C901, PLR0913
     self,
@@ -20,6 +69,8 @@ def fetch(  # noqa: C901, PLR0913
     all_commits=False,
     run_cache=False,
     revs=None,
+    max_size=None,
+    types=None,
 ) -> int:
     """Download data items from a cloud and imported repositories
 
@@ -51,39 +102,35 @@ def fetch(  # noqa: C901, PLR0913
     except DownloadError as exc:
         failed_count += exc.amount
 
-    indexes = []
-    index_keys = set()
-    for _ in self.brancher(
-        revs=revs,
+    indexes = _collect_indexes(
+        self,
+        targets=targets,
+        remote=remote,
         all_branches=all_branches,
+        with_deps=with_deps,
         all_tags=all_tags,
+        recursive=recursive,
         all_commits=all_commits,
-    ):
-        saved_remote = self.config["core"].get("remote")
-        try:
-            if remote:
-                self.config["core"]["remote"] = remote
+        revs=revs,
+        max_size=max_size,
+        types=types,
+    )
 
-            idx = self.index.targets_view(
-                targets,
-                with_deps=with_deps,
-                recursive=recursive,
-            )
-            index_keys.add(idx.data_tree.hash_info.value)
-            indexes.append(idx.data["repo"])
-        finally:
-            if remote:
-                self.config["core"]["remote"] = saved_remote
-
-    cache_key = ("fetch", tokenize(sorted(index_keys)))
+    cache_key = ("fetch", tokenize(sorted(indexes.keys())))
 
     with Callback.as_tqdm_callback(
         desc="Collecting",
         unit="entry",
     ) as cb:
         data = collect(
-            indexes, cache_index=self.data_index, cache_key=cache_key, callback=cb
+            indexes.values(),
+            "remote",
+            cache_index=self.data_index,
+            cache_key=cache_key,
+            callback=cb,
         )
+    data, unversioned_count = _log_unversioned(data)
+    failed_count += unversioned_count
 
     with Callback.as_tqdm_callback(
         desc="Fetching",
@@ -96,7 +143,7 @@ def fetch(  # noqa: C901, PLR0913
                 callback=cb,
             )  # pylint: disable=assignment-from-no-return
         finally:
-            for fs_index in data.values():
+            for fs_index in data:
                 fs_index.close()
 
     if fetch_transferred:
@@ -109,3 +156,34 @@ def fetch(  # noqa: C901, PLR0913
         raise DownloadError(failed_count)
 
     return transferred_count
+
+
+def _log_unversioned(data: List["DataIndex"]) -> Tuple[List["DataIndex"], int]:
+    ret: List["DataIndex"] = []
+    unversioned: List[str] = []
+    for fs_index in data:
+        remote = fs_index.storage_map[()].remote
+        if not isinstance(remote, FileStorage) or not remote.fs.version_aware:
+            ret.append(fs_index)
+            continue
+
+        fs = remote.fs
+        index = DataIndex()
+        index.storage_map = fs_index.storage_map
+        for key, entry in fs_index.iteritems():
+            if entry.meta and not entry.meta.isdir and entry.meta.version_id is None:
+                unversioned.append(fs.unstrip_protocol(fs.path.join(remote.path, *key)))
+            else:
+                index[key] = entry
+        fs_index.close()
+        ret.append(index)
+
+    if unversioned:
+        logger.warning(
+            (
+                "Some files are missing cloud version information and will not be "
+                "fetched from the remote:\n%s"
+            ),
+            "\n".join(unversioned),
+        )
+    return ret, len(unversioned)

dvc/repo/get.py

@@ -27,7 +27,7 @@ def get(url, path, out=None, rev=None, jobs=None, force=False):
     out = resolve_output(path, out, force=force)
 
     if is_valid_filename(out):
-        raise GetDVCFileError()
+        raise GetDVCFileError
 
     with Repo.open(
         url=url,

dvc/repo/graph.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Iterator, List, Set
+from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Set, TypeVar
 
 from dvc.fs import localfs
 from dvc.utils.fs import path_isin
@@ -8,6 +8,8 @@ if TYPE_CHECKING:
 
     from dvc.stage import Stage
 
+T = TypeVar("T")
+
 
 def check_acyclic(graph: "DiGraph") -> None:
     import networkx as nx
@@ -39,6 +41,21 @@ def get_pipelines(graph: "DiGraph"):
     return [graph.subgraph(c).copy() for c in nx.weakly_connected_components(graph)]
 
 
+def get_subgraph_of_nodes(
+    graph: "DiGraph", sources: Optional[List[Any]] = None, downstream: bool = False
+) -> "DiGraph":
+    from networkx import dfs_postorder_nodes, reverse_view
+
+    if not sources:
+        return graph
+
+    g = reverse_view(graph) if downstream else graph
+    nodes = []
+    for source in sources:
+        nodes.extend(dfs_postorder_nodes(g, source))
+    return graph.subgraph(nodes)
+
+
 def collect_pipeline(stage: "Stage", graph: "DiGraph") -> Iterator["Stage"]:
     import networkx as nx
 

dvc/repo/index.py

@@ -374,6 +374,50 @@ class Index:
 
         return dict(by_workspace)
 
+    @cached_property
+    def metric_keys(self) -> Dict[str, Set["DataIndexKey"]]:
+        from collections import defaultdict
+
+        from .metrics.show import _collect_top_level_metrics
+
+        by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
+
+        by_workspace["repo"] = set()
+
+        for out in self.outs:
+            if not out.metric:
+                continue
+
+            workspace, key = out.index_key
+            by_workspace[workspace].add(key)
+
+        for path in _collect_top_level_metrics(self.repo):
+            key = self.repo.fs.path.relparts(path, self.repo.root_dir)
+            by_workspace["repo"].add(key)
+
+        return dict(by_workspace)
+
+    @cached_property
+    def plot_keys(self) -> Dict[str, Set["DataIndexKey"]]:
+        from collections import defaultdict
+
+        by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
+
+        by_workspace["repo"] = set()
+
+        for out in self.outs:
+            if not out.plot:
+                continue
+
+            workspace, key = out.index_key
+            by_workspace[workspace].add(key)
+
+        for path in self._plot_sources:
+            key = self.repo.fs.path.relparts(path, self.repo.root_dir)
+            by_workspace["repo"].add(key)
+
+        return dict(by_workspace)
+
     @cached_property
     def data_tree(self):
         from dvc_data.hashfile.tree import Tree
@@ -487,11 +531,31 @@ class Index:
                 used[odb].update(objs)
         return used
 
+    def _types_filter(self, types, out):
+        ws, okey = out.index_key
+        for typ in types:
+            if typ == "plots":
+                keys = self.plot_keys
+            elif typ == "metrics":
+                keys = self.metric_keys
+            else:
+                raise ValueError(f"unsupported type {typ}")
+
+            for key in keys.get(ws, []):
+                if (len(key) >= len(okey) and key[: len(okey)] == okey) or (
+                    len(key) < len(okey) and okey[: len(key)] == key
+                ):
+                    return True
+
+        return False
+
     def targets_view(
         self,
         targets: Optional["TargetType"],
         stage_filter: Optional[Callable[["Stage"], bool]] = None,
         outs_filter: Optional[Callable[["Output"], bool]] = None,
+        max_size: Optional[int] = None,
+        types: Optional[List[str]] = None,
         **kwargs: Any,
     ) -> "IndexView":
         """Return read-only view of index for the specified targets.
@@ -514,7 +578,20 @@ class Index:
             for stage_info in self.collect_targets(targets, **kwargs)
             if not stage_filter or stage_filter(stage_info.stage)
         ]
-        return IndexView(self, stage_infos, outs_filter=outs_filter)
+
+        def _outs_filter(out):
+            if max_size and out.meta and out.meta.size and out.meta.size >= max_size:
+                return False
+
+            if types and not self._types_filter(types, out):
+                return False
+
+            if outs_filter:
+                return outs_filter(out)
+
+            return True
+
+        return IndexView(self, stage_infos, outs_filter=_outs_filter)
 
 
 class _DataPrefixes(NamedTuple):

dvc/repo/open_repo.py

@@ -2,8 +2,7 @@ import logging
 import os
 import tempfile
 import threading
-from contextlib import contextmanager
-from typing import TYPE_CHECKING, Dict, Iterator, Optional, Tuple
+from typing import TYPE_CHECKING, Dict, Optional, Tuple
 
 from funcy import retry, wrap_with
 
@@ -18,16 +17,14 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-@contextmanager
 @map_scm_exception()
 def _external_repo(
     url,
     rev: Optional[str] = None,
-    for_write: bool = False,
     **kwargs,
-) -> Iterator["Repo"]:
+) -> "Repo":
     logger.debug("Creating external repo %s@%s", url, rev)
-    path = _cached_clone(url, rev, for_write=for_write)
+    path = _cached_clone(url, rev)
     # Local HEAD points to the tip of whatever branch we first cloned from
     # (which may not be the default branch), use origin/HEAD here to get
     # the tip of the default branch
@@ -38,11 +35,6 @@ def _external_repo(
     config.update(kwargs.pop("config", None) or {})
 
     main_root = "/"
-    if for_write:
-        # we already checked out needed revision
-        rev = None
-        main_root = path
-
     repo_kwargs = dict(
         root_dir=path,
         url=url,
@@ -52,14 +44,7 @@ def _external_repo(
         **kwargs,
     )
 
-    repo = Repo(**repo_kwargs)
-
-    try:
-        yield repo
-    finally:
-        repo.close()
-        if for_write:
-            _remove(path)
+    return Repo(**repo_kwargs)
 
 
 def open_repo(url, *args, **kwargs):
@@ -67,6 +52,7 @@ def open_repo(url, *args, **kwargs):
         url = os.getcwd()
 
     if os.path.exists(url):
+        url = os.path.abspath(url)
         try:
             config = _get_remote_config(url)
             config.update(kwargs.get("config") or {})
@@ -139,20 +125,19 @@ def _get_remote_config(url):
         repo.close()
 
 
-def _cached_clone(url, rev, for_write=False):
+def _cached_clone(url, rev):
     """Clone an external git repo to a temporary directory.
 
     Returns the path to a local temporary directory with the specified
-    revision checked out. If for_write is set prevents reusing this dir via
-    cache.
+    revision checked out.
     """
     from shutil import copytree
 
     # even if we have already cloned this repo, we may need to
     # fetch/fast-forward to get specified rev
-    clone_path, shallow = _clone_default_branch(url, rev, for_write=for_write)
+    clone_path, shallow = _clone_default_branch(url, rev)
 
-    if not for_write and (url) in CLONES:
+    if url in CLONES:
         return CLONES[url][0]
 
     # Copy to a new dir to keep the clone clean
@@ -160,16 +145,12 @@ def _cached_clone(url, rev, for_write=False):
     logger.debug("erepo: making a copy of %s clone", url)
     copytree(clone_path, repo_path)
 
-    # Check out the specified revision
-    if for_write:
-        _git_checkout(repo_path, rev)
-    else:
-        CLONES[url] = (repo_path, shallow)
+    CLONES[url] = (repo_path, shallow)
     return repo_path
 
 
 @wrap_with(threading.Lock())
-def _clone_default_branch(url, rev, for_write=False):  # noqa: C901, PLR0912
+def _clone_default_branch(url, rev):  # noqa: C901, PLR0912
     """Get or create a clean clone of the url.
 
     The cloned is reactualized with git pull unless rev is a known sha.
@@ -203,7 +184,7 @@ def _clone_default_branch(url, rev, for_write=False):  # noqa: C901, PLR0912
 
             logger.debug("erepo: git clone '%s' to a temporary dir", url)
             clone_path = tempfile.mkdtemp("dvc-clone")
-            if not for_write and rev and not Git.is_sha(rev):
+            if rev and not Git.is_sha(rev):
                 # If rev is a tag or branch name try shallow clone first
 
                 try:
@@ -248,17 +229,6 @@ def _merge_upstream(git: "Git"):
         pass
 
 
-def _git_checkout(repo_path, rev):
-    from dvc.scm import Git
-
-    logger.debug("erepo: git checkout %s@%s", repo_path, rev)
-    git = Git(repo_path)
-    try:
-        git.checkout(rev)
-    finally:
-        git.close()
-
-
 def _remove(path):
     from dvc.utils.fs import remove
 

dvc/repo/plots/__init__.py

@@ -134,7 +134,7 @@ class Plots:
             if "workspace" in revs:
                 # reorder revs to match repo.brancher ordering
                 revs.remove("workspace")
-                revs = ["workspace"] + revs
+                revs = ["workspace", *revs]
         for rev in revs:
             with switch_repo(self.repo, rev) as (repo, _):
                 res: Dict = {}

dvc/repo/pull.py

@@ -1,5 +1,6 @@
 import logging
 
+from dvc.exceptions import CheckoutError
 from dvc.repo import locked
 from dvc.utils import glob_targets
 
@@ -38,13 +39,17 @@ def pull(  # noqa: PLR0913
         recursive=recursive,
         run_cache=run_cache,
     )
-    stats = self.checkout(
-        targets=expanded_targets,
-        with_deps=with_deps,
-        force=force,
-        recursive=recursive,
-        allow_missing=allow_missing,
-    )
+    try:
+        stats = self.checkout(
+            targets=expanded_targets,
+            with_deps=with_deps,
+            force=force,
+            recursive=recursive,
+            allow_missing=allow_missing,
+        )
+    except CheckoutError as exc:
+        exc.stats["fetched"] = processed_files_count
+        raise
 
     stats["fetched"] = processed_files_count
     return stats

dvc/repo/reproduce.py

@@ -1,129 +1,85 @@
 import logging
-from typing import TYPE_CHECKING, Iterator, List
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    NoReturn,
+    Optional,
+    Set,
+    Tuple,
+    TypeVar,
+    Union,
+    cast,
+)
+
+from funcy import ldistinct
 
 from dvc.exceptions import ReproductionError
 from dvc.repo.scm_context import scm_context
 from dvc.stage.cache import RunCacheNotSupported
+from dvc.utils import humanize
+from dvc.utils.collections import ensure_list
 
 from . import locked
 
 if TYPE_CHECKING:
+    from networkx import DiGraph
+
     from dvc.stage import Stage
 
     from . import Repo
 
 logger = logging.getLogger(__name__)
+T = TypeVar("T")
 
 
-def _reproduce_stage(stage: "Stage", **kwargs) -> List["Stage"]:
-    if stage.frozen and not stage.is_import:
-        logger.warning(
-            "%s is frozen. Its dependencies are not going to be reproduced.",
-            stage,
-        )
-
-    stage = stage.reproduce(**kwargs)
-    if not stage:
-        return []
-
-    if not kwargs.get("dry", False):
-        stage.dump(update_pipeline=False)
-        _track_stage(stage)
-
-    return [stage]
+def collect_stages(
+    repo: "Repo",
+    targets: Iterable[str],
+    recursive: bool = False,
+    glob: bool = False,
+) -> List["Stage"]:
+    stages: List["Stage"] = []
+    for target in targets:
+        stages.extend(repo.stage.collect(target, recursive=recursive, glob=glob))
+    return ldistinct(stages)
 
 
-def _get_stage_files(stage: "Stage") -> Iterator[str]:
-    yield stage.dvcfile.relpath
-    for dep in stage.deps:
-        if (
-            not dep.use_scm_ignore
-            and dep.is_in_repo
-            and not stage.repo.dvcfs.isdvc(stage.repo.dvcfs.from_os_path(str(dep)))
-        ):
-            yield dep.fs_path
-    for out in stage.outs:
-        if not out.use_scm_ignore and out.is_in_repo:
-            yield out.fs_path
+def get_subgraph(
+    graph: "DiGraph",
+    nodes: Optional[List] = None,
+    pipeline: bool = False,
+    downstream: bool = False,
+) -> "DiGraph":
+    import networkx as nx
+
+    from .graph import get_pipeline, get_pipelines, get_subgraph_of_nodes
+
+    if not pipeline or not nodes:
+        return get_subgraph_of_nodes(graph, nodes, downstream=downstream)
+
+    pipelines = get_pipelines(graph)
+    used_pipelines = [get_pipeline(pipelines, node) for node in nodes]
+    return nx.compose_all(used_pipelines)
 
 
-def _track_stage(stage: "Stage") -> None:
-    from dvc.utils import relpath
-
-    context = stage.repo.scm_context
-    for path in _get_stage_files(stage):
-        context.track_file(relpath(path))
-    return context.track_changed_files()
+def get_active_graph(graph: "DiGraph") -> "DiGraph":
+    g = cast("DiGraph", graph.copy())
+    for stage in graph:
+        if stage.frozen:
+            # NOTE: disconnect frozen stage from its dependencies
+            g.remove_edges_from(graph.out_edges(stage))
+    return g
 
 
-@locked
-@scm_context
-def reproduce(  # noqa: C901, PLR0912
-    self: "Repo",
-    targets=None,
-    recursive=False,
-    pipeline=False,
-    all_pipelines=False,
-    **kwargs,
-):
-    from .graph import get_pipeline, get_pipelines
-
-    glob = kwargs.pop("glob", False)
-
-    if isinstance(targets, str):
-        targets = [targets]
-
-    if not all_pipelines and not targets:
-        from dvc.dvcfile import PROJECT_FILE
-
-        targets = [PROJECT_FILE]
-
-    interactive = kwargs.get("interactive", False)
-    if not interactive:
-        kwargs["interactive"] = self.config["core"].get("interactive", False)
-
-    stages = set()
-    if pipeline or all_pipelines:
-        pipelines = get_pipelines(self.index.graph)
-        if all_pipelines:
-            used_pipelines = pipelines
-        else:
-            used_pipelines = []
-            for target in targets:
-                stage = self.stage.get_target(target)
-                used_pipelines.append(get_pipeline(pipelines, stage))
-
-        for pline in used_pipelines:
-            for stage in pline:
-                if pline.in_degree(stage) == 0:
-                    stages.add(stage)
-    else:
-        for target in targets:
-            stages.update(
-                self.stage.collect(
-                    target,
-                    recursive=recursive,
-                    glob=glob,
-                )
-            )
-
-    if kwargs.get("pull", False) and kwargs.get("run_cache", True):
-        logger.debug("Pulling run cache")
-        try:
-            self.stage_cache.pull(None)
-        except RunCacheNotSupported as e:
-            logger.warning("Failed to pull run cache: %s", e)
-
-    return _reproduce_stages(self.index.graph, list(stages), **kwargs)
-
-
-def _reproduce_stages(  # noqa: C901
-    graph,
-    stages,
-    downstream=False,
-    single_item=False,
-    **kwargs,
-):
+def plan_repro(
+    graph: "DiGraph",
+    stages: Optional[List["T"]] = None,
+    pipeline: bool = False,
+    downstream: bool = False,
+) -> List["T"]:
     r"""Derive the evaluation of the given node for the given graph.
 
     When you _reproduce a stage_, you want to _evaluate the descendants_
@@ -159,72 +115,146 @@ def _reproduce_stages(  # noqa: C901
 
     The derived evaluation of _downstream_ B would be: [B, D, E]
     """
-    steps = _get_steps(graph, stages, downstream, single_item)
+    import networkx as nx
+
+    sub = get_subgraph(graph, stages, pipeline=pipeline, downstream=downstream)
+    return list(nx.dfs_postorder_nodes(sub))
+
+
+def _reproduce_stage(stage: "Stage", **kwargs) -> Optional["Stage"]:
+    if stage.frozen and not stage.is_import:
+        msg = "%s is frozen. Its dependencies are not going to be reproduced."
+        logger.warning(msg, stage)
+
+    ret = stage.reproduce(**kwargs)
+    if ret and not kwargs.get("dry", False):
+        stage.dump(update_pipeline=False)
+    return ret
+
+
+def _get_upstream_downstream_nodes(
+    graph: Optional["DiGraph"], node: T
+) -> Tuple[List[T], List[T]]:
+    succ = list(graph.successors(node)) if graph else []
+    pre = list(graph.predecessors(node)) if graph else []
+    return succ, pre
+
+
+def _repr(stages: Iterable["Stage"]) -> str:
+    return humanize.join(repr(stage.addressing) for stage in stages)
+
+
+def handle_error(
+    graph: Optional["DiGraph"], on_error: str, exc: Exception, stage: "Stage"
+) -> Set["Stage"]:
+    import networkx as nx
+
+    logger.warning("%s%s", exc, " (ignored)" if on_error == "ignore" else "")
+    if not graph or on_error == "ignore":
+        return set()
+
+    dependents = set(nx.dfs_postorder_nodes(graph.reverse(), stage)) - {stage}
+    if dependents:
+        names = _repr(dependents)
+        msg = "%s %s will be skipped due to this failure"
+        logger.warning(msg, "Stages" if len(dependents) > 1 else "Stage", names)
+    return dependents
+
+
+def _raise_error(exc: Optional[Exception], *stages: "Stage") -> NoReturn:
+    names = _repr(stages)
+    segment = " stages:" if len(stages) > 1 else ""
+    raise ReproductionError(f"failed to reproduce{segment} {names}") from exc
+
+
+def _reproduce(
+    stages: List["Stage"],
+    graph: Optional["DiGraph"] = None,
+    force_downstream: bool = False,
+    on_error: str = "fail",
+    force: bool = False,
+    repro_fn: Callable = _reproduce_stage,
+    **kwargs,
+) -> List["Stage"]:
+    assert on_error in ("fail", "keep-going", "ignore")
 
-    force_downstream = kwargs.pop("force_downstream", False)
     result: List["Stage"] = []
-    unchanged: List["Stage"] = []
-    # `ret` is used to add a cosmetic newline.
-    ret: List["Stage"] = []
+    failed: List["Stage"] = []
+    to_skip: Dict["Stage", "Stage"] = {}
+    ret: Optional["Stage"] = None
+
+    force_state = {node: force for node in stages}
+
+    for stage in stages:
+        if stage in to_skip:
+            continue
 
-    for stage in steps:
         if ret:
-            logger.info("")
+            logger.info("")  # add a newline
+
+        upstream, downstream = _get_upstream_downstream_nodes(graph, stage)
+        force_stage = force_state[stage]
 
         try:
-            kwargs["upstream"] = result + unchanged
-            ret = _reproduce_stage(stage, **kwargs)
-            if not ret:
-                unchanged.extend([stage])
-                continue
+            ret = repro_fn(stage, upstream=upstream, force=force_stage, **kwargs)
+        except Exception as exc:  # noqa: BLE001, pylint: disable=broad-exception-caught
+            failed.append(stage)
+            if on_error == "fail":
+                _raise_error(exc, stage)
 
-            result.extend(ret)
-            if force_downstream:
-                # NOTE: we are walking our pipeline from the top to the
-                # bottom. If one stage is changed, it will be reproduced,
-                # which tells us that we should force reproducing all of
-                # the other stages down below, even if their direct
-                # dependencies didn't change.
-                kwargs["force"] = True
-        except Exception as exc:  # noqa: BLE001
-            raise ReproductionError(stage.addressing) from exc
+            dependents = handle_error(graph, on_error, exc, stage)
+            to_skip.update({node: stage for node in dependents})
+            continue
+
+        if force_downstream and (ret or force_stage):
+            force_state.update({node: True for node in downstream})
+
+        if ret:
+            result.append(ret)
+
+    if on_error != "ignore" and failed:
+        _raise_error(None, *failed)
     return result
 
 
-def _get_steps(graph, stages, downstream, single_item):
-    import networkx as nx
+@locked
+@scm_context
+def reproduce(
+    self: "Repo",
+    targets: Union[Iterable[str], str, None] = None,
+    recursive: bool = False,
+    pipeline: bool = False,
+    all_pipelines: bool = False,
+    downstream: bool = False,
+    single_item: bool = False,
+    glob: bool = False,
+    on_error: Optional[str] = "fail",
+    **kwargs,
+):
+    from dvc.dvcfile import PROJECT_FILE
 
-    active = graph.copy()
+    if all_pipelines or pipeline:
+        single_item = False
+        downstream = False
+
+    if not kwargs.get("interactive", False):
+        kwargs["interactive"] = self.config["core"].get("interactive", False)
+
+    stages: List["Stage"] = []
+    if not all_pipelines:
+        targets_list = ensure_list(targets or PROJECT_FILE)
+        stages = collect_stages(self, targets_list, recursive=recursive, glob=glob)
+
+    if kwargs.get("pull", False) and kwargs.get("run_cache", True):
+        logger.debug("Pulling run cache")
+        try:
+            self.stage_cache.pull(None)
+        except RunCacheNotSupported as e:
+            logger.warning("Failed to pull run cache: %s", e)
+
+    graph = None
+    steps = stages
     if not single_item:
-        # NOTE: frozen stages don't matter for single_item
-        for stage in graph:
-            if stage.frozen:
-                # NOTE: disconnect frozen stage from its dependencies
-                active.remove_edges_from(graph.out_edges(stage))
-
-    all_pipelines: List["Stage"] = []
-    for stage in stages:
-        if downstream:
-            # NOTE (py3 only):
-            # Python's `deepcopy` defaults to pickle/unpickle the object.
-            # Stages are complex objects (with references to `repo`,
-            # `outs`, and `deps`) that cause struggles when you try
-            # to serialize them. We need to create a copy of the graph
-            # itself, and then reverse it, instead of using
-            # graph.reverse() directly because it calls `deepcopy`
-            # underneath -- unless copy=False is specified.
-            nodes = nx.dfs_postorder_nodes(active.reverse(copy=False), stage)
-            all_pipelines += reversed(list(nodes))
-        else:
-            all_pipelines += nx.dfs_postorder_nodes(active, stage)
-
-    steps = []
-    for stage in all_pipelines:
-        if stage not in steps:
-            # NOTE: order of steps still matters for single_item
-            if single_item and stage not in stages:
-                continue
-
-            steps.append(stage)
-
-    return steps
+        graph = get_active_graph(self.index.graph)
+        steps = plan_repro(graph, stages, pipeline=pipeline, downstream=downstream)
+    return _reproduce(steps, graph=graph, on_error=on_error or "fail", **kwargs)

dvc/rwlock.py

@@ -31,15 +31,13 @@ RWLOCK_LOCK = "rwlock.lock"
 class RWLockFileCorruptedError(DvcException):
     def __init__(self, path):
         super().__init__(
-            "Unable to read RWLock-file '{}'. JSON structure is corrupted".format(
-                relpath(path)
-            )
+            f"Unable to read RWLock-file {relpath(path)!r}. JSON structure is corrupted"
         )
 
 
 class RWLockFileFormatError(DvcException):
     def __init__(self, path):
-        super().__init__(f"RWLock-file '{relpath(path)}' format error.")
+        super().__init__(f"RWLock-file {relpath(path)!r} format error.")
 
 
 @contextmanager

dvc/stage/__init__.py

@@ -426,10 +426,7 @@ class Stage(params.StageParams):
                 logger.info("Stage '%s' didn't change, skipping", self.addressing)
             return None
 
-        msg = "Going to reproduce {stage}. Are you sure you want to continue?".format(
-            stage=self
-        )
-
+        msg = f"Going to reproduce {self}. Are you sure you want to continue?"
         if interactive and not prompt.confirm(msg):
             raise DvcException("reproduction aborted by the user")
 

dvc/stage/cache.py

@@ -123,12 +123,15 @@ class StageCache:
 
     @contextmanager
     def _cache_type_copy(self):
-        cache_types = self.repo.cache.legacy.cache_types
+        cache_types = self.repo.cache.local.cache_types
+        legacy_cache_types = self.repo.cache.legacy.cache_types
+        self.repo.cache.local.cache_types = ["copy"]
         self.repo.cache.legacy.cache_types = ["copy"]
         try:
             yield
         finally:
-            self.repo.cache.legacy.cache_types = cache_types
+            self.repo.cache.local.cache_types = cache_types
+            self.repo.cache.legacy.cache_types = legacy_cache_types
 
     def _uncached_outs(self, stage, cache):
         # NOTE: using temporary stage to avoid accidentally modifying original

dvc/stage/run.py

@@ -80,7 +80,7 @@ def _run(executable, cmd, **kwargs):
     exec_cmd = _make_cmd(executable, cmd)
 
     try:
-        p = subprocess.Popen(exec_cmd, **kwargs)  # nosec B603
+        p = subprocess.Popen(exec_cmd, **kwargs)  # nosec B603 # noqa: S603
         if main_thread:
             old_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
 

dvc/stage/utils.py

@@ -102,6 +102,10 @@ def check_no_externals(stage):
     link = format_link(
         "https://dvc.org/doc/user-guide/pipelines/external-dependencies-and-outputs"
     )
+    if stage.is_data_source:
+        link = format_link(
+            "https://dvc.org/doc/user-guide/data-management/importing-external-data"
+        )
     raise StageExternalOutputsError(
         f"Cached output(s) outside of DVC project: {str_outs}. "
         f"See {link} for more info."
@@ -280,3 +284,27 @@ def validate_kwargs(
         kwargs.pop("name", None)
 
     return kwargs
+
+
+def _get_stage_files(stage: "Stage") -> List[str]:
+    from dvc.dvcfile import ProjectFile
+    from dvc.utils import relpath
+
+    ret: List[str] = []
+    file = stage.dvcfile
+    ret.append(file.relpath)
+    if isinstance(file, ProjectFile):
+        ret.append(file._lockfile.relpath)  # pylint: disable=protected-access
+
+    for dep in stage.deps:
+        if (
+            not dep.use_scm_ignore
+            and dep.is_in_repo
+            and not stage.repo.dvcfs.isdvc(stage.repo.dvcfs.from_os_path(str(dep)))
+        ):
+            ret.append(relpath(dep.fs_path))
+
+    for out in stage.outs:
+        if not out.use_scm_ignore and out.is_in_repo:
+            ret.append(relpath(out.fs_path))
+    return ret

dvc/testing/benchmarks/fixtures.py

@@ -90,7 +90,7 @@ def make_dvc_bin(
         dvc_bin = bench_config.dvc_bin
 
     def _dvc_bin(*args):
-        return check_output([dvc_bin, *args], text=True)  # nosec B603
+        return check_output([dvc_bin, *args], text=True)  # nosec B603  # noqa: S603
 
     _dvc_bin.version = parse_tuple(_dvc_bin("--version"))  # type: ignore[attr-defined]
     return _dvc_bin

dvc/testing/fixtures.py

@@ -233,8 +233,8 @@ def docker_services(
         pytest.skip("disabled for Windows on CI")
 
     try:
-        subprocess.check_output(  # nosec B607, B602
-            "docker ps", stderr=subprocess.STDOUT, shell=True
+        subprocess.check_output(  # nosec B607, B602,
+            "docker ps", stderr=subprocess.STDOUT, shell=True  # noqa: S602, S607
         )
     except subprocess.CalledProcessError as err:
         out = (err.output or b"").decode("utf-8")
@@ -242,7 +242,9 @@ def docker_services(
 
     try:
         cmd = "docker-compose version"
-        subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)  # nosec
+        subprocess.check_output(
+            cmd, stderr=subprocess.STDOUT, shell=True  # nosec B602 # noqa: S602
+        )
     except subprocess.CalledProcessError as err:
         out = (err.output or b"").decode("utf-8")
         pytest.skip(f"docker-compose is not installed: {out}")

dvc/ui/__init__.py

@@ -147,7 +147,7 @@ class Console:
                 return self.write(json.text, stderr=stderr)
         return self.rich_print(json, stderr=stderr, soft_wrap=True)
 
-    def rich_print(
+    def rich_print(  # noqa: PLR0913
         self,
         *objects: Any,
         sep: str = " ",

dvc/ui/pager.py

@@ -59,7 +59,8 @@ def find_pager():
     if not pager:
         pager = os.getenv(PAGER_ENV)
     if not pager:
-        if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") != 0:  # nosec B605
+        ret = os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}")  # nosec B605 # noqa: S605
+        if ret != 0:
             logger.warning(
                 "Unable to find `less` in the PATH. Check out %s for more info.",
                 format_link("https://man.dvc.org/pipeline/show"),

dvc/utils/__init__.py

@@ -319,7 +319,7 @@ def resolve_paths(repo, out, always_local=False):
 
 
 def format_link(link):
-    return "<{blue}{link}{nc}>".format(
+    return "<{blue}{link}{nc}>".format(  # noqa: UP032
         blue=colorama.Fore.CYAN, link=link, nc=colorama.Fore.RESET
     )
 

dvc/utils/threadpool.py

@@ -46,7 +46,7 @@ class ThreadPoolExecutor(futures.ThreadPoolExecutor):
         if sys.version_info > (3, 9):  # pylint: disable=no-else-return
             # pylint: disable=unexpected-keyword-arg
             return super().shutdown(wait=wait, cancel_futures=cancel_futures)
-        else:
+        else:  # noqa: RET505
             with self._shutdown_lock:
                 self._shutdown = True
                 if cancel_futures:

pyproject.toml

@@ -34,13 +34,14 @@ dependencies = [
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=2.3.2,<2.4.0",
+    "dvc-data>=2.6.0,<2.7.0",
     "dvc-http>=2.29.0",
     "dvc-render>=0.3.1,<1",
     "dvc-studio-client>=0.9.2,<1",
     "dvc-task>=0.3.0,<1",
     "flatten_dict<1,>=0.4.1",
-    "flufl.lock>=5",
+    # https://github.com/iterative/dvc/issues/9654
+    "flufl.lock>=5,<8",
     "funcy>=1.14",
     "grandalf<1,>=0.7",
     "hydra-core>=1.1",
@@ -74,15 +75,15 @@ gdrive = ["dvc-gdrive==2.20"]
 gs = ["dvc-gs==2.22.1"]
 hdfs = ["dvc-hdfs==2.19"]
 lint = [
-    "mypy==1.3.0",
+    "mypy==1.4.1",
     "pylint==2.17.4",
     "types-colorama",
     "types-psutil",
+    "types-pyinstaller",
     "types-requests",
     "types-tabulate",
     "types-toml",
     "types-tqdm",
-    "types-pyinstaller",
 ]
 oss = ["dvc-oss==2.19"]
 s3 = ["dvc-s3==2.23.0"]
@@ -267,8 +268,16 @@ ignored-argument-names = "_.*|args|kwargs"
 [tool.ruff]
 # external flake8 codes that should be preserved
 external = ["B301", "C901", "E302", "W601"]
-ignore = ["N818", "S101", "PT004", "PT007", "PT019", "SIM105", "SIM108", "SIM110", "SIM117", "TRY003", "TRY200", "TRY300", "TRY301", "PLR2004", "PLW2901", "RUF005"]
-select = ["F", "E", "W", "C90", "N", "UP", "YTT", "S", "BLE", "B", "A", "C4", "T10", "EXE", "ISC", "ICN", "G", "INP", "PIE", "T20", "PT", "Q", "RET501", "RET504", "SIM", "TID", "TCH", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY", "RUF"]
+ignore = [
+    "N818", "S101", "PT004", "PT007", "RET501", "RET502", "RET503", "SIM105", "SIM108",
+    "SIM117", "TRY003", "TRY200", "TRY300", "PLR2004", "PLW2901", "RUF012"
+]
+select = [
+    "F", "E", "W", "C90", "N", "UP", "YTT", "ASYNC", "S", "BLE", "B", "A", "C4", "T10",
+    "EXE", "ISC", "ICN", "G", "INP", "PIE", "T20", "PYI", "PT", "Q", "RSE", "RET",
+    "SLOT", "SIM", "TID", "TCH", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY",
+    "FLY", "PERF101", "RUF",
+]
 show-source = true
 
 [tool.ruff.flake8-pytest-style]
@@ -297,11 +306,11 @@ ignore-names = ["M", "SCM"]
 "dvc/commands/**" = ["N806"]
 "dvc/testing/**" = ["ARG002"]
 "dvc/testing/benchmarks/**" = ["ARG001"]
-"scripts/**" = ["T201", "INP001", "TRY002"]
+"tests/**" = ["S", "ARG001", "ARG002", "TRY002", "TRY301"]
 
 [tool.ruff.pylint]
 max-args = 10
 
 [tool.bandit]
-exclude_dirs = ["tests", "scripts", "dvc/repo/experiments/utils.py"]
+exclude_dirs = ["tests", "dvc/repo/experiments/utils.py"]
 skips = ["B101"]

scripts/build_package.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-set -e
-set -x
-
-if [ ! -d "dvc" ]; then
-  echo "Please run this script from repository root" >&2
-  exit 1
-fi
-
-echo 'PKG = "pip"' >dvc/_build.py
-
-python -m pip install build twine
-python -m build
-twine check dist/*

tests/func/api/test_data.py

@@ -198,11 +198,13 @@ def test_get_url_subrepos(tmp_dir, scm, local_cloud):
         local_cloud / "files" / "md5" / "ac" / "bd18db4cc2f85cedef654fccc4a4d8"
     )
     assert api.get_url(os.path.join("subrepo", "dir", "foo")) == expected_url
+    assert api.get_url(os.path.join("subrepo", "dir", "foo"), repo=".") == expected_url
 
     expected_url = os.fspath(
         local_cloud / "files" / "md5" / "37" / "b51d194a7513e45b56f6524f2d51f2"
     )
     assert api.get_url("subrepo/bar") == expected_url
+    assert api.get_url("subrepo/bar", repo=".") == expected_url
 
 
 def test_open_from_remote(tmp_dir, erepo_dir, cloud, local_cloud):

tests/func/api/test_show.py

@@ -187,7 +187,7 @@ def test_params_show_stages(params_repo):
 
 
 def test_params_show_stage_addressing(tmp_dir, dvc):
-    for subdir in {"subdir1", "subdir2"}:
+    for subdir in ("subdir1", "subdir2"):
         subdir = tmp_dir / subdir
         subdir.mkdir()
         with subdir.chdir():
@@ -195,7 +195,7 @@ def test_params_show_stage_addressing(tmp_dir, dvc):
 
             dvc.run(name="stage-0", cmd="echo stage-0", params=["foo"])
 
-    for s in {"subdir1", "subdir2"}:
+    for s in ("subdir1", "subdir2"):
         dvcyaml = os.path.join(s, "dvc.yaml")
         assert api.params_show(stages=f"{dvcyaml}:stage-0") == {"foo": 1}
 

tests/func/experiments/executor/test_ssh.py

@@ -119,7 +119,7 @@ def test_reproduce(tmp_dir, scm, dvc, cloud, exp_stage, mocker):
         rev,
         fs_factory=partial(_ssh_factory, cloud),
     )
-    assert mock_execute.called_once()
+    mock_execute.assert_called_once()
     _name, args, _kwargs = mock_execute.mock_calls[0]
     assert f"dvc exp exec-run --infofile {infofile}" in args[0]
 
@@ -148,7 +148,7 @@ def test_run_machine(tmp_dir, scm, dvc, cloud, exp_stage, mocker):
 
     tmp_dir.gen("params.yaml", "foo: 2")
     dvc.experiments.run(exp_stage.addressing, machine="foo")
-    assert mock_repro.called_once()
+    mock_repro.assert_called_once()
     _name, _args, kwargs = mock_repro.mock_calls[0]
     info = kwargs["info"]
     url = urlparse(info.git_url)

tests/func/experiments/test_experiments.py

@@ -488,16 +488,6 @@ def test_run_celery(tmp_dir, scm, dvc, exp_stage, mocker):
     assert expected == metrics
 
 
-def test_run_metrics(tmp_dir, scm, dvc, exp_stage, mocker):
-    from dvc.cli import main
-
-    mocker.patch.object(dvc.experiments, "run", return_value={"abc123": "abc123"})
-    show_mock = mocker.patch.object(dvc.metrics, "show", return_value={})
-
-    main(["exp", "run", "-m"])
-    assert show_mock.called_once()
-
-
 def test_checkout_targets_deps(tmp_dir, scm, dvc, exp_stage):
     from dvc.utils.fs import remove
 

tests/func/experiments/test_show.py

@@ -452,7 +452,7 @@ def test_show_csv(tmp_dir, scm, dvc, exp_stage, capsys):
     assert "metrics.yaml:foo,params.yaml:foo,copy.py" in cap.out
     assert f",workspace,baseline,,,3,3,{data_hash}" in cap.out
     assert (
-        ",master,baseline,{},,1,1,{}".format(
+        ",master,baseline,{},,1,1,{}".format(  # noqa: UP032
             _get_rev_isotimestamp(baseline_rev), data_hash
         )
         in cap.out
@@ -682,7 +682,7 @@ def test_metrics_renaming(tmp_dir, dvc, scm, capsys, copy_script):
 
     assert f",master,baseline,{_get_rev_isotimestamp(scores_rev)},,1,,1" in cap.out
     assert (
-        ",{},baseline,{},,,1,1".format(
+        ",{},baseline,{},,,1,1".format(  # noqa: UP032
             metrics_rev[:7], _get_rev_isotimestamp(metrics_rev)
         )
         in cap.out

tests/func/repro/test_repro.py

@@ -848,9 +848,7 @@ def test_repro_dry_no_exec(tmp_dir, dvc):
                 idir,
                 "-o",
                 odir,
-                'python -c \'import shutil; shutil.copytree("{}", "{}")\''.format(
-                    idir, odir
-                ),
+                f'python -c \'import shutil; shutil.copytree("{idir}", "{odir}")\'',
             ]
         )
         assert ret == 0
@@ -943,6 +941,18 @@ def test_repro_force_downstream(tmp_dir, dvc, copy_script):
     assert stages[1].addressing == file3_stage.addressing
 
 
+def test_repro_force_downstream_do_not_force_independent_stages(tmp_dir, dvc, run_copy):
+    tmp_dir.gen({"foo": "foo", "bar": "bar"})
+    foo1 = run_copy("foo", "foo1", name="foo1")
+    foo2 = run_copy("foo1", "foo2", name="foo2")
+    run_copy("bar", "bar1", name="bar1")
+    run_copy("bar1", "bar2", name="bar2")
+    cat = dvc.run(cmd="cat bar2 foo2", deps=["foo2", "bar2"], name="cat")
+
+    tmp_dir.gen("foo", "foobar")
+    assert dvc.reproduce(force_downstream=True) == [foo1, foo2, cat]
+
+
 def test_repro_pipeline(
     tmp_dir,
     dvc,
@@ -1282,3 +1292,67 @@ def test_repro_rm_recursive(tmp_dir, dvc):
     dvc.reproduce()
     assert (tmp_dir / "dir").exists()
     assert not (tmp_dir / "dir" / "foo").exists()
+
+
+def test_repro_single_item_with_multiple_targets(tmp_dir, dvc, copy_script):
+    stage1 = dvc.stage.add(cmd="echo foo > foo", outs=["foo"], name="gen-foo")
+    with dvc.lock:
+        stage1.run()
+
+    stage2 = dvc.stage.add(
+        cmd="python copy.py foo bar", deps=["foo"], outs=["bar"], name="copy-foo-bar"
+    )
+    assert dvc.reproduce(["copy-foo-bar", "gen-foo"], single_item=True) == [
+        stage2,
+        stage1,
+    ]
+
+
+def test_repro_keep_going(mocker, tmp_dir, dvc, copy_script):
+    from dvc.repo import reproduce
+
+    (bar_stage, foo_stage) = tmp_dir.dvc_gen({"bar": "bar", "foo": "foo"})
+    stage1 = dvc.stage.add(
+        cmd=["python copy.py bar foobar", "exit 1"],
+        deps=["bar"],
+        outs=["foobar"],
+        name="copy-bar-foobar",
+    )
+    dvc.stage.add(cmd="cat foobar foo", deps=["foobar", "foo"], name="cat")
+    spy = mocker.spy(reproduce, "_reproduce_stage")
+
+    with pytest.raises(ReproductionError):
+        dvc.reproduce(on_error="keep-going", repro_fn=spy)
+
+    assert spy.call_args_list == [
+        mocker.call(bar_stage, upstream=[], force=False, interactive=False),
+        mocker.call(stage1, upstream=[bar_stage], force=False, interactive=False),
+        mocker.call(foo_stage, upstream=[], force=False, interactive=False),
+    ]
+
+
+def test_repro_ignore_errors(M, mocker, tmp_dir, dvc, copy_script):
+    from dvc.repo import reproduce
+
+    (bar_stage, foo_stage) = tmp_dir.dvc_gen({"bar": "bar", "foo": "foo"})
+    stage1 = dvc.stage.add(
+        cmd=["python copy.py bar foobar", "exit 1"],
+        deps=["bar"],
+        outs=["foobar"],
+        name="copy-bar-foobar",
+    )
+    stage2 = dvc.stage.add(cmd="cat foobar foo", deps=["foobar", "foo"], name="cat")
+    spy = mocker.spy(reproduce, "_reproduce_stage")
+    dvc.reproduce(on_error="ignore", repro_fn=spy)
+
+    assert spy.call_args_list == [
+        mocker.call(bar_stage, upstream=[], force=False, interactive=False),
+        mocker.call(stage1, upstream=[bar_stage], force=False, interactive=False),
+        mocker.call(foo_stage, upstream=[], force=False, interactive=False),
+        mocker.call(
+            stage2,
+            upstream=[foo_stage, stage1],
+            force=False,
+            interactive=False,
+        ),
+    ]

tests/integration/plots/test_plots.py

@@ -377,7 +377,7 @@ def test_repo_with_removed_plots(tmp_dir, capsys, repo_with_plots):
     remove("confusion.json")
     remove("image.png")
 
-    for s in {"show", "diff"}:
+    for s in ("show", "diff"):
         _, json_result, split_json_result = call(capsys, subcommand=s)
         errors = [
             {

tests/ruff.toml

@@ -1,3 +0,0 @@
-extend = "../pyproject.toml"
-extend-select = ["RUF005"]
-extend-ignore = ["S", "RUF001", "ARG001", "ARG002", "TRY002"]

tests/unit/command/test_data_sync.py

@@ -18,6 +18,12 @@ def test_fetch(mocker, dvc):
             "--with-deps",
             "--recursive",
             "--run-cache",
+            "--max-size",
+            "10",
+            "--type",
+            "plots",
+            "--type",
+            "metrics",
         ]
     )
     assert cli_args.func == CmdDataFetch
@@ -37,6 +43,8 @@ def test_fetch(mocker, dvc):
         with_deps=True,
         recursive=True,
         run_cache=True,
+        max_size=10,
+        types=["plots", "metrics"],
     )
 
 

tests/unit/command/test_repro.py

@@ -14,6 +14,7 @@ common_arguments = {
     "pull": False,
     "allow_missing": False,
     "targets": [],
+    "on_error": "fail",
 }
 repro_arguments = {
     "run_cache": True,

tests/unit/command/test_stage.py

@@ -47,7 +47,6 @@ def test_stage_add(mocker, dvc, command, parsed_command):
             "file:param1,param2",
             "--params",
             "param3",
-            "--external",
             "--desc",
             "description",
             "--force",

tests/unit/fs/test_dvc.py

@@ -162,6 +162,16 @@ def test_isdir_mixed(tmp_dir, dvc):
     assert not fs.isfile("dir")
 
 
+def test_ls_dirty(tmp_dir, dvc):
+    tmp_dir.dvc_gen({"data": "data"})
+    (tmp_dir / "data").unlink()
+
+    tmp_dir.gen({"data": {"foo": "foo", "bar": "bar"}})
+
+    fs = DVCFileSystem(repo=dvc)
+    assert set(fs.ls("data")) == {"data/foo", "data/bar"}
+
+
 @pytest.mark.parametrize(
     "dvcfiles,extra_expected",
     [

tests/unit/repo/experiments/queue/test_celery.py

@@ -71,7 +71,7 @@ def test_post_run_after_kill(test_queue):
         except ProcessLookupError:
             time.sleep(0.1)
         if time.time() > timeout:
-            raise TimeoutError()
+            raise TimeoutError
 
     assert result_foo.get(timeout=10) == "foo"
 
@@ -132,9 +132,11 @@ def test_celery_queue_kill(test_queue, mocker, force):
     )
     with pytest.raises(CannotKillTasksError, match="Task 'foobar' is initializing,"):
         test_queue.kill(["bar", "foo", "foobar"], force=force)
-    assert kill_mock.called_once_with(mock_entry_foo.stash_rev)
-    assert kill_mock.called_once_with(mock_entry_bar.stash_rev)
-    assert kill_mock.called_once_with(mock_entry_foobar.stash_rev)
+    assert kill_mock.call_args_list == [
+        mocker.call(mock_entry_bar.stash_rev),
+        mocker.call(mock_entry_foo.stash_rev),
+        mocker.call(mock_entry_foobar.stash_rev),
+    ]
     mark_mocker.assert_called_once_with("bar", None)
 
 
@@ -157,7 +159,7 @@ def test_celery_queue_kill_invalid(test_queue, mocker, force):
 
     with pytest.raises(UnresolvedExpNamesError):
         test_queue.kill(["bar", "foo", "foobar"], force=force)
-    assert kill_mock.called_once_with(
+    kill_mock.assert_called_once_with(
         {mock_entry_foo: "foo", mock_entry_bar: "bar"}, force
     )
 

tests/unit/repo/experiments/test_utils.py

@@ -8,7 +8,7 @@ from dvc.repo.experiments.utils import check_ref_format, resolve_name, to_studio
 def commit_exp_ref(tmp_dir, scm, file="foo", contents="foo", name="foo"):
     tmp_dir.scm_gen(file, contents, commit="init")
     rev = scm.get_rev()
-    ref = "/".join([EXPS_NAMESPACE, "ab", "c123", name])
+    ref = f"{EXPS_NAMESPACE}/ab/c123/{name}"
     scm.gitpython.set_ref(ref, rev)
     return ref, rev
 

tests/unit/repo/test_graph.py

@@ -0,0 +1,38 @@
+import pytest
+from networkx import DiGraph
+from networkx.utils import graphs_equal
+
+from dvc.repo.graph import get_subgraph_of_nodes
+
+
+@pytest.mark.parametrize(
+    "nodes,downstream,expected_edges",
+    [
+        ([], False, {1: [2, 3], 2: [4, 5], 3: [6, 7], 8: [9]}),
+        ([1], False, {1: [2, 3], 2: [4, 5], 3: [6, 7]}),
+        ([2], False, {2: [4, 5]}),
+        ([3], False, {3: [6, 7]}),
+        ([8], False, [(8, 9)]),
+        ([2, 3, 8], False, {2: [4, 5], 3: [6, 7], 8: [9]}),
+        ([4], False, {4: []}),
+        ([], True, {1: [2, 3], 2: [4, 5], 3: [6, 7], 8: [9]}),
+        ([1], True, {1: []}),
+        ([9], True, [(8, 9)]),
+        ([2], True, [(1, 2)]),
+        ([6], True, [(1, 3), (3, 6)]),
+        ([2, 3, 8], True, {1: [2, 3], 8: []}),
+        ([4, 7], True, {1: [2, 3], 2: [4], 3: [7]}),
+    ],
+)
+def test_subgraph_of_nodes(nodes, downstream, expected_edges):
+    r"""
+             1
+           /   \
+          2     3      8
+         / \   / \     |
+        4   5 6   7    9
+    """
+    graph = DiGraph({1: [2, 3], 2: [4, 5], 3: [6, 7], 8: [9]})
+    subgraph = get_subgraph_of_nodes(graph, nodes, downstream=downstream)
+    expected = DiGraph(expected_edges)
+    assert graphs_equal(expected, subgraph)

tests/unit/repo/test_reproduce.py

@@ -1,60 +1,71 @@
-import os
+from itertools import chain
 
-from dvc.repo.reproduce import _get_stage_files
+from networkx import DiGraph
+from networkx.utils import graphs_equal
+
+from dvc.repo.reproduce import get_active_graph, plan_repro
+
+
+def test_active_graph(mocker):
+    n = mocker.sentinel
+    n1, n2, n3, n4, n5, n6, n7, n8, n9 = (getattr(n, f"n{i}") for i in range(1, 10))
+    edges = {n1: [n2, n3], n2: [n4, n5], n3: [n6, n7], n8: [n9]}
+    for node in chain.from_iterable([n, *v] for n, v in edges.items()):
+        node.frozen = False
+
+    g = DiGraph(edges)
+
+    active = get_active_graph(g)
+    assert graphs_equal(g, active)
+
+    n2.frozen = True
+    active = get_active_graph(g)
+    assert g.edges() - active.edges() == {(n2, n5), (n2, n4)}
+    assert n2 in active
+    assert not active.edges() - g.edges()
+    assert not graphs_equal(g, active)
+
+
+def test_repro_plan(M):
+    r"""
+             1
+           /  \
+          2    3    8
+         / \  / \   |
+        4  5 6  7   9
+    """
+    g = DiGraph({1: [2, 3], 2: [4, 5], 3: [6, 7], 8: [9]})
+
+    assert plan_repro(g) == [4, 5, 2, 6, 7, 3, 1, 9, 8]
+    assert plan_repro(g, [1]) == [4, 5, 2, 6, 7, 3, 1]
+    assert plan_repro(g, [4], downstream=True) == [4, 2, 1]
+    assert plan_repro(g, [8], True) == plan_repro(g, [9], True) == [9, 8]
+    assert plan_repro(g, [2, 8], True) == [4, 5, 2, 6, 7, 3, 1, 9, 8]
+    assert plan_repro(g, [2, 3], downstream=True) == [
+        M.any_of(2, 3),
+        M.any_of(2, 3),
+        1,
+    ]
 
 
 def test_number_reproduces(tmp_dir, dvc, mocker):
-    reproduce_stage_mock = mocker.patch(
-        "dvc.repo.reproduce._reproduce_stage", returns=[]
-    )
+    mock = mocker.Mock(return_value=None)
     tmp_dir.dvc_gen({"pre-foo": "pre-foo"})
 
-    dvc.run(name="echo-foo", outs=["foo"], cmd="echo foo > foo")
-    dvc.run(name="echo-bar", deps=["foo"], outs=["bar"], cmd="echo bar > bar")
-    dvc.run(name="echo-baz", deps=["foo"], outs=["baz"], cmd="echo baz > baz")
-    dvc.run(name="echo-boop", deps=["bar"], outs=["boop"], cmd="echo boop > boop")
-
-    reproduce_stage_mock.reset_mock()
-
-    dvc.reproduce(all_pipelines=True)
-
-    assert reproduce_stage_mock.call_count == 5
-
-
-def test_get_stage_files(tmp_dir, dvc):
-    tmp_dir.dvc_gen("dvc-dep", "dvc-dep")
-    tmp_dir.gen("other-dep", "other-dep")
-
-    stage = dvc.stage.add(
-        name="stage",
-        cmd="foo",
-        deps=["dvc-dep", "other-dep"],
-        outs=["dvc-out"],
-        outs_no_cache=["other-out"],
+    dvc.stage.add(name="echo-foo", outs=["foo"], cmd="echo foo > foo", verify=False)
+    dvc.stage.add(
+        name="echo-bar", deps=["foo"], outs=["bar"], cmd="echo bar > bar", verify=False
     )
-    result = set(_get_stage_files(stage))
-    assert result == {
-        stage.dvcfile.relpath,
-        str(tmp_dir / "other-dep"),
-        str(tmp_dir / "other-out"),
-    }
-
-
-def test_get_stage_files_wdir(tmp_dir, dvc):
-    tmp_dir.gen({"dir": {"dvc-dep": "dvc-dep", "other-dep": "other-dep"}})
-    dvc.add(os.path.join("dir", "dvc-dep"))
-
-    stage = dvc.stage.add(
-        name="stage",
-        cmd="foo",
-        wdir="dir",
-        deps=["dvc-dep", "other-dep"],
-        outs=["dvc-out"],
-        outs_no_cache=["other-out"],
+    dvc.stage.add(
+        name="echo-baz", deps=["foo"], outs=["baz"], cmd="echo baz > baz", verify=False
     )
-    result = set(_get_stage_files(stage))
-    assert result == {
-        stage.dvcfile.relpath,
-        str(tmp_dir / "dir" / "other-dep"),
-        str(tmp_dir / "dir" / "other-out"),
-    }
+    dvc.stage.add(
+        name="echo-boop",
+        deps=["bar"],
+        outs=["boop"],
+        cmd="echo boop > boop",
+        verify=False,
+    )
+
+    dvc.reproduce(all_pipelines=True, repro_fn=mock)
+    assert mock.call_count == 5

tests/unit/stage/test_cache.py

@@ -212,7 +212,7 @@ def test_unhashable(tmp_dir, dvc, mocker, kwargs):
     stage = create_stage(Stage, path="stage.dvc", repo=dvc, **kwargs)
     get_stage_hash = mocker.patch("dvc.stage.cache._get_stage_hash")
     assert cache.save(stage) is None
-    assert get_stage_hash.not_called
+    get_stage_hash.assert_not_called()
     with pytest.raises(RunCacheNotFoundError):
         cache.restore(stage)
-    assert get_stage_hash.not_called
+    get_stage_hash.assert_not_called()

tests/unit/stage/test_stage.py

@@ -69,7 +69,7 @@ def test_stage_update(dvc, mocker):
     is_repo_import.return_value = True
     with dvc.lock:
         stage.update()
-    assert reproduce.called_once_with()
+    reproduce.assert_called_once_with(no_download=None, jobs=None)
 
     is_repo_import.return_value = False
     with pytest.raises(StageUpdateError):
@@ -92,8 +92,8 @@ def test_stage_run_ignore_sigint(dvc, mocker):
 
     dvc.run(cmd="path", name="train")
 
-    assert popen.called_once()
-    assert communicate.called_once_with()
+    popen.assert_called_once()
+    communicate.assert_called_once_with()
     signal_mock.assert_any_call(signal.SIGINT, signal.SIG_IGN)
     assert signal.getsignal(signal.SIGINT) == signal.default_int_handler
 

tests/unit/stage/test_utils.py

@@ -1,7 +1,7 @@
 import os
 
 from dvc.fs import localfs
-from dvc.stage.utils import resolve_paths
+from dvc.stage.utils import _get_stage_files, resolve_paths
 
 
 def test_resolve_paths():
@@ -19,3 +19,40 @@ def test_resolve_paths():
     path, wdir = resolve_paths(fs=localfs, path=file_path, wdir="../../some-dir")
     assert path == os.path.abspath(file_path)
     assert wdir == os.path.abspath("some-dir")
+
+
+def test_get_stage_files(tmp_dir, dvc):
+    tmp_dir.dvc_gen("dvc-dep", "dvc-dep")
+    tmp_dir.gen("other-dep", "other-dep")
+    stage = dvc.stage.create(
+        name="stage",
+        cmd="foo",
+        deps=["dvc-dep", "other-dep"],
+        outs=["dvc-out"],
+        outs_no_cache=["other-out"],
+    )
+    assert _get_stage_files(stage) == [
+        "dvc.yaml",
+        "dvc.lock",
+        "other-dep",
+        "other-out",
+    ]
+
+
+def test_get_stage_files_wdir(tmp_dir, dvc):
+    tmp_dir.gen({"dir": {"dvc-dep": "dvc-dep", "other-dep": "other-dep"}})
+    dvc.add(os.path.join("dir", "dvc-dep"))
+    stage = dvc.stage.create(
+        name="stage",
+        cmd="foo",
+        wdir="dir",
+        deps=["dvc-dep", "other-dep"],
+        outs=["dvc-out"],
+        outs_no_cache=["other-out"],
+    )
+    assert _get_stage_files(stage) == [
+        "dvc.yaml",
+        "dvc.lock",
+        os.path.join("dir", "other-dep"),
+        os.path.join("dir", "other-out"),
+    ]