deps: limit flufl-lock to <8 (#9681)

Looks like flufl-lock switched to pdm in 8.0 which triggered a

```
pkg_resources.DistributionNotFound: The 'flufl.lock>=5' distribution was not found and is required by dvc
```

error in multiple scenarious. Can't put my finger on anything specific
in flufl-lock yet, so will just limit its version for now.

Fixes https://github.com/iterative/dvc/issues/9654

.dvc/.gitignore

@@ -1,12 +1,3 @@
-/state
-/lock
 /config.local
-/updater
-/updater.lock
-/state-journal
-/state-wal
-/cache
-/pkg
-/repos
 /tmp
-/experiments
+/cache

.dvc/config

@@ -1,4 +0,0 @@
-['remote "dvc"']
-url = https://remote.dvc.org/dvc
-[core]
-remote = dvc

.github/workflows/packages.yaml

@@ -11,66 +11,6 @@ permissions:  # added using https://github.com/step-security/secure-workflows
   contents: read
 
 jobs:
-  binary:
-    permissions:
-      contents: write  # for actions/upload-release-asset to upload release asset
-    strategy:
-      matrix:
-        include:
-         - {pkg: "deb",  os: "ubuntu-20.04", asset: "dvc_${{ github.event.release.tag_name }}_amd64.deb"}
-         - {pkg: "rpm", os: "ubuntu-20.04", asset: "dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm"}
-         - {pkg: "osxpkg", os: "macos-11", asset: "dvc-${{ github.event.release.tag_name }}.pkg"}
-         - {pkg: "exe",  os: "windows-2019", asset: "dvc-${{ github.event.release.tag_name }}.exe"}
-
-    name: ${{ matrix.pkg }}
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v3
-      with:
-          fetch-depth: 0
-
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-        cache: pip
-        cache-dependency-path: |
-          pyproject.toml
-          scripts/build-requirements.txt
-
-    - name: Set up Ruby 2.6
-      uses: ruby/setup-ruby@v1
-      if: matrix.pkg != 'exe'
-      with:
-        ruby-version: '2.6'
-
-    - name: Install fpm
-      if: matrix.pkg != 'exe'
-      run: gem install --no-document fpm
-
-    - name: Install deps
-      run: |
-        pip install --upgrade pip wheel setuptools
-        pip install .[all]
-        pip install -r scripts/build-requirements.txt
-
-    - name: Pull images
-      run: dvc pull
-
-    - name: Build ${{ matrix.pkg }}
-      run: python scripts/build.py ${{ matrix.pkg }}
-
-    - name: Publish ${{ matrix.pkg }}
-      if: github.event_name == 'release'
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ github.event.release.upload_url }}
-        asset_path: scripts/${{ matrix.pkg == 'exe' && 'innosetup' || 'fpm' }}/${{ matrix.asset }}
-        asset_name: ${{ matrix.asset }}
-        asset_content_type: binary/octet-stream
-
   pip:
     runs-on: ubuntu-20.04
     permissions:

dvc/__pyinstaller/__init__.py

@@ -0,0 +1,5 @@
+import os
+
+
+def get_hook_dirs():
+    return [os.path.dirname(__file__)]

dvc/__pyinstaller/hook-asyncssh.py

@@ -1 +1,2 @@
+# ruff: noqa: N999
 hiddenimports = ["win32timezone"]

dvc/__pyinstaller/hook-celery.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 # pylint: disable=import-error
 from PyInstaller.utils.hooks import collect_submodules, is_module_or_submodule
 

dvc/__pyinstaller/hook-dvc.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 from PyInstaller.utils.hooks import copy_metadata  # pylint:disable=import-error
 
 # needed for `dvc doctor` to show dep versions

dvc/__pyinstaller/hook-dvc.system.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 import os
 
 if os.name == "nt":

dvc/__pyinstaller/hook-dvc.tree.gs.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 from PyInstaller.utils.hooks import copy_metadata  # pylint:disable=import-error
 
 datas = copy_metadata("google-cloud-storage")

dvc/__pyinstaller/hook-dvc.utils.flatten.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 from PyInstaller.utils.hooks import copy_metadata  # pylint:disable=import-error
 
 datas = copy_metadata("flatten-dict")

dvc/__pyinstaller/hook-dvc_task.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 # pylint: disable=import-error
 from PyInstaller.utils.hooks import collect_submodules
 

dvc/__pyinstaller/hook-fsspec.py

@@ -1 +1,2 @@
+# ruff: noqa: N999
 hiddenimports = ["fsspec.implementations.memory"]

dvc/__pyinstaller/hook-google_compute_engine.logger.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 from PyInstaller.utils.hooks import copy_metadata  # pylint:disable=import-error
 
 datas = copy_metadata("google-compute-engine")

dvc/__pyinstaller/hook-pydrive2.py

@@ -1,3 +1,4 @@
+# ruff: noqa: N999
 from PyInstaller.utils.hooks import copy_metadata  # pylint:disable=import-error
 
 datas = copy_metadata("pydrive2")

dvc/api/data.py

@@ -73,6 +73,7 @@ def open(  # noqa, pylint: disable=redefined-builtin
     remote: Optional[str] = None,
     mode: str = "r",
     encoding: Optional[str] = None,
+    config: Optional[Dict[str, Any]] = None,
 ):
     """
     Opens a file tracked in a DVC project.
@@ -114,6 +115,8 @@ def open(  # noqa, pylint: disable=redefined-builtin
             Defaults to None.
             This should only be used in text mode.
             Mirrors the namesake parameter in builtin `open()`_.
+        config(dict, optional): config to be passed to the DVC repository.
+            Defaults to None.
 
     Returns:
         _OpenContextManager: A context manager that generatse a corresponding
@@ -209,14 +212,24 @@ def open(  # noqa, pylint: disable=redefined-builtin
         "rev": rev,
         "mode": mode,
         "encoding": encoding,
+        "config": config,
     }
     return _OpenContextManager(_open, args, kwargs)
 
 
-def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    repo_kwargs: Dict[str, Any] = {"subrepos": True, "uninitialized": True}
+def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None, config=None):
     if remote:
-        repo_kwargs["config"] = {"core": {"remote": remote}}
+        if config is not None:
+            raise ValueError(
+                "can't specify both `remote` and `config` at the same time"
+            )
+        config = {"core": {"remote": remote}}
+
+    repo_kwargs: Dict[str, Any] = {
+        "subrepos": True,
+        "uninitialized": True,
+        "config": config,
+    }
 
     with Repo.open(repo, rev=rev, **repo_kwargs) as _repo:
         with _wrap_exceptions(_repo, path):
@@ -251,13 +264,19 @@ def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
                 raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
 
 
-def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
+def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None, config=None):
     """
     Returns the contents of a tracked file (by DVC or Git). For Git repos, HEAD
     is used unless a rev argument is supplied. The default remote is tried
     unless a remote argument is supplied.
     """
     with open(
-        path, repo=repo, rev=rev, remote=remote, mode=mode, encoding=encoding
+        path,
+        repo=repo,
+        rev=rev,
+        remote=remote,
+        mode=mode,
+        encoding=encoding,
+        config=config,
     ) as fd:
         return fd.read()

dvc/api/experiments.py

@@ -65,6 +65,7 @@ def exp_show(
     num: int = 1,
     param_deps: bool = False,
     force: bool = False,
+    config: Optional[Dict] = None,
 ) -> List[Dict]:
     """Get DVC experiments tracked in `repo`.
 
@@ -99,13 +100,15 @@ def exp_show(
             When `force` is specified, DVC will reload all experiment data and
             ignore any previously cached results.
             Defaults to `False`.
+        config (dict, optional): config to be passed through to DVC project.
+            Defaults to `None`.
 
     Returns:
         List[Dict]: Each item in the list will contain a dictionary with
             the info for an individual experiment.
             See Examples below.
     """
-    with Repo.open(repo) as _repo:
+    with Repo.open(repo, config=config) as _repo:
         experiments = _repo.experiments.show(
             revs=revs,
             num=num,

dvc/api/show.py

@@ -41,6 +41,7 @@ def metrics_show(
     *targets: str,
     repo: Optional[str] = None,
     rev: Optional[str] = None,
+    config: Optional[Dict] = None,
 ) -> Dict:
     """Get metrics tracked in `repo`.
 
@@ -69,6 +70,8 @@ def metrics_show(
             hash or a dvc experiment name.
             If `repo` is not a Git repo, this option is ignored.
             If `None`, the current working tree will be used.
+        config (dict, optional): config to be passed through to DVC project.
+            Defaults to `None`.
 
     Returns:
         Dict: See Examples below.
@@ -136,7 +139,7 @@ def metrics_show(
         https://git-scm.com/docs/revisions
     """
 
-    with Repo.open(repo) as _repo:
+    with Repo.open(repo, config=config) as _repo:
         metrics = _repo.metrics.show(
             targets=targets,
             revs=rev if rev is None else [rev],
@@ -157,6 +160,7 @@ def params_show(
     stages: Optional[Union[str, Iterable[str]]] = None,
     rev: Optional[str] = None,
     deps: bool = False,
+    config: Optional[Dict] = None,
 ) -> Dict:
     """Get parameters tracked in `repo`.
 
@@ -195,6 +199,8 @@ def params_show(
         deps (bool, optional): Whether to retrieve only parameters that are
             stage dependencies or not.
             Defaults to `False`.
+        config (dict, optional): config to be passed through to DVC project.
+            Defaults to `None`.
 
     Returns:
         Dict: See Examples below.
@@ -379,7 +385,7 @@ def params_show(
     if isinstance(stages, str):
         stages = [stages]
 
-    with Repo.open(repo) as _repo:
+    with Repo.open(repo, config=config) as _repo:
         params = _repo.params.show(
             revs=rev if rev is None else [rev],
             targets=targets,

dvc/commands/experiments/ls.py

@@ -25,7 +25,14 @@ class CmdExperimentsList(CmdBase):
 
         for baseline in exps:
             if not (name_only or sha_only):
-                ui.write(f"{baseline[:7]}:")
+                tag_base = "refs/tags/"
+                branch_base = "refs/heads/"
+                name = baseline[:7]
+                if baseline.startswith(tag_base):
+                    name = baseline[len(tag_base) :]
+                elif baseline.startswith(branch_base):
+                    name = baseline[len(branch_base) :]
+                ui.write(f"{name}:")
             for exp_name, rev in exps[baseline]:
                 if name_only:
                     ui.write(exp_name)

dvc/commands/experiments/pull.py

@@ -3,23 +3,13 @@ import logging
 
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.exceptions import InvalidArgumentError
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsPull(CmdBase):
-    def raise_error_if_all_disabled(self):
-        if not any([self.args.experiment, self.args.all_commits, self.args.rev]):
-            raise InvalidArgumentError(
-                "Either provide an `experiment` argument, or use the "
-                "`--rev` or `--all-commits` flag."
-            )
-
     def run(self):
-        self.raise_error_if_all_disabled()
-
         pulled_exps = self.repo.experiments.pull(
             self.args.git_remote,
             self.args.experiment,
@@ -61,7 +51,7 @@ def add_parser(experiments_subparsers, parent_parser):
         help=EXPERIMENTS_PULL_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    add_rev_selection_flags(experiments_pull_parser, "Pull", False)
+    add_rev_selection_flags(experiments_pull_parser, "Pull", True)
     experiments_pull_parser.add_argument(
         "-f",
         "--force",

dvc/commands/experiments/push.py

@@ -5,20 +5,12 @@ from typing import Any, Dict
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.exceptions import InvalidArgumentError
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsPush(CmdBase):
-    def raise_error_if_all_disabled(self):
-        if not any([self.args.experiment, self.args.all_commits, self.args.rev]):
-            raise InvalidArgumentError(
-                "Either provide an `experiment` argument, or use the "
-                "`--rev` or `--all-commits` flag."
-            )
-
     @staticmethod
     def log_result(result: Dict[str, Any], remote: str):
         from dvc.utils import humanize
@@ -59,8 +51,6 @@ class CmdExperimentsPush(CmdBase):
     def run(self):
         from dvc.repo.experiments.push import UploadError
 
-        self.raise_error_if_all_disabled()
-
         try:
             result = self.repo.experiments.push(
                 self.args.git_remote,
@@ -102,7 +92,7 @@ def add_parser(experiments_subparsers, parent_parser):
         help=EXPERIMENTS_PUSH_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    add_rev_selection_flags(experiments_push_parser, "Push", False)
+    add_rev_selection_flags(experiments_push_parser, "Push", True)
     experiments_push_parser.add_argument(
         "-f",
         "--force",

dvc/dagascii.py

@@ -209,6 +209,30 @@ def draw(vertices, edges):
     Args:
         vertices (list): list of graph vertices.
         edges (list): list of graph edges.
+
+    Returns:
+        str: ASCII representation
+
+    Example:
+        >>> from dvc.dagascii import draw
+        >>> vertices = [1, 2, 3, 4]
+        >>> edges = [(1, 2), (2, 3), (2, 4), (1, 4)]
+        >>> print(draw(vertices, edges))
+        +---+     +---+
+        | 3 |     | 4 |
+        +---+    *+---+
+          *    **   *
+          *  **     *
+          * *       *
+        +---+       *
+        | 2 |      *
+        +---+     *
+             *    *
+              *  *
+               **
+             +---+
+             | 1 |
+             +---+
     """
     # pylint: disable=too-many-locals
     # NOTE: coordinates might me negative, so we need to shift

dvc/fs/callbacks.py

@@ -2,12 +2,9 @@
 from contextlib import ExitStack
 from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Union
 
+from dvc.progress import Tqdm
 from dvc.utils.objects import cached_property
-from dvc_objects.fs.callbacks import (  # noqa: F401
-    DEFAULT_CALLBACK,
-    Callback,
-    TqdmCallback,
-)
+from dvc_objects.fs.callbacks import DEFAULT_CALLBACK, Callback  # noqa: F401
 
 if TYPE_CHECKING:
     from rich.progress import TaskID
@@ -15,6 +12,59 @@ if TYPE_CHECKING:
     from dvc.ui._rich_progress import RichTransferProgress
 
 
+# NOTE: this is very similar to dvc-objects.fs.callbacks.TqdmCallback,
+# but it works with our own Tqdm instance.
+class TqdmCallback(Callback):
+    def __init__(
+        self,
+        size: Optional[int] = None,
+        value: int = 0,
+        progress_bar: Optional["Tqdm"] = None,
+        **tqdm_kwargs,
+    ):
+        tqdm_kwargs["total"] = size or -1
+        self._tqdm_kwargs = tqdm_kwargs
+        self._progress_bar = progress_bar
+        self._stack = ExitStack()
+        super().__init__(size=size, value=value)
+
+    @cached_property
+    def progress_bar(self):
+        progress_bar = (
+            self._progress_bar
+            if self._progress_bar is not None
+            else Tqdm(**self._tqdm_kwargs)
+        )
+        return self._stack.enter_context(progress_bar)
+
+    def __enter__(self):
+        return self
+
+    def close(self):
+        self._stack.close()
+
+    def set_size(self, size):
+        # Tqdm tries to be smart when to refresh,
+        # so we try to force it to re-render.
+        super().set_size(size)
+        self.progress_bar.refresh()
+
+    def call(self, hook_name=None, **kwargs):  # noqa: ARG002
+        self.progress_bar.update_to(self.value, total=self.size)
+
+    def branch(
+        self,
+        path_1: "Union[str, BinaryIO]",
+        path_2: str,
+        kwargs: Dict[str, Any],
+        child: Optional[Callback] = None,
+    ):
+        child = child or TqdmCallback(
+            bytes=True, desc=path_1 if isinstance(path_1, str) else path_2
+        )
+        return super().branch(path_1, path_2, kwargs, child=child)
+
+
 class RichCallback(Callback):
     def __init__(
         self,

dvc/output.py

@@ -20,6 +20,7 @@ from dvc.exceptions import (
     DvcException,
     MergeError,
 )
+from dvc.utils import format_link
 from dvc.utils.objects import cached_property
 from dvc_data.hashfile import check as ocheck
 from dvc_data.hashfile import load as oload
@@ -510,7 +511,7 @@ class Output:
             return False
 
         return self.repo and self.fs.path.isin(
-            self.fs.path.realpath(self.fs_path),
+            self.fs_path,
             self.repo.root_dir,
         )
 
@@ -678,6 +679,14 @@ class Output:
     # pylint: enable=no-member
 
     def save(self) -> None:
+        if self.use_cache and not self.is_in_repo:
+            raise DvcException(
+                f"Saving cached external output {str(self)} is not supported "
+                "since DVC 3.0. See "
+                f"{format_link('https://dvc.org/doc/user-guide/upgrade')} "
+                "for more info."
+            )
+
         if not self.exists:
             raise self.DoesNotExistError(self)
 
@@ -692,8 +701,7 @@ class Output:
         if self.metric:
             self.verify_metric()
 
-        if self.hash_name == "md5-dos2unix":
-            self.hash_name = "md5"
+        self._update_legacy_hash_name()
         if self.use_cache:
             _, self.meta, self.obj = build(
                 self.cache,
@@ -717,6 +725,10 @@ class Output:
         self.hash_info = self.obj.hash_info
         self.files = None
 
+    def _update_legacy_hash_name(self):
+        if self.hash_name == "md5-dos2unix" and self.changed_checksum():
+            self.hash_name = "md5"
+
     def set_exec(self) -> None:
         if self.isfile() and self.meta.isexec:
             self.cache.set_exec(self.fs_path)
@@ -965,7 +977,10 @@ class Output:
         return added, False if added else modified
 
     def remove(self, ignore_remove=False):
-        self.fs.remove(self.fs_path, recursive=True)
+        try:
+            self.fs.remove(self.fs_path, recursive=True)
+        except FileNotFoundError:
+            pass
         if self.protocol != Schemes.LOCAL:
             return
 
@@ -1264,7 +1279,7 @@ class Output:
             raise FileNotFoundError(  # noqa: B904
                 errno.ENOENT,
                 os.strerror(errno.ENOENT),
-                self.fs.path.relpath(path),
+                path,
             )
 
         new = tree.from_trie(trie)
@@ -1335,6 +1350,7 @@ class Output:
             )
 
         assert self.repo
+        self._update_legacy_hash_name()
         cache = self.cache if self.use_cache else self.local_cache
         assert isinstance(cache, HashFileDB)
 
@@ -1386,8 +1402,7 @@ class Output:
         otransfer(staging, self.cache, {obj.hash_info}, hardlink=relink, shallow=False)
 
         if relink:
-            rel = self.fs.path.relpath(path)
-            with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as callback:
+            with CheckoutCallback(desc="Checking out {path}", unit="files") as callback:
                 self._checkout(
                     path,
                     self.fs,

dvc/parsing/__init__.py

@@ -232,6 +232,18 @@ class EntryDefinition:
         self.definition = definition
         self.where = where
 
+    def _resolve_wdir(
+        self, context: Context, name: str, wdir: Optional[str] = None
+    ) -> str:
+        if not wdir:
+            return self.wdir
+
+        try:
+            wdir = to_str(context.resolve_str(wdir))
+        except (ContextError, ParseError) as exc:
+            format_and_raise(exc, f"'{self.where}.{name}.wdir'", self.relpath)
+        return self.resolver.fs.path.join(self.wdir, wdir)
+
     def resolve(self, **kwargs):
         try:
             return self.resolve_stage(**kwargs)
@@ -249,7 +261,33 @@ class EntryDefinition:
 
         # we need to pop vars from generated/evaluated data
         definition = deepcopy(self.definition)
+
+        wdir = self._resolve_wdir(context, name, definition.get(WDIR_KWD))
+        vars_ = definition.pop(VARS_KWD, [])
+        # FIXME: Should `vars` be templatized?
+        check_interpolations(vars_, f"{self.where}.{name}.vars", self.relpath)
+        if vars_:
+            # Optimization: Lookahead if it has any vars, if it does not, we
+            # don't need to clone them.
+            context = Context.clone(context)
+
+        try:
+            fs = self.resolver.fs
+            context.load_from_vars(fs, vars_, wdir, stage_name=name)
+        except VarsAlreadyLoaded as exc:
+            format_and_raise(exc, f"'{self.where}.{name}.vars'", self.relpath)
+
+        logger.trace(  # type: ignore[attr-defined]
+            "Context during resolution of stage %s:\n%s", name, context
+        )
+
         with context.track() as tracked_data:
+            # NOTE: we do not pop "wdir", and resolve it again
+            # this does not affect anything and is done to try to
+            # track the source of `wdir` interpolation.
+            # This works because of the side-effect that we do not
+            # allow overwriting and/or str interpolating complex objects.
+            # Fix if/when those assumptions are no longer valid.
             resolved = {
                 key: self._resolve(context, value, key, skip_checks)
                 for key, value in definition.items()

dvc/progress.py

@@ -2,12 +2,16 @@
 import logging
 import sys
 from threading import RLock
+from typing import TYPE_CHECKING
 
 from tqdm import tqdm
 
 from dvc.env import DVC_IGNORE_ISATTY
 from dvc.utils import env2bool
 
+if TYPE_CHECKING:
+    from dvc.fs.callbacks import TqdmCallback
+
 logger = logging.getLogger(__name__)
 tqdm.set_lock(RLock())
 
@@ -158,3 +162,8 @@ class Tqdm(tqdm):
             d["ncols_desc"] = d["ncols_info"] = 1
             d["prefix"] = ""
         return d
+
+    def as_callback(self) -> "TqdmCallback":
+        from dvc.fs.callbacks import TqdmCallback
+
+        return TqdmCallback(progress_bar=self)

dvc/repo/checkout.py

@@ -1,12 +1,22 @@
 import logging
 import os
-from typing import Dict, List
+from typing import TYPE_CHECKING, Dict, List
 
-from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit, NoOutputOrStageError
+from dvc.exceptions import (
+    CheckoutError,
+    CheckoutErrorSuggestGit,
+    DvcException,
+    NoOutputOrStageError,
+)
+from dvc.ui import ui
 from dvc.utils import relpath
 
 from . import locked
 
+if TYPE_CHECKING:
+    from dvc_data.index import BaseDataIndex, DataIndexEntry
+    from dvc_objects.fs.base import FileSystem
+
 logger = logging.getLogger(__name__)
 
 
@@ -59,6 +69,33 @@ def _build_out_changes(index, changes):
     return out_changes
 
 
+def _check_can_delete(
+    entries: List["DataIndexEntry"],
+    index: "BaseDataIndex",
+    path: str,
+    fs: "FileSystem",
+):
+    entry_paths = []
+    for entry in entries:
+        try:
+            cache_fs, cache_path = index.storage_map.get_cache(entry)
+        except ValueError:
+            continue
+
+        if cache_fs.exists(cache_path):
+            continue
+
+        entry_paths.append(fs.path.join(path, *entry.key))
+
+    if not entry_paths:
+        return
+
+    raise DvcException(
+        "Can't remove the following unsaved files without confirmation. "
+        "Use `--force` to force.\n" + "\n".join(entry_paths)
+    )
+
+
 @locked
 def checkout(  # noqa: C901
     self,
@@ -70,13 +107,9 @@ def checkout(  # noqa: C901
     allow_missing=False,
     **kwargs,
 ):
-    from dvc import prompt
-    from dvc.fs.callbacks import Callback
     from dvc.repo.index import build_data_index
     from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError
-    from dvc_data.hashfile.checkout import CheckoutError as IndexCheckoutError
-    from dvc_data.index.checkout import ADD, DELETE, MODIFY
-    from dvc_data.index.checkout import checkout as icheckout
+    from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare
 
     stats: Dict[str, List[str]] = {
         "added": [],
@@ -109,45 +142,69 @@ def checkout(  # noqa: C901
         onerror=onerror,
     )
 
-    with Callback.as_tqdm_callback(
+    with ui.progress(
         unit="entry",
         desc="Building data index",
-    ) as cb:
+    ) as pb:
         old = build_data_index(
-            view, self.root_dir, self.fs, compute_hash=True, callback=cb
+            view,
+            self.root_dir,
+            self.fs,
+            compute_hash=True,
+            callback=pb.as_callback(),
         )
 
     new = view.data["repo"]
 
-    with Callback.as_tqdm_callback(
+    with ui.progress(
+        desc="Comparing indexes",
+        unit="entry",
+    ) as pb:
+        diff = compare(old, new, relink=relink, delete=True, callback=pb.as_callback())
+
+    if not force:
+        _check_can_delete(diff.files_delete, new, self.root_dir, self.fs)
+
+    failed = set()
+    out_paths = [out.fs_path for out in view.outs if out.use_cache and out.is_in_repo]
+
+    def checkout_onerror(src_path, dest_path, _exc):
+        logger.debug(
+            "failed to create '%s' from '%s'", dest_path, src_path, exc_info=True
+        )
+
+        for out_path in out_paths:
+            if self.fs.path.isin_or_eq(dest_path, out_path):
+                failed.add(out_path)
+
+    with ui.progress(
         unit="file",
         desc="Checkout",
-    ) as cb:
-        try:
-            changes = icheckout(
-                new,
-                self.root_dir,
-                self.fs,
-                old=old,
-                callback=cb,
-                delete=True,
-                prompt=prompt.confirm,
-                update_meta=False,
-                relink=relink,
-                force=force,
-                allow_missing=allow_missing,
-                state=self.state,
-                **kwargs,
-            )
-        except IndexCheckoutError as exc:
-            raise CheckoutError(exc.paths, {}) from exc
+    ) as pb:
+        apply(
+            diff,
+            self.root_dir,
+            self.fs,
+            callback=pb.as_callback(),
+            update_meta=False,
+            onerror=checkout_onerror,
+            state=self.state,
+            **kwargs,
+        )
 
-    out_changes = _build_out_changes(view, changes)
+    out_changes = _build_out_changes(view, diff.changes)
 
     typ_map = {ADD: "added", DELETE: "deleted", MODIFY: "modified"}
     for key, typ in out_changes.items():
         out_path = self.fs.path.join(self.root_dir, *key)
-        self.state.save_link(out_path, self.fs)
-        stats[typ_map[typ]].append(_fspath_dir(out_path))
+
+        if out_path in failed:
+            self.fs.remove(out_path, recursive=True)
+        else:
+            self.state.save_link(out_path, self.fs)
+            stats[typ_map[typ]].append(_fspath_dir(out_path))
+
+    if failed and not allow_missing:
+        raise CheckoutError([relpath(out_path) for out_path in failed], stats)
 
     return stats

dvc/repo/data.py

@@ -2,9 +2,11 @@ import os
 import posixpath
 from typing import TYPE_CHECKING, Any, Dict, Iterable, List, TypedDict, Union
 
+from dvc.fs.callbacks import DEFAULT_CALLBACK
 from dvc.ui import ui
 
 if TYPE_CHECKING:
+    from dvc.fs.callbacks import Callback
     from dvc.repo import Repo
     from dvc.scm import Git, NoSCM
     from dvc_data.index import DataIndex
@@ -50,6 +52,7 @@ def _diff(
     not_in_cache: bool = False,
     not_in_remote: bool = False,
     remote_refresh: bool = False,
+    callback: "Callback" = DEFAULT_CALLBACK,
 ) -> Dict[str, List[str]]:
     from dvc_data.index import StorageError
     from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff
@@ -70,6 +73,7 @@ def _diff(
         shallow=not granular,
         hash_only=True,
         with_unknown=True,
+        callback=callback,
     ):
         if (
             change.typ == UNCHANGED
@@ -149,16 +153,27 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn
 def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
     from .index import build_data_index
 
-    with ui.status("Building workspace index"):
+    with ui.progress(
+        desc="Building workspace index",
+        unit="entry",
+    ) as pb:
         workspace = build_data_index(
-            repo.index, repo.root_dir, repo.fs, compute_hash=True
+            repo.index,
+            repo.root_dir,
+            repo.fs,
+            compute_hash=True,
+            callback=pb.as_callback(),
         )
 
-    with ui.status("Calculating diff between index/workspace"):
+    with ui.progress(
+        desc="Calculating diff between index/workspace",
+        unit="entry",
+    ) as pb:
         return _diff(
             repo.index.data["repo"],
             workspace,
             not_in_cache=True,
+            callback=pb.as_callback(),
             **kwargs,
         )
 
@@ -171,8 +186,11 @@ def _diff_head_to_index(
     with repo.switch(head):
         head_index = repo.index.data["repo"]
 
-    with ui.status("Calculating diff between head/index"):
-        return _diff(head_index, index, **kwargs)
+    with ui.progress(
+        desc="Calculating diff between head/index",
+        unit="entry",
+    ) as pb:
+        return _diff(head_index, index, callback=pb.as_callback(), **kwargs)
 
 
 class Status(TypedDict):

dvc/repo/diff.py

@@ -156,8 +156,13 @@ def diff(
         for target in old_missing & new_missing:
             raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), target)
 
-    old = indexes[a_rev]
-    new = indexes[b_rev]
+    if len(indexes.keys()) == 1:
+        # both a_rev and b_rev point to the same sha, nothing to compare
+        old = None
+        new = None
+    else:
+        old = indexes[a_rev]
+        new = indexes[b_rev]
 
     with ui.status("Calculating diff"):
         return _diff(old, new, with_missing=with_missing)

dvc/repo/experiments/collect.py

@@ -10,6 +10,7 @@ from typing import (
     Iterator,
     List,
     Optional,
+    Tuple,
     Union,
 )
 
@@ -369,12 +370,13 @@ def _describe(
 
 
 def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> List["ExpRange"]:
-    """Return list of ExpRange sorted by timestamp."""
+    """Return list of ExpRange sorted by (timestamp, rev)."""
 
-    def _head_timestamp(exp_range: "ExpRange") -> datetime:
+    def _head_timestamp(exp_range: "ExpRange") -> Tuple[datetime, str]:
         head_exp = first(exp_range.revs)
         if head_exp and head_exp.data and head_exp.data.timestamp:
-            return head_exp.data.timestamp
-        return datetime.fromtimestamp(0)
+            return head_exp.data.timestamp, head_exp.rev
+
+        return datetime.fromtimestamp(0), ""
 
     return sorted(exp_ranges, key=_head_timestamp, reverse=True)

dvc/repo/experiments/executor/base.py

@@ -459,25 +459,16 @@ class BaseExecutor(ABC):
             should force overwrite any existing duplicates.
         """
         from dvc.repo.checkout import checkout as dvc_checkout
-        from dvc.repo.reproduce import reproduce as dvc_reproduce
-        from dvc.stage import PipelineStage
         from dvc.ui import ui
 
         auto_push = env2bool(DVC_EXP_AUTO_PUSH)
         git_remote = os.getenv(DVC_EXP_GIT_REMOTE, None)
 
-        unchanged = []
-
         if queue is not None:
             queue.put((rev, os.getpid()))
         if log_errors and log_level is not None:
             cls._set_log_level(log_level)
 
-        def filter_pipeline(stages):
-            unchanged.extend(
-                [stage for stage in stages if isinstance(stage, PipelineStage)]
-            )
-
         exp_hash: Optional[str] = None
         exp_ref: Optional["ExpRefInfo"] = None
         repro_force: bool = False
@@ -515,17 +506,11 @@ class BaseExecutor(ABC):
                     targets=targets,
                     with_deps=targets is not None,
                     force=True,
-                    quiet=True,
                     allow_missing=True,
                     recursive=kwargs.get("recursive", False),
                 )
 
-            stages = dvc_reproduce(
-                dvc,
-                *args,
-                on_unchanged=filter_pipeline,
-                **kwargs,
-            )
+            stages = dvc.reproduce(*args, **kwargs)
             if paths := cls._get_top_level_paths(dvc):
                 logger.debug("Staging top-level files: %s", paths)
                 dvc.scm_context.add(paths)

dvc/repo/experiments/executor/ssh.py

@@ -210,8 +210,7 @@ class SSHExecutor(BaseExecutor):
         """Collect DVC cache."""
         from dvc.repo.experiments.pull import _pull_cache
 
-        with self.get_odb() as odb:
-            _pull_cache(repo, exp_ref, run_cache=run_cache, odb=odb)
+        _pull_cache(repo, exp_ref, run_cache=run_cache)
 
     @contextmanager
     def get_odb(self):

dvc/repo/experiments/ls.py

@@ -42,7 +42,7 @@ def ls(
     for baseline in ref_info_dict:
         name = baseline
         if tags[baseline] or ref_heads[baseline]:
-            name = tags[baseline] or ref_heads[baseline][len(base) + 1 :]
+            name = tags[baseline] or ref_heads[baseline]
         for info in ref_info_dict[baseline]:
             if git_remote:
                 exp_rev = None

dvc/repo/experiments/pull.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 def pull(  # noqa: C901
     repo,
     git_remote: str,
-    exp_names: Union[Iterable[str], str],
+    exp_names: Optional[Union[Iterable[str], str]] = None,
     all_commits=False,
     rev: Optional[Union[List[str], str]] = None,
     num=1,
@@ -32,30 +32,30 @@ def pull(  # noqa: C901
     exp_ref_set: Set["ExpRefInfo"] = set()
     if all_commits:
         exp_ref_set.update(exp_refs(repo.scm, git_remote))
+    elif exp_names:
+        if isinstance(exp_names, str):
+            exp_names = [exp_names]
+        exp_ref_dict = resolve_name(repo.scm, exp_names, git_remote)
+
+        unresolved_exp_names = []
+        for exp_name, exp_ref in exp_ref_dict.items():
+            if exp_ref is None:
+                unresolved_exp_names.append(exp_name)
+            else:
+                exp_ref_set.add(exp_ref)
+
+        if unresolved_exp_names:
+            raise UnresolvedExpNamesError(unresolved_exp_names)
+
     else:
-        if exp_names:
-            if isinstance(exp_names, str):
-                exp_names = [exp_names]
-            exp_ref_dict = resolve_name(repo.scm, exp_names, git_remote)
-
-            unresolved_exp_names = []
-            for exp_name, exp_ref in exp_ref_dict.items():
-                if exp_ref is None:
-                    unresolved_exp_names.append(exp_name)
-                else:
-                    exp_ref_set.add(exp_ref)
-
-            if unresolved_exp_names:
-                raise UnresolvedExpNamesError(unresolved_exp_names)
-
-        if rev:
-            if isinstance(rev, str):
-                rev = [rev]
-            rev_dict = iter_revs(repo.scm, rev, num)
-            rev_set = set(rev_dict.keys())
-            ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
-            for _, ref_info_list in ref_info_dict.items():
-                exp_ref_set.update(ref_info_list)
+        rev = rev or "HEAD"
+        if isinstance(rev, str):
+            rev = [rev]
+        rev_dict = iter_revs(repo.scm, rev, num)
+        rev_set = set(rev_dict.keys())
+        ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
+        for _, ref_info_list in ref_info_dict.items():
+            exp_ref_set.update(ref_info_list)
 
     pull_result = _pull(repo, git_remote, exp_ref_set, force)
 
@@ -107,10 +107,9 @@ def _pull_cache(
     dvc_remote=None,
     jobs=None,
     run_cache=False,
-    odb=None,
 ):
     if isinstance(refs, ExpRefInfo):
         refs = [refs]
     revs = list(exp_commits(repo.scm, refs))
     logger.debug("dvc fetch experiment '%s'", refs)
-    repo.fetch(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs, odb=odb)
+    repo.fetch(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs)

dvc/repo/experiments/push.py

@@ -97,7 +97,7 @@ def exp_refs_from_rev(scm: "Git", rev: List[str], num: int = 1) -> Set["ExpRefIn
 def push(
     repo: "Repo",
     git_remote: str,
-    exp_names: Union[List[str], str],
+    exp_names: Optional[Union[List[str], str]] = None,
     all_commits: bool = False,
     rev: Optional[Union[List[str], str]] = None,
     num: int = 1,
@@ -111,7 +111,8 @@ def push(
         exp_ref_set.update(exp_refs(repo.scm))
     if exp_names:
         exp_ref_set.update(exp_refs_from_names(repo.scm, ensure_list(exp_names)))
-    if rev:
+    else:
+        rev = rev or "HEAD"
         if isinstance(rev, str):
             rev = [rev]
         exp_ref_set.update(exp_refs_from_rev(repo.scm, rev, num=num))

dvc/repo/experiments/serialize.py

@@ -60,6 +60,7 @@ class SerializableExp:
         # supported in dataclasses.asdict() on all python releases
         # see https://bugs.python.org/issue35540
         params = dict(_gather_params(repo, deps=param_deps, onerror=onerror))
+        params = {k: dict(v) for k, v in params.items()}
         metrics = dict(
             _gather_metrics(
                 repo,

dvc/repo/fetch.py

@@ -36,6 +36,7 @@ def fetch(  # noqa: C901, PLR0913
     from fsspec.utils import tokenize
 
     from dvc.fs.callbacks import Callback
+    from dvc_data.index.fetch import collect
     from dvc_data.index.fetch import fetch as ifetch
 
     if isinstance(targets, str):
@@ -76,17 +77,27 @@ def fetch(  # noqa: C901, PLR0913
 
     cache_key = ("fetch", tokenize(sorted(index_keys)))
 
+    with Callback.as_tqdm_callback(
+        desc="Collecting",
+        unit="entry",
+    ) as cb:
+        data = collect(
+            indexes, cache_index=self.data_index, cache_key=cache_key, callback=cb
+        )
+
     with Callback.as_tqdm_callback(
         desc="Fetching",
         unit="file",
     ) as cb:
-        fetch_transferred, fetch_failed = ifetch(
-            indexes,
-            jobs=jobs,
-            callback=cb,
-            cache_index=self.data_index,
-            cache_key=cache_key,
-        )  # pylint: disable=assignment-from-no-return
+        try:
+            fetch_transferred, fetch_failed = ifetch(
+                data,
+                jobs=jobs,
+                callback=cb,
+            )  # pylint: disable=assignment-from-no-return
+        finally:
+            for fs_index in data.values():
+                fs_index.close()
 
     if fetch_transferred:
         # NOTE: dropping cached index to force reloading from newly saved cache

dvc/repo/imports.py

@@ -100,7 +100,8 @@ def save_imports(
         Objects which were downloaded from source location.
     """
     from dvc.stage.exceptions import DataSourceChanged
-    from dvc_data.index import checkout, md5, save
+    from dvc_data.index import md5, save
+    from dvc_data.index.checkout import apply, compare
     from dvc_objects.fs.callbacks import Callback
 
     downloaded: Set["HashInfo"] = set()
@@ -124,7 +125,8 @@ def save_imports(
                     desc="Downloading imports from source",
                     unit="files",
                 ) as cb:
-                    checkout(data_view, tmpdir, cache.fs, callback=cb, storage="remote")
+                    diff = compare(None, data_view)
+                    apply(diff, tmpdir, cache.fs, callback=cb, storage="remote")
                 md5(data_view, name=cache.hash_name)
                 save(data_view, odb=cache, hardlink=True)
 

dvc/repo/reproduce.py

@@ -122,7 +122,6 @@ def _reproduce_stages(  # noqa: C901
     stages,
     downstream=False,
     single_item=False,
-    on_unchanged=None,
     **kwargs,
 ):
     r"""Derive the evaluation of the given node for the given graph.
@@ -175,24 +174,20 @@ def _reproduce_stages(  # noqa: C901
         try:
             kwargs["upstream"] = result + unchanged
             ret = _reproduce_stage(stage, **kwargs)
-
-            if len(ret) == 0:
+            if not ret:
                 unchanged.extend([stage])
-            elif force_downstream:
+                continue
+
+            result.extend(ret)
+            if force_downstream:
                 # NOTE: we are walking our pipeline from the top to the
                 # bottom. If one stage is changed, it will be reproduced,
                 # which tells us that we should force reproducing all of
                 # the other stages down below, even if their direct
                 # dependencies didn't change.
                 kwargs["force"] = True
-
-            if ret:
-                result.extend(ret)
         except Exception as exc:  # noqa: BLE001
             raise ReproductionError(stage.addressing) from exc
-
-    if on_unchanged is not None:
-        on_unchanged(unchanged)
     return result
 
 
@@ -233,7 +228,3 @@ def _get_steps(graph, stages, downstream, single_item):
             steps.append(stage)
 
     return steps
-
-
-def _repro_callback(experiments_callback, unchanged, stages):
-    experiments_callback(unchanged, stages)

dvc/repo/worktree.py

@@ -112,8 +112,7 @@ def push_worktree(
     **kwargs: Any,
 ) -> int:
     from dvc.repo.index import build_data_index
-    from dvc_data.index import checkout
-    from dvc_data.index.checkout import VersioningNotSupported
+    from dvc_data.index.checkout import VersioningNotSupported, apply, compare
 
     pushed = 0
     stages: Set["Stage"] = set()
@@ -139,6 +138,18 @@ def push_worktree(
         else:
             diff_kwargs = {}
 
+        with Callback.as_tqdm_callback(
+            unit="entry",
+            desc=f"Comparing indexes for remote {remote_obj.name!r}",
+        ) as cb:
+            diff = compare(
+                old_index,
+                new_index,
+                callback=cb,
+                delete=remote_obj.worktree,
+                **diff_kwargs,
+            )
+
         total = len(new_index)
         with Callback.as_tqdm_callback(
             unit="file",
@@ -147,18 +158,15 @@ def push_worktree(
         ) as cb:
             cb.set_size(total)
             try:
-                stats = checkout(
-                    new_index,
+                apply(
+                    diff,
                     remote_obj.path,
                     remote_obj.fs,
-                    old=old_index,
-                    delete=remote_obj.worktree,
                     callback=cb,
                     latest_only=remote_obj.worktree,
                     jobs=jobs,
-                    **diff_kwargs,
                 )
-                pushed += sum(len(changes) for changes in stats.values())
+                pushed += sum(len(changes) for changes in diff.changes.values())
             except VersioningNotSupported:
                 logger.exception("")
                 raise DvcException(
@@ -316,23 +324,33 @@ def _fetch_out_changes(
     remote_index: Union["DataIndex", "DataIndexView"],
     remote: "Remote",
 ):
-    from dvc_data.index import checkout
+    from dvc_data.index.checkout import apply, compare
 
     old, new = _get_diff_indexes(out, local_index, remote_index)
+
+    with Callback.as_tqdm_callback(
+        unit="entry",
+        desc="Comparing indexes",
+    ) as cb:
+        diff = compare(
+            old,
+            new,
+            delete=True,
+            meta_only=True,
+            meta_cmp_key=partial(_meta_checksum, remote.fs),
+            callback=cb,
+        )
+
     total = len(new)
     with Callback.as_tqdm_callback(
         unit="file", desc=f"Updating '{out}'", disable=total == 0
     ) as cb:
         cb.set_size(total)
-        checkout(
-            new,
+        apply(
+            diff,
             out.repo.root_dir,
             out.fs,
-            old=old,
-            delete=True,
             update_meta=False,
-            meta_only=True,
-            meta_cmp_key=partial(_meta_checksum, remote.fs),
             storage="data",
             callback=cb,
         )

dvc/schema.py

@@ -53,6 +53,7 @@ OUT_PSTAGE_DETAILED_SCHEMA = {
         **ANNOTATION_SCHEMA,  # type: ignore[arg-type]
         Output.PARAM_CACHE: bool,
         Output.PARAM_PERSIST: bool,
+        "checkpoint": bool,
         Output.PARAM_REMOTE: str,
         Output.PARAM_PUSH: bool,
     }
@@ -73,12 +74,14 @@ PLOT_PSTAGE_SCHEMA = {str: Any(PLOT_PROPS_SCHEMA, [PLOT_PROPS_SCHEMA])}
 
 PARAM_PSTAGE_NON_DEFAULT_SCHEMA = {str: [str]}
 
+VARS_SCHEMA = [str, dict]
 
 STAGE_DEFINITION = {
     Required(StageParams.PARAM_CMD): Any(str, list),
     Optional(StageParams.PARAM_WDIR): str,
     Optional(StageParams.PARAM_DEPS): [str],
     Optional(StageParams.PARAM_PARAMS): [Any(str, dict)],
+    Optional(VARS_KWD): VARS_SCHEMA,
     Optional(StageParams.PARAM_FROZEN): bool,
     Optional(StageParams.PARAM_META): object,
     Optional(StageParams.PARAM_DESC): str,
@@ -120,7 +123,7 @@ SINGLE_PIPELINE_STAGE_SCHEMA = {
 MULTI_STAGE_SCHEMA = {
     PLOTS: [Any(str, SINGLE_PLOT_SCHEMA)],
     STAGES: SINGLE_PIPELINE_STAGE_SCHEMA,
-    VARS_KWD: [str, dict],
+    VARS_KWD: VARS_SCHEMA,
     StageParams.PARAM_PARAMS: [str],
     StageParams.PARAM_METRICS: [str],
     ARTIFACTS: SINGLE_ARTIFACT_SCHEMA,

dvc/stage/cache.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List, Optional, Tuple
 from funcy import first
 
 from dvc import fs
-from dvc.exceptions import DvcException
+from dvc.exceptions import CollectCacheError, DvcException
 from dvc.utils import dict_sha256, relpath
 
 if TYPE_CHECKING:
@@ -179,7 +179,7 @@ class StageCache:
         dump_yaml(tmp, cache)
         self.repo.cache.legacy.move(tmp, path)
 
-    def restore(self, stage, run_cache=True, pull=False, dry=False):
+    def restore(self, stage, run_cache=True, pull=False, dry=False):  # noqa: C901
         from .serialize import to_single_stage_lockfile
 
         if not _can_hash(stage):
@@ -203,8 +203,11 @@ class StageCache:
         cached_stage = self._create_stage(cache, wdir=stage.wdir)
 
         if pull and not dry:
-            for objs in cached_stage.get_used_objs().values():
-                self.repo.cloud.pull(objs)
+            try:
+                for objs in cached_stage.get_used_objs().values():
+                    self.repo.cloud.pull(objs)
+            except CollectCacheError as exc:
+                raise RunCacheNotFoundError(stage) from exc
 
         if not cached_stage.outs_cached():
             raise RunCacheNotFoundError(stage)

dvc/testing/benchmarks/cli/commands/test_exp_show.py

@@ -2,10 +2,10 @@
 import pytest
 
 
-@pytest.mark.requires(minversion=(2, 10, 0))
+@pytest.mark.requires(minversion=(2, 28, 0))
 def test_exp_show(make_project, monkeypatch, bench_dvc, dvc_bin):
-    url = "https://github.com/efiop/lstm_seq2seq"
-    rev = "dvc-bench"
+    url = "https://github.com/iterative/example-get-started"
+    rev = "main"
     path = make_project(url, rev=rev)
     monkeypatch.chdir(path)
 

dvc/testing/benchmarks/cli/commands/test_update.py

@@ -3,7 +3,7 @@ def test_update(bench_dvc, tmp_dir, dvc, make_dataset):
     dataset = make_dataset(
         cache=False, files=True, dvcfile=False, commit=False, remote=False
     )
-    dvc.imp_url(str(dataset), "new")
+    bench_dvc("import-url", str(dataset), "new")
     (dataset / "new").write_text("new")
     bench_dvc("update", "new.dvc")
     bench_dvc("update", "new.dvc", name="noop")

dvc/testing/benchmarks/cli/stories/test_modify_data.py

@@ -22,8 +22,8 @@ def test_partial_add(bench_dvc, tmp_dir, dvc, dataset, remote):
             shutil.move(dataset / f, tmp_dir / "partial-copy" / f)
 
     # Add/push partial dataset
-    dvc.add(str(dataset))
-    dvc.push()
+    bench_dvc("add", dataset)
+    bench_dvc("push")
 
     # Add more files to the dataset
     shutil.copytree("partial-copy", dataset, dirs_exist_ok=True)
@@ -39,8 +39,8 @@ def test_partial_add(bench_dvc, tmp_dir, dvc, dataset, remote):
 def test_partial_remove(bench_dvc, tmp_dir, dvc, dataset, remote):
     random.seed(5232)
     # Add/push full dataset
-    dvc.add(str(dataset))
-    dvc.push()
+    bench_dvc("add", dataset)
+    bench_dvc("push")
 
     # Remove some files
     for f in glob.glob("*", root_dir=dataset, recursive=True):  # type: ignore[call-arg]

pyproject.toml

@@ -34,13 +34,14 @@ dependencies = [
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=1.11.0,<1.12.0",
+    "dvc-data>=2.3.2,<2.4.0",
     "dvc-http>=2.29.0",
     "dvc-render>=0.3.1,<1",
     "dvc-studio-client>=0.9.2,<1",
     "dvc-task>=0.3.0,<1",
     "flatten_dict<1,>=0.4.1",
-    "flufl.lock>=5",
+    # https://github.com/iterative/dvc/issues/9654
+    "flufl.lock>=5,<8",
     "funcy>=1.14",
     "grandalf<1,>=0.7",
     "hydra-core>=1.1",
@@ -56,7 +57,7 @@ dependencies = [
     "requests>=2.22",
     "rich>=12",
     "ruamel.yaml>=0.17.11",
-    "scmrepo>=1.0.0,<2",
+    "scmrepo>=1.0.4,<2",
     "shortuuid>=0.5",
     "shtab<2,>=1.3.4",
     "tabulate>=0.8.7",
@@ -70,7 +71,7 @@ dependencies = [
 all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"]
 azure = ["dvc-azure>=2.21.2"]
 dev = ["dvc[azure,gdrive,gs,hdfs,lint,oss,s3,ssh,terraform,tests,webdav,webhdfs]"]
-gdrive = ["dvc-gdrive==2.19.2"]
+gdrive = ["dvc-gdrive==2.20"]
 gs = ["dvc-gs==2.22.1"]
 hdfs = ["dvc-hdfs==2.19"]
 lint = [
@@ -82,6 +83,7 @@ lint = [
     "types-tabulate",
     "types-toml",
     "types-tqdm",
+    "types-pyinstaller",
 ]
 oss = ["dvc-oss==2.19"]
 s3 = ["dvc-s3==2.23.0"]
@@ -128,6 +130,10 @@ dvc = "dvc.api:DVCFileSystem"
 [project.entry-points."pytest11"]
 dvc-testing = "dvc.testing.plugin"
 
+[project.entry-points."pyinstaller40"]
+hook-dirs = "dvc.__pyinstaller:get_hook_dirs"
+tests = "dvc.__pyinstaller:get_PyInstaller_tests"
+
 [tool.setuptools]
 license-files = ["LICENSE"]
 

scripts/build-requirements.txt

@@ -1,3 +0,0 @@
-# NOTE: using strict version due to
-# https://github.com/iterative/dvc/issues/7949
-PyInstaller==5.9.0

scripts/build.py

@@ -1,88 +0,0 @@
-import argparse
-import pathlib
-import sys
-from subprocess import STDOUT, check_call
-
-path = pathlib.Path(__file__).parent.absolute()
-dvc = path.parent / "dvc"
-pyinstaller = path / "pyinstaller"
-innosetup = path / "innosetup"
-fpm = path / "fpm"
-
-parser = argparse.ArgumentParser()
-parser.add_argument("pkg", choices=["deb", "rpm", "osxpkg", "exe"], help="package type")
-parser.add_argument("--sign-application", default=False, action="store_true")
-parser.add_argument("--application-id")
-parser.add_argument("--sign-installer", default=False, action="store_true")
-parser.add_argument("--installer-id")
-parser.add_argument("--notarize", default=False, action="store_true")
-parser.add_argument("--apple-id-username")
-parser.add_argument("--apple-id-password")
-args = parser.parse_args()
-
-(dvc / "_build.py").write_text(f'PKG = "{args.pkg}"')
-
-if not (dvc / "_dvc_version.py").exists():
-    raise Exception("no version info found")
-
-check_call(
-    ["python", "build.py"],
-    cwd=pyinstaller,
-    stderr=STDOUT,
-)
-
-if args.sign_application:
-    if args.pkg != "osxpkg":
-        raise NotImplementedError
-    if not args.application_id:
-        print("--sign-application requires --application-id")
-        sys.exit(1)
-    check_call(
-        ["python", "sign.py", "--application-id", args.application_id],
-        cwd=pyinstaller,
-        stderr=STDOUT,
-    )
-
-if args.pkg == "exe":
-    check_call(
-        ["python", "build.py"],
-        cwd=innosetup,
-        stderr=STDOUT,
-    )
-else:
-    check_call(
-        ["python", "build.py", args.pkg],
-        cwd=fpm,
-        stderr=STDOUT,
-    )
-
-if args.sign_installer:
-    if args.pkg != "osxpkg":
-        raise NotImplementedError
-    if not all([args.installer_id, args.apple_id_username, args.apple_id_password]):
-        print("--sign-installer requires --installer-id")
-        sys.exit(1)
-    check_call(
-        ["python", "sign.py", "--installer-id", args.installer_id],
-        cwd=fpm,
-        stderr=STDOUT,
-    )
-
-if args.notarize:
-    if args.pkg != "osxpkg":
-        raise NotImplementedError
-    if not all([args.apple_id_username, args.apple_id_password]):
-        print("--notarize requires --apple-id-username and --apple-id-password")
-        sys.exit(1)
-    check_call(
-        [
-            "python",
-            "notarize.py",
-            "--apple-id-username",
-            args.apple_id_username,
-            "--apple-id-password",
-            args.apple_id_password,
-        ],
-        cwd=fpm,
-        stderr=STDOUT,
-    )

scripts/fpm/.gitignore

@@ -1,4 +0,0 @@
-/build
-/*.deb
-/*.rpm
-/*.pkg

scripts/fpm/after-install.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-ln -sf /usr/local/lib/dvc/dvc /usr/local/bin/dvc

scripts/fpm/after-remove.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-rm -rf /usr/local/bin/dvc

scripts/fpm/build.py

@@ -1,98 +0,0 @@
-import argparse
-import os
-import pathlib
-import shutil
-from subprocess import STDOUT, check_call, check_output
-
-path = pathlib.Path(__file__).parent.absolute()
-dvc = path.parent.parent / "dvc"
-pyinstaller = path.parent / "pyinstaller"
-
-build = path / "build"
-install = build / "usr"
-
-parser = argparse.ArgumentParser()
-parser.add_argument("pkg", choices=["deb", "rpm", "osxpkg"], help="package type")
-args = parser.parse_args()
-
-flags = [
-    "--description",
-    '"Data Version Control | Git for Data & Models"',
-    "-n",
-    "dvc",
-    "-s",
-    "dir",
-    "-f",
-    "--license",
-    '"Apache License 2.0"',
-]
-
-if args.pkg == "osxpkg":
-    install /= "local"
-    bash_dir = install / "etc" / "bash_completion.d"
-    dirs = ["usr"]
-    flags.extend(
-        [
-            "--osxpkg-identifier-prefix",
-            "com.iterative",
-            "--after-install",
-            path / "after-install.sh",
-            "--after-remove",
-            path / "after-remove.sh",
-        ]
-    )
-else:
-    if args.pkg == "rpm":
-        # https://github.com/jordansissel/fpm/issues/1503
-        flags.extend(["--rpm-rpmbuild-define", "_build_id_links none"])
-
-    bash_dir = build / "etc" / "bash_completion.d"
-    dirs = ["usr", "etc"]
-    flags.extend(["--depends", "git >= 1.7.0"])  # needed for gitpython
-
-try:
-    shutil.rmtree(build)
-except FileNotFoundError:
-    pass
-
-lib = install / "lib"
-lib.mkdir(parents=True)
-shutil.copytree(pyinstaller / "dist" / "dvc", lib / "dvc")
-
-if args.pkg != "osxpkg":
-    # NOTE: in osxpkg fpm replaces symlinks with actual file that it
-    # points to, so we need to use after-install hook.
-    (install / "bin").mkdir()
-    os.symlink("../lib/dvc/dvc", install / "bin" / "dvc")
-
-bash_dir.mkdir(parents=True)
-bash_completion = check_output(
-    [lib / "dvc" / "dvc", "completion", "-s", "bash"], text=True
-)
-(bash_dir / "dvc").write_text(bash_completion)
-
-zsh_dir = install / "share" / "zsh" / "site-functions"
-zsh_dir.mkdir(parents=True)
-zsh_completion = check_output(
-    [lib / "dvc" / "dvc", "completion", "-s", "zsh"], text=True
-)
-(zsh_dir / "_dvc").write_text(zsh_completion)
-
-version = check_output([lib / "dvc" / "dvc", "--version"], text=True).strip()
-
-check_call(
-    [
-        "fpm",
-        "--verbose",
-        "-t",
-        args.pkg,
-        *flags,
-        "-v",
-        version,
-        "-C",
-        build,
-        *dirs,
-    ],
-    cwd=path,
-    stderr=STDOUT,
-)

scripts/fpm/notarize.py

@@ -1,64 +0,0 @@
-import argparse
-import json
-import os
-import pathlib
-import sys
-from subprocess import STDOUT, check_call
-
-if sys.platform != "darwin":
-    raise NotImplementedError
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "path",
-    nargs="?",
-    help="Path to the osxpkg to notarize. If not specified - try to find one.",
-)
-parser.add_argument("--apple-id-username", required=True, help="Apple ID username.")
-parser.add_argument(
-    "--apple-id-password",
-    required=True,
-    help=(
-        "Apple ID app-specific password. Note that this is not a regular "
-        "Apple ID password, so you need to generate one at "
-        "https://appleid.apple.com/account/manage"
-    ),
-)
-args = parser.parse_args()
-
-path = pathlib.Path(__file__).parent.absolute()
-
-if args.path:
-    pkg = pathlib.Path(args.path)
-else:
-    pkgs = list(path.glob("*.pkg"))
-    if not pkgs:
-        print("No pkgs found")
-        sys.exit(1)
-
-    if len(pkgs) > 1:
-        print("Too many packages")
-        sys.exit(1)
-
-    (pkg,) = pkgs
-
-
-config = {
-    "notarize": {
-        "path": os.fspath(pkg),
-        "bundle_id": "com.iterative.dvc",
-        "staple": True,
-    },
-    "apple_id": {
-        "username": args.apple_id_username,
-        "password": args.apple_id_password,
-    },
-}
-
-(path / "config.json").write_text(json.dumps(config))
-
-check_call(
-    ["gon", "config.json"],
-    cwd=path,
-    stderr=STDOUT,
-)

scripts/fpm/sign.py

@@ -1,55 +0,0 @@
-import argparse
-import os
-import pathlib
-import sys
-from subprocess import STDOUT, check_call
-
-if sys.platform != "darwin":
-    raise NotImplementedError
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "path",
-    nargs="?",
-    help="Path to the osxpkg to sign. If not specified - try to find one.",
-)
-parser.add_argument(
-    "--installer-id",
-    required=True,
-    help="Certificate ID (should be added to the keychain).",
-)
-args = parser.parse_args()
-
-path = pathlib.Path(__file__).parent.absolute()
-
-if args.path:
-    pkg = pathlib.Path(args.path)
-else:
-    pkgs = list(path.glob("*.pkg"))
-    if not pkgs:
-        print("No pkgs found")
-        sys.exit(1)
-
-    if len(pkgs) > 1:
-        print("Too many packages")
-        sys.exit(1)
-
-    (pkg,) = pkgs
-
-unsigned = pkg.with_suffix(".unsigned")
-os.rename(pkg, unsigned)
-check_call(
-    [
-        "productsign",
-        "--sign",
-        args.installer_id,
-        os.fspath(unsigned),
-        os.fspath(pkg),
-    ],
-    stderr=STDOUT,
-)
-
-check_call(
-    ["pkgutil", "--check-signature", os.fspath(pkg)],
-    stderr=STDOUT,
-)

scripts/innosetup/.gitignore

@@ -1,6 +0,0 @@
-/dvc.ico
-/dvc_left.bmp
-/dvc_up.bmp
-/config.ini
-/build
-/dvc*.exe

scripts/innosetup/addSymLinkPermissions.ps1

@@ -1,76 +0,0 @@
-param (
-	[string]$mytype = 'system'
-)
-
-function addSymLinkPermissions($accountToAdd){
-    Write-Host "Checking SymLink permissions.."
-    $sidstr = $null
-    if ( "$accountToAdd" -eq "Everyone" ) {
-    	$sidstr = "S-1-1-0"
-    } else {
-        try {
-            $ntprincipal = new-object System.Security.Principal.NTAccount "$accountToAdd"
-            $sid = $ntprincipal.Translate([System.Security.Principal.SecurityIdentifier])
-            $sidstr = $sid.Value.ToString()
-        } catch {
-            $sidstr = $null
-        }
-    }
-    Write-Host "Account: $($accountToAdd)" -ForegroundColor DarkCyan
-    if( [string]::IsNullOrEmpty($sidstr) ) {
-        Write-Host "Account not found!" -ForegroundColor Red
-        exit -1
-    }
-    Write-Host "Account SID: $($sidstr)" -ForegroundColor DarkCyan
-    $tmp = [System.IO.Path]::GetTempFileName()
-    Write-Host "Export current Local Security Policy" -ForegroundColor DarkCyan
-    secedit.exe /export /cfg "$($tmp)"
-    $c = Get-Content -Path $tmp
-    $currentSetting = ""
-    foreach($s in $c) {
-        if( $s -like "SECreateSymbolicLinkPrivilege*") {
-            $x = $s.split("=",[System.StringSplitOptions]::RemoveEmptyEntries)
-            $currentSetting = $x[1].Trim()
-        }
-    }
-    if( $currentSetting -notlike "*$($sidstr)*" ) {
-        Write-Host "Need to add permissions to SymLink" -ForegroundColor Yellow
-
-        Write-Host "Modify Setting ""Create SymLink""" -ForegroundColor DarkCyan
-
-        if( [string]::IsNullOrEmpty($currentSetting) ) {
-            $currentSetting = "*$($sidstr)"
-        } else {
-            $currentSetting = "*$($sidstr),$($currentSetting)"
-        }
-        Write-Host "$currentSetting"
-    $outfile = @"
-[Unicode]
-Unicode=yes
-[Version]
-signature="`$CHICAGO`$"
-Revision=1
-[Privilege Rights]
-SECreateSymbolicLinkPrivilege = $($currentSetting)
-"@
-    $tmp2 = [System.IO.Path]::GetTempFileName()
-        Write-Host "Import new settings to Local Security Policy" -ForegroundColor DarkCyan
-        $outfile | Set-Content -Path $tmp2 -Encoding Unicode -Force
-        Push-Location (Split-Path $tmp2)
-        try {
-            secedit.exe /configure /db "secedit.sdb" /cfg "$($tmp2)" /areas USER_RIGHTS
-        } finally {
-            Pop-Location
-        }
-    } else {
-        Write-Host "NO ACTIONS REQUIRED! Account already in ""Create SymLink""" -ForegroundColor DarkCyan
-        Write-Host "Account $accountToAdd already has permissions to SymLink" -ForegroundColor Green
-        return $true;
-    }
-}
-
-if ( "$mytype" -eq "user" ) {
-    addSymLinkPermissions $(Get-WMIObject -class Win32_ComputerSystem | select username).username
-} else {
-    addSymLinkPermissions Everyone
-}

scripts/innosetup/addsymlink.iss

@@ -1,14 +0,0 @@
-procedure AddSymLink();
-var
-    ErrorCode: Integer;
-    SRCdir: String;
-begin
-    SRCdir := ExpandConstant('{app}');
-    if isUninstaller() = false then begin
-        if not ShellExec('', 'powershell.exe', '-noninteractive -windowstyle hidden -executionpolicy bypass -File "' + SRCdir + '\addSymLinkPermissions.ps1" -mytype ' + SymLinkType, '', SW_HIDE, ewWaitUntilTerminated, ErrorCode) then begin
-            SuppressibleMsgBox('Failed to automatically grant SeCreateSymbolicLinkPrivilege. Please download Polsedit(www.southsoftware.com/polsedit.zip). Launch polseditx32.exe or polseeditx64.exe (depending on your Windows version), navigate to "Security Settings" -> "User Rights Assignment", add the account(s) to the list named "Create symbolic links", logoff and login back into your account. More info at https://github.com/git-for-windows/git/wiki/Symbolic-Links.', mbInformation, MB_OK, IDOK);
-        end else begin
-            SuppressibleMsgBox('Automatically added SeCreateSymbolicLinkPrivilege. Please logoff and login back into your account in order for the change to take effect.', mbInformation, MB_OK, IDOK);
-        end;
-    end;
-end;

scripts/innosetup/build.py

@@ -1,44 +0,0 @@
-import argparse
-import configparser
-import os
-import pathlib
-import shutil
-from subprocess import STDOUT, check_call, check_output
-
-path = pathlib.Path(__file__).parent.absolute()
-config = path / "config.ini"
-
-dvc = path.parent.parent / "dvc"
-pyinstaller = path.parent / "pyinstaller"
-
-build = path / "build"
-install = build / "usr"
-
-parser = argparse.ArgumentParser()
-args = parser.parse_args()
-
-try:
-    shutil.rmtree(build)
-except FileNotFoundError:
-    pass
-
-build.mkdir()
-shutil.copytree(pyinstaller / "dist" / "dvc", build / "dvc")
-shutil.copy(path / "addSymLinkPermissions.ps1", build)
-
-version = check_output(
-    [os.fspath(build / "dvc" / "dvc"), "--version"], text=True
-).strip()
-
-cfg = configparser.ConfigParser()
-cfg.add_section("Version")
-cfg.set("Version", "version", version)
-
-with (path / "config.ini").open("w") as fobj:
-    cfg.write(fobj)
-
-check_call(
-    ["iscc", "setup.iss"],
-    cwd=path,
-    stderr=STDOUT,
-)

scripts/innosetup/dvc.ico.dvc

@@ -1,3 +0,0 @@
-outs:
-- md5: 90104d9e83cfb825cf45507e90aadd27
-  path: dvc.ico

scripts/innosetup/dvc_left.bmp.dvc

@@ -1,3 +0,0 @@
-outs:
-- md5: 9106cda08aa427e73492389a0f17c72d
-  path: dvc_left.bmp

scripts/innosetup/dvc_up.bmp.dvc

@@ -1,3 +0,0 @@
-outs:
-- md5: 94614d6650e062655f9f77507dc9c1f2
-  path: dvc_up.bmp

scripts/innosetup/modpath.iss

@@ -1,215 +0,0 @@
-// ----------------------------------------------------------------------------
-// Script based on:
-//
-// Inno Setup Ver:	5.4.2
-// Script Version:	1.4.2
-// Author:			Jared Breland <[email protected]>
-// Homepage:		http://www.legroom.net/software
-// License:			GNU Lesser General Public License (LGPL), version 3
-//						http://www.gnu.org/licenses/lgpl.html
-//
-// Script Function:
-//	Allow modification of environmental path directly from Inno Setup installers
-//
-// Instructions:
-//	Copy modpath.iss to the same directory as your setup script
-//
-//	Add this statement to your [Setup] section
-//		ChangesEnvironment=true
-//
-//	Add this statement to your [Tasks] section
-//	You can change the Description or Flags
-//	You can change the Name, but it must match the ModPathName setting below
-//		Name: modifypath; Description: &Add application directory to your environmental path; Flags: unchecked
-//
-//	Add the following to the end of your [Code] section
-//	ModPathName defines the name of the task defined above
-//	ModPathType defines whether the 'user' or 'system' path will be modified;
-//		this will default to user if anything other than system is set
-//	setArrayLength must specify the total number of dirs to be added
-//	Result[0] contains first directory, Result[1] contains second, etc.
-//		const
-//			ModPathName = 'modifypath';
-//			ModPathType = 'user';
-//
-//		function ModPathDir(): TArrayOfString;
-//		begin
-//			setArrayLength(Result, 1);
-//			Result[0] := ExpandConstant('{app}');
-//		end;
-//		#include "modpath.iss"
-// ----------------------------------------------------------------------------
-
-function ModPathDir(): TArrayOfString;
-begin
-	setArrayLength(Result, 1)
-	Result[0] := ExpandConstant(ModPathPath);
-end;
-
-procedure ModPath();
-var
-	oldpath:	String;
-	newpath:	String;
-	updatepath:	Boolean;
-	pathArr:	TArrayOfString;
-	aExecFile:	String;
-	aExecArr:	TArrayOfString;
-	i, d:		Integer;
-	pathdir:	TArrayOfString;
-	regroot:	Integer;
-	regpath:	String;
-
-begin
-	// Get constants from main script and adjust behavior accordingly
-	// ModPathType MUST be 'system' or 'user'; force 'user' if invalid
-	if ModPathType = 'system' then begin
-		regroot := HKEY_LOCAL_MACHINE;
-		regpath := 'SYSTEM\CurrentControlSet\Control\Session Manager\Environment';
-	end else begin
-		regroot := HKEY_CURRENT_USER;
-		regpath := 'Environment';
-	end;
-
-	// Get array of new directories and act on each individually
-	pathdir := ModPathDir();
-	for d := 0 to GetArrayLength(pathdir)-1 do begin
-		updatepath := true;
-
-		// Modify WinNT path
-		if UsingWinNT() = true then begin
-
-			// Get current path, split into an array
-			RegQueryStringValue(regroot, regpath, 'Path', oldpath);
-			oldpath := oldpath + ';';
-			i := 0;
-
-			while (Pos(';', oldpath) > 0) do begin
-				SetArrayLength(pathArr, i+1);
-				pathArr[i] := Copy(oldpath, 0, Pos(';', oldpath)-1);
-				oldpath := Copy(oldpath, Pos(';', oldpath)+1, Length(oldpath));
-				i := i + 1;
-
-				// Check if current directory matches app dir
-				if pathdir[d] = pathArr[i-1] then begin
-					// if uninstalling, remove dir from path
-					if IsUninstaller() = true then begin
-						continue;
-					// if installing, flag that dir already exists in path
-					end else begin
-						updatepath := false;
-					end;
-				end;
-
-				// Add current directory to new path
-				if i = 1 then begin
-					newpath := pathArr[i-1];
-				end else begin
-					newpath := newpath + ';' + pathArr[i-1];
-				end;
-			end;
-
-			// Append app dir to path if not already included
-			if (IsUninstaller() = false) AND (updatepath = true) then
-				newpath := newpath + ';' + pathdir[d];
-
-			// Write new path
-			RegWriteStringValue(regroot, regpath, 'Path', newpath);
-
-		// Modify Win9x path
-		end else begin
-
-			// Convert to shortened dirname
-			pathdir[d] := GetShortName(pathdir[d]);
-
-			// If autoexec.bat exists, check if app dir already exists in path
-			aExecFile := 'C:\AUTOEXEC.BAT';
-			if FileExists(aExecFile) then begin
-				LoadStringsFromFile(aExecFile, aExecArr);
-				for i := 0 to GetArrayLength(aExecArr)-1 do begin
-					if IsUninstaller() = false then begin
-						// If app dir already exists while installing, skip add
-						if (Pos(pathdir[d], aExecArr[i]) > 0) then
-							updatepath := false;
-							break;
-					end else begin
-						// If app dir exists and = what we originally set, then delete at uninstall
-						if aExecArr[i] = 'SET PATH=%PATH%;' + pathdir[d] then
-							aExecArr[i] := '';
-					end;
-				end;
-			end;
-
-			// If app dir not found, or autoexec.bat didn't exist, then (create and) append to current path
-			if (IsUninstaller() = false) AND (updatepath = true) then begin
-				SaveStringToFile(aExecFile, #13#10 + 'SET PATH=%PATH%;' + pathdir[d], True);
-
-			// If uninstalling, write the full autoexec out
-			end else begin
-				SaveStringsToFile(aExecFile, aExecArr, False);
-			end;
-		end;
-	end;
-end;
-
-// Split a string into an array using passed delimiter
-procedure MPExplode(var Dest: TArrayOfString; Text: String; Separator: String);
-var
-	i: Integer;
-begin
-	i := 0;
-	repeat
-		SetArrayLength(Dest, i+1);
-		if Pos(Separator,Text) > 0 then	begin
-			Dest[i] := Copy(Text, 1, Pos(Separator, Text)-1);
-			Text := Copy(Text, Pos(Separator,Text) + Length(Separator), Length(Text));
-			i := i + 1;
-		end else begin
-			 Dest[i] := Text;
-			 Text := '';
-		end;
-	until Length(Text)=0;
-end;
-
-procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep);
-var
-	aSelectedTasks:	TArrayOfString;
-	i:				Integer;
-	taskname:		String;
-	regpath:		String;
-	regstring:		String;
-	appid:			String;
-begin
-	// only run during actual uninstall
-	if CurUninstallStep = usUninstall then begin
-		// get list of selected tasks saved in registry at install time
-		appid := '{#emit SetupSetting("AppId")}';
-		if appid = '' then appid := '{#emit SetupSetting("AppName")}';
-		regpath := ExpandConstant('Software\Microsoft\Windows\CurrentVersion\Uninstall\'+appid+'_is1');
-		RegQueryStringValue(HKLM, regpath, 'Inno Setup: Selected Tasks', regstring);
-		if regstring = '' then RegQueryStringValue(HKCU, regpath, 'Inno Setup: Selected Tasks', regstring);
-
-		// check each task; if matches modpath taskname, trigger patch removal
-		if regstring <> '' then begin
-			taskname := ModPathName;
-			MPExplode(aSelectedTasks, regstring, ',');
-			if GetArrayLength(aSelectedTasks) > 0 then begin
-				for i := 0 to GetArrayLength(aSelectedTasks)-1 do begin
-					if comparetext(aSelectedTasks[i], taskname) = 0 then
-						ModPath();
-				end;
-			end;
-		end;
-	end;
-end;
-
-function NeedRestart(): Boolean;
-var
-	taskname:	String;
-begin
-	taskname := ModPathName;
-	if IsTaskSelected(taskname) and not UsingWinNT() then begin
-		Result := True;
-	end else begin
-		Result := False;
-	end;
-end;

scripts/innosetup/setup.iss

@@ -1,83 +0,0 @@
-#define MyAppName "DVC (Data Version Control)"
-#define MyAppVersion ReadIni(SourcePath + "\config.ini", "Version", "version", "unknown")
-#define MyAppPublisher "Dmitry Petrov"
-#define MyAppURL "https://dvc.org"
-#define MyAppDir SourcePath + "\build\dvc"
-
-[Setup]
-AppId={{8258CE8A-110E-4E0D-AE60-FEE00B15F041}
-AppName={#MyAppName}
-AppVersion={#MyAppVersion}
-AppPublisher={#MyAppPublisher}
-AppPublisherURL={#MyAppURL}
-AppSupportURL={#MyAppURL}
-AppUpdatesURL={#MyAppURL}
-DefaultDirName={code:GetDefaultDirName}
-DefaultGroupName={#MyAppName}
-AllowNoIcons=yes
-LicenseFile=..\..\LICENSE
-OutputBaseFilename=dvc-{#MyAppVersion}
-Compression=lzma2/max
-SolidCompression=yes
-OutputDir=.
-ChangesEnvironment=yes
-SetupIconFile=dvc.ico
-WizardSmallImageFile=dvc_up.bmp
-WizardImageFile=dvc_left.bmp
-DisableWelcomePage=no
-
-[Languages]
-Name: "english"; MessagesFile: "compiler:Default.isl"
-
-[Files]
-Source: "{#MyAppDir}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs
-
-[Tasks]
-Name: modifypath; Description: Adds dvc's application directory to environmental path; Flags: checkablealone;
-Name: modifypath\system; Description: Adds dvc's application directory to environmental path for all users;
-Name: addsymlinkpermissions; Description: Add permission for creating symbolic links; Flags: checkablealone;
-Name: addsymlinkpermissions\system; Description: Add permissions for creating symbolic links for all users;
-
-[Code]
-const
-	ModPathName = 'modifypath';
-	ModPathPath = '{app}';
-	SymLinkName = 'addsymlinkpermissions';
-
-var
-	ModPathType: String;
-	SymLinkType: String;
-
-function GetDefaultDirName(Dummy: string): string;
-begin
-	if IsAdminLoggedOn then begin
-		Result := ExpandConstant('{pf}\{#MyAppName}');
-	end else begin
-		Result := ExpandConstant('{userpf}\{#MyAppName}');
-	end;
-end;
-
-#include "modpath.iss"
-#include "addsymlink.iss"
-
-procedure CurStepChanged(CurStep: TSetupStep);
-begin
-	if CurStep = ssPostInstall then begin
-		if IsTaskSelected(ModPathName + '\system') then begin
-			ModPathType := 'system';
-		end else begin
-			ModPathType := 'user';
-		end;
-
-		if IsTaskSelected(SymLinkName + '\system') then begin
-			SymLinkType := 'system';
-		end else begin
-			SymLinkType := 'user';
-		end;
-
-		if IsTaskSelected(ModPathName) then
-			ModPath();
-		if IsTaskSelected(SymLinkName) then
-			AddSymLink();
-	end;
-end;

scripts/pyinstaller/.gitignore

@@ -1,4 +0,0 @@
-/build
-/dist
-/dvc.spec
-/dvc

scripts/pyinstaller/build.py

@@ -1,47 +0,0 @@
-import os
-import pathlib
-from subprocess import STDOUT, check_call, check_output
-
-path = pathlib.Path(__file__).parent.absolute()
-hooks = path / "hooks"
-dvc = path.parent.parent / "dvc"
-entry = dvc / "__main__.py"
-
-check_call(
-    [
-        "pyinstaller",
-        "--additional-hooks-dir",
-        os.fspath(hooks),
-        "--name",
-        "dvc",
-        "-y",
-        os.fspath(entry),
-    ],
-    cwd=path,
-    stderr=STDOUT,
-)
-
-out = check_output(
-    [
-        path / "dist" / "dvc" / "dvc",
-        "doctor",
-    ],
-    stderr=STDOUT,
-).decode()
-
-remotes = [
-    "s3",
-    "oss",
-    "gdrive",
-    "gs",
-    "hdfs",
-    "http",
-    "webhdfs",
-    "azure",
-    "ssh",
-    "webdav",
-]
-
-print(out)
-for remote in remotes:
-    assert f"\t{remote}" in out, f"Missing support for {remote}"

scripts/pyinstaller/entitlements.plist

@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-	<!-- These are required for binaries built by PyInstaller -->
-	<key>com.apple.security.cs.allow-jit</key>
-	<true/>
-	<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
-	<true/>
-</dict>
-</plist>

scripts/pyinstaller/sign.py

@@ -1,39 +0,0 @@
-import argparse
-import os
-import pathlib
-import sys
-from subprocess import STDOUT, check_call
-
-if sys.platform != "darwin":
-    raise NotImplementedError
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "--application-id",
-    required=True,
-    help="Certificate ID (should be added to the keychain).",
-)
-args = parser.parse_args()
-
-path = pathlib.Path(__file__).parent.absolute()
-dvc = path / "dist" / "dvc"
-for root, _, fnames in os.walk(dvc):
-    for fname in fnames:
-        fpath = os.path.join(root, fname)
-        print(f"signing {fpath}")
-        check_call(
-            [
-                "codesign",
-                "--force",
-                "--verbose",
-                "-s",
-                args.application_id,
-                "-o",
-                "runtime",
-                "--entitlements",
-                "entitlements.plist",
-                fpath,
-            ],
-            stderr=STDOUT,
-            timeout=180,
-        )

tests/func/api/test_data.py

@@ -218,3 +218,36 @@ def test_open_from_remote(tmp_dir, erepo_dir, cloud, local_cloud):
         remote="other",
     ) as fd:
         assert fd.read() == "foo content"
+
+    with api.open(
+        os.path.join("dir", "foo"),
+        repo=f"file://{erepo_dir.as_posix()}",
+        config={"core": {"remote": "other"}},
+    ) as fd:
+        assert fd.read() == "foo content"
+
+
+def test_read_from_remote(tmp_dir, erepo_dir, cloud, local_cloud):
+    erepo_dir.add_remote(config=cloud.config, name="other")
+    erepo_dir.add_remote(config=local_cloud.config, default=True)
+    erepo_dir.dvc_gen({"dir": {"foo": "foo content"}}, commit="create file")
+    erepo_dir.dvc.push(remote="other")
+    remove(erepo_dir.dvc.cache.local.path)
+
+    assert (
+        api.read(
+            os.path.join("dir", "foo"),
+            repo=f"file://{erepo_dir.as_posix()}",
+            remote="other",
+        )
+        == "foo content"
+    )
+
+    assert (
+        api.read(
+            os.path.join("dir", "foo"),
+            repo=f"file://{erepo_dir.as_posix()}",
+            config={"core": {"remote": "other"}},
+        )
+        == "foo content"
+    )

tests/func/data/db/test_index.py

@@ -8,7 +8,7 @@ from dvc_data.hashfile.db import get_index
 
 
 @pytest.fixture
-def index(dvc, local_remote, mocker):
+def index(tmp_dir, dvc, local_remote):
     odb = dvc.cloud.get_remote_odb("upstream")
     return get_index(odb)
 
@@ -58,23 +58,13 @@ def test_clear_on_download_err(tmp_dir, dvc, index, mocker):
     out = tmp_dir.dvc_gen({"dir": {"foo": "foo content"}})[0].outs[0]
     dvc.push()
 
-    for _, _, hi in out.obj:
-        remove(dvc.cache.local.get(hi.value).path)
-    remove(out.fs_path)
-
     assert list(index.hashes())
 
-    def unreliable_download(_from_fs, from_info, _to_fs, to_info, **kwargs):
-        on_error = kwargs["on_error"]
-        assert on_error
-        if isinstance(from_info, str):
-            from_info = [from_info]
-        if isinstance(to_info, str):
-            to_info = [to_info]
-        for from_i, to_i in zip(from_info, to_info):
-            on_error(from_i, to_i, Exception())
+    for _, _, hi in out.obj:
+        remove(dvc.cache.local.get(hi.value).path)
+        remove(dvc.cloud.get_remote().odb.get(hi.value).path)
+    remove(out.fs_path)
 
-    mocker.patch("dvc_objects.fs.generic.transfer", unreliable_download)
     with pytest.raises(DownloadError):
         dvc.pull()
     assert not list(index.hashes())

tests/func/experiments/test_experiments.py

@@ -362,7 +362,7 @@ def test_list(tmp_dir, scm, dvc, exp_stage):
     exp_c = first(results)
     ref_info_c = first(exp_refs_by_rev(scm, exp_c))
 
-    assert dvc.experiments.ls() == {"master": [(ref_info_c.name, exp_c)]}
+    assert dvc.experiments.ls() == {"refs/heads/master": [(ref_info_c.name, exp_c)]}
 
     exp_list = dvc.experiments.ls(rev=ref_info_a.baseline_sha)
     assert {key: set(val) for key, val in exp_list.items()} == {
@@ -372,20 +372,20 @@ def test_list(tmp_dir, scm, dvc, exp_stage):
     exp_list = dvc.experiments.ls(rev=[baseline_a, scm.get_rev()])
     assert {key: set(val) for key, val in exp_list.items()} == {
         baseline_a: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)},
-        "master": {(ref_info_c.name, exp_c)},
+        "refs/heads/master": {(ref_info_c.name, exp_c)},
     }
 
     exp_list = dvc.experiments.ls(all_commits=True)
     assert {key: set(val) for key, val in exp_list.items()} == {
         baseline_a: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)},
-        "master": {(ref_info_c.name, exp_c)},
+        "refs/heads/master": {(ref_info_c.name, exp_c)},
     }
 
     scm.checkout("branch", True)
     exp_list = dvc.experiments.ls(all_commits=True)
     assert {key: set(val) for key, val in exp_list.items()} == {
         baseline_a: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)},
-        "branch": {(ref_info_c.name, exp_c)},
+        "refs/heads/branch": {(ref_info_c.name, exp_c)},
     }
 
 
@@ -676,7 +676,7 @@ def test_experiment_unchanged(tmp_dir, scm, dvc, exp_stage):
     dvc.experiments.run(exp_stage.addressing)
     dvc.experiments.run(exp_stage.addressing)
 
-    assert len(dvc.experiments.ls()["master"]) == 2
+    assert len(dvc.experiments.ls()["refs/heads/master"]) == 2
 
 
 def test_experiment_run_dry(tmp_dir, scm, dvc, exp_stage):

tests/func/experiments/test_remote.py

@@ -35,6 +35,9 @@ def test_push(tmp_dir, scm, dvc, git_upstream, exp_stage, use_url):
     dvc.experiments.push(remote, [ref_info1.name])
     assert git_upstream.tmp_dir.scm.get_ref(str(ref_info1)) == exp1
 
+    dvc.experiments.push(remote)
+    assert git_upstream.tmp_dir.scm.get_ref(str(ref_info3)) == exp3
+
 
 @pytest.mark.parametrize("all_,rev,result3", [(True, False, True), (False, True, None)])
 def test_push_args(tmp_dir, scm, dvc, git_upstream, exp_stage, all_, rev, result3):
@@ -165,7 +168,7 @@ def test_list_remote(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
     exp_list = downstream_exp.ls(all_commits=True, git_remote=remote)
     assert {key: set(val) for key, val in exp_list.items()} == {
         baseline_a: {(ref_info_a.name, None), (ref_info_b.name, None)},
-        "master": {(ref_info_c.name, None)},
+        "refs/heads/master": {(ref_info_c.name, None)},
     }
 
 
@@ -173,6 +176,11 @@ def test_list_remote(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
 def test_pull(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
     from dvc.exceptions import InvalidArgumentError
 
+    # fetch and checkout to downstream so both repos start from same commit
+    downstream_repo = git_downstream.tmp_dir.scm.gitpython.repo
+    fetched = downstream_repo.remote(git_downstream.remote).fetch()
+    downstream_repo.git.checkout(fetched)
+
     remote = git_downstream.url if use_url else git_downstream.remote
     downstream_exp = git_downstream.tmp_dir.dvc.experiments
     with pytest.raises(InvalidArgumentError):
@@ -200,6 +208,9 @@ def test_pull(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
     downstream_exp.pull(remote, [str(ref_info1)])
     assert git_downstream.tmp_dir.scm.get_ref(str(ref_info1)) == exp1
 
+    downstream_exp.pull(remote)
+    assert git_downstream.tmp_dir.scm.get_ref(str(ref_info3)) == exp3
+
 
 @pytest.mark.parametrize("all_,rev,result3", [(True, False, True), (False, True, None)])
 def test_pull_args(tmp_dir, scm, dvc, git_downstream, exp_stage, all_, rev, result3):

tests/func/experiments/test_save.py

@@ -71,7 +71,7 @@ def test_exp_save_after_commit(tmp_dir, dvc, scm):
 
     all_exps = dvc.experiments.ls(all_commits=True)
     assert all_exps[baseline][0][0] == "exp-1"
-    assert all_exps["master"][0][0] == "exp-2"
+    assert all_exps["refs/heads/master"][0][0] == "exp-2"
 
 
 def test_exp_save_with_staged_changes(tmp_dir, dvc, scm):

tests/func/parsing/test_errors.py

@@ -9,6 +9,7 @@ import pytest
 from dvc.parsing import ResolveError
 from dvc.parsing.context import Context
 from dvc.parsing.interpolate import embrace
+from dvc.utils.humanize import join
 
 from . import make_entry_definition, make_foreach_def
 
@@ -21,6 +22,18 @@ def escape_ansi(line):
 # Tests for the interpolated entries
 
 
+@pytest.mark.parametrize("vars_", ["${file}_params.yaml", {"foo": "${foo}"}])
+def test_vars_interpolation_errors(tmp_dir, dvc, vars_):
+    definition = make_entry_definition(tmp_dir, "build", {"vars": [vars_]})
+    with pytest.raises(ResolveError) as exc_info:
+        definition.resolve()
+
+    assert (
+        str(exc_info.value) == "failed to parse 'stages.build.vars' in 'dvc.yaml': "
+        "interpolating is not allowed"
+    )
+
+
 def test_failed_to_interpolate(tmp_dir, dvc):
     context = Context(models={"foo": "bar"})
     definition = make_entry_definition(
@@ -41,6 +54,23 @@ def test_failed_to_interpolate(tmp_dir, dvc):
     assert definition.context == {"models": {"foo": "bar"}}
 
 
+def test_local_vars_params_file_not_exist(tmp_dir, dvc):
+    definition = make_entry_definition(
+        tmp_dir,
+        "build",
+        {"vars": ["not_existing_params.yaml"], "cmd": "echo ${models.foo}"},
+    )
+
+    with pytest.raises(ResolveError) as exc_info:
+        definition.resolve()
+
+    assert (
+        str(exc_info.value) == "failed to parse stage 'build' in 'dvc.yaml': "
+        "'not_existing_params.yaml' does not exist"
+    )
+    assert not definition.context
+
+
 def test_specified_key_does_not_exist(tmp_dir, dvc):
     definition = make_entry_definition(
         tmp_dir,
@@ -119,6 +149,25 @@ def test_interpolate_nested_iterable(tmp_dir, dvc):
     )
 
 
+def test_partial_vars_doesnot_exist(tmp_dir, dvc):
+    (tmp_dir / "test_params.yaml").dump({"sub1": "sub1", "sub2": "sub2"})
+
+    definition = make_entry_definition(
+        tmp_dir,
+        "build",
+        {"vars": ["test_params.yaml:sub3"], "cmd": "echo ${sub1} ${sub2}"},
+    )
+
+    with pytest.raises(ResolveError) as exc_info:
+        definition.resolve()
+
+    assert (
+        str(exc_info.value) == "failed to parse stage 'build' in 'dvc.yaml': "
+        "could not find 'sub3' in 'test_params.yaml'"
+    )
+    assert not definition.context
+
+
 # Tests foreach generated stages and their error messages
 
 
@@ -235,6 +284,47 @@ def test_foreach_do_definition_item_does_not_exist(tmp_dir, dvc, key, loc):
     assert context == {"foo": "bar"}
 
 
+@pytest.mark.parametrize(
+    "redefine",
+    [
+        {"item": 5},
+        {"key": 5},
+        {"item": 5, "key": 10},
+        {"item": {"epochs": 10}},
+    ],
+)
+@pytest.mark.parametrize("from_file", [True, False])
+def test_item_key_in_generated_stage_vars(tmp_dir, dvc, redefine, from_file):
+    context = Context(foo="bar")
+    vars_ = [redefine]
+    if from_file:
+        (tmp_dir / "test_params.yaml").dump(redefine)
+        vars_ = ["test_params.yaml"]
+
+    definition = make_foreach_def(
+        tmp_dir,
+        "build",
+        {"model1": {"thresh": "10"}, "model2": {"thresh": 5}},
+        {"vars": vars_, "cmd": "${item}"},
+        context,
+    )
+
+    with pytest.raises(ResolveError) as exc_info:
+        definition.resolve_all()
+
+    message = str(exc_info.value)
+    assert (
+        "failed to parse stage 'build@model1' in 'dvc.yaml': "
+        "attempted to modify reserved" in message
+    )
+
+    key_or_keys = "keys" if len(redefine) > 1 else "key"
+    assert f"{key_or_keys} {join(redefine)}" in message
+    if from_file:
+        assert "in 'test_params.yaml'" in message
+    assert context == {"foo": "bar"}
+
+
 def test_foreach_wdir_key_does_not_exist(tmp_dir, dvc):
     definition = make_foreach_def(
         tmp_dir,

tests/func/parsing/test_foreach.py

@@ -1,8 +1,9 @@
 """Testing happy paths for the foreach."""
+import os
 
 import pytest
 
-from dvc.parsing import DataResolver, ForeachDefinition
+from dvc.parsing import DEFAULT_PARAMS_FILE, DataResolver, ForeachDefinition
 from dvc.parsing.context import Context
 
 
@@ -269,6 +270,134 @@ def test_foreach_with_interpolated_wdir(tmp_dir, dvc):
     assert not any(item for item in resolver.tracked_vars.values())
 
 
+def test_foreach_with_local_vars(tmp_dir, dvc):
+    resolver = DataResolver(dvc, (tmp_dir / "data").fs_path, {})
+    foreach_data = ["foo", "bar"]
+    data = {
+        "foreach": foreach_data,
+        "do": {
+            "vars": [{"foobar": "foobar"}],
+            "cmd": "echo ${item} ${foobar}",
+        },
+    }
+    definition = ForeachDefinition(resolver, resolver.context, "build", data)
+
+    assert definition.resolve_all() == {
+        # note that the resolver generates `wdir` relative to file's wdir
+        # so, this is just `foo`, not `data/foo`.
+        # figuring out `wdir` is the responsibility of the `load_stage`/`Stage`
+        "build@foo": {"cmd": "echo foo foobar"},
+        "build@bar": {"cmd": "echo bar foobar"},
+    }
+    assert not resolver.context
+    assert not any(item for item in resolver.tracked_vars.values())
+
+
+@pytest.mark.parametrize(
+    "local_import",
+    [
+        "test_params.yaml",
+        "test_params.yaml:train",
+        "test_params.yaml:train,prepare",
+    ],
+)
+def test_foreach_with_imported_vars(tmp_dir, dvc, local_import):
+    (tmp_dir / "params.yaml").dump({"models": {"model1": {"thresh": "foo"}}})
+    (tmp_dir / "test_params.yaml").dump(
+        {"train": {"epochs": 10}, "prepare": {"nums": 25}}
+    )
+    resolver = DataResolver(dvc, tmp_dir.fs_path, {})
+    foreach_data = ["foo", "bar"]
+    data = {
+        "foreach": foreach_data,
+        "do": {"vars": [local_import], "cmd": "echo ${item} ${train.epochs}"},
+    }
+    definition = ForeachDefinition(resolver, resolver.context, "build", data)
+
+    assert definition.resolve_all() == {
+        # note that the resolver generates `wdir` relative to file's wdir
+        # so, this is just `foo`, not `data/foo`.
+        # figuring out `wdir` is the responsibility of the `load_stage`/`Stage`
+        "build@foo": {"cmd": "echo foo 10"},
+        "build@bar": {"cmd": "echo bar 10"},
+    }
+
+    assert resolver.context == {"models": {"model1": {"thresh": "foo"}}}
+    assert resolver.tracked_vars == {
+        "build@foo": {"test_params.yaml": {"train.epochs": 10}},
+        "build@bar": {"test_params.yaml": {"train.epochs": 10}},
+    }
+
+
+@pytest.mark.parametrize("local_import", ["params.yaml", "params.yaml:train,prepare"])
+def test_foreach_with_interpolated_wdir_and_local_vars(tmp_dir, dvc, local_import):
+    (tmp_dir / "params.yaml").dump({"models": {"model1": {"thresh": "foo"}}})
+
+    for i in range(5):
+        build_dir = tmp_dir / ("model-" + str(i))
+        build_dir.mkdir()
+        (build_dir / "params.yaml").dump(
+            {"train": {"epochs": 1 + i}, "prepare": {"nums": 10 * i}},
+        )
+
+    resolver = DataResolver(dvc, tmp_dir.fs_path, {})
+    data = {
+        "foreach": [0, 1, 2, 3, 4],
+        "do": {
+            "wdir": "model-${item}",
+            "vars": [local_import],
+            "cmd": "echo ${item} ${train.epochs} ${prepare.nums}",
+        },
+    }
+    definition = ForeachDefinition(resolver, resolver.context, "build", data)
+
+    assert definition.resolve_all() == {
+        # note that the resolver generates `wdir` relative to file's wdir
+        # so, this is just `foo`, not `data/foo`.
+        # figuring out `wdir` is the responsibility of the `load_stage`/`Stage`
+        "build@0": {"wdir": "model-0", "cmd": "echo 0 1 0"},
+        "build@1": {"wdir": "model-1", "cmd": "echo 1 2 10"},
+        "build@2": {"wdir": "model-2", "cmd": "echo 2 3 20"},
+        "build@3": {"wdir": "model-3", "cmd": "echo 3 4 30"},
+        "build@4": {"wdir": "model-4", "cmd": "echo 4 5 40"},
+    }
+
+    assert resolver.context == {"models": {"model1": {"thresh": "foo"}}}
+    assert resolver.tracked_vars == {
+        "build@0": {
+            os.path.join("model-0", "params.yaml"): {
+                "train.epochs": 1,
+                "prepare.nums": 0,
+            }
+        },
+        "build@1": {
+            os.path.join("model-1", "params.yaml"): {
+                "train.epochs": 2,
+                "prepare.nums": 10,
+            }
+        },
+        "build@2": {
+            os.path.join("model-2", "params.yaml"): {
+                "train.epochs": 3,
+                "prepare.nums": 20,
+            }
+        },
+        "build@3": {
+            os.path.join("model-3", "params.yaml"): {
+                "train.epochs": 4,
+                "prepare.nums": 30,
+            }
+        },
+        "build@4": {
+            os.path.join("model-4", "params.yaml"): {
+                "train.epochs": 5,
+                "prepare.nums": 40,
+            }
+        },
+    }
+    assert resolver.context.imports == {DEFAULT_PARAMS_FILE: None}
+
+
 def test_foreach_do_syntax_is_checked_once(tmp_dir, dvc, mocker):
     do_def = {"cmd": "python script.py --epochs ${item}"}
     data = {"foreach": [0, 1, 2, 3, 4], "do": do_def}

tests/func/parsing/test_interpolated_entry.py

@@ -1,3 +1,4 @@
+import os
 from copy import deepcopy
 
 import pytest
@@ -71,6 +72,147 @@ def test_vars_and_params_import(tmp_dir, dvc):
     }
 
 
+def test_stage_with_wdir(tmp_dir, dvc):
+    """
+    Test that params file from wdir are also loaded
+    """
+    d = {
+        "stages": {
+            "stage1": {
+                "cmd": "echo ${dict.foo} ${dict.bar}",
+                "params": ["value1"],
+                "wdir": "data",
+                "vars": [DEFAULT_PARAMS_FILE],
+            }
+        }
+    }
+
+    data_dir = tmp_dir / "data"
+    data_dir.mkdir()
+    (tmp_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"bar": "bar"}})
+    (data_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"foo": "foo"}})
+    resolver = DataResolver(dvc, tmp_dir.fs_path, d)
+
+    assert_stage_equal(
+        resolver.resolve(),
+        {
+            "stages": {
+                "stage1": {
+                    "cmd": "echo foo bar",
+                    "wdir": "data",
+                    "params": ["value1"],
+                }
+            }
+        },
+    )
+    assert resolver.tracked_vars == {
+        "stage1": {
+            os.path.join("data", DEFAULT_PARAMS_FILE): {"dict.foo": "foo"},
+            DEFAULT_PARAMS_FILE: {"dict.bar": "bar"},
+        }
+    }
+
+
+def test_with_templated_wdir(tmp_dir, dvc):
+    """
+    Test that params from the resolved wdir are still loaded
+    and is used in the interpolation.
+    """
+    d = {
+        "stages": {
+            "stage1": {
+                "cmd": "echo ${dict.foo} ${dict.bar}",
+                "params": ["value1"],
+                "wdir": "${dict.ws}",
+                "vars": [DEFAULT_PARAMS_FILE],
+            }
+        }
+    }
+    (tmp_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"bar": "bar", "ws": "data"}})
+    data_dir = tmp_dir / "data"
+    data_dir.mkdir()
+    (data_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"foo": "foo"}})
+    resolver = DataResolver(dvc, tmp_dir.fs_path, d)
+
+    assert_stage_equal(
+        resolver.resolve(),
+        {
+            "stages": {
+                "stage1": {
+                    "cmd": "echo foo bar",
+                    "wdir": "data",
+                    "params": ["value1"],
+                }
+            }
+        },
+    )
+    assert resolver.tracked_vars == {
+        "stage1": {
+            os.path.join("data", DEFAULT_PARAMS_FILE): {"dict.foo": "foo"},
+            DEFAULT_PARAMS_FILE: {"dict.bar": "bar", "dict.ws": "data"},
+        }
+    }
+    assert resolver.context.imports == {"params.yaml": None}
+    assert resolver.context == {"dict": {"bar": "bar", "ws": "data"}}
+
+
+def test_resolve_local_tries_to_load_globally_used_files(tmp_dir, dvc):
+    iterable = {"bar": "bar", "foo": "foo"}
+    (tmp_dir / "params.json").dump(iterable)
+
+    d = {
+        "vars": ["params.json"],
+        "stages": {
+            "build": {
+                "cmd": "command --value ${bar}",
+                "params": [{"params.json": ["foo"]}],
+                "vars": ["params.json"],
+            }
+        },
+    }
+    resolver = DataResolver(dvc, tmp_dir.fs_path, d)
+    assert_stage_equal(
+        resolver.resolve(),
+        {
+            "stages": {
+                "build": {
+                    "cmd": "command --value bar",
+                    "params": [{"params.json": ["foo"]}],
+                }
+            }
+        },
+    )
+    assert resolver.tracked_vars == {"build": {"params.json": {"bar": "bar"}}}
+
+
+def test_resolve_local_tries_to_load_globally_used_params_yaml(tmp_dir, dvc):
+    iterable = {"bar": "bar", "foo": "foo"}
+    (tmp_dir / "params.yaml").dump(iterable)
+
+    d = {
+        "stages": {
+            "build": {
+                "cmd": "command --value ${bar}",
+                "params": [{"params.yaml": ["foo"]}],
+                "vars": ["params.yaml"],
+            }
+        }
+    }
+    resolver = DataResolver(dvc, tmp_dir.fs_path, d)
+    assert_stage_equal(
+        resolver.resolve(),
+        {
+            "stages": {
+                "build": {
+                    "cmd": "command --value bar",
+                    "params": [{"params.yaml": ["foo"]}],
+                }
+            }
+        },
+    )
+    assert resolver.tracked_vars == {"build": {"params.yaml": {"bar": "bar"}}}
+
+
 def test_vars_relpath_overwrite(tmp_dir, dvc):
     iterable = {"bar": "bar", "foo": "foo"}
     (tmp_dir / "params.yaml").dump(iterable)
@@ -89,6 +231,7 @@ def test_vars_relpath_overwrite(tmp_dir, dvc):
     assert resolver.context.imports == {"params.yaml": None}
 
 
+@pytest.mark.parametrize("local", [True, False])
 @pytest.mark.parametrize(
     "vars_",
     [
@@ -98,10 +241,14 @@ def test_vars_relpath_overwrite(tmp_dir, dvc):
         ["test_params.yaml", "test_params.yaml"],
     ],
 )
-def test_vars_load_partial(tmp_dir, dvc, vars_):
+def test_vars_load_partial(tmp_dir, dvc, local, vars_):
     iterable = {"bar": "bar", "foo": "foo"}
     (tmp_dir / "test_params.yaml").dump(iterable)
-    d = {"vars": vars_, "stages": {"build": {"cmd": "echo ${bar}"}}}
+    d = {"stages": {"build": {"cmd": "echo ${bar}"}}}
+    if local:
+        d["stages"]["build"]["vars"] = vars_
+    else:
+        d["vars"] = vars_
     resolver = DataResolver(dvc, tmp_dir.fs_path, d)
     resolver.resolve()
 

tests/func/parsing/test_resolver.py

@@ -144,6 +144,7 @@ def test_specified_params_file_not_exist(tmp_dir, dvc):
     )
 
 
+@pytest.mark.parametrize("local", [True, False])
 @pytest.mark.parametrize(
     "vars_",
     [
@@ -152,11 +153,35 @@ def test_specified_params_file_not_exist(tmp_dir, dvc):
         ["test_params.yaml:sub1", "test_params.yaml:sub1,sub2"],
     ],
 )
-def test_vars_already_loaded_message(tmp_dir, dvc, vars_):
-    d = {"vars": vars_, "stages": {"build": {"cmd": "echo ${sub1} ${sub2}"}}}
+def test_vars_already_loaded_message(tmp_dir, dvc, local, vars_):
+    d = {"stages": {"build": {"cmd": "echo ${sub1} ${sub2}"}}}
     (tmp_dir / "test_params.yaml").dump({"sub1": "sub1", "sub2": "sub2"})
+    if not local:
+        d["vars"] = vars_
+    else:
+        d["stages"]["build"]["vars"] = vars_
 
     with pytest.raises(ResolveError) as exc_info:  # noqa: PT012
         resolver = DataResolver(dvc, tmp_dir.fs_path, d)
         resolver.resolve()
     assert "partially" in str(exc_info.value)
+
+
+@pytest.mark.parametrize(
+    "vars_, loc", [(DATA, "build.vars[0]"), ("params.json", "params.json")]
+)
+def test_local_overwrite_error(tmp_dir, dvc, vars_, loc):
+    (tmp_dir / DEFAULT_PARAMS_FILE).dump(DATA)
+    (tmp_dir / "params.json").dump(DATA)
+
+    d = {"stages": {"build": {"cmd": "echo ${models.foo}", "vars": [vars_]}}}
+
+    resolver = DataResolver(dvc, tmp_dir.fs_path, d)
+    with pytest.raises(ResolveError) as exc_info:
+        resolver.resolve()
+
+    assert (
+        str(exc_info.value) == "failed to parse stage 'build' in 'dvc.yaml':\n"
+        f"cannot redefine 'models.bar' from '{loc}' "
+        "as it already exists in 'params.yaml'"
+    )

tests/func/repro/test_repro.py

@@ -1273,3 +1273,12 @@ def test_repro_missing_lock_info(tmp_dir, dvc, copy_script):
 
     stages = dvc.reproduce(stage.addressing)
     assert len(stages) == 1
+
+
+def test_repro_rm_recursive(tmp_dir, dvc):
+    # check that dir output recursively removes files in the dir
+    tmp_dir.gen({"dir": {"foo": "foo"}})
+    dvc.stage.add(name="dir", cmd="mkdir dir", outs=["dir"])
+    dvc.reproduce()
+    assert (tmp_dir / "dir").exists()
+    assert not (tmp_dir / "dir" / "foo").exists()

tests/func/test_add.py

@@ -4,7 +4,6 @@ import os
 import shutil
 import stat
 import textwrap
-from unittest.mock import call
 
 import pytest
 
@@ -631,8 +630,8 @@ def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker):
     stages = dvc.add(["foo", "bar", "lorem", "ipsum"])
 
     assert m.call_count == 8
-    assert m.call_args != call(tmp_dir / "foo")
-    assert m.call_args != call(tmp_dir / "bar")
+    assert m.call_args != mocker.call(tmp_dir / "foo")
+    assert m.call_args != mocker.call(tmp_dir / "bar")
 
     for stage in stages[:2]:
         # hardlinks are not created for empty files
@@ -785,6 +784,7 @@ def test_add_preserve_fields(tmp_dir, dvc):
           remote: testremote
           md5: acbd18db4cc2f85cedef654fccc4a4d8
           size: 3
+          hash: md5
         meta: some metadata
     """
     )
@@ -976,6 +976,7 @@ def test_add_updates_to_cloud_versioning_dir(tmp_dir, dvc):
             "outs": [
                 {
                     "path": "data",
+                    "hash": "md5",
                     "files": [
                         {
                             "size": 3,
@@ -1008,6 +1009,7 @@ def test_add_updates_to_cloud_versioning_dir(tmp_dir, dvc):
         "outs": [
             {
                 "path": "data",
+                "hash": "md5",
                 "files": [
                     {
                         "size": 4,

tests/func/test_analytics.py

@@ -1,5 +1,4 @@
 import os
-from unittest.mock import call
 
 import pytest
 
@@ -40,7 +39,7 @@ def test_collect_and_send_report(mocker, dvc, mock_daemon):
 
     assert mock_daemon.call_count == 1
     assert mock_post.call_count == 1
-    assert mock_post.call_args == call(
+    assert mock_post.call_args == mocker.call(
         "https://analytics.dvc.org",
         json=ANY(dict),
         headers={"content-type": "application/json"},

tests/func/test_checkout.py

@@ -84,23 +84,19 @@ def test_remove_files_when_checkout(tmp_dir, dvc, scm):
 
 class TestCheckoutCleanWorkingDir:
     def test(self, mocker, tmp_dir, dvc):
-        mock_prompt = mocker.patch("dvc.prompt.confirm", return_value=True)
         (stage,) = tmp_dir.dvc_gen("data", {"foo": "foo"})
 
         # change working directory
         (tmp_dir / "data").gen("not_cached.txt", "not_cached")
-        assert main(["checkout", stage.relpath]) == 0
-        mock_prompt.assert_called()
+        assert main(["checkout", stage.relpath, "--force"]) == 0
         assert not (tmp_dir / "data" / "not_cached.txt").exists()
 
     def test_force(self, mocker, tmp_dir, dvc):
-        mock_prompt = mocker.patch("dvc.prompt.confirm", return_value=False)
         (stage,) = tmp_dir.dvc_gen("data", {"foo": "foo"})
 
         # change working directory
         (tmp_dir / "data").gen("not_cached.txt", "not_cached")
         assert main(["checkout", stage.relpath]) != 0
-        mock_prompt.assert_called()
         assert (tmp_dir / "data" / "not_cached.txt").exists()
 
 

tests/func/test_commit.py

@@ -316,9 +316,12 @@ def test_commit_dos2unix(tmp_dir, dvc):
         }
     )
     legacy_content = (tmp_dir / "foo.dvc").read_text()
+    assert "hash: md5" not in legacy_content
 
-    dvc.commit("foo.dvc")
+    dvc.commit("foo.dvc", force=True)
     assert (tmp_dir / "foo.dvc").read_text() == legacy_content
+
+    tmp_dir.gen("foo", "modified")
     dvc.commit("foo.dvc", force=True)
     content = (tmp_dir / "foo.dvc").read_text()
     assert "hash: md5" in content

tests/func/test_diff.py

@@ -21,6 +21,14 @@ def test_no_scm(tmp_dir, dvc):
         dvc.diff()
 
 
+def test_same_rev(tmp_dir, scm, dvc):
+    tmp_dir.dvc_gen("file", "text", commit="add file")
+
+    assert not dvc.diff("HEAD", "HEAD")
+    assert not dvc.diff("HEAD", "master")
+    assert not dvc.diff("master", "HEAD")
+
+
 def test_added(tmp_dir, scm, dvc):
     tmp_dir.dvc_gen("file", "text")
 

tests/func/test_external_repo.py

@@ -1,5 +1,4 @@
 import os
-from unittest.mock import ANY
 
 from dvc.repo.open_repo import CLONES
 from dvc.repo.open_repo import _external_repo as external_repo
@@ -140,7 +139,9 @@ def test_shallow_clone_branch(erepo_dir, mocker):
         with repo.dvcfs.open("file") as fd:
             assert fd.read() == "branch"
 
-    clone_spy.assert_called_with(url, ANY, shallow_branch="branch", progress=ANY)
+    clone_spy.assert_called_with(
+        url, mocker.ANY, shallow_branch="branch", progress=mocker.ANY
+    )
 
     path, _ = CLONES[url]
     CLONES[url] = (path, True)
@@ -165,7 +166,9 @@ def test_shallow_clone_tag(erepo_dir, mocker):
         with repo.dvcfs.open("file") as fd:
             assert fd.read() == "foo"
 
-    clone_spy.assert_called_with(url, ANY, shallow_branch="v1", progress=ANY)
+    clone_spy.assert_called_with(
+        url, mocker.ANY, shallow_branch="v1", progress=mocker.ANY
+    )
 
     path, _ = CLONES[url]
     CLONES[url] = (path, True)

tests/func/test_remote.py

@@ -2,7 +2,6 @@ import errno
 import itertools
 import os
 import stat
-from unittest.mock import patch
 
 import configobj
 import pytest
@@ -128,7 +127,7 @@ def test_upper_case_remote(tmp_dir, dvc, local_cloud):
     assert ret == 0
 
 
-def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
+def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc, mocker):
     from dvc_data.hashfile.build import build
     from dvc_data.hashfile.tree import Tree
 
@@ -138,24 +137,27 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
 
     tree = Tree.from_list([{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}])
     tree.digest()
-    with patch("dvc_data.hashfile.build._build_tree", return_value=(None, tree)):
-        _, _, obj = build(dvc.cache.local, path, dvc.cache.local.fs, "md5")
-        hash1 = obj.hash_info
+
+    mocker.patch("dvc_data.hashfile.build._build_tree", return_value=(None, tree))
+
+    _, _, obj = build(dvc.cache.local, path, dvc.cache.local.fs, "md5")
+    hash1 = obj.hash_info
 
     # remove the raw dir obj to force building the tree on the next build call
     dvc.cache.local.fs.remove(dvc.cache.local.oid_to_path(hash1.as_raw().value))
 
     tree = Tree.from_list([{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}])
     tree.digest()
-    with patch("dvc_data.hashfile.build._build_tree", return_value=(None, tree)):
-        _, _, obj = build(dvc.cache.local, path, dvc.cache.local.fs, "md5")
-        hash2 = obj.hash_info
+
+    # NOTE: _build_tree already patched above
+    _, _, obj = build(dvc.cache.local, path, dvc.cache.local.fs, "md5")
+    hash2 = obj.hash_info
 
     assert hash1 == hash2
 
 
 def test_partial_push_n_pull(  # noqa: C901
-    tmp_dir, dvc, tmp_path_factory, local_remote
+    tmp_dir, dvc, tmp_path_factory, local_remote, mocker
 ):
     from dvc_objects.fs import generic
 
@@ -190,14 +192,15 @@ def test_partial_push_n_pull(  # noqa: C901
                 del to_info[i]
         return original(from_fs, from_info, to_fs, to_info, **kwargs)
 
-    with patch.object(generic, "transfer", unreliable_upload):
-        with pytest.raises(UploadError) as upload_error_info:
-            dvc.push()
-        assert upload_error_info.value.amount == 2
+    mock_upload = mocker.patch.object(generic, "transfer", unreliable_upload)
+    with pytest.raises(UploadError) as upload_error_info:
+        dvc.push()
+    assert upload_error_info.value.amount == 2
 
-        assert not odb.exists(foo.hash_info.value)
-        assert odb.exists(bar.hash_info.value)
-        assert not odb.exists(baz.hash_info.value)
+    assert not odb.exists(foo.hash_info.value)
+    assert odb.exists(bar.hash_info.value)
+    assert not odb.exists(baz.hash_info.value)
+    mocker.stop(mock_upload)
 
     # Push everything and delete local cache
     dvc.push()
@@ -215,12 +218,13 @@ def test_partial_push_n_pull(  # noqa: C901
         for from_i, to_i in zip(from_info, to_info):
             on_error(from_i, to_i, Exception())
 
-    with patch.object(generic, "transfer", unreliable_download):
-        with pytest.raises(DownloadError) as download_error_info:
-            dvc.pull()
-        # error count should be len(.dir + standalone file checksums)
-        # since files inside dir are ignored if dir cache entry is missing
-        assert download_error_info.value.amount == 2
+    mock_download = mocker.patch.object(generic, "transfer", unreliable_download)
+    with pytest.raises(DownloadError) as download_error_info:
+        dvc.pull()
+    # error count should be len(.dir + standalone file checksums)
+    # since files inside dir are ignored if dir cache entry is missing
+    assert download_error_info.value.amount == 2
+    mocker.stop(mock_download)
 
 
 def test_raise_on_too_many_open_files(

tests/unit/command/test_add.py

@@ -32,7 +32,7 @@ def test_add(mocker, dvc):
     )
 
 
-def test_add_to_remote(mocker):
+def test_add_to_remote(mocker, dvc):
     cli_args = parse_args(
         [
             "add",
@@ -63,7 +63,7 @@ def test_add_to_remote(mocker):
     )
 
 
-def test_add_to_remote_invalid_combinations(mocker, caplog):
+def test_add_to_remote_invalid_combinations(mocker, caplog, dvc):
     cli_args = parse_args(["add", "s3://bucket/foo", "s3://bucket/bar", "--to-remote"])
     assert cli_args.func == CmdAdd
 
@@ -83,7 +83,7 @@ def test_add_to_remote_invalid_combinations(mocker, caplog):
             assert expected_msg in caplog.text
 
 
-def test_add_to_cache_invalid_combinations(mocker, caplog):
+def test_add_to_cache_invalid_combinations(mocker, caplog, dvc):
     cli_args = parse_args(["add", "s3://bucket/foo", "s3://bucket/bar", "-o", "foo"])
     assert cli_args.func == CmdAdd
 

tests/unit/command/test_data_status.py

@@ -75,7 +75,8 @@ def test_json(dvc, mocker, capsys, mocked_status, args, to_omit):
 
 def test_no_changes_repo(dvc, scm, capsys):
     assert main(["data", "status"]) == 0
-    assert capsys.readouterr() == ("No changes.\n", "")
+    out, _ = capsys.readouterr()
+    assert out == "No changes.\n"
 
 
 def test_empty_scm_repo(tmp_dir, capsys):
@@ -83,7 +84,7 @@ def test_empty_scm_repo(tmp_dir, capsys):
     Repo.init()
 
     assert main(["data", "status"]) == 0
-    out, err = capsys.readouterr()
+    out, _ = capsys.readouterr()
     assert (
         out
         == """\
@@ -91,7 +92,6 @@ No changes in an empty git repo.
 (there are changes not tracked by dvc, use "git status" to see)
 """
     )
-    assert not err
 
 
 @pytest.mark.parametrize(

tests/unit/command/test_data_sync.py

@@ -2,7 +2,7 @@ from dvc.cli import parse_args
 from dvc.commands.data_sync import CmdDataFetch, CmdDataPull, CmdDataPush
 
 
-def test_fetch(mocker):
+def test_fetch(mocker, dvc):
     cli_args = parse_args(
         [
             "fetch",
@@ -40,7 +40,7 @@ def test_fetch(mocker):
     )
 
 
-def test_pull(mocker):
+def test_pull(mocker, dvc):
     cli_args = parse_args(
         [
             "pull",
@@ -84,7 +84,7 @@ def test_pull(mocker):
     )
 
 
-def test_push(mocker):
+def test_push(mocker, dvc):
     cli_args = parse_args(
         [
             "push",

tests/unit/command/test_diff.py

@@ -19,7 +19,7 @@ def test_digest(checksum, expected):
     assert expected == _digest(checksum)
 
 
-def test_default(mocker, capsys):
+def test_default(mocker, capsys, dvc):
     args = parse_args(["diff"])
     cmd = args.func(args)
     diff = {
@@ -51,7 +51,7 @@ def test_default(mocker, capsys):
     ).format(sep=os.path.sep) in capsys.readouterr()[0]
 
 
-def test_show_hash(mocker, capsys):
+def test_show_hash(mocker, capsys, dvc):
     args = parse_args(["diff", "--show-hash"])
     cmd = args.func(args)
     diff = {
@@ -101,7 +101,7 @@ def test_show_hash(mocker, capsys):
     ) in out
 
 
-def test_show_json(mocker, capsys):
+def test_show_json(mocker, capsys, dvc):
     args = parse_args(["diff", "--json"])
     cmd = args.func(args)
     diff = {
@@ -123,7 +123,7 @@ def test_show_json(mocker, capsys):
     assert '"not in cache": []' in out
 
 
-def test_show_json_and_hash(mocker, capsys):
+def test_show_json_and_hash(mocker, capsys, dvc):
     args = parse_args(["diff", "--json", "--show-hash"])
     cmd = args.func(args)
 
@@ -160,7 +160,7 @@ def test_show_json_and_hash(mocker, capsys):
     assert '"not in cache": []' in out
 
 
-def test_show_json_hide_missing(mocker, capsys):
+def test_show_json_hide_missing(mocker, capsys, dvc):
     args = parse_args(["diff", "--json", "--hide-missing"])
     cmd = args.func(args)
     diff = {
@@ -190,7 +190,7 @@ def test_show_json_hide_missing(mocker, capsys):
 
 
 @pytest.mark.parametrize("show_hash", [None, True, False])
-def test_diff_show_markdown_and_hash(mocker, show_hash):
+def test_diff_show_markdown_and_hash(mocker, show_hash, dvc):
     options = ["diff", "--md"] + (["--show-hash"] if show_hash else [])
     args = parse_args(options)
     cmd = args.func(args)
@@ -221,7 +221,7 @@ def test_diff_show_markdown_and_hash(mocker, show_hash):
         (["--md"], "| Status   | Path   |\n|----------|--------|"),
     ),
 )
-def test_no_changes(mocker, capsys, opts, show, expected):
+def test_no_changes(mocker, capsys, opts, show, expected, dvc):
     args = parse_args(["diff", *opts, *show])
     cmd = args.func(args)
     mocker.patch("dvc.repo.Repo.diff", return_value={})
@@ -330,7 +330,7 @@ def test_show_markdown_hide_missing(capsys):
     ).format(sep=os.path.sep)
 
 
-def test_hide_missing(mocker, capsys):
+def test_hide_missing(mocker, capsys, dvc):
     args = parse_args(["diff", "--hide-missing"])
     cmd = args.func(args)
     diff = {

tests/unit/command/test_experiments.py

@@ -56,7 +56,7 @@ def test_experiments_diff(dvc, scm, mocker):
     )
 
 
-def test_experiments_diff_revs(mocker, capsys):
+def test_experiments_diff_revs(mocker, capsys, dvc, scm):
     mocker.patch(
         "dvc.repo.experiments.diff.diff",
         return_value={
@@ -194,11 +194,11 @@ def test_experiments_list(dvc, scm, mocker):
         (["--sha-only"], "sha-a\n"),
     ],
 )
-def test_experiments_list_format(mocker, capsys, args, expected):
+def test_experiments_list_format(mocker, capsys, args, expected, dvc, scm):
     mocker.patch(
         "dvc.repo.experiments.ls.ls",
         return_value={
-            "main": [
+            "refs/heads/main": [
                 ("exp-a", "sha-a"),
             ]
         },
@@ -214,7 +214,7 @@ def test_experiments_list_format(mocker, capsys, args, expected):
     assert cap.out == expected
 
 
-def test_experiments_list_remote(mocker, capsys):
+def test_experiments_list_remote(mocker, capsys, dvc, scm):
     mocker.patch(
         "dvc.repo.experiments.ls.ls",
         return_value={
@@ -294,12 +294,7 @@ def test_experiments_push(dvc, scm, mocker):
     )
     cmd = cli_args.func(cli_args)
 
-    with pytest.raises(InvalidArgumentError) as exp_info:
-        cmd.run()
-    assert (
-        str(exp_info.value) == "Either provide an `experiment` argument"
-        ", or use the `--rev` or `--all-commits` flag."
-    )
+    assert cmd.run() == 0
 
 
 def test_experiments_pull(dvc, scm, mocker):
@@ -351,12 +346,7 @@ def test_experiments_pull(dvc, scm, mocker):
     )
     cmd = cli_args.func(cli_args)
 
-    with pytest.raises(InvalidArgumentError) as exp_info:
-        cmd.run()
-    assert (
-        str(exp_info.value) == "Either provide an `experiment` argument"
-        ", or use the `--rev` or `--all-commits` flag."
-    )
+    assert cmd.run() == 0
 
 
 def test_experiments_remove_flag(dvc, scm, mocker, capsys, caplog):

tests/unit/command/test_imp.py

@@ -2,7 +2,7 @@ from dvc.cli import parse_args
 from dvc.commands.imp import CmdImport
 
 
-def test_import(mocker):
+def test_import(mocker, dvc):
     cli_args = parse_args(
         [
             "import",
@@ -34,7 +34,7 @@ def test_import(mocker):
     )
 
 
-def test_import_no_exec(mocker):
+def test_import_no_exec(mocker, dvc):
     cli_args = parse_args(
         [
             "import",
@@ -64,7 +64,7 @@ def test_import_no_exec(mocker):
     )
 
 
-def test_import_no_download(mocker):
+def test_import_no_download(mocker, dvc):
     cli_args = parse_args(
         [
             "import",

tests/unit/command/test_imp_url.py

@@ -7,7 +7,7 @@ from dvc.commands.imp_url import CmdImportUrl
 from dvc.exceptions import DvcException
 
 
-def test_import_url(mocker):
+def test_import_url(mocker, dvc):
     cli_args = parse_args(
         [
             "import-url",
@@ -37,7 +37,7 @@ def test_import_url(mocker):
     )
 
 
-def test_failed_import_url(mocker, caplog):
+def test_failed_import_url(mocker, caplog, dvc):
     cli_args = parse_args(["import-url", "http://somesite.com/file_name"])
     assert cli_args.func == CmdImportUrl
 
@@ -60,7 +60,7 @@ def test_failed_import_url(mocker, caplog):
         ("--no-download", {"no_download": True, "no_exec": False}),
     ],
 )
-def test_import_url_no_exec_download_flags(mocker, flag, expected):
+def test_import_url_no_exec_download_flags(mocker, flag, expected, dvc):
     cli_args = parse_args(
         [
             "import-url",
@@ -87,7 +87,7 @@ def test_import_url_no_exec_download_flags(mocker, flag, expected):
     )
 
 
-def test_import_url_to_remote(mocker):
+def test_import_url_to_remote(mocker, dvc):
     cli_args = parse_args(
         [
             "import-url",

tests/unit/command/test_machine.py

@@ -1,5 +1,4 @@
 import os
-from unittest.mock import call
 
 import configobj
 import pytest
@@ -77,7 +76,7 @@ def test_status(tmp_dir, scm, dvc, mocker):
     m = mocker.patch.object(cmd.repo.machine, "status", autospec=True, return_value=[])
     assert cmd.run() == 0
     assert m.call_count == 2
-    m.assert_has_calls([call("foo"), call("myaws")])
+    m.assert_has_calls([mocker.call("foo"), mocker.call("myaws")])
 
 
 def test_destroy(tmp_dir, dvc, mocker):

tests/unit/fs/test_dvc.py

@@ -1,7 +1,6 @@
 import os
 import posixpath
 import shutil
-from unittest import mock
 
 import pytest
 
@@ -346,33 +345,35 @@ def test_subrepos(tmp_dir, scm, dvc, mocker):
 
         return f
 
-    with mock.patch.object(
+    mock_subrepo1 = mocker.patch.object(
         fs.fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo1.dvc)
-    ):
-        assert fs.exists("dir/repo/foo") is True
-        assert fs.exists("dir/repo/bar") is False
+    )
+    assert fs.exists("dir/repo/foo") is True
+    assert fs.exists("dir/repo/bar") is False
 
-        assert fs.isfile("dir/repo/foo") is True
-        assert fs.isfile("dir/repo/dir1/bar") is True
-        assert fs.isfile("dir/repo/dir1") is False
+    assert fs.isfile("dir/repo/foo") is True
+    assert fs.isfile("dir/repo/dir1/bar") is True
+    assert fs.isfile("dir/repo/dir1") is False
 
-        assert fs.isdir("dir/repo/dir1") is True
-        assert fs.isdir("dir/repo/dir1/bar") is False
-        assert fs.isdvc("dir/repo/foo") is True
+    assert fs.isdir("dir/repo/dir1") is True
+    assert fs.isdir("dir/repo/dir1/bar") is False
+    assert fs.isdvc("dir/repo/foo") is True
+    mocker.stop(mock_subrepo1)
 
-    with mock.patch.object(
+    mock_subrepo2 = mocker.patch.object(
         fs.fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo2.dvc)
-    ):
-        assert fs.exists("dir/repo2/lorem") is True
-        assert fs.exists("dir/repo2/ipsum") is False
+    )
+    assert fs.exists("dir/repo2/lorem") is True
+    assert fs.exists("dir/repo2/ipsum") is False
 
-        assert fs.isfile("dir/repo2/lorem") is True
-        assert fs.isfile("dir/repo2/dir2/ipsum") is True
-        assert fs.isfile("dir/repo2/dir2") is False
+    assert fs.isfile("dir/repo2/lorem") is True
+    assert fs.isfile("dir/repo2/dir2/ipsum") is True
+    assert fs.isfile("dir/repo2/dir2") is False
 
-        assert fs.isdir("dir/repo2/dir2") is True
-        assert fs.isdir("dir/repo2/dir2/ipsum") is False
-        assert fs.isdvc("dir/repo2/lorem") is True
+    assert fs.isdir("dir/repo2/dir2") is True
+    assert fs.isdir("dir/repo2/dir2/ipsum") is False
+    assert fs.isdvc("dir/repo2/lorem") is True
+    mocker.stop(mock_subrepo2)
 
 
 @pytest.mark.parametrize(

tests/unit/repo/experiments/queue/test_remove.py

@@ -1,5 +1,3 @@
-from unittest.mock import call
-
 from dvc.repo.experiments.queue.base import QueueDoneResult
 
 
@@ -37,7 +35,11 @@ def test_remove_queued(test_queue, mocker):
     assert test_queue.clear(queued=True) == queued_test
     remove_revs_mocker.assert_called_once_with(list(stash_dict.values()))
     reject_mocker.assert_has_calls(
-        [call("msg_queue1"), call("msg_queue2"), call("msg_queue3")]
+        [
+            mocker.call("msg_queue1"),
+            mocker.call("msg_queue2"),
+            mocker.call("msg_queue3"),
+        ]
     )
 
 
@@ -84,7 +86,9 @@ def test_remove_done(test_queue, mocker):
         "success2",
     ]
     remove_revs_mocker.assert_called_once_with([stash_dict["failed3"]])
-    purge_mocker.assert_has_calls([call("msg_failed3"), call("msg_success2")])
+    purge_mocker.assert_has_calls(
+        [mocker.call("msg_failed3"), mocker.call("msg_success2")]
+    )
 
     remove_revs_mocker.reset_mock()
     purge_mocker.reset_mock()
@@ -94,12 +98,12 @@ def test_remove_done(test_queue, mocker):
     )
     purge_mocker.assert_has_calls(
         [
-            call("msg_failed1"),
-            call("msg_failed2"),
-            call("msg_failed3"),
-            call("msg_success1"),
-            call("msg_success2"),
-            call("msg_success3"),
+            mocker.call("msg_failed1"),
+            mocker.call("msg_failed2"),
+            mocker.call("msg_failed3"),
+            mocker.call("msg_success1"),
+            mocker.call("msg_success2"),
+            mocker.call("msg_success3"),
         ],
         any_order=True,
     )

tests/unit/repo/experiments/test_collect.py

@@ -0,0 +1,69 @@
+import datetime
+import random
+from typing import Dict, List
+
+import pytest
+
+from dvc.repo.experiments.collect import ExpRange, ExpState, SerializableExp, collect
+
+
+@pytest.mark.vscode
+def test_collect_stable_sorting(dvc, scm, mocker):
+    """
+    Check that output is deterministically sorted even for
+    commits with the same timestamp. This affects the experience
+    in vs-code to avoid experiments "bouncing around" when "exp show"
+    is called repeatedly
+    """
+    expected_revs = [
+        "c" * 40,
+        "b" * 40,
+        "a" * 40,
+        "7" * 40,
+    ]
+
+    def collect_queued_patched(_, baseline_revs) -> Dict[str, List["ExpRange"]]:
+        single_timestamp = datetime.datetime(2023, 6, 20, 0, 0, 0)
+
+        exp_ranges = [
+            ExpRange(
+                revs=[
+                    ExpState(
+                        rev=rev,
+                        name=f"exp-state-{rev[0]}",
+                        data=SerializableExp(rev=rev, timestamp=single_timestamp),
+                    )
+                ],
+                name=f"exp-range-{rev[0]}",
+            )
+            for rev in expected_revs
+        ]
+
+        # shuffle collection order
+        random.shuffle(exp_ranges)
+
+        return {baseline_rev: exp_ranges for baseline_rev in baseline_revs}
+
+    mocker.patch("dvc.repo.experiments.collect.collect_queued", collect_queued_patched)
+    mocker.patch("dvc.repo.experiments.collect.collect_active", return_value={})
+    mocker.patch("dvc.repo.experiments.collect.collect_failed", return_value={})
+    mocker.patch("dvc.repo.experiments.collect.collect_successful", return_value={})
+
+    # repeat (shuffling collection order in collect_queued_patched)
+    for _ in range(20):
+        collected = collect(repo=dvc, all_commits=True)
+        assert collected[0].rev == "workspace"
+        assert collected[0].experiments is None
+        assert collected[1].rev == scm.get_rev()
+        _assert_experiment_rev_order(collected[1].experiments, expected_revs)
+
+
+def _assert_experiment_rev_order(
+    actual: List["ExpRange"],
+    expected_revs: List[str],
+):
+    expected_revs = expected_revs.copy()
+
+    for actual_exp_range in actual:
+        for exp_state in actual_exp_range.revs:
+            assert exp_state.rev == expected_revs.pop(0)

tests/unit/repo/test_open_repo.py

@@ -1,5 +1,4 @@
 import os
-from unittest.mock import call
 
 import pytest
 
@@ -34,7 +33,7 @@ def test_hook_is_called(tmp_dir, erepo_dir, mocker):
         paths = ["/" + path.replace("\\", "/") for path in subrepo_paths]
         spy.assert_has_calls(
             [
-                call(
+                mocker.call(
                     path,
                     fs=repo.fs,
                     scm=repo.scm,

tests/unit/test_ignore.py

@@ -1,21 +1,10 @@
 import os
-from unittest.mock import MagicMock, mock_open, patch
 
 import pytest
 
 from dvc.ignore import DvcIgnorePatterns
 
 
-def mock_dvcignore(dvcignore_path, patterns):
-    from dvc.fs import localfs
-
-    fs = MagicMock()
-    fs.path = localfs.path
-    fs.sep = localfs.sep
-    with patch.object(fs, "open", mock_open(read_data="\n".join(patterns))):
-        return DvcIgnorePatterns.from_file(dvcignore_path, fs, "mocked")
-
-
 @pytest.mark.parametrize(
     "file_to_ignore_relpath, patterns,  expected_match",
     [
@@ -174,13 +163,21 @@ def mock_dvcignore(dvcignore_path, patterns):
         ),
     ],
 )
-def test_match_ignore_from_file(file_to_ignore_relpath, patterns, expected_match):
+def test_match_ignore_from_file(
+    file_to_ignore_relpath, patterns, expected_match, mocker
+):
+    from dvc.fs import localfs
+
     dvcignore_path = os.path.join(
         os.path.sep, "full", "path", "to", "ignore", "file", ".dvcignore"
     )
     dvcignore_dirname = os.path.dirname(dvcignore_path)
 
-    ignore_file = mock_dvcignore(dvcignore_path, patterns)
+    fs = mocker.MagicMock()
+    fs.path = localfs.path
+    fs.sep = localfs.sep
+    mocker.patch.object(fs, "open", mocker.mock_open(read_data="\n".join(patterns)))
+    ignore_file = DvcIgnorePatterns.from_file(dvcignore_path, fs, "mocked")
 
     assert (
         ignore_file.matches(dvcignore_dirname, file_to_ignore_relpath) == expected_match