exp pull: update for index fetch

* secondary sorting by rev alphanumerically. to introduce determinism

* add UT

.dvc/.gitignore

@@ -1,12 +0,0 @@
-/state
-/lock
-/config.local
-/updater
-/updater.lock
-/state-journal
-/state-wal
-/cache
-/pkg
-/repos
-/tmp
-/experiments

.dvc/config

@@ -1,4 +0,0 @@
-['remote "dvc"']
-url = https://remote.dvc.org/dvc
-[core]
-remote = dvc

.github/workflows/packages.yaml

@@ -11,66 +11,6 @@ permissions:  # added using https://github.com/step-security/secure-workflows
   contents: read
 
 jobs:
-  binary:
-    permissions:
-      contents: write  # for actions/upload-release-asset to upload release asset
-    strategy:
-      matrix:
-        include:
-         - {pkg: "deb",  os: "ubuntu-20.04", asset: "dvc_${{ github.event.release.tag_name }}_amd64.deb"}
-         - {pkg: "rpm", os: "ubuntu-20.04", asset: "dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm"}
-         - {pkg: "osxpkg", os: "macos-11", asset: "dvc-${{ github.event.release.tag_name }}.pkg"}
-         - {pkg: "exe",  os: "windows-2019", asset: "dvc-${{ github.event.release.tag_name }}.exe"}
-
-    name: ${{ matrix.pkg }}
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v3
-      with:
-          fetch-depth: 0
-
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-        cache: pip
-        cache-dependency-path: |
-          pyproject.toml
-          scripts/build-requirements.txt
-
-    - name: Set up Ruby 2.6
-      uses: ruby/setup-ruby@v1
-      if: matrix.pkg != 'exe'
-      with:
-        ruby-version: '2.6'
-
-    - name: Install fpm
-      if: matrix.pkg != 'exe'
-      run: gem install --no-document fpm
-
-    - name: Install deps
-      run: |
-        pip install --upgrade pip wheel setuptools
-        pip install .[all]
-        pip install -r scripts/build-requirements.txt
-
-    - name: Pull images
-      run: dvc pull
-
-    - name: Build ${{ matrix.pkg }}
-      run: python scripts/build.py ${{ matrix.pkg }}
-
-    - name: Publish ${{ matrix.pkg }}
-      if: github.event_name == 'release'
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ github.event.release.upload_url }}
-        asset_path: scripts/${{ matrix.pkg == 'exe' && 'innosetup' || 'fpm' }}/${{ matrix.asset }}
-        asset_name: ${{ matrix.asset }}
-        asset_content_type: binary/octet-stream
-
   pip:
     runs-on: ubuntu-20.04
     permissions:

dvc/commands/experiments/ls.py

@@ -25,7 +25,14 @@ class CmdExperimentsList(CmdBase):
 
         for baseline in exps:
             if not (name_only or sha_only):
-                ui.write(f"{baseline[:7]}:")
+                tag_base = "refs/tags/"
+                branch_base = "refs/heads/"
+                name = baseline[:7]
+                if baseline.startswith(tag_base):
+                    name = baseline[len(tag_base) :]
+                elif baseline.startswith(branch_base):
+                    name = baseline[len(branch_base) :]
+                ui.write(f"{name}:")
             for exp_name, rev in exps[baseline]:
                 if name_only:
                     ui.write(exp_name)

dvc/commands/experiments/pull.py

@@ -3,23 +3,13 @@ import logging
 
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.exceptions import InvalidArgumentError
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsPull(CmdBase):
-    def raise_error_if_all_disabled(self):
-        if not any([self.args.experiment, self.args.all_commits, self.args.rev]):
-            raise InvalidArgumentError(
-                "Either provide an `experiment` argument, or use the "
-                "`--rev` or `--all-commits` flag."
-            )
-
     def run(self):
-        self.raise_error_if_all_disabled()
-
         pulled_exps = self.repo.experiments.pull(
             self.args.git_remote,
             self.args.experiment,
@@ -61,7 +51,7 @@ def add_parser(experiments_subparsers, parent_parser):
         help=EXPERIMENTS_PULL_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    add_rev_selection_flags(experiments_pull_parser, "Pull", False)
+    add_rev_selection_flags(experiments_pull_parser, "Pull", True)
     experiments_pull_parser.add_argument(
         "-f",
         "--force",

dvc/commands/experiments/push.py

@@ -5,20 +5,12 @@ from typing import Any, Dict
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.exceptions import InvalidArgumentError
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsPush(CmdBase):
-    def raise_error_if_all_disabled(self):
-        if not any([self.args.experiment, self.args.all_commits, self.args.rev]):
-            raise InvalidArgumentError(
-                "Either provide an `experiment` argument, or use the "
-                "`--rev` or `--all-commits` flag."
-            )
-
     @staticmethod
     def log_result(result: Dict[str, Any], remote: str):
         from dvc.utils import humanize
@@ -59,8 +51,6 @@ class CmdExperimentsPush(CmdBase):
     def run(self):
         from dvc.repo.experiments.push import UploadError
 
-        self.raise_error_if_all_disabled()
-
         try:
             result = self.repo.experiments.push(
                 self.args.git_remote,
@@ -102,7 +92,7 @@ def add_parser(experiments_subparsers, parent_parser):
         help=EXPERIMENTS_PUSH_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    add_rev_selection_flags(experiments_push_parser, "Push", False)
+    add_rev_selection_flags(experiments_push_parser, "Push", True)
     experiments_push_parser.add_argument(
         "-f",
         "--force",

dvc/fs/callbacks.py

@@ -2,12 +2,9 @@
 from contextlib import ExitStack
 from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Union
 
+from dvc.progress import Tqdm
 from dvc.utils.objects import cached_property
-from dvc_objects.fs.callbacks import (  # noqa: F401
-    DEFAULT_CALLBACK,
-    Callback,
-    TqdmCallback,
-)
+from dvc_objects.fs.callbacks import DEFAULT_CALLBACK, Callback  # noqa: F401
 
 if TYPE_CHECKING:
     from rich.progress import TaskID
@@ -15,6 +12,59 @@ if TYPE_CHECKING:
     from dvc.ui._rich_progress import RichTransferProgress
 
 
+# NOTE: this is very similar to dvc-objects.fs.callbacks.TqdmCallback,
+# but it works with our own Tqdm instance.
+class TqdmCallback(Callback):
+    def __init__(
+        self,
+        size: Optional[int] = None,
+        value: int = 0,
+        progress_bar: Optional["Tqdm"] = None,
+        **tqdm_kwargs,
+    ):
+        tqdm_kwargs["total"] = size or -1
+        self._tqdm_kwargs = tqdm_kwargs
+        self._progress_bar = progress_bar
+        self._stack = ExitStack()
+        super().__init__(size=size, value=value)
+
+    @cached_property
+    def progress_bar(self):
+        progress_bar = (
+            self._progress_bar
+            if self._progress_bar is not None
+            else Tqdm(**self._tqdm_kwargs)
+        )
+        return self._stack.enter_context(progress_bar)
+
+    def __enter__(self):
+        return self
+
+    def close(self):
+        self._stack.close()
+
+    def set_size(self, size):
+        # Tqdm tries to be smart when to refresh,
+        # so we try to force it to re-render.
+        super().set_size(size)
+        self.progress_bar.refresh()
+
+    def call(self, hook_name=None, **kwargs):  # noqa: ARG002
+        self.progress_bar.update_to(self.value, total=self.size)
+
+    def branch(
+        self,
+        path_1: "Union[str, BinaryIO]",
+        path_2: str,
+        kwargs: Dict[str, Any],
+        child: Optional[Callback] = None,
+    ):
+        child = child or TqdmCallback(
+            bytes=True, desc=path_1 if isinstance(path_1, str) else path_2
+        )
+        return super().branch(path_1, path_2, kwargs, child=child)
+
+
 class RichCallback(Callback):
     def __init__(
         self,

dvc/output.py

@@ -692,8 +692,7 @@ class Output:
         if self.metric:
             self.verify_metric()
 
-        if self.hash_name == "md5-dos2unix":
-            self.hash_name = "md5"
+        self._update_legacy_hash_name()
         if self.use_cache:
             _, self.meta, self.obj = build(
                 self.cache,
@@ -717,6 +716,10 @@ class Output:
         self.hash_info = self.obj.hash_info
         self.files = None
 
+    def _update_legacy_hash_name(self):
+        if self.hash_name == "md5-dos2unix" and self.changed_checksum():
+            self.hash_name = "md5"
+
     def set_exec(self) -> None:
         if self.isfile() and self.meta.isexec:
             self.cache.set_exec(self.fs_path)
@@ -965,7 +968,10 @@ class Output:
         return added, False if added else modified
 
     def remove(self, ignore_remove=False):
-        self.fs.remove(self.fs_path, recursive=True)
+        try:
+            self.fs.remove(self.fs_path, recursive=True)
+        except FileNotFoundError:
+            pass
         if self.protocol != Schemes.LOCAL:
             return
 
@@ -1335,6 +1341,7 @@ class Output:
             )
 
         assert self.repo
+        self._update_legacy_hash_name()
         cache = self.cache if self.use_cache else self.local_cache
         assert isinstance(cache, HashFileDB)
 

dvc/parsing/__init__.py

@@ -232,6 +232,18 @@ class EntryDefinition:
         self.definition = definition
         self.where = where
 
+    def _resolve_wdir(
+        self, context: Context, name: str, wdir: Optional[str] = None
+    ) -> str:
+        if not wdir:
+            return self.wdir
+
+        try:
+            wdir = to_str(context.resolve_str(wdir))
+        except (ContextError, ParseError) as exc:
+            format_and_raise(exc, f"'{self.where}.{name}.wdir'", self.relpath)
+        return self.resolver.fs.path.join(self.wdir, wdir)
+
     def resolve(self, **kwargs):
         try:
             return self.resolve_stage(**kwargs)
@@ -249,7 +261,33 @@ class EntryDefinition:
 
         # we need to pop vars from generated/evaluated data
         definition = deepcopy(self.definition)
+
+        wdir = self._resolve_wdir(context, name, definition.get(WDIR_KWD))
+        vars_ = definition.pop(VARS_KWD, [])
+        # FIXME: Should `vars` be templatized?
+        check_interpolations(vars_, f"{self.where}.{name}.vars", self.relpath)
+        if vars_:
+            # Optimization: Lookahead if it has any vars, if it does not, we
+            # don't need to clone them.
+            context = Context.clone(context)
+
+        try:
+            fs = self.resolver.fs
+            context.load_from_vars(fs, vars_, wdir, stage_name=name)
+        except VarsAlreadyLoaded as exc:
+            format_and_raise(exc, f"'{self.where}.{name}.vars'", self.relpath)
+
+        logger.trace(  # type: ignore[attr-defined]
+            "Context during resolution of stage %s:\n%s", name, context
+        )
+
         with context.track() as tracked_data:
+            # NOTE: we do not pop "wdir", and resolve it again
+            # this does not affect anything and is done to try to
+            # track the source of `wdir` interpolation.
+            # This works because of the side-effect that we do not
+            # allow overwriting and/or str interpolating complex objects.
+            # Fix if/when those assumptions are no longer valid.
             resolved = {
                 key: self._resolve(context, value, key, skip_checks)
                 for key, value in definition.items()

dvc/progress.py

@@ -2,12 +2,16 @@
 import logging
 import sys
 from threading import RLock
+from typing import TYPE_CHECKING
 
 from tqdm import tqdm
 
 from dvc.env import DVC_IGNORE_ISATTY
 from dvc.utils import env2bool
 
+if TYPE_CHECKING:
+    from dvc.fs.callbacks import TqdmCallback
+
 logger = logging.getLogger(__name__)
 tqdm.set_lock(RLock())
 
@@ -158,3 +162,8 @@ class Tqdm(tqdm):
             d["ncols_desc"] = d["ncols_info"] = 1
             d["prefix"] = ""
         return d
+
+    def as_callback(self) -> "TqdmCallback":
+        from dvc.fs.callbacks import TqdmCallback
+
+        return TqdmCallback(progress_bar=self)

dvc/repo/checkout.py

@@ -1,12 +1,22 @@
 import logging
 import os
-from typing import Dict, List
+from typing import TYPE_CHECKING, Dict, List
 
-from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit, NoOutputOrStageError
+from dvc.exceptions import (
+    CheckoutError,
+    CheckoutErrorSuggestGit,
+    DvcException,
+    NoOutputOrStageError,
+)
+from dvc.ui import ui
 from dvc.utils import relpath
 
 from . import locked
 
+if TYPE_CHECKING:
+    from dvc_data.index import BaseDataIndex, DataIndexEntry
+    from dvc_objects.fs.base import FileSystem
+
 logger = logging.getLogger(__name__)
 
 
@@ -59,6 +69,33 @@ def _build_out_changes(index, changes):
     return out_changes
 
 
+def _check_can_delete(
+    entries: List["DataIndexEntry"],
+    index: "BaseDataIndex",
+    path: str,
+    fs: "FileSystem",
+):
+    entry_paths = []
+    for entry in entries:
+        try:
+            cache_fs, cache_path = index.storage_map.get_cache(entry)
+        except ValueError:
+            continue
+
+        if cache_fs.exists(cache_path):
+            continue
+
+        entry_paths.append(fs.path.join(path, *entry.key))
+
+    if not entry_paths:
+        return
+
+    raise DvcException(
+        "Can't remove the following unsaved files without confirmation. "
+        "Use `--force` to force.\n" + "\n".join(entry_paths)
+    )
+
+
 @locked
 def checkout(  # noqa: C901
     self,
@@ -70,8 +107,6 @@ def checkout(  # noqa: C901
     allow_missing=False,
     **kwargs,
 ):
-    from dvc import prompt
-    from dvc.fs.callbacks import Callback
     from dvc.repo.index import build_data_index
     from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError
     from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare
@@ -107,56 +142,69 @@ def checkout(  # noqa: C901
         onerror=onerror,
     )
 
-    with Callback.as_tqdm_callback(
+    with ui.progress(
         unit="entry",
         desc="Building data index",
-    ) as cb:
+    ) as pb:
         old = build_data_index(
-            view, self.root_dir, self.fs, compute_hash=True, callback=cb
+            view,
+            self.root_dir,
+            self.fs,
+            compute_hash=True,
+            callback=pb.as_callback(),
         )
 
     new = view.data["repo"]
 
-    with Callback.as_tqdm_callback(
+    with ui.progress(
         desc="Comparing indexes",
         unit="entry",
-    ) as cb:
-        diff = compare(old, new, relink=relink, delete=True, callback=cb)
+    ) as pb:
+        diff = compare(old, new, relink=relink, delete=True, callback=pb.as_callback())
 
-    failed = []
+    if not force:
+        _check_can_delete(diff.files_delete, new, self.root_dir, self.fs)
+
+    failed = set()
+    out_paths = [out.fs_path for out in view.outs if out.use_cache and out.is_in_repo]
 
     def checkout_onerror(src_path, dest_path, _exc):
         logger.debug(
-            "failed to create {%s} from {%s}", dest_path, src_path, exc_info=True
+            "failed to create '%s' from '%s'", dest_path, src_path, exc_info=True
         )
-        failed.append(dest_path)
 
-    with Callback.as_tqdm_callback(
+        for out_path in out_paths:
+            if self.fs.path.isin_or_eq(dest_path, out_path):
+                failed.add(out_path)
+
+    with ui.progress(
         unit="file",
         desc="Checkout",
-    ) as cb:
-        changes = apply(
+    ) as pb:
+        apply(
             diff,
             self.root_dir,
             self.fs,
-            callback=cb,
-            prompt=prompt.confirm,
+            callback=pb.as_callback(),
             update_meta=False,
-            force=force,
             onerror=checkout_onerror,
             state=self.state,
             **kwargs,
         )
 
-    out_changes = _build_out_changes(view, changes)
+    out_changes = _build_out_changes(view, diff.changes)
 
     typ_map = {ADD: "added", DELETE: "deleted", MODIFY: "modified"}
     for key, typ in out_changes.items():
         out_path = self.fs.path.join(self.root_dir, *key)
-        self.state.save_link(out_path, self.fs)
-        stats[typ_map[typ]].append(_fspath_dir(out_path))
+
+        if out_path in failed:
+            self.fs.remove(out_path, recursive=True)
+        else:
+            self.state.save_link(out_path, self.fs)
+            stats[typ_map[typ]].append(_fspath_dir(out_path))
 
     if failed and not allow_missing:
-        raise CheckoutError(failed, stats)
+        raise CheckoutError([relpath(out_path) for out_path in failed], stats)
 
     return stats

dvc/repo/data.py

@@ -2,9 +2,11 @@ import os
 import posixpath
 from typing import TYPE_CHECKING, Any, Dict, Iterable, List, TypedDict, Union
 
+from dvc.fs.callbacks import DEFAULT_CALLBACK
 from dvc.ui import ui
 
 if TYPE_CHECKING:
+    from dvc.fs.callbacks import Callback
     from dvc.repo import Repo
     from dvc.scm import Git, NoSCM
     from dvc_data.index import DataIndex
@@ -50,6 +52,7 @@ def _diff(
     not_in_cache: bool = False,
     not_in_remote: bool = False,
     remote_refresh: bool = False,
+    callback: "Callback" = DEFAULT_CALLBACK,
 ) -> Dict[str, List[str]]:
     from dvc_data.index import StorageError
     from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff
@@ -70,6 +73,7 @@ def _diff(
         shallow=not granular,
         hash_only=True,
         with_unknown=True,
+        callback=callback,
     ):
         if (
             change.typ == UNCHANGED
@@ -149,16 +153,27 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn
 def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
     from .index import build_data_index
 
-    with ui.status("Building workspace index"):
+    with ui.progress(
+        desc="Building workspace index",
+        unit="entry",
+    ) as pb:
         workspace = build_data_index(
-            repo.index, repo.root_dir, repo.fs, compute_hash=True
+            repo.index,
+            repo.root_dir,
+            repo.fs,
+            compute_hash=True,
+            callback=pb.as_callback(),
         )
 
-    with ui.status("Calculating diff between index/workspace"):
+    with ui.progress(
+        desc="Calculating diff between index/workspace",
+        unit="entry",
+    ) as pb:
         return _diff(
             repo.index.data["repo"],
             workspace,
             not_in_cache=True,
+            callback=pb.as_callback(),
             **kwargs,
         )
 
@@ -171,8 +186,11 @@ def _diff_head_to_index(
     with repo.switch(head):
         head_index = repo.index.data["repo"]
 
-    with ui.status("Calculating diff between head/index"):
-        return _diff(head_index, index, **kwargs)
+    with ui.progress(
+        desc="Calculating diff between head/index",
+        unit="entry",
+    ) as pb:
+        return _diff(head_index, index, callback=pb.as_callback(), **kwargs)
 
 
 class Status(TypedDict):

dvc/repo/experiments/collect.py

@@ -10,6 +10,7 @@ from typing import (
     Iterator,
     List,
     Optional,
+    Tuple,
     Union,
 )
 
@@ -369,12 +370,13 @@ def _describe(
 
 
 def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> List["ExpRange"]:
-    """Return list of ExpRange sorted by timestamp."""
+    """Return list of ExpRange sorted by (timestamp, rev)."""
 
-    def _head_timestamp(exp_range: "ExpRange") -> datetime:
+    def _head_timestamp(exp_range: "ExpRange") -> Tuple[datetime, str]:
         head_exp = first(exp_range.revs)
         if head_exp and head_exp.data and head_exp.data.timestamp:
-            return head_exp.data.timestamp
-        return datetime.fromtimestamp(0)
+            return head_exp.data.timestamp, head_exp.rev
+
+        return datetime.fromtimestamp(0), ""
 
     return sorted(exp_ranges, key=_head_timestamp, reverse=True)

dvc/repo/experiments/executor/ssh.py

@@ -210,8 +210,7 @@ class SSHExecutor(BaseExecutor):
         """Collect DVC cache."""
         from dvc.repo.experiments.pull import _pull_cache
 
-        with self.get_odb() as odb:
-            _pull_cache(repo, exp_ref, run_cache=run_cache, odb=odb)
+        _pull_cache(repo, exp_ref, run_cache=run_cache)
 
     @contextmanager
     def get_odb(self):

dvc/repo/experiments/ls.py

@@ -42,7 +42,7 @@ def ls(
     for baseline in ref_info_dict:
         name = baseline
         if tags[baseline] or ref_heads[baseline]:
-            name = tags[baseline] or ref_heads[baseline][len(base) + 1 :]
+            name = tags[baseline] or ref_heads[baseline]
         for info in ref_info_dict[baseline]:
             if git_remote:
                 exp_rev = None

dvc/repo/experiments/pull.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 def pull(  # noqa: C901
     repo,
     git_remote: str,
-    exp_names: Union[Iterable[str], str],
+    exp_names: Optional[Union[Iterable[str], str]] = None,
     all_commits=False,
     rev: Optional[Union[List[str], str]] = None,
     num=1,
@@ -32,30 +32,30 @@ def pull(  # noqa: C901
     exp_ref_set: Set["ExpRefInfo"] = set()
     if all_commits:
         exp_ref_set.update(exp_refs(repo.scm, git_remote))
+    elif exp_names:
+        if isinstance(exp_names, str):
+            exp_names = [exp_names]
+        exp_ref_dict = resolve_name(repo.scm, exp_names, git_remote)
+
+        unresolved_exp_names = []
+        for exp_name, exp_ref in exp_ref_dict.items():
+            if exp_ref is None:
+                unresolved_exp_names.append(exp_name)
+            else:
+                exp_ref_set.add(exp_ref)
+
+        if unresolved_exp_names:
+            raise UnresolvedExpNamesError(unresolved_exp_names)
+
     else:
-        if exp_names:
-            if isinstance(exp_names, str):
-                exp_names = [exp_names]
-            exp_ref_dict = resolve_name(repo.scm, exp_names, git_remote)
-
-            unresolved_exp_names = []
-            for exp_name, exp_ref in exp_ref_dict.items():
-                if exp_ref is None:
-                    unresolved_exp_names.append(exp_name)
-                else:
-                    exp_ref_set.add(exp_ref)
-
-            if unresolved_exp_names:
-                raise UnresolvedExpNamesError(unresolved_exp_names)
-
-        if rev:
-            if isinstance(rev, str):
-                rev = [rev]
-            rev_dict = iter_revs(repo.scm, rev, num)
-            rev_set = set(rev_dict.keys())
-            ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
-            for _, ref_info_list in ref_info_dict.items():
-                exp_ref_set.update(ref_info_list)
+        rev = rev or "HEAD"
+        if isinstance(rev, str):
+            rev = [rev]
+        rev_dict = iter_revs(repo.scm, rev, num)
+        rev_set = set(rev_dict.keys())
+        ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
+        for _, ref_info_list in ref_info_dict.items():
+            exp_ref_set.update(ref_info_list)
 
     pull_result = _pull(repo, git_remote, exp_ref_set, force)
 
@@ -107,10 +107,9 @@ def _pull_cache(
     dvc_remote=None,
     jobs=None,
     run_cache=False,
-    odb=None,
 ):
     if isinstance(refs, ExpRefInfo):
         refs = [refs]
     revs = list(exp_commits(repo.scm, refs))
     logger.debug("dvc fetch experiment '%s'", refs)
-    repo.fetch(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs, odb=odb)
+    repo.fetch(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs)

dvc/repo/experiments/push.py

@@ -97,7 +97,7 @@ def exp_refs_from_rev(scm: "Git", rev: List[str], num: int = 1) -> Set["ExpRefIn
 def push(
     repo: "Repo",
     git_remote: str,
-    exp_names: Union[List[str], str],
+    exp_names: Optional[Union[List[str], str]] = None,
     all_commits: bool = False,
     rev: Optional[Union[List[str], str]] = None,
     num: int = 1,
@@ -111,7 +111,8 @@ def push(
         exp_ref_set.update(exp_refs(repo.scm))
     if exp_names:
         exp_ref_set.update(exp_refs_from_names(repo.scm, ensure_list(exp_names)))
-    if rev:
+    else:
+        rev = rev or "HEAD"
         if isinstance(rev, str):
             rev = [rev]
         exp_ref_set.update(exp_refs_from_rev(repo.scm, rev, num=num))

dvc/repo/worktree.py

@@ -158,7 +158,7 @@ def push_worktree(
         ) as cb:
             cb.set_size(total)
             try:
-                stats = apply(
+                apply(
                     diff,
                     remote_obj.path,
                     remote_obj.fs,
@@ -166,7 +166,7 @@ def push_worktree(
                     latest_only=remote_obj.worktree,
                     jobs=jobs,
                 )
-                pushed += sum(len(changes) for changes in stats.values())
+                pushed += sum(len(changes) for changes in diff.changes.values())
             except VersioningNotSupported:
                 logger.exception("")
                 raise DvcException(

dvc/schema.py

@@ -53,6 +53,7 @@ OUT_PSTAGE_DETAILED_SCHEMA = {
         **ANNOTATION_SCHEMA,  # type: ignore[arg-type]
         Output.PARAM_CACHE: bool,
         Output.PARAM_PERSIST: bool,
+        "checkpoint": bool,
         Output.PARAM_REMOTE: str,
         Output.PARAM_PUSH: bool,
     }
@@ -73,12 +74,14 @@ PLOT_PSTAGE_SCHEMA = {str: Any(PLOT_PROPS_SCHEMA, [PLOT_PROPS_SCHEMA])}
 
 PARAM_PSTAGE_NON_DEFAULT_SCHEMA = {str: [str]}
 
+VARS_SCHEMA = [str, dict]
 
 STAGE_DEFINITION = {
     Required(StageParams.PARAM_CMD): Any(str, list),
     Optional(StageParams.PARAM_WDIR): str,
     Optional(StageParams.PARAM_DEPS): [str],
     Optional(StageParams.PARAM_PARAMS): [Any(str, dict)],
+    Optional(VARS_KWD): VARS_SCHEMA,
     Optional(StageParams.PARAM_FROZEN): bool,
     Optional(StageParams.PARAM_META): object,
     Optional(StageParams.PARAM_DESC): str,
@@ -120,7 +123,7 @@ SINGLE_PIPELINE_STAGE_SCHEMA = {
 MULTI_STAGE_SCHEMA = {
     PLOTS: [Any(str, SINGLE_PLOT_SCHEMA)],
     STAGES: SINGLE_PIPELINE_STAGE_SCHEMA,
-    VARS_KWD: [str, dict],
+    VARS_KWD: VARS_SCHEMA,
     StageParams.PARAM_PARAMS: [str],
     StageParams.PARAM_METRICS: [str],
     ARTIFACTS: SINGLE_ARTIFACT_SCHEMA,

pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=2.0.2,<2.1.0",
+    "dvc-data>=2.3.0,<2.4.0",
     "dvc-http>=2.29.0",
     "dvc-render>=0.3.1,<1",
     "dvc-studio-client>=0.9.2,<1",
@@ -56,7 +56,7 @@ dependencies = [
     "requests>=2.22",
     "rich>=12",
     "ruamel.yaml>=0.17.11",
-    "scmrepo>=1.0.0,<2",
+    "scmrepo>=1.0.4,<2",
     "shortuuid>=0.5",
     "shtab<2,>=1.3.4",
     "tabulate>=0.8.7",
@@ -70,7 +70,7 @@ dependencies = [
 all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"]
 azure = ["dvc-azure>=2.21.2"]
 dev = ["dvc[azure,gdrive,gs,hdfs,lint,oss,s3,ssh,terraform,tests,webdav,webhdfs]"]
-gdrive = ["dvc-gdrive==2.19.2"]
+gdrive = ["dvc-gdrive==2.20"]
 gs = ["dvc-gs==2.22.1"]
 hdfs = ["dvc-hdfs==2.19"]
 lint = [

scripts/build.py

@@ -1,88 +0,0 @@
-import argparse
-import pathlib
-import sys
-from subprocess import STDOUT, check_call
-
-path = pathlib.Path(__file__).parent.absolute()
-dvc = path.parent / "dvc"
-pyinstaller = path / "pyinstaller"
-innosetup = path / "innosetup"
-fpm = path / "fpm"
-
-parser = argparse.ArgumentParser()
-parser.add_argument("pkg", choices=["deb", "rpm", "osxpkg", "exe"], help="package type")
-parser.add_argument("--sign-application", default=False, action="store_true")
-parser.add_argument("--application-id")
-parser.add_argument("--sign-installer", default=False, action="store_true")
-parser.add_argument("--installer-id")
-parser.add_argument("--notarize", default=False, action="store_true")
-parser.add_argument("--apple-id-username")
-parser.add_argument("--apple-id-password")
-args = parser.parse_args()
-
-(dvc / "_build.py").write_text(f'PKG = "{args.pkg}"')
-
-if not (dvc / "_dvc_version.py").exists():
-    raise Exception("no version info found")
-
-check_call(
-    ["python", "build.py"],
-    cwd=pyinstaller,
-    stderr=STDOUT,
-)
-
-if args.sign_application:
-    if args.pkg != "osxpkg":
-        raise NotImplementedError
-    if not args.application_id:
-        print("--sign-application requires --application-id")
-        sys.exit(1)
-    check_call(
-        ["python", "sign.py", "--application-id", args.application_id],
-        cwd=pyinstaller,
-        stderr=STDOUT,
-    )
-
-if args.pkg == "exe":
-    check_call(
-        ["python", "build.py"],
-        cwd=innosetup,
-        stderr=STDOUT,
-    )
-else:
-    check_call(
-        ["python", "build.py", args.pkg],
-        cwd=fpm,
-        stderr=STDOUT,
-    )
-
-if args.sign_installer:
-    if args.pkg != "osxpkg":
-        raise NotImplementedError
-    if not all([args.installer_id, args.apple_id_username, args.apple_id_password]):
-        print("--sign-installer requires --installer-id")
-        sys.exit(1)
-    check_call(
-        ["python", "sign.py", "--installer-id", args.installer_id],
-        cwd=fpm,
-        stderr=STDOUT,
-    )
-
-if args.notarize:
-    if args.pkg != "osxpkg":
-        raise NotImplementedError
-    if not all([args.apple_id_username, args.apple_id_password]):
-        print("--notarize requires --apple-id-username and --apple-id-password")
-        sys.exit(1)
-    check_call(
-        [
-            "python",
-            "notarize.py",
-            "--apple-id-username",
-            args.apple_id_username,
-            "--apple-id-password",
-            args.apple_id_password,
-        ],
-        cwd=fpm,
-        stderr=STDOUT,
-    )

scripts/fpm/.gitignore

@@ -1,4 +0,0 @@
-/build
-/*.deb
-/*.rpm
-/*.pkg

scripts/fpm/after-install.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-ln -sf /usr/local/lib/dvc/dvc /usr/local/bin/dvc

scripts/fpm/after-remove.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-rm -rf /usr/local/bin/dvc

scripts/fpm/build.py

@@ -1,98 +0,0 @@
-import argparse
-import os
-import pathlib
-import shutil
-from subprocess import STDOUT, check_call, check_output
-
-path = pathlib.Path(__file__).parent.absolute()
-dvc = path.parent.parent / "dvc"
-pyinstaller = path.parent / "pyinstaller"
-
-build = path / "build"
-install = build / "usr"
-
-parser = argparse.ArgumentParser()
-parser.add_argument("pkg", choices=["deb", "rpm", "osxpkg"], help="package type")
-args = parser.parse_args()
-
-flags = [
-    "--description",
-    '"Data Version Control | Git for Data & Models"',
-    "-n",
-    "dvc",
-    "-s",
-    "dir",
-    "-f",
-    "--license",
-    '"Apache License 2.0"',
-]
-
-if args.pkg == "osxpkg":
-    install /= "local"
-    bash_dir = install / "etc" / "bash_completion.d"
-    dirs = ["usr"]
-    flags.extend(
-        [
-            "--osxpkg-identifier-prefix",
-            "com.iterative",
-            "--after-install",
-            path / "after-install.sh",
-            "--after-remove",
-            path / "after-remove.sh",
-        ]
-    )
-else:
-    if args.pkg == "rpm":
-        # https://github.com/jordansissel/fpm/issues/1503
-        flags.extend(["--rpm-rpmbuild-define", "_build_id_links none"])
-
-    bash_dir = build / "etc" / "bash_completion.d"
-    dirs = ["usr", "etc"]
-    flags.extend(["--depends", "git >= 1.7.0"])  # needed for gitpython
-
-try:
-    shutil.rmtree(build)
-except FileNotFoundError:
-    pass
-
-lib = install / "lib"
-lib.mkdir(parents=True)
-shutil.copytree(pyinstaller / "dist" / "dvc", lib / "dvc")
-
-if args.pkg != "osxpkg":
-    # NOTE: in osxpkg fpm replaces symlinks with actual file that it
-    # points to, so we need to use after-install hook.
-    (install / "bin").mkdir()
-    os.symlink("../lib/dvc/dvc", install / "bin" / "dvc")
-
-bash_dir.mkdir(parents=True)
-bash_completion = check_output(
-    [lib / "dvc" / "dvc", "completion", "-s", "bash"], text=True
-)
-(bash_dir / "dvc").write_text(bash_completion)
-
-zsh_dir = install / "share" / "zsh" / "site-functions"
-zsh_dir.mkdir(parents=True)
-zsh_completion = check_output(
-    [lib / "dvc" / "dvc", "completion", "-s", "zsh"], text=True
-)
-(zsh_dir / "_dvc").write_text(zsh_completion)
-
-version = check_output([lib / "dvc" / "dvc", "--version"], text=True).strip()
-
-check_call(
-    [
-        "fpm",
-        "--verbose",
-        "-t",
-        args.pkg,
-        *flags,
-        "-v",
-        version,
-        "-C",
-        build,
-        *dirs,
-    ],
-    cwd=path,
-    stderr=STDOUT,
-)

scripts/fpm/notarize.py

@@ -1,64 +0,0 @@
-import argparse
-import json
-import os
-import pathlib
-import sys
-from subprocess import STDOUT, check_call
-
-if sys.platform != "darwin":
-    raise NotImplementedError
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "path",
-    nargs="?",
-    help="Path to the osxpkg to notarize. If not specified - try to find one.",
-)
-parser.add_argument("--apple-id-username", required=True, help="Apple ID username.")
-parser.add_argument(
-    "--apple-id-password",
-    required=True,
-    help=(
-        "Apple ID app-specific password. Note that this is not a regular "
-        "Apple ID password, so you need to generate one at "
-        "https://appleid.apple.com/account/manage"
-    ),
-)
-args = parser.parse_args()
-
-path = pathlib.Path(__file__).parent.absolute()
-
-if args.path:
-    pkg = pathlib.Path(args.path)
-else:
-    pkgs = list(path.glob("*.pkg"))
-    if not pkgs:
-        print("No pkgs found")
-        sys.exit(1)
-
-    if len(pkgs) > 1:
-        print("Too many packages")
-        sys.exit(1)
-
-    (pkg,) = pkgs
-
-
-config = {
-    "notarize": {
-        "path": os.fspath(pkg),
-        "bundle_id": "com.iterative.dvc",
-        "staple": True,
-    },
-    "apple_id": {
-        "username": args.apple_id_username,
-        "password": args.apple_id_password,
-    },
-}
-
-(path / "config.json").write_text(json.dumps(config))
-
-check_call(
-    ["gon", "config.json"],
-    cwd=path,
-    stderr=STDOUT,
-)

scripts/fpm/sign.py

@@ -1,55 +0,0 @@
-import argparse
-import os
-import pathlib
-import sys
-from subprocess import STDOUT, check_call
-
-if sys.platform != "darwin":
-    raise NotImplementedError
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "path",
-    nargs="?",
-    help="Path to the osxpkg to sign. If not specified - try to find one.",
-)
-parser.add_argument(
-    "--installer-id",
-    required=True,
-    help="Certificate ID (should be added to the keychain).",
-)
-args = parser.parse_args()
-
-path = pathlib.Path(__file__).parent.absolute()
-
-if args.path:
-    pkg = pathlib.Path(args.path)
-else:
-    pkgs = list(path.glob("*.pkg"))
-    if not pkgs:
-        print("No pkgs found")
-        sys.exit(1)
-
-    if len(pkgs) > 1:
-        print("Too many packages")
-        sys.exit(1)
-
-    (pkg,) = pkgs
-
-unsigned = pkg.with_suffix(".unsigned")
-os.rename(pkg, unsigned)
-check_call(
-    [
-        "productsign",
-        "--sign",
-        args.installer_id,
-        os.fspath(unsigned),
-        os.fspath(pkg),
-    ],
-    stderr=STDOUT,
-)
-
-check_call(
-    ["pkgutil", "--check-signature", os.fspath(pkg)],
-    stderr=STDOUT,
-)

scripts/innosetup/.gitignore

@@ -1,6 +0,0 @@
-/dvc.ico
-/dvc_left.bmp
-/dvc_up.bmp
-/config.ini
-/build
-/dvc*.exe

scripts/innosetup/addSymLinkPermissions.ps1

@@ -1,76 +0,0 @@
-param (
-	[string]$mytype = 'system'
-)
-
-function addSymLinkPermissions($accountToAdd){
-    Write-Host "Checking SymLink permissions.."
-    $sidstr = $null
-    if ( "$accountToAdd" -eq "Everyone" ) {
-    	$sidstr = "S-1-1-0"
-    } else {
-        try {
-            $ntprincipal = new-object System.Security.Principal.NTAccount "$accountToAdd"
-            $sid = $ntprincipal.Translate([System.Security.Principal.SecurityIdentifier])
-            $sidstr = $sid.Value.ToString()
-        } catch {
-            $sidstr = $null
-        }
-    }
-    Write-Host "Account: $($accountToAdd)" -ForegroundColor DarkCyan
-    if( [string]::IsNullOrEmpty($sidstr) ) {
-        Write-Host "Account not found!" -ForegroundColor Red
-        exit -1
-    }
-    Write-Host "Account SID: $($sidstr)" -ForegroundColor DarkCyan
-    $tmp = [System.IO.Path]::GetTempFileName()
-    Write-Host "Export current Local Security Policy" -ForegroundColor DarkCyan
-    secedit.exe /export /cfg "$($tmp)"
-    $c = Get-Content -Path $tmp
-    $currentSetting = ""
-    foreach($s in $c) {
-        if( $s -like "SECreateSymbolicLinkPrivilege*") {
-            $x = $s.split("=",[System.StringSplitOptions]::RemoveEmptyEntries)
-            $currentSetting = $x[1].Trim()
-        }
-    }
-    if( $currentSetting -notlike "*$($sidstr)*" ) {
-        Write-Host "Need to add permissions to SymLink" -ForegroundColor Yellow
-
-        Write-Host "Modify Setting ""Create SymLink""" -ForegroundColor DarkCyan
-
-        if( [string]::IsNullOrEmpty($currentSetting) ) {
-            $currentSetting = "*$($sidstr)"
-        } else {
-            $currentSetting = "*$($sidstr),$($currentSetting)"
-        }
-        Write-Host "$currentSetting"
-    $outfile = @"
-[Unicode]
-Unicode=yes
-[Version]
-signature="`$CHICAGO`$"
-Revision=1
-[Privilege Rights]
-SECreateSymbolicLinkPrivilege = $($currentSetting)
-"@
-    $tmp2 = [System.IO.Path]::GetTempFileName()
-        Write-Host "Import new settings to Local Security Policy" -ForegroundColor DarkCyan
-        $outfile | Set-Content -Path $tmp2 -Encoding Unicode -Force
-        Push-Location (Split-Path $tmp2)
-        try {
-            secedit.exe /configure /db "secedit.sdb" /cfg "$($tmp2)" /areas USER_RIGHTS
-        } finally {
-            Pop-Location
-        }
-    } else {
-        Write-Host "NO ACTIONS REQUIRED! Account already in ""Create SymLink""" -ForegroundColor DarkCyan
-        Write-Host "Account $accountToAdd already has permissions to SymLink" -ForegroundColor Green
-        return $true;
-    }
-}
-
-if ( "$mytype" -eq "user" ) {
-    addSymLinkPermissions $(Get-WMIObject -class Win32_ComputerSystem | select username).username
-} else {
-    addSymLinkPermissions Everyone
-}

scripts/innosetup/addsymlink.iss

@@ -1,14 +0,0 @@
-procedure AddSymLink();
-var
-    ErrorCode: Integer;
-    SRCdir: String;
-begin
-    SRCdir := ExpandConstant('{app}');
-    if isUninstaller() = false then begin
-        if not ShellExec('', 'powershell.exe', '-noninteractive -windowstyle hidden -executionpolicy bypass -File "' + SRCdir + '\addSymLinkPermissions.ps1" -mytype ' + SymLinkType, '', SW_HIDE, ewWaitUntilTerminated, ErrorCode) then begin
-            SuppressibleMsgBox('Failed to automatically grant SeCreateSymbolicLinkPrivilege. Please download Polsedit(www.southsoftware.com/polsedit.zip). Launch polseditx32.exe or polseeditx64.exe (depending on your Windows version), navigate to "Security Settings" -> "User Rights Assignment", add the account(s) to the list named "Create symbolic links", logoff and login back into your account. More info at https://github.com/git-for-windows/git/wiki/Symbolic-Links.', mbInformation, MB_OK, IDOK);
-        end else begin
-            SuppressibleMsgBox('Automatically added SeCreateSymbolicLinkPrivilege. Please logoff and login back into your account in order for the change to take effect.', mbInformation, MB_OK, IDOK);
-        end;
-    end;
-end;

scripts/innosetup/build.py

@@ -1,44 +0,0 @@
-import argparse
-import configparser
-import os
-import pathlib
-import shutil
-from subprocess import STDOUT, check_call, check_output
-
-path = pathlib.Path(__file__).parent.absolute()
-config = path / "config.ini"
-
-dvc = path.parent.parent / "dvc"
-pyinstaller = path.parent / "pyinstaller"
-
-build = path / "build"
-install = build / "usr"
-
-parser = argparse.ArgumentParser()
-args = parser.parse_args()
-
-try:
-    shutil.rmtree(build)
-except FileNotFoundError:
-    pass
-
-build.mkdir()
-shutil.copytree(pyinstaller / "dist" / "dvc", build / "dvc")
-shutil.copy(path / "addSymLinkPermissions.ps1", build)
-
-version = check_output(
-    [os.fspath(build / "dvc" / "dvc"), "--version"], text=True
-).strip()
-
-cfg = configparser.ConfigParser()
-cfg.add_section("Version")
-cfg.set("Version", "version", version)
-
-with (path / "config.ini").open("w") as fobj:
-    cfg.write(fobj)
-
-check_call(
-    ["iscc", "setup.iss"],
-    cwd=path,
-    stderr=STDOUT,
-)

scripts/innosetup/dvc.ico.dvc

@@ -1,3 +0,0 @@
-outs:
-- md5: 90104d9e83cfb825cf45507e90aadd27
-  path: dvc.ico

scripts/innosetup/dvc_left.bmp.dvc

@@ -1,3 +0,0 @@
-outs:
-- md5: 9106cda08aa427e73492389a0f17c72d
-  path: dvc_left.bmp

scripts/innosetup/dvc_up.bmp.dvc

@@ -1,3 +0,0 @@
-outs:
-- md5: 94614d6650e062655f9f77507dc9c1f2
-  path: dvc_up.bmp

scripts/innosetup/modpath.iss

@@ -1,215 +0,0 @@
-// ----------------------------------------------------------------------------
-// Script based on:
-//
-// Inno Setup Ver:	5.4.2
-// Script Version:	1.4.2
-// Author:			Jared Breland <[email protected]>
-// Homepage:		http://www.legroom.net/software
-// License:			GNU Lesser General Public License (LGPL), version 3
-//						http://www.gnu.org/licenses/lgpl.html
-//
-// Script Function:
-//	Allow modification of environmental path directly from Inno Setup installers
-//
-// Instructions:
-//	Copy modpath.iss to the same directory as your setup script
-//
-//	Add this statement to your [Setup] section
-//		ChangesEnvironment=true
-//
-//	Add this statement to your [Tasks] section
-//	You can change the Description or Flags
-//	You can change the Name, but it must match the ModPathName setting below
-//		Name: modifypath; Description: &Add application directory to your environmental path; Flags: unchecked
-//
-//	Add the following to the end of your [Code] section
-//	ModPathName defines the name of the task defined above
-//	ModPathType defines whether the 'user' or 'system' path will be modified;
-//		this will default to user if anything other than system is set
-//	setArrayLength must specify the total number of dirs to be added
-//	Result[0] contains first directory, Result[1] contains second, etc.
-//		const
-//			ModPathName = 'modifypath';
-//			ModPathType = 'user';
-//
-//		function ModPathDir(): TArrayOfString;
-//		begin
-//			setArrayLength(Result, 1);
-//			Result[0] := ExpandConstant('{app}');
-//		end;
-//		#include "modpath.iss"
-// ----------------------------------------------------------------------------
-
-function ModPathDir(): TArrayOfString;
-begin
-	setArrayLength(Result, 1)
-	Result[0] := ExpandConstant(ModPathPath);
-end;
-
-procedure ModPath();
-var
-	oldpath:	String;
-	newpath:	String;
-	updatepath:	Boolean;
-	pathArr:	TArrayOfString;
-	aExecFile:	String;
-	aExecArr:	TArrayOfString;
-	i, d:		Integer;
-	pathdir:	TArrayOfString;
-	regroot:	Integer;
-	regpath:	String;
-
-begin
-	// Get constants from main script and adjust behavior accordingly
-	// ModPathType MUST be 'system' or 'user'; force 'user' if invalid
-	if ModPathType = 'system' then begin
-		regroot := HKEY_LOCAL_MACHINE;
-		regpath := 'SYSTEM\CurrentControlSet\Control\Session Manager\Environment';
-	end else begin
-		regroot := HKEY_CURRENT_USER;
-		regpath := 'Environment';
-	end;
-
-	// Get array of new directories and act on each individually
-	pathdir := ModPathDir();
-	for d := 0 to GetArrayLength(pathdir)-1 do begin
-		updatepath := true;
-
-		// Modify WinNT path
-		if UsingWinNT() = true then begin
-
-			// Get current path, split into an array
-			RegQueryStringValue(regroot, regpath, 'Path', oldpath);
-			oldpath := oldpath + ';';
-			i := 0;
-
-			while (Pos(';', oldpath) > 0) do begin
-				SetArrayLength(pathArr, i+1);
-				pathArr[i] := Copy(oldpath, 0, Pos(';', oldpath)-1);
-				oldpath := Copy(oldpath, Pos(';', oldpath)+1, Length(oldpath));
-				i := i + 1;
-
-				// Check if current directory matches app dir
-				if pathdir[d] = pathArr[i-1] then begin
-					// if uninstalling, remove dir from path
-					if IsUninstaller() = true then begin
-						continue;
-					// if installing, flag that dir already exists in path
-					end else begin
-						updatepath := false;
-					end;
-				end;
-
-				// Add current directory to new path
-				if i = 1 then begin
-					newpath := pathArr[i-1];
-				end else begin
-					newpath := newpath + ';' + pathArr[i-1];
-				end;
-			end;
-
-			// Append app dir to path if not already included
-			if (IsUninstaller() = false) AND (updatepath = true) then
-				newpath := newpath + ';' + pathdir[d];
-
-			// Write new path
-			RegWriteStringValue(regroot, regpath, 'Path', newpath);
-
-		// Modify Win9x path
-		end else begin
-
-			// Convert to shortened dirname
-			pathdir[d] := GetShortName(pathdir[d]);
-
-			// If autoexec.bat exists, check if app dir already exists in path
-			aExecFile := 'C:\AUTOEXEC.BAT';
-			if FileExists(aExecFile) then begin
-				LoadStringsFromFile(aExecFile, aExecArr);
-				for i := 0 to GetArrayLength(aExecArr)-1 do begin
-					if IsUninstaller() = false then begin
-						// If app dir already exists while installing, skip add
-						if (Pos(pathdir[d], aExecArr[i]) > 0) then
-							updatepath := false;
-							break;
-					end else begin
-						// If app dir exists and = what we originally set, then delete at uninstall
-						if aExecArr[i] = 'SET PATH=%PATH%;' + pathdir[d] then
-							aExecArr[i] := '';
-					end;
-				end;
-			end;
-
-			// If app dir not found, or autoexec.bat didn't exist, then (create and) append to current path
-			if (IsUninstaller() = false) AND (updatepath = true) then begin
-				SaveStringToFile(aExecFile, #13#10 + 'SET PATH=%PATH%;' + pathdir[d], True);
-
-			// If uninstalling, write the full autoexec out
-			end else begin
-				SaveStringsToFile(aExecFile, aExecArr, False);
-			end;
-		end;
-	end;
-end;
-
-// Split a string into an array using passed delimiter
-procedure MPExplode(var Dest: TArrayOfString; Text: String; Separator: String);
-var
-	i: Integer;
-begin
-	i := 0;
-	repeat
-		SetArrayLength(Dest, i+1);
-		if Pos(Separator,Text) > 0 then	begin
-			Dest[i] := Copy(Text, 1, Pos(Separator, Text)-1);
-			Text := Copy(Text, Pos(Separator,Text) + Length(Separator), Length(Text));
-			i := i + 1;
-		end else begin
-			 Dest[i] := Text;
-			 Text := '';
-		end;
-	until Length(Text)=0;
-end;
-
-procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep);
-var
-	aSelectedTasks:	TArrayOfString;
-	i:				Integer;
-	taskname:		String;
-	regpath:		String;
-	regstring:		String;
-	appid:			String;
-begin
-	// only run during actual uninstall
-	if CurUninstallStep = usUninstall then begin
-		// get list of selected tasks saved in registry at install time
-		appid := '{#emit SetupSetting("AppId")}';
-		if appid = '' then appid := '{#emit SetupSetting("AppName")}';
-		regpath := ExpandConstant('Software\Microsoft\Windows\CurrentVersion\Uninstall\'+appid+'_is1');
-		RegQueryStringValue(HKLM, regpath, 'Inno Setup: Selected Tasks', regstring);
-		if regstring = '' then RegQueryStringValue(HKCU, regpath, 'Inno Setup: Selected Tasks', regstring);
-
-		// check each task; if matches modpath taskname, trigger patch removal
-		if regstring <> '' then begin
-			taskname := ModPathName;
-			MPExplode(aSelectedTasks, regstring, ',');
-			if GetArrayLength(aSelectedTasks) > 0 then begin
-				for i := 0 to GetArrayLength(aSelectedTasks)-1 do begin
-					if comparetext(aSelectedTasks[i], taskname) = 0 then
-						ModPath();
-				end;
-			end;
-		end;
-	end;
-end;
-
-function NeedRestart(): Boolean;
-var
-	taskname:	String;
-begin
-	taskname := ModPathName;
-	if IsTaskSelected(taskname) and not UsingWinNT() then begin
-		Result := True;
-	end else begin
-		Result := False;
-	end;
-end;

scripts/innosetup/setup.iss

@@ -1,83 +0,0 @@
-#define MyAppName "DVC (Data Version Control)"
-#define MyAppVersion ReadIni(SourcePath + "\config.ini", "Version", "version", "unknown")
-#define MyAppPublisher "Dmitry Petrov"
-#define MyAppURL "https://dvc.org"
-#define MyAppDir SourcePath + "\build\dvc"
-
-[Setup]
-AppId={{8258CE8A-110E-4E0D-AE60-FEE00B15F041}
-AppName={#MyAppName}
-AppVersion={#MyAppVersion}
-AppPublisher={#MyAppPublisher}
-AppPublisherURL={#MyAppURL}
-AppSupportURL={#MyAppURL}
-AppUpdatesURL={#MyAppURL}
-DefaultDirName={code:GetDefaultDirName}
-DefaultGroupName={#MyAppName}
-AllowNoIcons=yes
-LicenseFile=..\..\LICENSE
-OutputBaseFilename=dvc-{#MyAppVersion}
-Compression=lzma2/max
-SolidCompression=yes
-OutputDir=.
-ChangesEnvironment=yes
-SetupIconFile=dvc.ico
-WizardSmallImageFile=dvc_up.bmp
-WizardImageFile=dvc_left.bmp
-DisableWelcomePage=no
-
-[Languages]
-Name: "english"; MessagesFile: "compiler:Default.isl"
-
-[Files]
-Source: "{#MyAppDir}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs
-
-[Tasks]
-Name: modifypath; Description: Adds dvc's application directory to environmental path; Flags: checkablealone;
-Name: modifypath\system; Description: Adds dvc's application directory to environmental path for all users;
-Name: addsymlinkpermissions; Description: Add permission for creating symbolic links; Flags: checkablealone;
-Name: addsymlinkpermissions\system; Description: Add permissions for creating symbolic links for all users;
-
-[Code]
-const
-	ModPathName = 'modifypath';
-	ModPathPath = '{app}';
-	SymLinkName = 'addsymlinkpermissions';
-
-var
-	ModPathType: String;
-	SymLinkType: String;
-
-function GetDefaultDirName(Dummy: string): string;
-begin
-	if IsAdminLoggedOn then begin
-		Result := ExpandConstant('{pf}\{#MyAppName}');
-	end else begin
-		Result := ExpandConstant('{userpf}\{#MyAppName}');
-	end;
-end;
-
-#include "modpath.iss"
-#include "addsymlink.iss"
-
-procedure CurStepChanged(CurStep: TSetupStep);
-begin
-	if CurStep = ssPostInstall then begin
-		if IsTaskSelected(ModPathName + '\system') then begin
-			ModPathType := 'system';
-		end else begin
-			ModPathType := 'user';
-		end;
-
-		if IsTaskSelected(SymLinkName + '\system') then begin
-			SymLinkType := 'system';
-		end else begin
-			SymLinkType := 'user';
-		end;
-
-		if IsTaskSelected(ModPathName) then
-			ModPath();
-		if IsTaskSelected(SymLinkName) then
-			AddSymLink();
-	end;
-end;

scripts/pyinstaller/entitlements.plist

@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-	<!-- These are required for binaries built by PyInstaller -->
-	<key>com.apple.security.cs.allow-jit</key>
-	<true/>
-	<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
-	<true/>
-</dict>
-</plist>

scripts/pyinstaller/sign.py

@@ -1,39 +0,0 @@
-import argparse
-import os
-import pathlib
-import sys
-from subprocess import STDOUT, check_call
-
-if sys.platform != "darwin":
-    raise NotImplementedError
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "--application-id",
-    required=True,
-    help="Certificate ID (should be added to the keychain).",
-)
-args = parser.parse_args()
-
-path = pathlib.Path(__file__).parent.absolute()
-dvc = path / "dist" / "dvc"
-for root, _, fnames in os.walk(dvc):
-    for fname in fnames:
-        fpath = os.path.join(root, fname)
-        print(f"signing {fpath}")
-        check_call(
-            [
-                "codesign",
-                "--force",
-                "--verbose",
-                "-s",
-                args.application_id,
-                "-o",
-                "runtime",
-                "--entitlements",
-                "entitlements.plist",
-                fpath,
-            ],
-            stderr=STDOUT,
-            timeout=180,
-        )

tests/func/experiments/test_experiments.py

@@ -362,7 +362,7 @@ def test_list(tmp_dir, scm, dvc, exp_stage):
     exp_c = first(results)
     ref_info_c = first(exp_refs_by_rev(scm, exp_c))
 
-    assert dvc.experiments.ls() == {"master": [(ref_info_c.name, exp_c)]}
+    assert dvc.experiments.ls() == {"refs/heads/master": [(ref_info_c.name, exp_c)]}
 
     exp_list = dvc.experiments.ls(rev=ref_info_a.baseline_sha)
     assert {key: set(val) for key, val in exp_list.items()} == {
@@ -372,20 +372,20 @@ def test_list(tmp_dir, scm, dvc, exp_stage):
     exp_list = dvc.experiments.ls(rev=[baseline_a, scm.get_rev()])
     assert {key: set(val) for key, val in exp_list.items()} == {
         baseline_a: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)},
-        "master": {(ref_info_c.name, exp_c)},
+        "refs/heads/master": {(ref_info_c.name, exp_c)},
     }
 
     exp_list = dvc.experiments.ls(all_commits=True)
     assert {key: set(val) for key, val in exp_list.items()} == {
         baseline_a: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)},
-        "master": {(ref_info_c.name, exp_c)},
+        "refs/heads/master": {(ref_info_c.name, exp_c)},
     }
 
     scm.checkout("branch", True)
     exp_list = dvc.experiments.ls(all_commits=True)
     assert {key: set(val) for key, val in exp_list.items()} == {
         baseline_a: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)},
-        "branch": {(ref_info_c.name, exp_c)},
+        "refs/heads/branch": {(ref_info_c.name, exp_c)},
     }
 
 
@@ -676,7 +676,7 @@ def test_experiment_unchanged(tmp_dir, scm, dvc, exp_stage):
     dvc.experiments.run(exp_stage.addressing)
     dvc.experiments.run(exp_stage.addressing)
 
-    assert len(dvc.experiments.ls()["master"]) == 2
+    assert len(dvc.experiments.ls()["refs/heads/master"]) == 2
 
 
 def test_experiment_run_dry(tmp_dir, scm, dvc, exp_stage):

tests/func/experiments/test_remote.py

@@ -35,6 +35,9 @@ def test_push(tmp_dir, scm, dvc, git_upstream, exp_stage, use_url):
     dvc.experiments.push(remote, [ref_info1.name])
     assert git_upstream.tmp_dir.scm.get_ref(str(ref_info1)) == exp1
 
+    dvc.experiments.push(remote)
+    assert git_upstream.tmp_dir.scm.get_ref(str(ref_info3)) == exp3
+
 
 @pytest.mark.parametrize("all_,rev,result3", [(True, False, True), (False, True, None)])
 def test_push_args(tmp_dir, scm, dvc, git_upstream, exp_stage, all_, rev, result3):
@@ -165,7 +168,7 @@ def test_list_remote(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
     exp_list = downstream_exp.ls(all_commits=True, git_remote=remote)
     assert {key: set(val) for key, val in exp_list.items()} == {
         baseline_a: {(ref_info_a.name, None), (ref_info_b.name, None)},
-        "master": {(ref_info_c.name, None)},
+        "refs/heads/master": {(ref_info_c.name, None)},
     }
 
 
@@ -173,6 +176,11 @@ def test_list_remote(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
 def test_pull(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
     from dvc.exceptions import InvalidArgumentError
 
+    # fetch and checkout to downstream so both repos start from same commit
+    downstream_repo = git_downstream.tmp_dir.scm.gitpython.repo
+    fetched = downstream_repo.remote(git_downstream.remote).fetch()
+    downstream_repo.git.checkout(fetched)
+
     remote = git_downstream.url if use_url else git_downstream.remote
     downstream_exp = git_downstream.tmp_dir.dvc.experiments
     with pytest.raises(InvalidArgumentError):
@@ -200,6 +208,9 @@ def test_pull(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
     downstream_exp.pull(remote, [str(ref_info1)])
     assert git_downstream.tmp_dir.scm.get_ref(str(ref_info1)) == exp1
 
+    downstream_exp.pull(remote)
+    assert git_downstream.tmp_dir.scm.get_ref(str(ref_info3)) == exp3
+
 
 @pytest.mark.parametrize("all_,rev,result3", [(True, False, True), (False, True, None)])
 def test_pull_args(tmp_dir, scm, dvc, git_downstream, exp_stage, all_, rev, result3):

tests/func/experiments/test_save.py

@@ -71,7 +71,7 @@ def test_exp_save_after_commit(tmp_dir, dvc, scm):
 
     all_exps = dvc.experiments.ls(all_commits=True)
     assert all_exps[baseline][0][0] == "exp-1"
-    assert all_exps["master"][0][0] == "exp-2"
+    assert all_exps["refs/heads/master"][0][0] == "exp-2"
 
 
 def test_exp_save_with_staged_changes(tmp_dir, dvc, scm):

tests/func/parsing/test_errors.py

@@ -9,6 +9,7 @@ import pytest
 from dvc.parsing import ResolveError
 from dvc.parsing.context import Context
 from dvc.parsing.interpolate import embrace
+from dvc.utils.humanize import join
 
 from . import make_entry_definition, make_foreach_def
 
@@ -21,6 +22,18 @@ def escape_ansi(line):
 # Tests for the interpolated entries
 
 
+@pytest.mark.parametrize("vars_", ["${file}_params.yaml", {"foo": "${foo}"}])
+def test_vars_interpolation_errors(tmp_dir, dvc, vars_):
+    definition = make_entry_definition(tmp_dir, "build", {"vars": [vars_]})
+    with pytest.raises(ResolveError) as exc_info:
+        definition.resolve()
+
+    assert (
+        str(exc_info.value) == "failed to parse 'stages.build.vars' in 'dvc.yaml': "
+        "interpolating is not allowed"
+    )
+
+
 def test_failed_to_interpolate(tmp_dir, dvc):
     context = Context(models={"foo": "bar"})
     definition = make_entry_definition(
@@ -41,6 +54,23 @@ def test_failed_to_interpolate(tmp_dir, dvc):
     assert definition.context == {"models": {"foo": "bar"}}
 
 
+def test_local_vars_params_file_not_exist(tmp_dir, dvc):
+    definition = make_entry_definition(
+        tmp_dir,
+        "build",
+        {"vars": ["not_existing_params.yaml"], "cmd": "echo ${models.foo}"},
+    )
+
+    with pytest.raises(ResolveError) as exc_info:
+        definition.resolve()
+
+    assert (
+        str(exc_info.value) == "failed to parse stage 'build' in 'dvc.yaml': "
+        "'not_existing_params.yaml' does not exist"
+    )
+    assert not definition.context
+
+
 def test_specified_key_does_not_exist(tmp_dir, dvc):
     definition = make_entry_definition(
         tmp_dir,
@@ -119,6 +149,25 @@ def test_interpolate_nested_iterable(tmp_dir, dvc):
     )
 
 
+def test_partial_vars_doesnot_exist(tmp_dir, dvc):
+    (tmp_dir / "test_params.yaml").dump({"sub1": "sub1", "sub2": "sub2"})
+
+    definition = make_entry_definition(
+        tmp_dir,
+        "build",
+        {"vars": ["test_params.yaml:sub3"], "cmd": "echo ${sub1} ${sub2}"},
+    )
+
+    with pytest.raises(ResolveError) as exc_info:
+        definition.resolve()
+
+    assert (
+        str(exc_info.value) == "failed to parse stage 'build' in 'dvc.yaml': "
+        "could not find 'sub3' in 'test_params.yaml'"
+    )
+    assert not definition.context
+
+
 # Tests foreach generated stages and their error messages
 
 
@@ -235,6 +284,47 @@ def test_foreach_do_definition_item_does_not_exist(tmp_dir, dvc, key, loc):
     assert context == {"foo": "bar"}
 
 
+@pytest.mark.parametrize(
+    "redefine",
+    [
+        {"item": 5},
+        {"key": 5},
+        {"item": 5, "key": 10},
+        {"item": {"epochs": 10}},
+    ],
+)
+@pytest.mark.parametrize("from_file", [True, False])
+def test_item_key_in_generated_stage_vars(tmp_dir, dvc, redefine, from_file):
+    context = Context(foo="bar")
+    vars_ = [redefine]
+    if from_file:
+        (tmp_dir / "test_params.yaml").dump(redefine)
+        vars_ = ["test_params.yaml"]
+
+    definition = make_foreach_def(
+        tmp_dir,
+        "build",
+        {"model1": {"thresh": "10"}, "model2": {"thresh": 5}},
+        {"vars": vars_, "cmd": "${item}"},
+        context,
+    )
+
+    with pytest.raises(ResolveError) as exc_info:
+        definition.resolve_all()
+
+    message = str(exc_info.value)
+    assert (
+        "failed to parse stage 'build@model1' in 'dvc.yaml': "
+        "attempted to modify reserved" in message
+    )
+
+    key_or_keys = "keys" if len(redefine) > 1 else "key"
+    assert f"{key_or_keys} {join(redefine)}" in message
+    if from_file:
+        assert "in 'test_params.yaml'" in message
+    assert context == {"foo": "bar"}
+
+
 def test_foreach_wdir_key_does_not_exist(tmp_dir, dvc):
     definition = make_foreach_def(
         tmp_dir,

tests/func/parsing/test_foreach.py

@@ -1,8 +1,9 @@
 """Testing happy paths for the foreach."""
+import os
 
 import pytest
 
-from dvc.parsing import DataResolver, ForeachDefinition
+from dvc.parsing import DEFAULT_PARAMS_FILE, DataResolver, ForeachDefinition
 from dvc.parsing.context import Context
 
 
@@ -269,6 +270,134 @@ def test_foreach_with_interpolated_wdir(tmp_dir, dvc):
     assert not any(item for item in resolver.tracked_vars.values())
 
 
+def test_foreach_with_local_vars(tmp_dir, dvc):
+    resolver = DataResolver(dvc, (tmp_dir / "data").fs_path, {})
+    foreach_data = ["foo", "bar"]
+    data = {
+        "foreach": foreach_data,
+        "do": {
+            "vars": [{"foobar": "foobar"}],
+            "cmd": "echo ${item} ${foobar}",
+        },
+    }
+    definition = ForeachDefinition(resolver, resolver.context, "build", data)
+
+    assert definition.resolve_all() == {
+        # note that the resolver generates `wdir` relative to file's wdir
+        # so, this is just `foo`, not `data/foo`.
+        # figuring out `wdir` is the responsibility of the `load_stage`/`Stage`
+        "build@foo": {"cmd": "echo foo foobar"},
+        "build@bar": {"cmd": "echo bar foobar"},
+    }
+    assert not resolver.context
+    assert not any(item for item in resolver.tracked_vars.values())
+
+
+@pytest.mark.parametrize(
+    "local_import",
+    [
+        "test_params.yaml",
+        "test_params.yaml:train",
+        "test_params.yaml:train,prepare",
+    ],
+)
+def test_foreach_with_imported_vars(tmp_dir, dvc, local_import):
+    (tmp_dir / "params.yaml").dump({"models": {"model1": {"thresh": "foo"}}})
+    (tmp_dir / "test_params.yaml").dump(
+        {"train": {"epochs": 10}, "prepare": {"nums": 25}}
+    )
+    resolver = DataResolver(dvc, tmp_dir.fs_path, {})
+    foreach_data = ["foo", "bar"]
+    data = {
+        "foreach": foreach_data,
+        "do": {"vars": [local_import], "cmd": "echo ${item} ${train.epochs}"},
+    }
+    definition = ForeachDefinition(resolver, resolver.context, "build", data)
+
+    assert definition.resolve_all() == {
+        # note that the resolver generates `wdir` relative to file's wdir
+        # so, this is just `foo`, not `data/foo`.
+        # figuring out `wdir` is the responsibility of the `load_stage`/`Stage`
+        "build@foo": {"cmd": "echo foo 10"},
+        "build@bar": {"cmd": "echo bar 10"},
+    }
+
+    assert resolver.context == {"models": {"model1": {"thresh": "foo"}}}
+    assert resolver.tracked_vars == {
+        "build@foo": {"test_params.yaml": {"train.epochs": 10}},
+        "build@bar": {"test_params.yaml": {"train.epochs": 10}},
+    }
+
+
+@pytest.mark.parametrize("local_import", ["params.yaml", "params.yaml:train,prepare"])
+def test_foreach_with_interpolated_wdir_and_local_vars(tmp_dir, dvc, local_import):
+    (tmp_dir / "params.yaml").dump({"models": {"model1": {"thresh": "foo"}}})
+
+    for i in range(5):
+        build_dir = tmp_dir / ("model-" + str(i))
+        build_dir.mkdir()
+        (build_dir / "params.yaml").dump(
+            {"train": {"epochs": 1 + i}, "prepare": {"nums": 10 * i}},
+        )
+
+    resolver = DataResolver(dvc, tmp_dir.fs_path, {})
+    data = {
+        "foreach": [0, 1, 2, 3, 4],
+        "do": {
+            "wdir": "model-${item}",
+            "vars": [local_import],
+            "cmd": "echo ${item} ${train.epochs} ${prepare.nums}",
+        },
+    }
+    definition = ForeachDefinition(resolver, resolver.context, "build", data)
+
+    assert definition.resolve_all() == {
+        # note that the resolver generates `wdir` relative to file's wdir
+        # so, this is just `foo`, not `data/foo`.
+        # figuring out `wdir` is the responsibility of the `load_stage`/`Stage`
+        "build@0": {"wdir": "model-0", "cmd": "echo 0 1 0"},
+        "build@1": {"wdir": "model-1", "cmd": "echo 1 2 10"},
+        "build@2": {"wdir": "model-2", "cmd": "echo 2 3 20"},
+        "build@3": {"wdir": "model-3", "cmd": "echo 3 4 30"},
+        "build@4": {"wdir": "model-4", "cmd": "echo 4 5 40"},
+    }
+
+    assert resolver.context == {"models": {"model1": {"thresh": "foo"}}}
+    assert resolver.tracked_vars == {
+        "build@0": {
+            os.path.join("model-0", "params.yaml"): {
+                "train.epochs": 1,
+                "prepare.nums": 0,
+            }
+        },
+        "build@1": {
+            os.path.join("model-1", "params.yaml"): {
+                "train.epochs": 2,
+                "prepare.nums": 10,
+            }
+        },
+        "build@2": {
+            os.path.join("model-2", "params.yaml"): {
+                "train.epochs": 3,
+                "prepare.nums": 20,
+            }
+        },
+        "build@3": {
+            os.path.join("model-3", "params.yaml"): {
+                "train.epochs": 4,
+                "prepare.nums": 30,
+            }
+        },
+        "build@4": {
+            os.path.join("model-4", "params.yaml"): {
+                "train.epochs": 5,
+                "prepare.nums": 40,
+            }
+        },
+    }
+    assert resolver.context.imports == {DEFAULT_PARAMS_FILE: None}
+
+
 def test_foreach_do_syntax_is_checked_once(tmp_dir, dvc, mocker):
     do_def = {"cmd": "python script.py --epochs ${item}"}
     data = {"foreach": [0, 1, 2, 3, 4], "do": do_def}

tests/func/parsing/test_interpolated_entry.py

@@ -1,3 +1,4 @@
+import os
 from copy import deepcopy
 
 import pytest
@@ -71,6 +72,147 @@ def test_vars_and_params_import(tmp_dir, dvc):
     }
 
 
+def test_stage_with_wdir(tmp_dir, dvc):
+    """
+    Test that params file from wdir are also loaded
+    """
+    d = {
+        "stages": {
+            "stage1": {
+                "cmd": "echo ${dict.foo} ${dict.bar}",
+                "params": ["value1"],
+                "wdir": "data",
+                "vars": [DEFAULT_PARAMS_FILE],
+            }
+        }
+    }
+
+    data_dir = tmp_dir / "data"
+    data_dir.mkdir()
+    (tmp_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"bar": "bar"}})
+    (data_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"foo": "foo"}})
+    resolver = DataResolver(dvc, tmp_dir.fs_path, d)
+
+    assert_stage_equal(
+        resolver.resolve(),
+        {
+            "stages": {
+                "stage1": {
+                    "cmd": "echo foo bar",
+                    "wdir": "data",
+                    "params": ["value1"],
+                }
+            }
+        },
+    )
+    assert resolver.tracked_vars == {
+        "stage1": {
+            os.path.join("data", DEFAULT_PARAMS_FILE): {"dict.foo": "foo"},
+            DEFAULT_PARAMS_FILE: {"dict.bar": "bar"},
+        }
+    }
+
+
+def test_with_templated_wdir(tmp_dir, dvc):
+    """
+    Test that params from the resolved wdir are still loaded
+    and is used in the interpolation.
+    """
+    d = {
+        "stages": {
+            "stage1": {
+                "cmd": "echo ${dict.foo} ${dict.bar}",
+                "params": ["value1"],
+                "wdir": "${dict.ws}",
+                "vars": [DEFAULT_PARAMS_FILE],
+            }
+        }
+    }
+    (tmp_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"bar": "bar", "ws": "data"}})
+    data_dir = tmp_dir / "data"
+    data_dir.mkdir()
+    (data_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"foo": "foo"}})
+    resolver = DataResolver(dvc, tmp_dir.fs_path, d)
+
+    assert_stage_equal(
+        resolver.resolve(),
+        {
+            "stages": {
+                "stage1": {
+                    "cmd": "echo foo bar",
+                    "wdir": "data",
+                    "params": ["value1"],
+                }
+            }
+        },
+    )
+    assert resolver.tracked_vars == {
+        "stage1": {
+            os.path.join("data", DEFAULT_PARAMS_FILE): {"dict.foo": "foo"},
+            DEFAULT_PARAMS_FILE: {"dict.bar": "bar", "dict.ws": "data"},
+        }
+    }
+    assert resolver.context.imports == {"params.yaml": None}
+    assert resolver.context == {"dict": {"bar": "bar", "ws": "data"}}
+
+
+def test_resolve_local_tries_to_load_globally_used_files(tmp_dir, dvc):
+    iterable = {"bar": "bar", "foo": "foo"}
+    (tmp_dir / "params.json").dump(iterable)
+
+    d = {
+        "vars": ["params.json"],
+        "stages": {
+            "build": {
+                "cmd": "command --value ${bar}",
+                "params": [{"params.json": ["foo"]}],
+                "vars": ["params.json"],
+            }
+        },
+    }
+    resolver = DataResolver(dvc, tmp_dir.fs_path, d)
+    assert_stage_equal(
+        resolver.resolve(),
+        {
+            "stages": {
+                "build": {
+                    "cmd": "command --value bar",
+                    "params": [{"params.json": ["foo"]}],
+                }
+            }
+        },
+    )
+    assert resolver.tracked_vars == {"build": {"params.json": {"bar": "bar"}}}
+
+
+def test_resolve_local_tries_to_load_globally_used_params_yaml(tmp_dir, dvc):
+    iterable = {"bar": "bar", "foo": "foo"}
+    (tmp_dir / "params.yaml").dump(iterable)
+
+    d = {
+        "stages": {
+            "build": {
+                "cmd": "command --value ${bar}",
+                "params": [{"params.yaml": ["foo"]}],
+                "vars": ["params.yaml"],
+            }
+        }
+    }
+    resolver = DataResolver(dvc, tmp_dir.fs_path, d)
+    assert_stage_equal(
+        resolver.resolve(),
+        {
+            "stages": {
+                "build": {
+                    "cmd": "command --value bar",
+                    "params": [{"params.yaml": ["foo"]}],
+                }
+            }
+        },
+    )
+    assert resolver.tracked_vars == {"build": {"params.yaml": {"bar": "bar"}}}
+
+
 def test_vars_relpath_overwrite(tmp_dir, dvc):
     iterable = {"bar": "bar", "foo": "foo"}
     (tmp_dir / "params.yaml").dump(iterable)
@@ -89,6 +231,7 @@ def test_vars_relpath_overwrite(tmp_dir, dvc):
     assert resolver.context.imports == {"params.yaml": None}
 
 
+@pytest.mark.parametrize("local", [True, False])
 @pytest.mark.parametrize(
     "vars_",
     [
@@ -98,10 +241,14 @@ def test_vars_relpath_overwrite(tmp_dir, dvc):
         ["test_params.yaml", "test_params.yaml"],
     ],
 )
-def test_vars_load_partial(tmp_dir, dvc, vars_):
+def test_vars_load_partial(tmp_dir, dvc, local, vars_):
     iterable = {"bar": "bar", "foo": "foo"}
     (tmp_dir / "test_params.yaml").dump(iterable)
-    d = {"vars": vars_, "stages": {"build": {"cmd": "echo ${bar}"}}}
+    d = {"stages": {"build": {"cmd": "echo ${bar}"}}}
+    if local:
+        d["stages"]["build"]["vars"] = vars_
+    else:
+        d["vars"] = vars_
     resolver = DataResolver(dvc, tmp_dir.fs_path, d)
     resolver.resolve()
 

tests/func/parsing/test_resolver.py

@@ -144,6 +144,7 @@ def test_specified_params_file_not_exist(tmp_dir, dvc):
     )
 
 
+@pytest.mark.parametrize("local", [True, False])
 @pytest.mark.parametrize(
     "vars_",
     [
@@ -152,11 +153,35 @@ def test_specified_params_file_not_exist(tmp_dir, dvc):
         ["test_params.yaml:sub1", "test_params.yaml:sub1,sub2"],
     ],
 )
-def test_vars_already_loaded_message(tmp_dir, dvc, vars_):
-    d = {"vars": vars_, "stages": {"build": {"cmd": "echo ${sub1} ${sub2}"}}}
+def test_vars_already_loaded_message(tmp_dir, dvc, local, vars_):
+    d = {"stages": {"build": {"cmd": "echo ${sub1} ${sub2}"}}}
     (tmp_dir / "test_params.yaml").dump({"sub1": "sub1", "sub2": "sub2"})
+    if not local:
+        d["vars"] = vars_
+    else:
+        d["stages"]["build"]["vars"] = vars_
 
     with pytest.raises(ResolveError) as exc_info:  # noqa: PT012
         resolver = DataResolver(dvc, tmp_dir.fs_path, d)
         resolver.resolve()
     assert "partially" in str(exc_info.value)
+
+
+@pytest.mark.parametrize(
+    "vars_, loc", [(DATA, "build.vars[0]"), ("params.json", "params.json")]
+)
+def test_local_overwrite_error(tmp_dir, dvc, vars_, loc):
+    (tmp_dir / DEFAULT_PARAMS_FILE).dump(DATA)
+    (tmp_dir / "params.json").dump(DATA)
+
+    d = {"stages": {"build": {"cmd": "echo ${models.foo}", "vars": [vars_]}}}
+
+    resolver = DataResolver(dvc, tmp_dir.fs_path, d)
+    with pytest.raises(ResolveError) as exc_info:
+        resolver.resolve()
+
+    assert (
+        str(exc_info.value) == "failed to parse stage 'build' in 'dvc.yaml':\n"
+        f"cannot redefine 'models.bar' from '{loc}' "
+        "as it already exists in 'params.yaml'"
+    )

tests/func/repro/test_repro.py

@@ -1273,3 +1273,12 @@ def test_repro_missing_lock_info(tmp_dir, dvc, copy_script):
 
     stages = dvc.reproduce(stage.addressing)
     assert len(stages) == 1
+
+
+def test_repro_rm_recursive(tmp_dir, dvc):
+    # check that dir output recursively removes files in the dir
+    tmp_dir.gen({"dir": {"foo": "foo"}})
+    dvc.stage.add(name="dir", cmd="mkdir dir", outs=["dir"])
+    dvc.reproduce()
+    assert (tmp_dir / "dir").exists()
+    assert not (tmp_dir / "dir" / "foo").exists()

tests/func/test_add.py

@@ -784,6 +784,7 @@ def test_add_preserve_fields(tmp_dir, dvc):
           remote: testremote
           md5: acbd18db4cc2f85cedef654fccc4a4d8
           size: 3
+          hash: md5
         meta: some metadata
     """
     )
@@ -975,6 +976,7 @@ def test_add_updates_to_cloud_versioning_dir(tmp_dir, dvc):
             "outs": [
                 {
                     "path": "data",
+                    "hash": "md5",
                     "files": [
                         {
                             "size": 3,
@@ -1007,6 +1009,7 @@ def test_add_updates_to_cloud_versioning_dir(tmp_dir, dvc):
         "outs": [
             {
                 "path": "data",
+                "hash": "md5",
                 "files": [
                     {
                         "size": 4,

tests/func/test_checkout.py

@@ -84,23 +84,19 @@ def test_remove_files_when_checkout(tmp_dir, dvc, scm):
 
 class TestCheckoutCleanWorkingDir:
     def test(self, mocker, tmp_dir, dvc):
-        mock_prompt = mocker.patch("dvc.prompt.confirm", return_value=True)
         (stage,) = tmp_dir.dvc_gen("data", {"foo": "foo"})
 
         # change working directory
         (tmp_dir / "data").gen("not_cached.txt", "not_cached")
-        assert main(["checkout", stage.relpath]) == 0
-        mock_prompt.assert_called()
+        assert main(["checkout", stage.relpath, "--force"]) == 0
         assert not (tmp_dir / "data" / "not_cached.txt").exists()
 
     def test_force(self, mocker, tmp_dir, dvc):
-        mock_prompt = mocker.patch("dvc.prompt.confirm", return_value=False)
         (stage,) = tmp_dir.dvc_gen("data", {"foo": "foo"})
 
         # change working directory
         (tmp_dir / "data").gen("not_cached.txt", "not_cached")
         assert main(["checkout", stage.relpath]) != 0
-        mock_prompt.assert_called()
         assert (tmp_dir / "data" / "not_cached.txt").exists()
 
 

tests/func/test_commit.py

@@ -316,9 +316,12 @@ def test_commit_dos2unix(tmp_dir, dvc):
         }
     )
     legacy_content = (tmp_dir / "foo.dvc").read_text()
+    assert "hash: md5" not in legacy_content
 
-    dvc.commit("foo.dvc")
+    dvc.commit("foo.dvc", force=True)
     assert (tmp_dir / "foo.dvc").read_text() == legacy_content
+
+    tmp_dir.gen("foo", "modified")
     dvc.commit("foo.dvc", force=True)
     content = (tmp_dir / "foo.dvc").read_text()
     assert "hash: md5" in content

tests/unit/command/test_add.py

@@ -32,7 +32,7 @@ def test_add(mocker, dvc):
     )
 
 
-def test_add_to_remote(mocker):
+def test_add_to_remote(mocker, dvc):
     cli_args = parse_args(
         [
             "add",
@@ -63,7 +63,7 @@ def test_add_to_remote(mocker):
     )
 
 
-def test_add_to_remote_invalid_combinations(mocker, caplog):
+def test_add_to_remote_invalid_combinations(mocker, caplog, dvc):
     cli_args = parse_args(["add", "s3://bucket/foo", "s3://bucket/bar", "--to-remote"])
     assert cli_args.func == CmdAdd
 
@@ -83,7 +83,7 @@ def test_add_to_remote_invalid_combinations(mocker, caplog):
             assert expected_msg in caplog.text
 
 
-def test_add_to_cache_invalid_combinations(mocker, caplog):
+def test_add_to_cache_invalid_combinations(mocker, caplog, dvc):
     cli_args = parse_args(["add", "s3://bucket/foo", "s3://bucket/bar", "-o", "foo"])
     assert cli_args.func == CmdAdd
 

tests/unit/command/test_data_status.py

@@ -75,7 +75,8 @@ def test_json(dvc, mocker, capsys, mocked_status, args, to_omit):
 
 def test_no_changes_repo(dvc, scm, capsys):
     assert main(["data", "status"]) == 0
-    assert capsys.readouterr() == ("No changes.\n", "")
+    out, _ = capsys.readouterr()
+    assert out == "No changes.\n"
 
 
 def test_empty_scm_repo(tmp_dir, capsys):
@@ -83,7 +84,7 @@ def test_empty_scm_repo(tmp_dir, capsys):
     Repo.init()
 
     assert main(["data", "status"]) == 0
-    out, err = capsys.readouterr()
+    out, _ = capsys.readouterr()
     assert (
         out
         == """\
@@ -91,7 +92,6 @@ No changes in an empty git repo.
 (there are changes not tracked by dvc, use "git status" to see)
 """
     )
-    assert not err
 
 
 @pytest.mark.parametrize(

tests/unit/command/test_data_sync.py

@@ -2,7 +2,7 @@ from dvc.cli import parse_args
 from dvc.commands.data_sync import CmdDataFetch, CmdDataPull, CmdDataPush
 
 
-def test_fetch(mocker):
+def test_fetch(mocker, dvc):
     cli_args = parse_args(
         [
             "fetch",
@@ -40,7 +40,7 @@ def test_fetch(mocker):
     )
 
 
-def test_pull(mocker):
+def test_pull(mocker, dvc):
     cli_args = parse_args(
         [
             "pull",
@@ -84,7 +84,7 @@ def test_pull(mocker):
     )
 
 
-def test_push(mocker):
+def test_push(mocker, dvc):
     cli_args = parse_args(
         [
             "push",

tests/unit/command/test_diff.py

@@ -19,7 +19,7 @@ def test_digest(checksum, expected):
     assert expected == _digest(checksum)
 
 
-def test_default(mocker, capsys):
+def test_default(mocker, capsys, dvc):
     args = parse_args(["diff"])
     cmd = args.func(args)
     diff = {
@@ -51,7 +51,7 @@ def test_default(mocker, capsys):
     ).format(sep=os.path.sep) in capsys.readouterr()[0]
 
 
-def test_show_hash(mocker, capsys):
+def test_show_hash(mocker, capsys, dvc):
     args = parse_args(["diff", "--show-hash"])
     cmd = args.func(args)
     diff = {
@@ -101,7 +101,7 @@ def test_show_hash(mocker, capsys):
     ) in out
 
 
-def test_show_json(mocker, capsys):
+def test_show_json(mocker, capsys, dvc):
     args = parse_args(["diff", "--json"])
     cmd = args.func(args)
     diff = {
@@ -123,7 +123,7 @@ def test_show_json(mocker, capsys):
     assert '"not in cache": []' in out
 
 
-def test_show_json_and_hash(mocker, capsys):
+def test_show_json_and_hash(mocker, capsys, dvc):
     args = parse_args(["diff", "--json", "--show-hash"])
     cmd = args.func(args)
 
@@ -160,7 +160,7 @@ def test_show_json_and_hash(mocker, capsys):
     assert '"not in cache": []' in out
 
 
-def test_show_json_hide_missing(mocker, capsys):
+def test_show_json_hide_missing(mocker, capsys, dvc):
     args = parse_args(["diff", "--json", "--hide-missing"])
     cmd = args.func(args)
     diff = {
@@ -190,7 +190,7 @@ def test_show_json_hide_missing(mocker, capsys):
 
 
 @pytest.mark.parametrize("show_hash", [None, True, False])
-def test_diff_show_markdown_and_hash(mocker, show_hash):
+def test_diff_show_markdown_and_hash(mocker, show_hash, dvc):
     options = ["diff", "--md"] + (["--show-hash"] if show_hash else [])
     args = parse_args(options)
     cmd = args.func(args)
@@ -221,7 +221,7 @@ def test_diff_show_markdown_and_hash(mocker, show_hash):
         (["--md"], "| Status   | Path   |\n|----------|--------|"),
     ),
 )
-def test_no_changes(mocker, capsys, opts, show, expected):
+def test_no_changes(mocker, capsys, opts, show, expected, dvc):
     args = parse_args(["diff", *opts, *show])
     cmd = args.func(args)
     mocker.patch("dvc.repo.Repo.diff", return_value={})
@@ -330,7 +330,7 @@ def test_show_markdown_hide_missing(capsys):
     ).format(sep=os.path.sep)
 
 
-def test_hide_missing(mocker, capsys):
+def test_hide_missing(mocker, capsys, dvc):
     args = parse_args(["diff", "--hide-missing"])
     cmd = args.func(args)
     diff = {

tests/unit/command/test_experiments.py

@@ -56,7 +56,7 @@ def test_experiments_diff(dvc, scm, mocker):
     )
 
 
-def test_experiments_diff_revs(mocker, capsys):
+def test_experiments_diff_revs(mocker, capsys, dvc, scm):
     mocker.patch(
         "dvc.repo.experiments.diff.diff",
         return_value={
@@ -194,11 +194,11 @@ def test_experiments_list(dvc, scm, mocker):
         (["--sha-only"], "sha-a\n"),
     ],
 )
-def test_experiments_list_format(mocker, capsys, args, expected):
+def test_experiments_list_format(mocker, capsys, args, expected, dvc, scm):
     mocker.patch(
         "dvc.repo.experiments.ls.ls",
         return_value={
-            "main": [
+            "refs/heads/main": [
                 ("exp-a", "sha-a"),
             ]
         },
@@ -214,7 +214,7 @@ def test_experiments_list_format(mocker, capsys, args, expected):
     assert cap.out == expected
 
 
-def test_experiments_list_remote(mocker, capsys):
+def test_experiments_list_remote(mocker, capsys, dvc, scm):
     mocker.patch(
         "dvc.repo.experiments.ls.ls",
         return_value={
@@ -294,12 +294,7 @@ def test_experiments_push(dvc, scm, mocker):
     )
     cmd = cli_args.func(cli_args)
 
-    with pytest.raises(InvalidArgumentError) as exp_info:
-        cmd.run()
-    assert (
-        str(exp_info.value) == "Either provide an `experiment` argument"
-        ", or use the `--rev` or `--all-commits` flag."
-    )
+    assert cmd.run() == 0
 
 
 def test_experiments_pull(dvc, scm, mocker):
@@ -351,12 +346,7 @@ def test_experiments_pull(dvc, scm, mocker):
     )
     cmd = cli_args.func(cli_args)
 
-    with pytest.raises(InvalidArgumentError) as exp_info:
-        cmd.run()
-    assert (
-        str(exp_info.value) == "Either provide an `experiment` argument"
-        ", or use the `--rev` or `--all-commits` flag."
-    )
+    assert cmd.run() == 0
 
 
 def test_experiments_remove_flag(dvc, scm, mocker, capsys, caplog):

tests/unit/command/test_imp.py

@@ -2,7 +2,7 @@ from dvc.cli import parse_args
 from dvc.commands.imp import CmdImport
 
 
-def test_import(mocker):
+def test_import(mocker, dvc):
     cli_args = parse_args(
         [
             "import",
@@ -34,7 +34,7 @@ def test_import(mocker):
     )
 
 
-def test_import_no_exec(mocker):
+def test_import_no_exec(mocker, dvc):
     cli_args = parse_args(
         [
             "import",
@@ -64,7 +64,7 @@ def test_import_no_exec(mocker):
     )
 
 
-def test_import_no_download(mocker):
+def test_import_no_download(mocker, dvc):
     cli_args = parse_args(
         [
             "import",

tests/unit/command/test_imp_url.py

@@ -7,7 +7,7 @@ from dvc.commands.imp_url import CmdImportUrl
 from dvc.exceptions import DvcException
 
 
-def test_import_url(mocker):
+def test_import_url(mocker, dvc):
     cli_args = parse_args(
         [
             "import-url",
@@ -37,7 +37,7 @@ def test_import_url(mocker):
     )
 
 
-def test_failed_import_url(mocker, caplog):
+def test_failed_import_url(mocker, caplog, dvc):
     cli_args = parse_args(["import-url", "http://somesite.com/file_name"])
     assert cli_args.func == CmdImportUrl
 
@@ -60,7 +60,7 @@ def test_failed_import_url(mocker, caplog):
         ("--no-download", {"no_download": True, "no_exec": False}),
     ],
 )
-def test_import_url_no_exec_download_flags(mocker, flag, expected):
+def test_import_url_no_exec_download_flags(mocker, flag, expected, dvc):
     cli_args = parse_args(
         [
             "import-url",
@@ -87,7 +87,7 @@ def test_import_url_no_exec_download_flags(mocker, flag, expected):
     )
 
 
-def test_import_url_to_remote(mocker):
+def test_import_url_to_remote(mocker, dvc):
     cli_args = parse_args(
         [
             "import-url",

tests/unit/repo/experiments/test_collect.py

@@ -0,0 +1,69 @@
+import datetime
+import random
+from typing import Dict, List
+
+import pytest
+
+from dvc.repo.experiments.collect import ExpRange, ExpState, SerializableExp, collect
+
+
+@pytest.mark.vscode
+def test_collect_stable_sorting(dvc, scm, mocker):
+    """
+    Check that output is deterministically sorted even for
+    commits with the same timestamp. This affects the experience
+    in vs-code to avoid experiments "bouncing around" when "exp show"
+    is called repeatedly
+    """
+    expected_revs = [
+        "c" * 40,
+        "b" * 40,
+        "a" * 40,
+        "7" * 40,
+    ]
+
+    def collect_queued_patched(_, baseline_revs) -> Dict[str, List["ExpRange"]]:
+        single_timestamp = datetime.datetime(2023, 6, 20, 0, 0, 0)
+
+        exp_ranges = [
+            ExpRange(
+                revs=[
+                    ExpState(
+                        rev=rev,
+                        name=f"exp-state-{rev[0]}",
+                        data=SerializableExp(rev=rev, timestamp=single_timestamp),
+                    )
+                ],
+                name=f"exp-range-{rev[0]}",
+            )
+            for rev in expected_revs
+        ]
+
+        # shuffle collection order
+        random.shuffle(exp_ranges)
+
+        return {baseline_rev: exp_ranges for baseline_rev in baseline_revs}
+
+    mocker.patch("dvc.repo.experiments.collect.collect_queued", collect_queued_patched)
+    mocker.patch("dvc.repo.experiments.collect.collect_active", return_value={})
+    mocker.patch("dvc.repo.experiments.collect.collect_failed", return_value={})
+    mocker.patch("dvc.repo.experiments.collect.collect_successful", return_value={})
+
+    # repeat (shuffling collection order in collect_queued_patched)
+    for _ in range(20):
+        collected = collect(repo=dvc, all_commits=True)
+        assert collected[0].rev == "workspace"
+        assert collected[0].experiments is None
+        assert collected[1].rev == scm.get_rev()
+        _assert_experiment_rev_order(collected[1].experiments, expected_revs)
+
+
+def _assert_experiment_rev_order(
+    actual: List["ExpRange"],
+    expected_revs: List[str],
+):
+    expected_revs = expected_revs.copy()
+
+    for actual_exp_range in actual:
+        for exp_state in actual_exp_range.revs:
+            assert exp_state.rev == expected_revs.pop(0)