logger: add DVC_SHOW_TRACEBACK envvar to show traceback on exception (#9310)

This will come useful in debugging, where you don't have to add --verbose just to
see where it failed or go through verbose logs (which might be useful when working
with UI issues as you want to see what the user sees).

Ideally this should be handled in main when we exit with error instead of using
logger.{warning,error,exception} to log exception as the intention is to only log
last exception that makes it exit. But since we don't have that capability and
we don't abuse logging.{warning,error,exception} to log and suppress exceptions, this is a good
enough solution for now without complicating `dvc.cli.main()`.

.github/ISSUE_TEMPLATE/epic_story.md

@@ -0,0 +1,60 @@
+---
+name: Epic/Story Issue Template
+about: Template for "top" level issues - Epics (>2 weeks) / Stories (<2 weeks)
+title: 'Epic: New Feature'
+labels: epic
+assignees:
+
+---
+
+## Summary / Background
+What do you want to achieve, why? business context
+...
+
+## Scope
+
+What will be impacted and what won't be?
+What needs implementation and what is invariant?
+e.g.
+- user should be able to run workflow X from UI
+- enable workflow Y, Z from CLI
+
+## Assumptions
+Product / UX assumptions as well as technical assumptions / limitations
+e.g.
+* Support only Python Runtime
+* Focus on DVC experiments only
+* Deployment environments don't change often and can be picked up from shared configuration
+
+## Open Questions
+e.g.
+- How should access control work for shared artifacts (workflow X)
+- Python runtime assumption - is it really valid? in light of <...>
+
+## Blockers / Dependencies
+List issues or other conditions / blockers
+
+## General Approach
+Invocation example:
+```shell
+$ mapper-run task.tar.gz --ray-cluster <ip>:<port>
+```
+
+## Steps
+
+### Must have (p1)
+- [ ] subissue2
+- [ ] step 2
+ - info
+ - info
+
+### Optional / followup (p2)
+- [ ] ⌛ step 3 wip
+- [ ] step 4
+
+## Timelines
+
+Put your estimations here. Update once certainty changes
+- end of week (Feb 3) for prototype with workflows X, Y
+- Feb 15 - MVP in production
+- Low priority followups can be done later

.pre-commit-config.yaml

@@ -35,7 +35,7 @@ repos:
         args:
           - --py38-plus
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.3.0
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/isort
@@ -43,11 +43,11 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.254'
+    rev: 'v0.0.260'
     hooks:
       - id: ruff
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.2
+    rev: v2.2.4
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]
@@ -61,6 +61,13 @@ repos:
           - flake8-debugger
           - flake8-string-format
           - flake8-typing-imports
+  - repo: https://github.com/PyCQA/bandit
+    rev: '1.7.5'
+    hooks:
+    - id: bandit
+      args: ["-c", "pyproject.toml", "--quiet", "--format=custom"]
+      files: ^dvc/
+      additional_dependencies: [".[toml]"]
   - repo: local
     hooks:
     - id: mypy

dvc/analytics.py

@@ -47,7 +47,7 @@ def is_enabled():
     enabled = not os.getenv(DVC_NO_ANALYTICS)
     if enabled:
         enabled = to_bool(
-            Config(validate=False).get("core", {}).get("analytics", "true")
+            Config.from_cwd(validate=False).get("core", {}).get("analytics", "true")
         )
 
     logger.debug("Analytics is %sabled.", "en" if enabled else "dis")

dvc/annotations.py

@@ -2,6 +2,7 @@ from dataclasses import asdict, dataclass, field, fields
 from typing import Any, ClassVar, Dict, List, Optional
 
 from funcy import compact
+from voluptuous import Required
 
 
 @dataclass
@@ -26,6 +27,12 @@ class Annotation:
         return compact(asdict(self))
 
 
+@dataclass
+class Artifact(Annotation):
+    PARAM_PATH: ClassVar[str] = "path"
+    path: Optional[str] = None
+
+
 ANNOTATION_FIELDS = [field.name for field in fields(Annotation)]
 ANNOTATION_SCHEMA = {
     Annotation.PARAM_DESC: str,
@@ -33,3 +40,7 @@ ANNOTATION_SCHEMA = {
     Annotation.PARAM_LABELS: [str],
     Annotation.PARAM_META: object,
 }
+ARTIFACT_SCHEMA = {
+    Required(Artifact.PARAM_PATH): str,
+    **ANNOTATION_SCHEMA,  # type: ignore[arg-type]
+}

dvc/api/__init__.py

@@ -2,10 +2,11 @@ from dvc.fs.dvc import _DVCFileSystem as DVCFileSystem
 
 from .data import open  # pylint: disable=redefined-builtin
 from .data import get_url, read
-from .experiments import make_checkpoint
+from .experiments import exp_save, make_checkpoint
 from .show import metrics_show, params_show
 
 __all__ = [
+    "exp_save",
     "get_url",
     "make_checkpoint",
     "open",

dvc/api/data.py

@@ -1,12 +1,30 @@
 from contextlib import _GeneratorContextManager as GCM
-from typing import Optional
+from contextlib import contextmanager
+from typing import Any, Dict, Optional
 
 from funcy import reraise
 
-from dvc.exceptions import OutputNotFoundError, PathMissingError
+from dvc.exceptions import FileMissingError, OutputNotFoundError, PathMissingError
 from dvc.repo import Repo
 
 
+@contextmanager
+def _wrap_exceptions(repo, url):
+    from dvc.config import NoRemoteError
+    from dvc.exceptions import NoOutputInExternalRepoError, NoRemoteInExternalRepoError
+
+    try:
+        yield
+    except NoRemoteError as exc:
+        raise NoRemoteInExternalRepoError(url) from exc
+    except OutputNotFoundError as exc:
+        if exc.repo is repo:
+            raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc
+        raise
+    except FileMissingError as exc:
+        raise PathMissingError(exc.path, url) from exc
+
+
 def get_url(path, repo=None, rev=None, remote=None):
     """
     Returns the URL to the storage location of a data file or directory tracked
@@ -19,18 +37,20 @@ def get_url(path, repo=None, rev=None, remote=None):
     directory in the remote storage.
     """
     with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
-        fs_path = _repo.dvcfs.from_os_path(path)
-        with reraise(FileNotFoundError, PathMissingError(path, repo)):
-            info = _repo.dvcfs.info(fs_path)
+        with _wrap_exceptions(_repo, path):
+            fs_path = _repo.dvcfs.from_os_path(path)
 
-        dvc_info = info.get("dvc_info")
-        if not dvc_info:
-            raise OutputNotFoundError(path, repo)
+            with reraise(FileNotFoundError, PathMissingError(path, repo)):
+                info = _repo.dvcfs.info(fs_path)
 
-        dvc_repo = info["repo"]  # pylint: disable=unsubscriptable-object
-        md5 = dvc_info["md5"]
+            dvc_info = info.get("dvc_info")
+            if not dvc_info:
+                raise OutputNotFoundError(path, repo)
 
-        return dvc_repo.cloud.get_url_for(remote, checksum=md5)
+            dvc_repo = info["repo"]  # pylint: disable=unsubscriptable-object
+            md5 = dvc_info["md5"]
+
+            return dvc_repo.cloud.get_url_for(remote, checksum=md5)
 
 
 class _OpenContextManager(GCM):
@@ -194,11 +214,41 @@ def open(  # noqa, pylint: disable=redefined-builtin
 
 
 def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
-        with _repo.open_by_relpath(
-            path, remote=remote, mode=mode, encoding=encoding
-        ) as fd:
-            yield fd
+    repo_kwargs: Dict[str, Any] = {"subrepos": True, "uninitialized": True}
+    if remote:
+        repo_kwargs["config"] = {"core": {"remote": remote}}
+
+    with Repo.open(repo, rev=rev, **repo_kwargs) as _repo:
+        with _wrap_exceptions(_repo, path):
+            import os
+            from typing import TYPE_CHECKING, Union
+
+            from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
+            from dvc.fs.data import DataFileSystem
+            from dvc.fs.dvc import DVCFileSystem
+
+            if TYPE_CHECKING:
+                from dvc.fs import FileSystem
+
+            fs: Union["FileSystem", DataFileSystem, DVCFileSystem]
+            if os.path.isabs(path):
+                fs = DataFileSystem(index=_repo.index.data["local"])
+                fs_path = path
+            else:
+                fs = DVCFileSystem(repo=_repo, subrepos=True)
+                fs_path = fs.from_os_path(path)
+
+            try:
+                with fs.open(
+                    fs_path,
+                    mode=mode,
+                    encoding=encoding,
+                ) as fobj:
+                    yield fobj
+            except FileNotFoundError as exc:
+                raise FileMissingError(path) from exc
+            except IsADirectoryError as exc:
+                raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
 
 
 def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):

dvc/api/experiments.py

@@ -1,6 +1,7 @@
 import builtins
 import os
 from time import sleep
+from typing import List, Optional
 
 from dvc.env import DVC_CHECKPOINT, DVC_ROOT
 from dvc.repo import Repo
@@ -31,3 +32,40 @@ def make_checkpoint():
         os.fsync(fobj.fileno())
     while os.path.exists(signal_file):
         sleep(0.1)
+
+
+def exp_save(
+    name: Optional[str] = None,
+    force: bool = False,
+    include_untracked: Optional[List[str]] = None,
+):
+    """
+    Create a new DVC experiment using `exp save`.
+
+    See https://dvc.org/doc/command-reference/exp/save.
+
+    Args:
+        name (str, optional): specify a name for this experiment.
+            If `None`, a default one will be generated, such as `urban-sign`.
+            Defaults to `None`.
+        force (bool):  overwrite the experiment if an experiment with the same
+            name already exists.
+            Defaults to `False`.
+        include_untracked (List[str], optional): specify untracked file(s) to
+            be included in the saved experiment.
+            Defaults to `None`.
+
+    Returns:
+        str: The `Git revision`_ of the created experiment.
+
+    Raises:
+        ExperimentExistsError: If an experiment with `name` already exists and
+            `force=False`.
+
+    .. _Git revision:
+        https://git-scm.com/docs/revisions
+    """
+    with Repo() as repo:
+        return repo.experiments.save(
+            name=name, force=force, include_untracked=include_untracked
+        )

dvc/commands/check_ignore.py

@@ -39,7 +39,7 @@ class CmdCheckIgnore(CmdBase):
                 target = input()
             except (KeyboardInterrupt, EOFError):
                 break
-            if target == "":
+            if not target:
                 break
             if not self._check_one_file(target):
                 ret = 0

dvc/commands/completion.py

@@ -15,7 +15,7 @@ class CmdCompletion(CmdBaseNoRepo):
 
         shell = self.args.shell
         parser = self.args.parser
-        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
+        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)  # nosec B604
         ui.write(script, force=True)
         return 0
 

dvc/commands/config.py

@@ -32,7 +32,7 @@ class CmdConfig(CmdBaseNoRepo):
 
         super().__init__(args)
 
-        self.config = Config(validate=False)
+        self.config = Config.from_cwd(validate=False)
 
     def run(self):
         if self.args.show_origin and (self.args.value or self.args.unset):

dvc/commands/experiments/branch.py

@@ -26,5 +26,13 @@ def add_parser(experiments_subparsers, parent_parser):
     experiments_branch_parser.add_argument(
         "experiment", help="Experiment to be promoted."
     )
-    experiments_branch_parser.add_argument("branch", help="Name for the new Git branch")
+    experiments_branch_parser.add_argument(
+        "branch",
+        nargs="?",
+        default=None,
+        help=(
+            "Optional name for the new Git branch. "
+            "Defaults to '{experiment-name}-branch'."
+        ),
+    )
     experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)

dvc/commands/experiments/exec_run.py

@@ -19,6 +19,7 @@ class CmdExecutorRun(CmdBaseNoRepo):
             queue=None,
             log_level=logger.getEffectiveLevel(),
             infofile=self.args.infofile,
+            copy_paths=self.args.copy_paths,
         )
         return 0
 
@@ -36,4 +37,14 @@ def add_parser(experiments_subparsers, parent_parser):
         help="Path to executor info file",
         default=None,
     )
+    exec_run_parser.add_argument(
+        "-C",
+        "--copy-paths",
+        action="append",
+        default=[],
+        help=(
+            "List of ignored or untracked paths to copy into the temp directory."
+            " Only used if `--temp` or `--queue` is specified."
+        ),
+    )
     exec_run_parser.set_defaults(func=CmdExecutorRun)

dvc/commands/experiments/push.py

@@ -1,5 +1,6 @@
 import argparse
 import logging
+from typing import Any, Dict
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
@@ -18,29 +19,64 @@ class CmdExperimentsPush(CmdBase):
                 "`--rev` or `--all-commits` flag."
             )
 
+    @staticmethod
+    def log_result(result: Dict[str, Any], remote: str):
+        from dvc.utils import humanize
+
+        def join_exps(exps):
+            return humanize.join([f"[bold]{e}[/]" for e in exps])
+
+        if diverged_exps := result.get("diverged"):
+            exps = join_exps(diverged_exps)
+            ui.error_write(
+                f"[yellow]Local experiment {exps} has diverged "
+                "from remote experiment with the same name.\n"
+                "To override the remote experiment re-run with '--force'.",
+                styled=True,
+            )
+        if uptodate_exps := result.get("up_to_date"):
+            exps = join_exps(uptodate_exps)
+            verb = "are" if len(uptodate_exps) > 1 else "is"
+            ui.write(
+                f"Experiment {exps} {verb} up to date on Git remote {remote!r}.",
+                styled=True,
+            )
+        if pushed_exps := result.get("success"):
+            exps = join_exps(pushed_exps)
+            ui.write(f"Pushed experiment {exps} to Git remote {remote!r}.", styled=True)
+        if not uptodate_exps and not pushed_exps:
+            ui.write("No experiments to push.")
+
+        if uploaded := result.get("uploaded"):
+            stats = {"uploaded": uploaded}
+            ui.write(humanize.get_summary(stats.items()))
+
+        if project_url := result.get("url"):
+            ui.write("[yellow]View your experiments at", project_url, styled=True)
+
     def run(self):
+        from dvc.repo.experiments.push import UploadError
+
         self.raise_error_if_all_disabled()
 
-        pushed_exps = self.repo.experiments.push(
-            self.args.git_remote,
-            self.args.experiment,
-            all_commits=self.args.all_commits,
-            rev=self.args.rev,
-            num=self.args.num,
-            force=self.args.force,
-            push_cache=self.args.push_cache,
-            dvc_remote=self.args.dvc_remote,
-            jobs=self.args.jobs,
-            run_cache=self.args.run_cache,
-        )
-
-        if pushed_exps:
-            ui.write(
-                f"Pushed experiment '{pushed_exps}'"
-                f"to Git remote '{self.args.git_remote}'."
+        try:
+            result = self.repo.experiments.push(
+                self.args.git_remote,
+                self.args.experiment,
+                all_commits=self.args.all_commits,
+                rev=self.args.rev,
+                num=self.args.num,
+                force=self.args.force,
+                push_cache=self.args.push_cache,
+                dvc_remote=self.args.dvc_remote,
+                jobs=self.args.jobs,
+                run_cache=self.args.run_cache,
             )
-        else:
-            ui.write("No experiments to push.")
+        except UploadError as e:
+            self.log_result(e.result, self.args.git_remote)
+            raise
+
+        self.log_result(result, self.args.git_remote)
         if not self.args.push_cache:
             ui.write(
                 "To push cached outputs",

dvc/commands/experiments/remove.py

@@ -24,9 +24,11 @@ class CmdExperimentsRemove(CmdBase):
             )
 
     def run(self):
+        from dvc.utils import humanize
+
         self.check_arguments()
 
-        removed_list = self.repo.experiments.remove(
+        removed = self.repo.experiments.remove(
             exp_names=self.args.experiment,
             all_commits=self.args.all_commits,
             rev=self.args.rev,
@@ -34,8 +36,10 @@ class CmdExperimentsRemove(CmdBase):
             queue=self.args.queue,
             git_remote=self.args.git_remote,
         )
-        removed = ",".join(removed_list)
-        ui.write(f"Removed experiments: {removed}")
+        if removed:
+            ui.write(f"Removed experiments: {humanize.join(map(repr, removed))}")
+        else:
+            ui.write("No experiments to remove.")
 
         return 0
 

dvc/commands/experiments/run.py

@@ -36,6 +36,7 @@ class CmdExperimentsRun(CmdRepro):
             reset=self.args.reset,
             tmp_dir=self.args.tmp_dir,
             machine=self.args.machine,
+            copy_paths=self.args.copy_paths,
             **self._common_kwargs,
         )
 
@@ -136,3 +137,13 @@ def _add_run_common(parser):
         # )
         # metavar="<name>",
     )
+    parser.add_argument(
+        "-C",
+        "--copy-paths",
+        action="append",
+        default=[],
+        help=(
+            "List of ignored or untracked paths to copy into the temp directory."
+            " Only used if `--temp` or `--queue` is specified."
+        ),
+    )

dvc/commands/get.py

@@ -14,12 +14,8 @@ class CmdGet(CmdBaseNoRepo):
         from dvc.api import get_url
         from dvc.ui import ui
 
-        try:
-            url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
-            ui.write(url, force=True)
-        except DvcException:
-            logger.exception("failed to show URL")
-            return 1
+        url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
+        ui.write(url, force=True)
 
         return 0
 

dvc/commands/plots.py

@@ -1,9 +1,9 @@
 import argparse
-import json
 import logging
 import os
+from typing import TYPE_CHECKING, Dict, List, Optional
 
-from funcy import first
+from funcy import compact, first, get_in
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
@@ -12,14 +12,51 @@ from dvc.exceptions import DvcException
 from dvc.ui import ui
 from dvc.utils import format_link
 
+if TYPE_CHECKING:
+    from dvc.render.match import RendererWithErrors
+
+
 logger = logging.getLogger(__name__)
 
 
-def _show_json(renderers, split=False):
+def _show_json(
+    renderers_with_errors: List["RendererWithErrors"],
+    split=False,
+    errors: Optional[Dict[str, Exception]] = None,
+):
     from dvc.render.convert import to_json
+    from dvc.utils.serialize import encode_exception
 
-    result = {renderer.name: to_json(renderer, split) for renderer in renderers}
-    ui.write_json(result, highlight=False)
+    all_errors: List[Dict] = []
+    data = {}
+
+    for renderer, src_errors, def_errors in renderers_with_errors:
+        name = renderer.name
+        data[name] = to_json(renderer, split)
+        all_errors.extend(
+            {
+                "name": name,
+                "rev": rev,
+                "source": source,
+                **encode_exception(e),
+            }
+            for rev, per_rev_src_errors in src_errors.items()
+            for source, e in per_rev_src_errors.items()
+        )
+        all_errors.extend(
+            {
+                "name": name,
+                "rev": rev,
+                **encode_exception(e),
+            }
+            for rev, e in def_errors.items()
+        )
+
+    # these errors are not tied to any renderers
+    errors = errors or {}
+    all_errors.extend({"rev": rev, **encode_exception(e)} for rev, e in errors.items())
+
+    ui.write_json(compact({"errors": all_errors, "data": data}), highlight=False)
 
 
 def _adjust_vega_renderers(renderers):
@@ -85,7 +122,7 @@ class CmdPlots(CmdBase):
             if html_template_path and not os.path.isabs(html_template_path):
                 assert self.repo.dvc_dir
                 html_template_path = os.path.join(self.repo.dvc_dir, html_template_path)
-        return html_template_path  # noqa: RET504
+        return html_template_path
 
     def run(self) -> int:  # noqa: C901, PLR0911, PLR0912
         from pathlib import Path
@@ -122,21 +159,27 @@ class CmdPlots(CmdBase):
             )
 
             renderers_out = out if self.args.json else os.path.join(out, "static")
-
-            renderers = match_defs_renderers(
+            renderers_with_errors = match_defs_renderers(
                 data=plots_data,
                 out=renderers_out,
                 templates_dir=self.repo.plots.templates_dir,
             )
             if self.args.json:
-                _show_json(renderers, self.args.split)
+                errors = compact(
+                    {
+                        rev: get_in(data, ["definitions", "error"])
+                        for rev, data in plots_data.items()
+                    }
+                )
+                _show_json(renderers_with_errors, self.args.split, errors=errors)
                 return 0
 
+            renderers = [r.renderer for r in renderers_with_errors]
             _adjust_vega_renderers(renderers)
             if self.args.show_vega:
                 renderer = first(filter(lambda r: r.TYPE == "vega", renderers))
                 if renderer:
-                    ui.write_json(json.loads(renderer.get_filled_template()))
+                    ui.write_json(renderer.get_filled_template(as_string=False))
                 return 0
 
             output_file: Path = (Path.cwd() / out).resolve() / "index.html"

dvc/config.py

@@ -103,18 +103,22 @@ class Config(dict):
         self.wfs = LocalFileSystem()
         self.fs = fs or self.wfs
 
-        if not dvc_dir:
-            try:
-                from dvc.repo import Repo
-
-                self.dvc_dir = Repo.find_dvc_dir()
-            except NotDvcRepoError:
-                self.dvc_dir = None
-        else:
+        if dvc_dir:
             self.dvc_dir = self.fs.path.abspath(self.fs.path.realpath(dvc_dir))
 
         self.load(validate=validate, config=config)
 
+    @classmethod
+    def from_cwd(cls, fs: Optional["FileSystem"] = None, **kwargs):
+        from dvc.repo import Repo
+
+        try:
+            dvc_dir = Repo.find_dvc_dir(fs=fs)
+        except NotDvcRepoError:
+            dvc_dir = None
+
+        return cls(dvc_dir=dvc_dir, fs=fs, **kwargs)
+
     @classmethod
     def get_dir(cls, level):
         from platformdirs import site_config_dir, user_config_dir

dvc/config_schema.py

@@ -62,9 +62,9 @@ def ByUrl(mapping):  # noqa: N802
 
         parsed = urlparse(data["url"])
         # Windows absolute paths should really have scheme == "" (local)
-        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+        if os.name == "nt" and len(parsed.scheme) == 1 and not parsed.netloc:
             return schemas[""](data)
-        if parsed.netloc == "":
+        if not parsed.netloc:
             return schemas[""](data)
         if parsed.scheme not in schemas:
             raise Invalid(f"Unsupported URL type {parsed.scheme}://")
@@ -150,6 +150,7 @@ SCHEMA = {
         Optional("autostage", default=False): Bool,
         Optional("experiments"): Bool,  # obsoleted
         Optional("check_update", default=True): Bool,
+        "site_cache_dir": str,
         "machine": Lower,
     },
     "cache": {
@@ -300,6 +301,9 @@ SCHEMA = {
     "feature": FeatureSchema(
         {
             Optional("machine", default=False): Bool,
+            Optional("push_exp_to_studio", default=False): Bool,
+            "studio_token": str,
+            "studio_url": str,
         },
     ),
     "plots": {

dvc/daemon.py

@@ -5,7 +5,7 @@ import logging
 import os
 import platform
 import sys
-from subprocess import Popen
+from subprocess import Popen  # nosec B404
 from typing import List
 
 from dvc.env import DVC_DAEMON
@@ -28,12 +28,12 @@ def _popen(cmd, **kwargs) -> Popen:
             os.path.abspath(os.path.dirname(__file__)), "__main__.py"
         )
         prefix += [main_entrypoint]
-    return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
+    return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)  # nosec B603
 
 
 def _spawn_windows(cmd, env):
     if sys.platform == "win32":
-        from subprocess import (
+        from subprocess import (  # nosec B404
             CREATE_NEW_PROCESS_GROUP,
             CREATE_NO_WINDOW,
             STARTF_USESHOWWINDOW,
@@ -56,6 +56,13 @@ def _spawn_windows(cmd, env):
 def _spawn_posix(cmd, env):
     from dvc.cli import main
 
+    # `fork` will copy buffers, so we need to flush them before forking.
+    # Otherwise, we will get duplicated outputs.
+    if sys.stdout and not sys.stdout.closed:
+        sys.stdout.flush()
+    if sys.stderr and not sys.stderr.closed:
+        sys.stderr.flush()
+
     # NOTE: using os._exit instead of sys.exit, because dvc built
     # with PyInstaller has trouble with SystemExit exception and throws
     # errors such as "[26338] Failed to execute script __main__"
@@ -82,6 +89,7 @@ def _spawn_posix(cmd, env):
     sys.stdin.close()
     sys.stdout.close()
     sys.stderr.close()
+    os.closerange(0, 3)
 
     if platform.system() == "Darwin":
         # workaround for MacOS bug

dvc/data_cloud.py

@@ -84,14 +84,14 @@ class DataCloud:
 
         if bool(self.repo.config["remote"]):
             error_msg = (
-                "no remote specified. Setup default remote with\n"
+                f"no remote specified in {self.repo}. Setup default remote with\n"
                 "    dvc remote default <remote name>\n"
                 "or use:\n"
-                "    dvc {} -r <remote name>".format(command)
+                f"    dvc {command} -r <remote name>"
             )
         else:
             error_msg = (
-                "no remote specified. Create a default remote with\n"
+                f"no remote specified in {self.repo}. Create a default remote with\n"
                 "    dvc remote add -d <remote name> <remote url>"
             )
 

dvc/dependency/repo.py

@@ -1,24 +1,14 @@
-import os
-from collections import defaultdict
-from copy import copy
-from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple, Union
+from typing import TYPE_CHECKING, Dict, Optional, Union
 
 from voluptuous import Required
 
-from dvc.prompt import confirm
+from dvc.utils import as_posix
 
 from .base import Dependency
 
 if TYPE_CHECKING:
-    from typing import ContextManager
-
-    from dvc.output import Output
-    from dvc.repo import Repo
+    from dvc.fs import DVCFileSystem
     from dvc.stage import Stage
-    from dvc_data.hashfile.hash_info import HashInfo
-    from dvc_data.hashfile.meta import Meta
-    from dvc_data.hashfile.obj import HashFile
-    from dvc_objects.db import ObjectDB
 
 
 class RepoDependency(Dependency):
@@ -37,10 +27,11 @@ class RepoDependency(Dependency):
 
     def __init__(self, def_repo: Dict[str, str], stage: "Stage", *args, **kwargs):
         self.def_repo = def_repo
-        self._objs: Dict[str, "HashFile"] = {}
-        self._meta: Dict[str, "Meta"] = {}
         super().__init__(stage, *args, **kwargs)
 
+        self.fs = self._make_fs()
+        self.fs_path = as_posix(self.def_path)
+
     def _parse_path(self, fs, fs_path):  # noqa: ARG002
         return None
 
@@ -52,8 +43,8 @@ class RepoDependency(Dependency):
         return f"{self.def_path} ({self.def_repo[self.PARAM_URL]})"
 
     def workspace_status(self):
-        current = self.get_obj(locked=True).hash_info
-        updated = self.get_obj(locked=False).hash_info
+        current = self._make_fs(locked=True).repo.get_rev()
+        updated = self._make_fs(locked=False).repo.get_rev()
 
         if current != updated:
             return {str(self): "update available"}
@@ -64,33 +55,18 @@ class RepoDependency(Dependency):
         return self.workspace_status()
 
     def save(self):
-        pass
+        rev = self.fs.repo.get_rev()
+        if self.def_repo.get(self.PARAM_REV_LOCK) is None:
+            self.def_repo[self.PARAM_REV_LOCK] = rev
 
     def dumpd(self, **kwargs) -> Dict[str, Union[str, Dict[str, str]]]:
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
-    def download(self, to: "Output", jobs: Optional[int] = None):
-        from dvc_data.hashfile.checkout import checkout
-
-        for odb, objs in self.get_used_objs().items():
-            self.repo.cloud.pull(objs, jobs=jobs, odb=odb)
-
-        obj = self.get_obj()
-        checkout(
-            to.fs_path,
-            to.fs,
-            obj,
-            self.repo.cache.local,
-            ignore=None,
-            state=self.repo.state,
-            prompt=confirm,
-        )
-
     def update(self, rev: Optional[str] = None):
         if rev:
             self.def_repo[self.PARAM_REV] = rev
-        with self._make_repo(locked=False) as repo:
-            self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
+        self.fs = self._make_fs(rev=rev, locked=False)
+        self.def_repo[self.PARAM_REV_LOCK] = self.fs.repo.get_rev()
 
     def changed_checksum(self) -> bool:
         # From current repo point of view what describes RepoDependency is its
@@ -98,124 +74,16 @@ class RepoDependency(Dependency):
         # immutable, hence its impossible for checksum to change.
         return False
 
-    def get_used_objs(self, **kwargs) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
-        used, _, _ = self._get_used_and_obj(**kwargs)
-        return used
+    def _make_fs(
+        self, rev: Optional[str] = None, locked: bool = True
+    ) -> "DVCFileSystem":
+        from dvc.fs import DVCFileSystem
 
-    def _get_used_and_obj(
-        self, obj_only: bool = False, **kwargs
-    ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile"]:
-        from dvc.config import NoRemoteError
-        from dvc.exceptions import NoOutputOrStageError, PathMissingError
-        from dvc.utils import as_posix
-        from dvc_data.hashfile.build import build
-        from dvc_data.hashfile.tree import Tree, TreeError
-
-        local_odb = self.repo.cache.local
-        locked = kwargs.pop("locked", True)
-        with self._make_repo(locked=locked, cache_dir=local_odb.path) as repo:
-            used_obj_ids = defaultdict(set)
-            rev = repo.get_rev()
-            if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
-                self.def_repo[self.PARAM_REV_LOCK] = rev
-
-            if not obj_only:
-                try:
-                    for odb, obj_ids in repo.used_objs(
-                        [os.path.join(repo.root_dir, self.def_path)],
-                        force=True,
-                        jobs=kwargs.get("jobs"),
-                        recursive=True,
-                    ).items():
-                        if odb is None:
-                            odb = repo.cloud.get_remote_odb()
-                            odb.read_only = True
-                        self._check_circular_import(odb, obj_ids)
-                        used_obj_ids[odb].update(obj_ids)
-                except (NoRemoteError, NoOutputOrStageError):
-                    pass
-
-            try:
-                object_store, meta, obj = build(
-                    local_odb,
-                    as_posix(self.def_path),
-                    repo.dvcfs,
-                    local_odb.fs.PARAM_CHECKSUM,
-                )
-            except (FileNotFoundError, TreeError) as exc:
-                raise PathMissingError(
-                    self.def_path, self.def_repo[self.PARAM_URL]
-                ) from exc
-            object_store = copy(object_store)
-            object_store.read_only = True
-
-            self._objs[rev] = obj
-            self._meta[rev] = meta
-
-            used_obj_ids[object_store].add(obj.hash_info)
-            if isinstance(obj, Tree):
-                used_obj_ids[object_store].update(oid for _, _, oid in obj)
-            return used_obj_ids, meta, obj
-
-    def _check_circular_import(self, odb: "ObjectDB", obj_ids: Set["HashInfo"]) -> None:
-        from dvc.exceptions import CircularImportError
-        from dvc.fs.dvc import DVCFileSystem
-        from dvc_data.hashfile.db.reference import ReferenceHashFileDB
-        from dvc_data.hashfile.tree import Tree
-
-        if not isinstance(odb, ReferenceHashFileDB):
-            return
-
-        def iter_objs():
-            for hash_info in obj_ids:
-                if hash_info.isdir:
-                    tree = Tree.load(odb, hash_info)
-                    yield from (odb.get(hi.value) for _, _, hi in tree)
-                else:
-                    assert hash_info.value
-                    yield odb.get(hash_info.value)
-
-        checked_urls = set()
-        for obj in iter_objs():
-            if not isinstance(obj.fs, DVCFileSystem):
-                continue
-            if obj.fs.repo_url in checked_urls or obj.fs.repo.root_dir in checked_urls:
-                continue
-            self_url = self.repo.url or self.repo.root_dir
-            if (
-                obj.fs.repo_url is not None
-                and obj.fs.repo_url == self_url
-                or obj.fs.repo.root_dir == self.repo.root_dir
-            ):
-                raise CircularImportError(self, obj.fs.repo_url, self_url)
-            checked_urls.update([obj.fs.repo_url, obj.fs.repo.root_dir])
-
-    def get_obj(self, filter_info=None, **kwargs):
-        locked = kwargs.get("locked", True)
-        rev = self._get_rev(locked=locked)
-        if rev in self._objs:
-            return self._objs[rev]
-        _, _, obj = self._get_used_and_obj(
-            obj_only=True, filter_info=filter_info, **kwargs
+        return DVCFileSystem(
+            url=self.def_repo[self.PARAM_URL],
+            rev=rev or self._get_rev(locked=locked),
+            subrepos=True,
         )
-        return obj
-
-    def get_meta(self, filter_info=None, **kwargs):
-        locked = kwargs.get("locked", True)
-        rev = self._get_rev(locked=locked)
-        if rev in self._meta:
-            return self._meta[rev]
-        _, meta, _ = self._get_used_and_obj(
-            obj_only=True, filter_info=filter_info, **kwargs
-        )
-        return meta
-
-    def _make_repo(self, locked: bool = True, **kwargs) -> "ContextManager[Repo]":
-        from dvc.external_repo import external_repo
-
-        d = self.def_repo
-        rev = self._get_rev(locked=locked)
-        return external_repo(d[self.PARAM_URL], rev=rev, **kwargs)
 
     def _get_rev(self, locked: bool = True):
         d = self.def_repo

dvc/dvcfile.py

@@ -192,6 +192,7 @@ class SingleStageFile(FileMixin):
     metrics: List[str] = []
     plots: Any = {}
     params: List[str] = []
+    artifacts: Dict[str, Optional[Dict[str, Any]]] = {}
 
     @property
     def stage(self) -> "Stage":
@@ -323,6 +324,10 @@ class ProjectFile(FileMixin):
     def params(self) -> List[str]:
         return self.contents.get("params", [])
 
+    @property
+    def artifacts(self) -> Dict[str, Optional[Dict[str, Any]]]:
+        return self.contents.get("artifacts", {})
+
     def remove(self, force=False):
         if not force:
             logger.warning("Cannot remove pipeline file.")
@@ -366,7 +371,7 @@ def get_lockfile_schema(d):
 
 
 def migrate_lock_v1_to_v2(d, version_info):
-    stages = {k: v for k, v in d.items()}
+    stages = dict(d)
 
     for key in stages:
         d.pop(key)

dvc/env.py

@@ -9,3 +9,4 @@ DVC_EXP_NAME = "DVC_EXP_NAME"
 DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE"
 DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH"
 DVC_NO_ANALYTICS = "DVC_NO_ANALYTICS"
+DVC_SHOW_TRACEBACK = "DVC_SHOW_TRACEBACK"

dvc/exceptions.py

@@ -31,14 +31,19 @@ class OutputDuplicationError(DvcException):
 
         assert isinstance(output, str)
         assert all(hasattr(stage, "relpath") for stage in stages)
+        msg = ""
+        stage_names = [s.addressing for s in stages]
+        stages_str = " ".join(stage_names)
         if len(stages) == 1:
-            msg = "output '{}' is already specified in {}.".format(
-                output, first(stages)
-            )
+            stage_name = first(stages)
+            msg = f"output '{output}' is already specified in {stage_name}."
         else:
             msg = "output '{}' is already specified in stages:\n{}".format(
-                output, "\n".join(f"\t- {s.addressing}" for s in stages)
+                output, "\n".join(f"\t- {s}" for s in stage_names)
             )
+        msg += (
+            f"\nUse `dvc remove {stages_str}` to stop tracking the overlapping output."
+        )
         super().__init__(msg)
         self.stages = stages
         self.output = output
@@ -333,14 +338,6 @@ class CacheLinkError(DvcException):
         self.fs_paths = fs_paths
 
 
-class CircularImportError(DvcException):
-    def __init__(self, dep, a, b):
-        super().__init__(
-            f"'{dep}' contains invalid circular import. "
-            f"DVC repo '{a}' already imports from '{b}'."
-        )
-
-
 class PrettyDvcException(DvcException):
     def __pretty_exc__(self, **kwargs):
         """Print prettier exception message."""

dvc/external_repo.py

@@ -7,14 +7,7 @@ from typing import TYPE_CHECKING, Collection, Dict, Iterator, Optional, Tuple
 
 from funcy import retry, wrap_with
 
-from dvc.exceptions import (
-    FileMissingError,
-    NoOutputInExternalRepoError,
-    NoRemoteInExternalRepoError,
-    NotDvcRepoError,
-    OutputNotFoundError,
-    PathMissingError,
-)
+from dvc.exceptions import NotDvcRepoError
 from dvc.repo import Repo
 from dvc.scm import CloneError, map_scm_exception
 from dvc.utils import relpath
@@ -37,10 +30,6 @@ def external_repo(
     cache_types: Optional[Collection[str]] = None,
     **kwargs,
 ) -> Iterator["Repo"]:
-    from dvc.config import NoRemoteError
-    from dvc.fs import GitFileSystem
-    from dvc.scm import Git
-
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
     # Local HEAD points to the tip of whatever branch we first cloned from
@@ -54,44 +43,27 @@ def external_repo(
 
     config = _get_remote_config(url) if os.path.isdir(url) else {}
     config.update(cache_config)
+    config.update(kwargs.pop("config", None) or {})
 
+    main_root = "/"
     if for_write:
-        scm = None
-        root_dir = path
-        fs = None
-    else:
-        scm = Git(path)
-        fs = GitFileSystem(scm=scm, rev=rev)
-        root_dir = "/"
+        # we already checked out needed revision
+        rev = None
+        main_root = path
 
     repo_kwargs = dict(
-        root_dir=root_dir,
+        root_dir=path,
         url=url,
-        fs=fs,
         config=config,
-        repo_factory=erepo_factory(url, root_dir, cache_config),
-        scm=scm,
+        repo_factory=erepo_factory(url, main_root, cache_config),
+        rev=rev,
         **kwargs,
     )
 
-    if "subrepos" not in repo_kwargs:
-        repo_kwargs["subrepos"] = True
-
-    if "uninitialized" not in repo_kwargs:
-        repo_kwargs["uninitialized"] = True
-
     repo = Repo(**repo_kwargs)
 
     try:
         yield repo
-    except NoRemoteError as exc:
-        raise NoRemoteInExternalRepoError(url) from exc
-    except OutputNotFoundError as exc:
-        if exc.repo is repo:
-            raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc
-        raise
-    except FileMissingError as exc:
-        raise PathMissingError(exc.path, url) from exc
     finally:
         repo.close()
         if for_write:

dvc/fs/dvc.py

@@ -251,12 +251,15 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         try:
             return self.repo.fs.open(fs_path, mode=mode)
         except FileNotFoundError:
-            _, dvc_fs, subkey = self._get_subrepo_info(key)
+            repo, dvc_fs, subkey = self._get_subrepo_info(key)
             if not dvc_fs:
                 raise
 
         dvc_path = _get_dvc_path(dvc_fs, subkey)
-        return dvc_fs.open(dvc_path, mode=mode)
+        kw = {}
+        if kwargs.get("cache_remote_stream", False):
+            kw["cache_odb"] = repo.cache.local
+        return dvc_fs.open(dvc_path, mode=mode, **kw)
 
     def isdvc(self, path, **kwargs) -> bool:
         """Is this entry dvc-tracked?"""
@@ -265,19 +268,22 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         dvc_path = _get_dvc_path(dvc_fs, subkey)
         return dvc_fs is not None and dvc_fs.isdvc(dvc_path, **kwargs)
 
-    def ls(  # pylint: disable=arguments-differ
+    def ls(  # pylint: disable=arguments-differ # noqa: C901
         self, path, detail=True, dvc_only=False, **kwargs
     ):
         key = self._get_key_from_relative(path)
         repo, dvc_fs, subkey = self._get_subrepo_info(key)
 
+        dvc_exists = False
         dvc_infos = {}
         if dvc_fs:
+            dvc_path = _get_dvc_path(dvc_fs, subkey)
             with suppress(FileNotFoundError):
-                dvc_path = _get_dvc_path(dvc_fs, subkey)
                 for info in dvc_fs.ls(dvc_path, detail=True):
                     dvc_infos[dvc_fs.path.name(info["name"])] = info
+            dvc_exists = bool(dvc_infos) or dvc_fs.exists(dvc_path)
 
+        fs_exists = False
         fs_infos = {}
         ignore_subrepos = kwargs.get("ignore_subrepos", True)
         if not dvc_only:
@@ -291,11 +297,18 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
             except (FileNotFoundError, NotADirectoryError):
                 pass
 
+            fs_exists = bool(fs_infos) or fs.exists(fs_path)
+
         dvcfiles = kwargs.get("dvcfiles", False)
 
         infos = []
         paths = []
         names = set(dvc_infos.keys()) | set(fs_infos.keys())
+
+        if not names and (dvc_exists or fs_exists):
+            # broken symlink or TreeError
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
+
         for name in names:
             if not dvcfiles and _is_dvc_file(name):
                 continue

dvc/info.py

@@ -139,7 +139,7 @@ def _get_supported_remotes():
 def get_fs_type(path):
     partition = {}
     for part in psutil.disk_partitions(all=True):
-        if part.fstype != "":
+        if part.fstype:
             try:
                 mountpoint = pathlib.Path(part.mountpoint).resolve()
                 partition[mountpoint] = part.fstype + " on " + part.device

dvc/lock.py

@@ -187,7 +187,9 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
 
         if self._tmp_dir is not None:
             # Under Windows file path length is limited so we hash it
-            hasher = hashlib.md5(self._claimfile.encode())  # noqa: S324
+            hasher = hashlib.md5(  # nosec B324, B303  # noqa: S324
+                self._claimfile.encode()
+            )
             filename = hasher.hexdigest()
             self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
 

dvc/logger.py

@@ -3,10 +3,12 @@
 import logging
 import logging.config
 import logging.handlers
+import os
 import sys
 
 import colorama
 
+from dvc.env import DVC_SHOW_TRACEBACK
 from dvc.progress import Tqdm
 
 
@@ -86,9 +88,10 @@ class ColorFormatter(logging.Formatter):
         "CRITICAL": colorama.Fore.RED,
     }
 
-    def __init__(self, log_colors: bool = True) -> None:
+    def __init__(self, log_colors: bool = True, show_traceback: bool = False) -> None:
         super().__init__()
         self.log_colors = log_colors
+        self.show_traceback = show_traceback
 
     def format(self, record) -> str:  # noqa: A003, C901
         record.message = record.getMessage()
@@ -106,8 +109,10 @@ class ColorFormatter(logging.Formatter):
             msg = msg + sep + cause
 
         asctime = ""
-        if _is_verbose():
+        verbose = _is_verbose()
+        if verbose:
             asctime = self.formatTime(record, self.datefmt)
+        if verbose or self.show_traceback:
             if ei and not record.exc_text:
                 record.exc_text = self.formatException(ei)
             if record.exc_text:
@@ -146,7 +151,8 @@ class LoggerHandler(logging.StreamHandler):
                         self.emit_pretty_exception(exc, verbose=_is_verbose())
                         if not _is_verbose():
                             return
-                    except Exception:  # noqa, pylint: disable=broad-except
+                    # pylint: disable-next=broad-except
+                    except Exception:  # noqa: BLE001, S110  # nosec B110
                         pass
 
             msg = self.format(record)
@@ -197,7 +203,10 @@ def setup(level: int = logging.INFO, log_colors: bool = True) -> None:
     console_trace.setFormatter(formatter)
     console_trace.addFilter(exclude_filter(logging.DEBUG))
 
-    err_formatter = ColorFormatter(log_colors=log_colors and sys.stderr.isatty())
+    show_traceback = bool(os.environ.get(DVC_SHOW_TRACEBACK))
+    err_formatter = ColorFormatter(
+        log_colors=log_colors and sys.stderr.isatty(), show_traceback=show_traceback
+    )
     console_errors = LoggerHandler(sys.stderr)
     console_errors.setLevel(logging.WARNING)
     console_errors.setFormatter(err_formatter)

dvc/output.py

@@ -427,7 +427,7 @@ class Output:
             self.hash_info = tree.hash_info
             self.meta.isdir = True
             self.meta.nfiles = len(self.files)
-            self.meta.size = sum(f.get("size") for f in self.files)
+            self.meta.size = sum(filter(None, (f.get("size") for f in self.files)))
             self.meta.remote = first(f.get("remote") for f in self.files)
         elif self.meta.nfiles or self.hash_info and self.hash_info.isdir:
             self.meta.isdir = True
@@ -1066,7 +1066,7 @@ class Output:
             return obj.filter(prefix)
         return obj
 
-    def get_used_objs(  # noqa: C901, PLR0911
+    def get_used_objs(  # noqa: C901
         self, **kwargs
     ) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
         """Return filtered set of used object IDs for this out."""
@@ -1076,9 +1076,7 @@ class Output:
 
         push: bool = kwargs.pop("push", False)
         if self.stage.is_repo_import:
-            if push:
-                return {}
-            return self.get_used_external(**kwargs)
+            return {}
 
         if push and not self.can_push:
             return {}
@@ -1130,15 +1128,6 @@ class Output:
                 oids.add(oid)
         return oids
 
-    def get_used_external(
-        self, **kwargs
-    ) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
-        if not self.use_cache or not self.stage.is_repo_import:
-            return {}
-
-        (dep,) = self.stage.deps
-        return dep.get_used_objs(**kwargs)
-
     def _validate_output_path(self, path, stage=None):
         from dvc.dvcfile import is_valid_filename
 

dvc/parsing/__init__.py

@@ -392,7 +392,7 @@ class ForeachDefinition:
         """Convert sequence to Mapping with keys normalized."""
         iterable = self.resolved_iterable
         if isinstance(iterable, Mapping):
-            return iterable
+            return {to_str(k): v for k, v in iterable.items()}
 
         assert isinstance(iterable, Sequence)
         if any(map(is_map_or_seq, iterable)):

dvc/parsing/interpolate.py

@@ -73,7 +73,7 @@ def embrace(s: str):
 
 def escape_str(value):
     if os.name == "nt":
-        from subprocess import list2cmdline
+        from subprocess import list2cmdline  # nosec B404
 
         return list2cmdline([value])
     from shlex import quote
@@ -95,7 +95,7 @@ def _(obj: bool):
 def _(obj: dict):  # noqa: C901
     from dvc.config import Config
 
-    config = Config().get("parsing", {})
+    config = Config.from_cwd().get("parsing", {})
 
     result = ""
     for k, v in flatten(obj).items():

dvc/render/convert.py

@@ -1,4 +1,3 @@
-import json
 from collections import defaultdict
 from typing import Dict, List, Union
 
@@ -32,15 +31,17 @@ def to_json(renderer, split: bool = False) -> List[Dict]:
     if renderer.TYPE == "vega":
         grouped = _group_by_rev(renderer.datapoints)
         if split:
-            content = renderer.get_filled_template(skip_anchors=["data"])
+            content = renderer.get_filled_template(
+                skip_anchors=["data"], as_string=False
+            )
         else:
-            content = renderer.get_filled_template()
+            content = renderer.get_filled_template(as_string=False)
         if grouped:
             return [
                 {
                     TYPE_KEY: renderer.TYPE,
                     REVISIONS_KEY: sorted(grouped.keys()),
-                    "content": json.loads(content),
+                    "content": content,
                     "datapoints": grouped,
                 }
             ]

dvc/render/converter/vega.py

@@ -56,7 +56,6 @@ def _verify_field(file2datapoints: Dict[str, List], filename: str, field: str):
         datapoint = first(file2datapoints[filename])
         if field not in datapoint:
             raise FieldNotFoundError(field, datapoint.keys())
-    return
 
 
 def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]):
@@ -245,7 +244,7 @@ class VegaConverter(Converter):
         for i, (y_file, y_field) in enumerate(ys):
             if num_xs > 1:
                 x_file, x_field = xs[i]
-            datapoints = [d.copy() for d in file2datapoints.get(y_file, [])]
+            datapoints = [{**d} for d in file2datapoints.get(y_file, [])]
 
             if props_update.get("y", None) == "dvc_inferred_y_value":
                 _update_from_field(

dvc/render/match.py

@@ -1,10 +1,10 @@
 import os
 from collections import defaultdict
-from typing import TYPE_CHECKING, Dict, List, Optional
+from typing import TYPE_CHECKING, DefaultDict, Dict, List, NamedTuple, Optional
 
 import dpath
 import dpath.options
-from funcy import last
+from funcy import get_in, last
 
 from dvc.repo.plots import _normpath, infer_data_sources
 from dvc.utils.plots import get_plot_id
@@ -13,6 +13,8 @@ from .convert import _get_converter
 
 if TYPE_CHECKING:
     from dvc.types import StrPath
+    from dvc_render.base import Renderer
+
 
 dpath.options.ALLOW_EMPTY_STRING_KEYS = True
 
@@ -61,21 +63,31 @@ class PlotsData:
         return result
 
 
-def match_defs_renderers(
+class RendererWithErrors(NamedTuple):
+    renderer: "Renderer"
+    source_errors: Dict[str, Dict[str, Exception]]
+    definition_errors: Dict[str, Exception]
+
+
+def match_defs_renderers(  # noqa: C901, PLR0912
     data,
     out=None,
     templates_dir: Optional["StrPath"] = None,
-):
+) -> List[RendererWithErrors]:
     from dvc_render import ImageRenderer, VegaRenderer
 
     plots_data = PlotsData(data)
     renderers = []
     renderer_cls = None
+
     for plot_id, group in plots_data.group_definitions().items():
         plot_datapoints: List[Dict] = []
         props = _squash_plots_properties(group)
         final_props: Dict = {}
 
+        def_errors: Dict[str, Exception] = {}
+        src_errors: DefaultDict[str, Dict[str, Exception]] = defaultdict(dict)
+
         if out is not None:
             props["out"] = out
         if templates_dir is not None:
@@ -94,13 +106,24 @@ def match_defs_renderers(
 
             converter = _get_converter(renderer_cls, inner_id, props, definitions_data)
 
-            dps, rev_props = converter.flat_datapoints(rev)
+            for src in plot_sources:
+                if error := get_in(data, [rev, "sources", "data", src, "error"]):
+                    src_errors[rev][src] = error
+
+            try:
+                dps, rev_props = converter.flat_datapoints(rev)
+            except Exception as e:  # noqa: BLE001, pylint: disable=broad-except
+                def_errors[rev] = e
+                continue
+
             if not final_props and rev_props:
                 final_props = rev_props
             plot_datapoints.extend(dps)
 
         if "title" not in final_props:
             final_props["title"] = renderer_id
+
         if renderer_cls is not None:
-            renderers.append(renderer_cls(plot_datapoints, renderer_id, **final_props))
+            renderer = renderer_cls(plot_datapoints, renderer_id, **final_props)
+            renderers.append(RendererWithErrors(renderer, dict(src_errors), def_errors))
     return renderers

dvc/repo/__init__.py

@@ -14,8 +14,6 @@ from typing import (
     Union,
 )
 
-from dvc.exceptions import FileMissingError
-from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
 from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
 from dvc.ignore import DvcIgnoreFilter
 from dvc.utils import env2bool
@@ -125,6 +123,8 @@ class Repo:
             if not scm:
                 try:
                     scm = SCM(root_dir or os.curdir)
+                    if scm.dulwich.repo.bare:
+                        raise NotDvcRepoError(f"{scm.root_dir} is a bare git repo")
                 except SCMError:
                     scm = SCM(os.curdir, no_scm=True)
 
@@ -151,6 +151,7 @@ class Repo:
         from dvc.data_cloud import DataCloud
         from dvc.fs import GitFileSystem, LocalFileSystem, localfs
         from dvc.lock import LockNoop, make_lock
+        from dvc.repo.artifacts import Artifacts
         from dvc.repo.metrics import Metrics
         from dvc.repo.params import Params
         from dvc.repo.plots import Plots
@@ -224,6 +225,7 @@ class Repo:
         self.metrics: Metrics = Metrics(self)
         self.plots: Plots = Plots(self)
         self.params: Params = Params(self)
+        self.artifacts: Artifacts = Artifacts(self)
 
         self.stage_collection_error_handler: Optional[
             Callable[[str, Exception], None]
@@ -408,9 +410,12 @@ class Repo:
         raise NotDvcRepoError(msg)
 
     @classmethod
-    def find_dvc_dir(cls, root=None) -> str:
-        root_dir = cls.find_root(root)
-        return os.path.join(root_dir, cls.DVC_DIR)
+    def find_dvc_dir(cls, root=None, fs=None) -> str:
+        from dvc.fs import localfs
+
+        fs = fs or localfs
+        root_dir = cls.find_root(root, fs=fs)
+        return fs.path.join(root_dir, cls.DVC_DIR)
 
     @staticmethod
     def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False) -> "Repo":
@@ -555,13 +560,15 @@ class Repo:
 
     @cached_property
     def site_cache_dir(self) -> str:
+        import getpass
         import hashlib
 
         import platformdirs
 
         from dvc.fs import GitFileSystem
 
-        cache_dir = platformdirs.site_cache_dir("dvc", "iterative", opinion=True)
+        default = platformdirs.site_cache_dir("dvc", "iterative", opinion=True)
+        cache_dir = self.config["core"].get("site_cache_dir") or default
 
         if isinstance(self.fs, GitFileSystem):
             relparts = ()
@@ -572,41 +579,19 @@ class Repo:
         else:
             root_dir = self.root_dir
 
-        repo_token = hashlib.md5(os.fsencode(root_dir)).hexdigest()  # noqa: S324
-
-        return os.path.join(cache_dir, "repo", repo_token)
-
-    @contextmanager
-    def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
-        """Opens a specified resource as a file descriptor"""
-        from dvc.fs.data import DataFileSystem
-        from dvc.fs.dvc import DVCFileSystem
-
-        fs: Union["FileSystem", DataFileSystem, DVCFileSystem]
-        if os.path.isabs(path):
-            fs = DataFileSystem(index=self.index.data["local"])
-            fs_path = path
-        else:
-            fs = DVCFileSystem(repo=self, subrepos=True)
-            fs_path = fs.from_os_path(path)
+        repos_dir = os.path.join(cache_dir, "repo")
 
+        umask = os.umask(0)
         try:
-            if remote:
-                remote_odb = self.cloud.get_remote_odb(name=remote)
-                oid = fs.info(fs_path)["dvc_info"]["md5"]
-                fs = remote_odb.fs
-                fs_path = remote_odb.oid_to_path(oid)
+            os.makedirs(repos_dir, mode=0o777, exist_ok=True)
+        finally:
+            os.umask(umask)
 
-            with fs.open(
-                fs_path,
-                mode=mode,
-                encoding=encoding,
-            ) as fobj:
-                yield fobj
-        except FileNotFoundError as exc:
-            raise FileMissingError(path) from exc
-        except IsADirectoryError as exc:
-            raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
+        md5 = hashlib.md5(  # noqa: S324  # nosec B324, B303
+            str((root_dir, getpass.getuser())).encode()
+        )
+        repo_token = md5.hexdigest()
+        return os.path.join(repos_dir, repo_token)
 
     def close(self):
         self.scm.close()

dvc/repo/artifacts.py

@@ -0,0 +1,67 @@
+import logging
+import os
+import re
+from typing import TYPE_CHECKING, Dict
+
+from dvc.annotations import Artifact
+from dvc.dvcfile import FileMixin
+from dvc.utils import relpath
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+logger = logging.getLogger(__name__)
+
+
+NAME_RE = re.compile(r"^[a-z]([a-z0-9-]*[a-z0-9])?$")
+
+
+def name_is_compatible(name: str) -> bool:
+    return bool(NAME_RE.search(name))
+
+
+def check_name_format(name: str) -> None:
+    if not name_is_compatible(name):
+        logger.warning(
+            "Can't use '%s' as artifact name (ID)."
+            " You can use letters and numbers, and use '-' as separator"
+            " (but not at the start or end). The first character must be a letter.",
+            name,
+        )
+
+
+class ArtifactsFile(FileMixin):
+    from dvc.schema import SINGLE_ARTIFACT_SCHEMA as SCHEMA
+
+    def dump(self, stage, **kwargs):
+        raise NotImplementedError
+
+    def merge(self, ancestor, other, allowed=None):
+        raise NotImplementedError
+
+
+class Artifacts:
+    def __init__(self, repo: "Repo") -> None:
+        self.repo = repo
+
+    def read(self) -> Dict[str, Dict[str, Artifact]]:
+        artifacts: Dict[str, Dict[str, Artifact]] = {}
+        for (
+            dvcfile,
+            dvcfile_artifacts,
+        ) in self.repo.index._artifacts.items():  # pylint: disable=protected-access
+            # read the artifacts.yaml file if needed
+            if isinstance(dvcfile_artifacts, str):
+                dvcfile_artifacts = ArtifactsFile(
+                    self.repo,
+                    os.path.join(os.path.dirname(dvcfile), dvcfile_artifacts),
+                    verify=False,
+                ).load()
+            if not dvcfile_artifacts:
+                continue
+            dvcyaml = relpath(dvcfile, self.repo.root_dir)
+            artifacts[dvcyaml] = {}
+            for name, value in dvcfile_artifacts.items():
+                check_name_format(name)
+                artifacts[dvcyaml][name] = Artifact(**value)
+        return artifacts

dvc/repo/diff.py

@@ -1,9 +1,9 @@
+import errno
 import logging
 import os
 from collections import defaultdict
 from typing import Dict, List, Optional
 
-from dvc.exceptions import PathMissingError
 from dvc.repo import locked
 from dvc.ui import ui
 
@@ -154,7 +154,7 @@ def diff(
 
         # check for overlapping missing targets between a_rev and b_rev
         for target in old_missing & new_missing:
-            raise PathMissingError(target, self)
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), target)
 
     old = indexes[a_rev]
     new = indexes[b_rev]

dvc/repo/experiments/__init__.py

@@ -1,8 +1,7 @@
 import logging
 import os
 import re
-import time
-from typing import TYPE_CHECKING, Dict, Iterable, Optional
+from typing import TYPE_CHECKING, Dict, Iterable, List, Optional
 
 from funcy import chain, first
 
@@ -18,10 +17,6 @@ from .exceptions import (
     InvalidExpRefError,
     MultipleBranchError,
 )
-from .executor.base import BaseExecutor
-from .queue.celery import LocalCeleryQueue
-from .queue.tempdir import TempDirQueue
-from .queue.workspace import WorkspaceQueue
 from .refs import (
     APPLY_STASH,
     CELERY_FAILED_STASH,
@@ -38,6 +33,9 @@ from .utils import check_ref_format, exp_refs_by_rev, unlocked_repo
 
 if TYPE_CHECKING:
     from .queue.base import BaseStashQueue, QueueEntry
+    from .queue.celery import LocalCeleryQueue
+    from .queue.tempdir import TempDirQueue
+    from .queue.workspace import WorkspaceQueue
     from .stash import ExpStashEntry
 
 logger = logging.getLogger(__name__)
@@ -78,20 +76,28 @@ class Experiments:
 
     @cached_property
     def args_file(self) -> str:
+        from .executor.base import BaseExecutor
+
         return os.path.join(self.repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE)
 
     @cached_property
-    def workspace_queue(self) -> WorkspaceQueue:
+    def workspace_queue(self) -> "WorkspaceQueue":
+        from .queue.workspace import WorkspaceQueue
+
         return WorkspaceQueue(self.repo, WORKSPACE_STASH)
 
     @cached_property
-    def tempdir_queue(self) -> TempDirQueue:
+    def tempdir_queue(self) -> "TempDirQueue":
+        from .queue.tempdir import TempDirQueue
+
         # NOTE: tempdir and workspace stash is shared since both
         # implementations immediately push -> pop (queue length is only 0 or 1)
         return TempDirQueue(self.repo, WORKSPACE_STASH)
 
     @cached_property
-    def celery_queue(self) -> LocalCeleryQueue:
+    def celery_queue(self) -> "LocalCeleryQueue":
+        from .queue.celery import LocalCeleryQueue
+
         return LocalCeleryQueue(self.repo, CELERY_STASH, CELERY_FAILED_STASH)
 
     @cached_property
@@ -112,6 +118,7 @@ class Experiments:
     def reproduce_one(
         self,
         tmp_dir: bool = False,
+        copy_paths: Optional[List[str]] = None,
         **kwargs,
     ):
         """Reproduce and checkout a single (standalone) experiment."""
@@ -119,7 +126,7 @@ class Experiments:
             self.tempdir_queue if tmp_dir else self.workspace_queue
         )
         self.queue_one(exp_queue, **kwargs)
-        results = self._reproduce_queue(exp_queue)
+        results = self._reproduce_queue(exp_queue, copy_paths=copy_paths)
         exp_rev = first(results)
         if exp_rev is not None:
             self._log_reproduced(results, tmp_dir=tmp_dir)
@@ -198,8 +205,7 @@ class Experiments:
             )
             for entry in entries:
                 # wait for task execution to start
-                while not self.celery_queue.proc.get(entry.stash_rev):
-                    time.sleep(1)
+                self.celery_queue.wait_for_start(entry, sleep_interval=1)
                 self.celery_queue.follow(entry)
                 # wait for task collection to complete
                 try:
@@ -342,7 +348,9 @@ class Experiments:
         self.scm.remove_ref(EXEC_APPLY)
 
     @unlocked_repo
-    def _reproduce_queue(self, queue: "BaseStashQueue", **kwargs) -> Dict[str, str]:
+    def _reproduce_queue(
+        self, queue: "BaseStashQueue", copy_paths: Optional[List[str]] = None, **kwargs
+    ) -> Dict[str, str]:
         """Reproduce queued experiments.
 
         Arguments:
@@ -352,7 +360,7 @@ class Experiments:
             dict mapping successfully reproduced experiment revs to their
             results.
         """
-        exec_results = queue.reproduce()
+        exec_results = queue.reproduce(copy_paths=copy_paths)
 
         results: Dict[str, str] = {}
         for _, exp_result in exec_results.items():

dvc/repo/experiments/branch.py

@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
 
 @locked
 @scm_context
-def branch(repo, exp_rev, branch_name, *args, **kwargs):
+def branch(repo, exp_rev, branch_name=None, **kwargs):
     from dvc.scm import resolve_rev
 
     try:
@@ -46,6 +46,8 @@ def branch(repo, exp_rev, branch_name, *args, **kwargs):
     if not ref_info:
         raise InvalidExpRevError(exp_rev)
 
+    branch_name = branch_name or f"{ref_info.name}-branch"
+
     branch_ref = f"refs/heads/{branch_name}"
     if repo.scm.get_ref(branch_ref):
         raise InvalidArgumentError(f"Git branch '{branch_name}' already exists.")

dvc/repo/experiments/executor/base.py

@@ -1,6 +1,7 @@
 import logging
 import os
-import pickle
+import pickle  # nosec B403
+import shutil
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
 from dataclasses import asdict, dataclass
@@ -34,6 +35,8 @@ from dvc.repo.experiments.refs import (
     ExpRefInfo,
 )
 from dvc.repo.experiments.utils import to_studio_params
+from dvc.repo.metrics.show import _collect_top_level_metrics
+from dvc.repo.params.show import _collect_top_level_params
 from dvc.stage.serialize import to_lockfile
 from dvc.ui import ui
 from dvc.utils import dict_sha256, env2bool, relpath
@@ -256,6 +259,72 @@ class BaseExecutor(ABC):
             **kwargs,
         )
 
+    @classmethod
+    def save(
+        cls,
+        info: "ExecutorInfo",
+        force: bool = False,
+        include_untracked: Optional[List[str]] = None,
+    ) -> ExecutorResult:
+        from dvc.dvcfile import LOCK_FILE
+        from dvc.repo import Repo
+
+        exp_hash: Optional[str] = None
+        exp_ref: Optional[ExpRefInfo] = None
+
+        dvc = Repo(os.path.join(info.root_dir, info.dvc_dir))
+        old_cwd = os.getcwd()
+        if info.wdir:
+            os.chdir(os.path.join(dvc.scm.root_dir, info.wdir))
+        else:
+            os.chdir(dvc.root_dir)
+
+        include_untracked = include_untracked or []
+        include_untracked.extend(_collect_top_level_metrics(dvc))
+        include_untracked.extend(_collect_top_level_params(dvc))
+        include_untracked.extend(
+            dvc.index._plot_sources  # pylint: disable=protected-access
+        )
+        # dvc repro automatically stages dvc.lock. Running redundant `git add`
+        # on it causes an error when exiting the detached head context.
+        if LOCK_FILE in dvc.scm.untracked_files():
+            include_untracked.append(LOCK_FILE)
+
+        try:
+            stages = dvc.commit([], force=True, relink=False)
+            exp_hash = cls.hash_exp(stages)
+            if include_untracked:
+                dvc.scm.add(include_untracked)
+            cls.commit(
+                dvc.scm,  # type: ignore[arg-type]
+                exp_hash,
+                exp_name=info.name,
+                force=force,
+            )
+            ref: Optional[str] = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
+            exp_ref = ExpRefInfo.from_ref(ref) if ref else None
+            untracked = dvc.scm.untracked_files()
+            if untracked:
+                logger.warning(
+                    "The following untracked files were present in "
+                    "the workspace before saving but "
+                    "will not be included in the experiment commit:\n"
+                    "\t%s",
+                    ", ".join(untracked),
+                )
+            info.result_hash = exp_hash
+            info.result_ref = ref
+            info.result_force = False
+            info.status = TaskStatus.SUCCESS
+        except DvcException:
+            info.status = TaskStatus.FAILED
+            raise
+        finally:
+            dvc.close()
+            os.chdir(old_cwd)
+
+        return ExecutorResult(ref, exp_ref, info.result_force)
+
     @staticmethod
     def hash_exp(stages: Iterable["PipelineStage"]) -> str:
         from dvc.stage import PipelineStage
@@ -266,7 +335,7 @@ class BaseExecutor(ABC):
                 exp_data.update(to_lockfile(stage))
         return dict_sha256(exp_data)
 
-    def cleanup(self, infofile: str):
+    def cleanup(self, infofile: Optional[str] = None):
         if infofile is not None:
             info = ExecutorInfo.load_json(infofile)
             if info.status < TaskStatus.FAILED:
@@ -293,7 +362,7 @@ class BaseExecutor(ABC):
     @staticmethod
     def unpack_repro_args(path):
         with open(path, "rb") as fobj:
-            data = pickle.load(fobj)
+            data = pickle.load(fobj)  # noqa: S301 # nosec B301
         return data["args"], data["kwargs"]
 
     def fetch_exps(
@@ -383,6 +452,7 @@ class BaseExecutor(ABC):
         infofile: Optional[str] = None,
         log_errors: bool = True,
         log_level: Optional[int] = None,
+        copy_paths: Optional[List[str]] = None,
         **kwargs,
     ) -> "ExecutorResult":
         """Run dvc repro and return the result.
@@ -419,6 +489,7 @@ class BaseExecutor(ABC):
             info,
             infofile,
             log_errors=log_errors,
+            copy_paths=copy_paths,
             **kwargs,
         ) as dvc:
             if auto_push:
@@ -541,6 +612,7 @@ class BaseExecutor(ABC):
         info: "ExecutorInfo",
         infofile: Optional[str] = None,
         log_errors: bool = True,
+        copy_paths: Optional[List[str]] = None,
         **kwargs,
     ):
         from dvc_studio_client.post_live_metrics import post_live_metrics
@@ -555,6 +627,10 @@ class BaseExecutor(ABC):
             if cls.QUIET:
                 dvc.scm_context.quiet = cls.QUIET
             old_cwd = os.getcwd()
+
+            for path in copy_paths or []:
+                cls._copy_path(os.path.realpath(path), os.path.join(dvc.root_dir, path))
+
             if info.wdir:
                 os.chdir(os.path.join(dvc.scm.root_dir, info.wdir))
             else:
@@ -724,6 +800,20 @@ class BaseExecutor(ABC):
         if level is not None:
             dvc_logger.setLevel(level)
 
+    @staticmethod
+    def _copy_path(src, dst):
+        try:
+            if os.path.isfile(src):
+                shutil.copy(src, dst)
+            elif os.path.isdir(src):
+                shutil.copytree(src, dst)
+            else:
+                raise DvcException(
+                    f"Unable to copy '{src}'. It is not a file or directory."
+                )
+        except OSError as exc:
+            raise DvcException(f"Unable to copy '{src}' to '{dst}'.") from exc
+
     @contextmanager
     def set_temp_refs(self, scm: "Git", temp_dict: Dict[str, str]):
         try:

dvc/repo/experiments/executor/local.py

@@ -4,6 +4,7 @@ from contextlib import ExitStack
 from tempfile import mkdtemp
 from typing import TYPE_CHECKING, Optional, Union
 
+from configobj import ConfigObj
 from funcy import retry
 from shortuuid import uuid
 
@@ -48,7 +49,7 @@ class BaseLocalExecutor(BaseExecutor):
     def scm(self) -> Union["Git", "NoSCM"]:
         return SCM(self.root_dir)
 
-    def cleanup(self, infofile: str):
+    def cleanup(self, infofile: Optional[str] = None):
         self.scm.close()
         del self.scm
         super().cleanup(infofile)
@@ -134,24 +135,31 @@ class TempDirExecutor(BaseLocalExecutor):
         merge_rev = self.scm.get_ref(EXEC_MERGE)
 
         self.scm.stash.apply(merge_rev)
+        self._update_config(repo.config.read("local"))
 
-    def _config(self, cache_dir):
+    def _update_config(self, update):
         local_config = os.path.join(
             self.root_dir,
             self.dvc_dir,
             "config.local",
         )
         logger.debug("Writing experiments local config '%s'", local_config)
-        with open(local_config, "w", encoding="utf-8") as fobj:
-            fobj.write(f"[cache]\n    dir = {cache_dir}")
+        if os.path.exists(local_config):
+            conf_obj = ConfigObj(local_config)
+            conf_obj.merge(update)
+        else:
+            conf_obj = ConfigObj(update)
+        if conf_obj:
+            with open(local_config, "wb") as fobj:
+                conf_obj.write(fobj)
 
     def init_cache(
         self, repo: "Repo", rev: str, run_cache: bool = True  # noqa: ARG002
     ):
         """Initialize DVC cache."""
-        self._config(repo.cache.repo.path)
+        self._update_config({"cache": {"dir": repo.cache.repo.path}})
 
-    def cleanup(self, infofile: str):
+    def cleanup(self, infofile: Optional[str] = None):
         super().cleanup(infofile)
         logger.debug("Removing tmpdir '%s'", self.root_dir)
         remove(self.root_dir)
@@ -234,9 +242,10 @@ class WorkspaceExecutor(BaseLocalExecutor):
     def init_cache(self, repo: "Repo", rev: str, run_cache: bool = True):
         pass
 
-    def cleanup(self, infofile: str):
+    def cleanup(self, infofile: Optional[str] = None):
         super().cleanup(infofile)
-        remove(os.path.dirname(infofile))
+        if infofile:
+            remove(os.path.dirname(infofile))
         with self._detach_stack:
             self.scm.remove_ref(EXEC_BASELINE)
             self.scm.remove_ref(EXEC_MERGE)

dvc/repo/experiments/executor/ssh.py

@@ -3,7 +3,7 @@ import os
 import posixpath
 import sys
 from contextlib import contextmanager
-from typing import TYPE_CHECKING, Callable, Iterable, Optional
+from typing import TYPE_CHECKING, Callable, Iterable, List, Optional
 
 from dvc_ssh import SSHFileSystem
 from funcy import first
@@ -242,6 +242,7 @@ class SSHExecutor(BaseExecutor):
         infofile: Optional[str] = None,
         log_errors: bool = True,
         log_level: Optional[int] = None,
+        copy_paths: Optional[List[str]] = None,  # noqa: ARG003
         **kwargs,
     ) -> "ExecutorResult":
         """Reproduce an experiment on a remote machine over SSH.

dvc/repo/experiments/push.py

@@ -1,35 +1,88 @@
 import logging
-from typing import Iterable, List, Mapping, Optional, Set, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Set,
+    Union,
+)
 
-from funcy import group_by
+from funcy import compact, group_by
 from scmrepo.git.backend.base import SyncStatus
 
+from dvc.exceptions import DvcException
 from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
-from dvc.scm import TqdmGit, iter_revs
-from dvc.ui import ui
+from dvc.scm import Git, TqdmGit, iter_revs
+from dvc.utils import env2bool
 
 from .exceptions import UnresolvedExpNamesError
 from .refs import ExpRefInfo
 from .utils import exp_commits, exp_refs, exp_refs_by_baseline, resolve_name
 
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
 logger = logging.getLogger(__name__)
 
 
+class UploadError(DvcException):
+    def __init__(self, msg, result):
+        self.result = result
+        super().__init__(msg)
+
+
+def notify_refs_to_studio(
+    repo: "Repo", git_remote: str, **refs: List[str]
+) -> Optional[str]:
+    config = repo.config["feature"]
+    refs = compact(refs)
+    if not refs or env2bool("DVC_TEST"):
+        return None
+
+    if not (config.get("studio_token") or config["push_exp_to_studio"]):
+        logger.debug(
+            "Either feature.studio_token or feature.push_exp_to_studio config "
+            "needs to be set."
+        )
+        return None
+
+    import os
+
+    token = os.environ.get("STUDIO_TOKEN") or config.get("studio_token")
+    if not token:
+        logger.debug("Studio token not found.")
+        return None
+
+    from dulwich.porcelain import get_remote_repo
+
+    from dvc.utils import studio
+
+    _, repo_url = get_remote_repo(repo.scm.dulwich.repo, git_remote)
+    studio_url = config.get("studio_url")
+    d = studio.notify_refs(repo_url, token, base_url=studio_url, **refs)
+    return d.get("url")
+
+
 @locked
 @scm_context
 def push(  # noqa: C901
-    repo,
+    repo: "Repo",
     git_remote: str,
     exp_names: Union[Iterable[str], str],
-    all_commits=False,
+    all_commits: bool = False,
     rev: Optional[str] = None,
-    num=1,
+    num: int = 1,
     force: bool = False,
     push_cache: bool = False,
-    **kwargs,
-) -> Iterable[str]:
+    **kwargs: Any,
+) -> Dict[str, Any]:
     exp_ref_set: Set["ExpRefInfo"] = set()
+    assert isinstance(repo.scm, Git)
     if all_commits:
         exp_ref_set.update(exp_refs(repo.scm))
 
@@ -57,23 +110,29 @@ def push(  # noqa: C901
                 exp_ref_set.update(ref_info_list)
 
     push_result = _push(repo, git_remote, exp_ref_set, force)
-    if push_result[SyncStatus.DIVERGED]:
-        diverged_refs = [ref.name for ref in push_result[SyncStatus.DIVERGED]]
-        ui.warn(
-            f"Local experiment '{diverged_refs}' has diverged from remote "
-            "experiment with the same name. To override the remote experiment "
-            "re-run with '--force'."
-        )
+
+    refs = {
+        status.name.lower(): [ref.name for ref in ref_list]
+        for status, ref_list in push_result.items()
+    }
+    result: Dict[str, Any] = {**refs, "uploaded": 0}
+
+    pushed_refs_info = (
+        push_result[SyncStatus.UP_TO_DATE] + push_result[SyncStatus.SUCCESS]
+    )
     if push_cache:
-        push_cache_ref = (
-            push_result[SyncStatus.UP_TO_DATE] + push_result[SyncStatus.SUCCESS]
-        )
-        _push_cache(repo, push_cache_ref, **kwargs)
-    return [ref.name for ref in push_result[SyncStatus.SUCCESS]]
+        try:
+            result["uploaded"] = _push_cache(repo, pushed_refs_info, **kwargs)
+        except Exception as exc:  # noqa: BLE001, pylint: disable=broad-except
+            raise UploadError("failed to push cache", result) from exc
+
+    pushed_refs = [str(r) for r in pushed_refs_info]
+    url = notify_refs_to_studio(repo, git_remote, pushed=pushed_refs)
+    return {**result, "url": url}
 
 
 def _push(
-    repo,
+    repo: "Repo",
     git_remote: str,
     refs: Iterable["ExpRefInfo"],
     force: bool,
@@ -83,7 +142,7 @@ def _push(
     from dvc.scm import GitAuthError
 
     refspec_list = [f"{exp_ref}:{exp_ref}" for exp_ref in refs]
-    logger.debug("git push experiment '%s' -> '%s'", refspec_list, git_remote)
+    logger.debug("git push experiment %s -> '%s'", refspec_list, git_remote)
 
     with TqdmGit(desc="Pushing git refs") as pbar:
         try:
@@ -105,14 +164,15 @@ def _push(
 
 
 def _push_cache(
-    repo,
+    repo: "Repo",
     refs: Union[ExpRefInfo, Iterable["ExpRefInfo"]],
-    dvc_remote=None,
-    jobs=None,
-    run_cache=False,
-):
+    dvc_remote: Optional[str] = None,
+    jobs: Optional[int] = None,
+    run_cache: bool = False,
+) -> int:
     if isinstance(refs, ExpRefInfo):
         refs = [refs]
+    assert isinstance(repo.scm, Git)
     revs = list(exp_commits(repo.scm, refs))
     logger.debug("dvc push experiment '%s'", refs)
-    repo.push(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs)
+    return repo.push(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs)

dvc/repo/experiments/queue/base.py

@@ -252,7 +252,9 @@ class BaseStashQueue(ABC):
         """Iterate over items which been failed."""
 
     @abstractmethod
-    def reproduce(self) -> Mapping[str, Mapping[str, str]]:
+    def reproduce(
+        self, copy_paths: Optional[List[str]] = None
+    ) -> Mapping[str, Mapping[str, str]]:
         """Reproduce queued experiments sequentially."""
 
     @abstractmethod
@@ -496,7 +498,7 @@ class BaseStashQueue(ABC):
         return msg
 
     def _pack_args(self, *args, **kwargs) -> None:
-        import pickle
+        import pickle  # nosec B403
 
         if os.path.exists(self.args_file) and self.scm.is_tracked(self.args_file):
             logger.warning(
@@ -511,7 +513,7 @@ class BaseStashQueue(ABC):
             )
             with open(self.args_file, "rb") as fobj:
                 try:
-                    data = pickle.load(fobj)
+                    data = pickle.load(fobj)  # noqa: S301  # nosec B301
                 except Exception:  # noqa: BLE001, pylint: disable=broad-except
                     data = {}
             extra = int(data.get("extra", 0)) + 1

dvc/repo/experiments/queue/celery.py

@@ -174,11 +174,13 @@ class LocalCeleryQueue(BaseStashQueue):
 
         return started
 
-    def put(self, *args, **kwargs) -> QueueEntry:
+    def put(
+        self, *args, copy_paths: Optional[List[str]] = None, **kwargs
+    ) -> QueueEntry:
         """Stash an experiment and add it to the queue."""
         with get_exp_rwlock(self.repo, writes=["workspace", CELERY_STASH]):
             entry = self._stash_exp(*args, **kwargs)
-        self.celery.signature(run_exp.s(entry.asdict())).delay()
+        self.celery.signature(run_exp.s(entry.asdict(), copy_paths=copy_paths)).delay()
         return entry
 
     # NOTE: Queue consumption should not be done directly. Celery worker(s)
@@ -250,7 +252,9 @@ class LocalCeleryQueue(BaseStashQueue):
             if exp_result is None:
                 yield QueueDoneResult(queue_entry, exp_result)
 
-    def reproduce(self) -> Mapping[str, Mapping[str, str]]:
+    def reproduce(
+        self, copy_paths: Optional[List[str]] = None
+    ) -> Mapping[str, Mapping[str, str]]:
         raise NotImplementedError
 
     def _load_info(self, rev: str) -> ExecutorInfo:
@@ -295,6 +299,28 @@ class LocalCeleryQueue(BaseStashQueue):
         # out of the active task list, and needs to be loaded here.
         return self._get_done_result(entry, timeout)
 
+    def wait(self, revs: Collection[str], **kwargs) -> None:
+        """Block until the specified tasks have completed."""
+        revs = [revs] if isinstance(revs, str) else revs
+        results = self.match_queue_entry_by_name(
+            revs, self.iter_queued(), self.iter_done(), self.iter_failed()
+        )
+        for entry in results.values():
+            if not entry:
+                continue
+            self.wait_for_start(entry, **kwargs)
+            try:
+                self.get_result(entry)
+            except FileNotFoundError:
+                pass
+
+    def wait_for_start(self, entry: QueueEntry, sleep_interval: float = 0.001) -> None:
+        """Block until the specified task has been started."""
+        import time
+
+        while not self.proc.get(entry.stash_rev):
+            time.sleep(sleep_interval)
+
     def _get_running_task_ids(self) -> Set[str]:
         running_task_ids: Set[str] = set()
         active_workers = self.worker_status()

dvc/repo/experiments/queue/tasks.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Any, Dict
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 from celery import shared_task
 from celery.utils.log import get_task_logger
@@ -91,7 +91,7 @@ def cleanup_exp(executor: TempDirExecutor, infofile: str) -> None:
 
 
 @shared_task
-def run_exp(entry_dict: Dict[str, Any]) -> None:
+def run_exp(entry_dict: Dict[str, Any], copy_paths: Optional[List[str]] = None) -> None:
     """Run a full experiment.
 
     Experiment subtasks are executed inline as one atomic operation.
@@ -108,6 +108,9 @@ def run_exp(entry_dict: Dict[str, Any]) -> None:
     executor = setup_exp.s(entry_dict)()
     try:
         cmd = ["dvc", "exp", "exec-run", "--infofile", infofile]
+        if copy_paths:
+            for path in copy_paths:
+                cmd.extend(["--copy-paths", path])
         proc_dict = queue.proc.run_signature(cmd, name=entry.stash_rev)()
         collect_exp.s(proc_dict, entry_dict)()
     finally:

dvc/repo/experiments/queue/tempdir.py

@@ -1,7 +1,7 @@
 import logging
 import os
 from collections import defaultdict
-from typing import TYPE_CHECKING, Dict, Generator, Optional
+from typing import TYPE_CHECKING, Dict, Generator, List, Optional
 
 from funcy import first
 
@@ -92,7 +92,11 @@ class TempDirQueue(WorkspaceQueue):
                 )
 
     def _reproduce_entry(
-        self, entry: QueueEntry, executor: "BaseExecutor"
+        self,
+        entry: QueueEntry,
+        executor: "BaseExecutor",
+        copy_paths: Optional[List[str]] = None,
+        **kwargs,
     ) -> Dict[str, Dict[str, str]]:
         from dvc.stage.monitor import CheckpointKilledError
 
@@ -107,6 +111,7 @@ class TempDirQueue(WorkspaceQueue):
                 infofile=infofile,
                 log_level=logger.getEffectiveLevel(),
                 log_errors=True,
+                copy_paths=copy_paths,
             )
             if not exec_result.exp_hash:
                 raise DvcException(f"Failed to reproduce experiment '{rev[:7]}'")

dvc/repo/experiments/queue/workspace.py

@@ -33,6 +33,7 @@ class WorkspaceQueue(BaseStashQueue):
     _EXEC_NAME: Optional[str] = "workspace"
 
     def put(self, *args, **kwargs) -> QueueEntry:
+        kwargs.pop("copy_paths", None)
         with get_exp_rwlock(self.repo, writes=["workspace", WORKSPACE_STASH]):
             return self._stash_exp(*args, **kwargs)
 
@@ -81,19 +82,24 @@ class WorkspaceQueue(BaseStashQueue):
     def iter_success(self) -> Generator["QueueDoneResult", None, None]:
         raise NotImplementedError
 
-    def reproduce(self) -> Dict[str, Dict[str, str]]:
+    def reproduce(
+        self, copy_paths: Optional[List[str]] = None
+    ) -> Dict[str, Dict[str, str]]:
         results: Dict[str, Dict[str, str]] = defaultdict(dict)
         try:
             while True:
                 entry, executor = self.get()
-                results.update(self._reproduce_entry(entry, executor))
+                results.update(
+                    self._reproduce_entry(entry, executor, copy_paths=copy_paths)
+                )
         except ExpQueueEmptyError:
             pass
         return results
 
     def _reproduce_entry(
-        self, entry: QueueEntry, executor: "BaseExecutor"
+        self, entry: QueueEntry, executor: "BaseExecutor", **kwargs
     ) -> Dict[str, Dict[str, str]]:
+        kwargs.pop("copy_paths", None)
         from dvc.stage.monitor import CheckpointKilledError
 
         results: Dict[str, Dict[str, str]] = defaultdict(dict)

dvc/repo/experiments/remove.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 
 @locked
 @scm_context
-def remove(  # noqa: C901
+def remove(  # noqa: C901, PLR0912
     repo: "Repo",
     exp_names: Union[None, str, List[str]] = None,
     rev: Optional[str] = None,
@@ -39,13 +39,6 @@ def remove(  # noqa: C901
         removed.extend(celery_queue.clear(queued=True))
 
     assert isinstance(repo.scm, Git)
-    if all_commits:
-        removed.extend(
-            _remove_commited_exps(
-                repo.scm, list(exp_refs(repo.scm, git_remote)), git_remote
-            )
-        )
-        return removed
 
     exp_ref_list: List["ExpRefInfo"] = []
     queue_entry_list: List["QueueEntry"] = []
@@ -65,11 +58,14 @@ def remove(  # noqa: C901
                 queue_entry_list.append(result.queue_entry)
 
         if remained:
-            raise UnresolvedExpNamesError(remained)
+            raise UnresolvedExpNamesError(remained, git_remote=git_remote)
     elif rev:
         exp_ref_dict = _resolve_exp_by_baseline(repo, rev, num, git_remote)
         removed.extend(exp_ref_dict.keys())
         exp_ref_list.extend(exp_ref_dict.values())
+    elif all_commits:
+        exp_ref_list.extend(exp_refs(repo.scm, git_remote))
+        removed = [ref.name for ref in exp_ref_list]
 
     if exp_ref_list:
         _remove_commited_exps(repo.scm, exp_ref_list, git_remote)
@@ -79,15 +75,22 @@ def remove(  # noqa: C901
 
         remove_tasks(celery_queue, queue_entry_list)
 
+    if git_remote:
+        from .push import notify_refs_to_studio
+
+        removed_refs = [str(r) for r in exp_ref_list]
+        notify_refs_to_studio(repo, git_remote, removed=removed_refs)
     return removed
 
 
 def _resolve_exp_by_baseline(
-    repo,
+    repo: "Repo",
     rev: str,
     num: int,
     git_remote: Optional[str] = None,
 ) -> Dict[str, "ExpRefInfo"]:
+    assert isinstance(repo.scm, Git)
+
     commit_ref_dict: Dict[str, "ExpRefInfo"] = {}
     rev_dict = iter_revs(repo.scm, [rev], num)
     rev_set = set(rev_dict.keys())

dvc/repo/experiments/run.py

@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
 
 
 @locked
-def run(  # noqa: C901
+def run(  # noqa: C901, PLR0912
     repo,
     targets: Optional[Iterable[str]] = None,
     params: Optional[Iterable[str]] = None,
@@ -19,6 +19,7 @@ def run(  # noqa: C901
     jobs: int = 1,
     tmp_dir: bool = False,
     queue: bool = False,
+    copy_paths: Optional[Iterable[str]] = None,
     **kwargs,
 ) -> Dict[str, str]:
     """Reproduce the specified targets as an experiment.
@@ -36,6 +37,18 @@ def run(  # noqa: C901
         from dvc.utils.hydra import to_hydra_overrides
 
         path_overrides = to_path_overrides(params)
+
+        if tmp_dir or queue:
+            untracked = repo.scm.untracked_files()
+            for path in path_overrides:
+                if path in untracked:
+                    logger.debug(
+                        "'%s' is currently untracked but will be modified by DVC. "
+                        "Adding it to git.",
+                        path,
+                    )
+                    repo.scm.add([path])
+
         hydra_sweep = any(
             x.is_sweep_override()
             for param_file in path_overrides
@@ -57,7 +70,11 @@ def run(  # noqa: C901
 
     if not queue:
         return repo.experiments.reproduce_one(
-            targets=targets, params=path_overrides, tmp_dir=tmp_dir, **kwargs
+            targets=targets,
+            params=path_overrides,
+            tmp_dir=tmp_dir,
+            copy_paths=copy_paths,
+            **kwargs,
         )
 
     if hydra_sweep:
@@ -78,6 +95,7 @@ def run(  # noqa: C901
             repo.experiments.celery_queue,
             targets=targets,
             params=sweep_overrides,
+            copy_paths=copy_paths,
             **kwargs,
         )
         if sweep_overrides:

dvc/repo/experiments/save.py

@@ -2,13 +2,7 @@ import logging
 import os
 from typing import TYPE_CHECKING, List, Optional
 
-from pathspec import PathSpec
-
-from dvc.scm import Git
-
-from .exceptions import ExperimentExistsError
-from .refs import ExpRefInfo
-from .utils import check_ref_format, get_random_exp_name
+from funcy import first
 
 if TYPE_CHECKING:
     from dvc.repo import Repo
@@ -17,34 +11,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-def _save_experiment(
-    repo: "Repo",
-    baseline_rev: str,
-    force: bool,
-    name: Optional[str],
-    include_untracked: Optional[List[str]],
-) -> str:
-    repo.commit([], force=True, relink=False)
-
-    name = name or get_random_exp_name(repo.scm, baseline_rev)
-    ref_info = ExpRefInfo(baseline_rev, name)
-    check_ref_format(repo.scm.dulwich, ref_info)
-    ref = str(ref_info)
-    if repo.scm.get_ref(ref) and not force:
-        raise ExperimentExistsError(ref_info.name, command="save")
-
-    assert isinstance(repo.scm, Git)
-
-    repo.scm.add([], update=True)
-    if include_untracked:
-        repo.scm.add(include_untracked)
-    repo.scm.commit(f"dvc: commit experiment {name}", no_verify=True)
-    exp_rev = repo.scm.get_rev()
-    repo.scm.set_ref(ref, exp_rev, old_ref=None)
-
-    return exp_rev
-
-
 def save(
     repo: "Repo",
     name: Optional[str] = None,
@@ -57,33 +23,16 @@ def save(
     """
     logger.debug("Saving workspace in %s", os.getcwd())
 
-    assert isinstance(repo.scm, Git)
+    queue = repo.experiments.workspace_queue
+    entry = repo.experiments.new(queue=queue, name=name, force=force)
+    executor = queue.init_executor(repo.experiments, entry)
 
-    _, _, untracked = repo.scm.status()
-    if include_untracked:
-        spec = PathSpec.from_lines("gitwildmatch", include_untracked)
-        untracked = [file for file in untracked if not spec.match_file(file)]
-    if untracked:
-        logger.warning(
-            (
-                "The following untracked files were present in "
-                "the workspace before saving but "
-                "will not be included in the experiment commit:\n"
-                "\t%s"
-            ),
-            ", ".join(untracked),
+    try:
+        save_result = executor.save(
+            executor.info, force=force, include_untracked=include_untracked
         )
+        result = queue.collect_executor(repo.experiments, executor, save_result)
+    finally:
+        executor.cleanup()
 
-    with repo.scm.detach_head(client="dvc") as orig_head:
-        with repo.scm.stash_workspace() as workspace:
-            try:
-                if workspace is not None:
-                    repo.scm.stash.apply(workspace)
-
-                exp_rev = _save_experiment(
-                    repo, orig_head, force, name, include_untracked
-                )
-            finally:
-                repo.scm.reset(hard=True)
-
-    return exp_rev
+    return first(result)

dvc/repo/get.py

@@ -52,7 +52,12 @@ def get(url, path, out=None, rev=None, jobs=None):
     cache_types = ["reflink", "hardlink", "copy"]
     try:
         with external_repo(
-            url=url, rev=rev, cache_dir=tmp_dir, cache_types=cache_types
+            url=url,
+            rev=rev,
+            subrepos=True,
+            uninitialized=True,
+            cache_dir=tmp_dir,
+            cache_types=cache_types,
         ) as repo:
             from dvc.fs.data import DataFileSystem
 

dvc/repo/graph.py

@@ -110,10 +110,10 @@ def build_graph(stages, outs_trie=None):
     # Building graph
     graph.add_nodes_from(stages)
     for stage in stages:
-        for dep in stage.deps:
-            if dep.fs_path is None:
-                continue
+        if stage.is_repo_import:
+            continue
 
+        for dep in stage.deps:
             dep_key = dep.fs.path.parts(dep.fs_path)
             overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
             if outs_trie.has_subtrie(dep_key):

dvc/repo/imports.py

@@ -28,11 +28,7 @@ def unfetched_view(
     changed_deps: List["Dependency"] = []
 
     def need_fetch(stage: "Stage") -> bool:
-        if (
-            not stage.is_import
-            or stage.is_repo_import
-            or (stage.is_partial_import and not unpartial)
-        ):
+        if not stage.is_import or (stage.is_partial_import and not unpartial):
             return False
 
         out = stage.outs[0]
@@ -65,6 +61,7 @@ def unpartial_imports(index: Union["Index", "IndexView"]) -> int:
         Total number of files which were unpartialed.
     """
     from dvc_data.hashfile.hash_info import HashInfo
+    from dvc_data.hashfile.meta import Meta
 
     updated = 0
     for out in index.outs:
@@ -73,14 +70,8 @@ def unpartial_imports(index: Union["Index", "IndexView"]) -> int:
         workspace, key = out.index_key
         entry = index.data[workspace][key]
         if out.stage.is_partial_import:
-            if out.stage.is_repo_import:
-                dep = out.stage.deps[0]
-                out.hash_info = dep.get_obj().hash_info
-                out.meta = dep.get_meta()
-            else:
-                assert isinstance(entry.hash_info, HashInfo)
-                out.hash_info = entry.hash_info
-                out.meta = entry.meta
+            out.hash_info = entry.hash_info or HashInfo()
+            out.meta = entry.meta or Meta()
             out.stage.md5 = out.stage.compute_md5()
             out.stage.dump()
             updated += out.meta.nfiles if out.meta.nfiles is not None else 1

dvc/repo/index.py

@@ -161,7 +161,7 @@ def _load_storage_from_out(storage_map, key, out):
     except NoRemoteError:
         pass
 
-    if out.stage.is_import and not out.stage.is_repo_import:
+    if out.stage.is_import:
         dep = out.stage.deps[0]
         storage_map.add_data(FileStorage(key, dep.fs, dep.fs_path))
 
@@ -174,12 +174,14 @@ class Index:
         metrics: Optional[Dict[str, List[str]]] = None,
         plots: Optional[Dict[str, List[str]]] = None,
         params: Optional[Dict[str, Any]] = None,
+        artifacts: Optional[Dict[str, Any]] = None,
     ) -> None:
         self.repo = repo
         self.stages = stages or []
         self._metrics = metrics or {}
         self._plots = plots or {}
         self._params = params or {}
+        self._artifacts = artifacts or {}
         self._collected_targets: Dict[int, List["StageInfo"]] = {}
 
     @cached_property
@@ -202,6 +204,7 @@ class Index:
         metrics = {}
         plots = {}
         params = {}
+        artifacts = {}
 
         onerror = onerror or repo.stage_collection_error_handler
         for _, idx in collect_files(repo, onerror=onerror):
@@ -210,12 +213,14 @@ class Index:
             metrics.update(idx._metrics)
             plots.update(idx._plots)
             params.update(idx._params)
+            artifacts.update(idx._artifacts)
         return cls(
             repo,
             stages=stages,
             metrics=metrics,
             plots=plots,
             params=params,
+            artifacts=artifacts,
         )
 
     @classmethod
@@ -229,6 +234,7 @@ class Index:
             metrics={path: dvcfile.metrics} if dvcfile.metrics else {},
             plots={path: dvcfile.plots} if dvcfile.plots else {},
             params={path: dvcfile.params} if dvcfile.params else {},
+            artifacts={path: dvcfile.artifacts} if dvcfile.artifacts else {},
         )
 
     def update(self, stages: Iterable["Stage"]) -> "Index":
@@ -242,6 +248,7 @@ class Index:
             metrics=self._metrics,
             plots=self._plots,
             params=self._params,
+            artifacts=self._artifacts,
         )
 
     @cached_property
@@ -264,7 +271,7 @@ class Index:
 
     def check_graph(self) -> None:
         if not getattr(self.repo, "_skip_graph_checks", False):
-            self.graph  # pylint: disable=pointless-statement
+            self.graph  # noqa: B018, pylint: disable=pointless-statement
 
     @property
     def params(self) -> Iterator["ParamsDependency"]:
@@ -530,7 +537,8 @@ class IndexView:
             workspace, key = out.index_key
             if filter_info and out.fs.path.isin(filter_info, out.fs_path):
                 key = key + out.fs.path.relparts(filter_info, out.fs_path)
-            if out.meta.isdir or out.stage.is_import and out.stage.deps[0].meta.isdir:
+            entry = self._index.data[workspace][key]
+            if entry and entry.meta and entry.meta.isdir:
                 prefixes[workspace].recursive.add(key)
             prefixes[workspace].explicit.update(key[:i] for i in range(len(key), 0, -1))
         return prefixes

dvc/repo/init.py

@@ -77,7 +77,10 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):  # noqa:
 
     if os.path.isdir(proj.site_cache_dir):
         proj.close()
-        remove(proj.site_cache_dir)
+        try:
+            remove(proj.site_cache_dir)
+        except OSError:
+            logger.debug("failed to remove %s", dvc_dir, exc_info=True)
         proj = Repo(root_dir)
 
     with proj.scm_context(autostage=True) as context:

dvc/repo/ls.py

@@ -1,8 +1,6 @@
 import os
 from typing import TYPE_CHECKING, Optional
 
-from dvc.exceptions import PathMissingError
-
 if TYPE_CHECKING:
     from dvc.fs.dvc import DVCFileSystem
 
@@ -61,10 +59,7 @@ def _ls(
     fs: "DVCFileSystem" = repo.dvcfs
     fs_path = fs.from_os_path(path)
 
-    try:
-        fs_path = fs.info(fs_path)["name"]
-    except FileNotFoundError:
-        raise PathMissingError(path, repo, dvc_only=dvc_only)  # noqa: B904
+    fs_path = fs.info(fs_path)["name"]
 
     infos = {}
     for root, dirs, files in fs.walk(

dvc/repo/plots/__init__.py

@@ -20,13 +20,14 @@ from typing import (
 
 import dpath
 import dpath.options
-from funcy import distinct, first, project
+from funcy import first, ldistinct, project, reraise
 
 from dvc.exceptions import DvcException
 from dvc.utils import error_handler, errored_revisions, onerror_collect
 from dvc.utils.objects import cached_property
-from dvc.utils.serialize import LOADERS
+from dvc.utils.serialize import PARSERS, EncodingError
 from dvc.utils.threadpool import ThreadPoolExecutor
+from dvc_render.image import ImageRenderer
 
 if TYPE_CHECKING:
     from dvc.fs import FileSystem
@@ -39,6 +40,9 @@ dpath.options.ALLOW_EMPTY_STRING_KEYS = True
 logger = logging.getLogger(__name__)
 
 
+SUPPORTED_IMAGE_EXTENSIONS = ImageRenderer.EXTENSIONS
+
+
 class PlotMetricTypeError(DvcException):
     def __init__(self, file):
         super().__init__(
@@ -60,7 +64,11 @@ class PropsNotFoundError(DvcException):
 
 @error_handler
 def _unpack_dir_files(fs, path, **kwargs):
-    return list(fs.find(path))
+    ret = list(fs.find(path))
+    if not ret:
+        # This will raise FileNotFoundError if it is a broken symlink or TreeError
+        next(iter(fs.ls(path)), None)
+    return ret
 
 
 class Plots:
@@ -153,9 +161,7 @@ class Plots:
         props: Optional[Dict] = None,
         onerror: Optional[Callable] = None,
     ):
-        from dvc.fs.dvc import DVCFileSystem
-
-        fs = DVCFileSystem(repo=self.repo)
+        fs = self.repo.dvcfs
 
         props = props or {}
 
@@ -196,7 +202,10 @@ class Plots:
             onerror=onerror,
             props=props,
         ):
-            _resolve_data_sources(data)
+            short_rev = "workspace"
+            if rev := getattr(self.repo.fs, "rev", None):
+                short_rev = rev[:7]
+            _resolve_data_sources(data, short_rev, cache_remote_stream=True)
             result.update(data)
 
         errored = errored_revisions(result)
@@ -265,7 +274,11 @@ def _is_plot(out: "Output") -> bool:
     return bool(out.plot)
 
 
-def _resolve_data_sources(plots_data: Dict):
+def _resolve_data_sources(
+    plots_data: Dict, rev: str, cache_remote_stream: bool = False
+):
+    from dvc.progress import Tqdm
+
     values = list(plots_data.values())
     to_resolve = []
     while values:
@@ -278,16 +291,26 @@ def _resolve_data_sources(plots_data: Dict):
     def resolve(value):
         data_source = value.pop("data_source")
         assert callable(data_source)
-        value.update(data_source())
+        value.update(data_source(cache_remote_stream=cache_remote_stream))
+
+    if not to_resolve:
+        return
 
     executor = ThreadPoolExecutor(
-        max_workers=4 * cpu_count(),
+        max_workers=min(16, 4 * cpu_count()),
         thread_name_prefix="resolve_data",
         cancel_on_error=True,
     )
     with executor:
-        # imap_unordered is lazy, wrapping to trigger it
-        list(executor.imap_unordered(resolve, to_resolve))
+        iterable = executor.imap_unordered(resolve, to_resolve)
+        with Tqdm(
+            iterable,
+            total=len(to_resolve),
+            desc=f"Reading plot's data from {rev}",
+            unit="files",
+            unit_scale=False,
+        ) as progress_iterable:
+            list(progress_iterable)
 
 
 def _collect_plots(
@@ -333,7 +356,7 @@ def infer_data_sources(plot_id, config=None):
     if isinstance(x, dict):
         sources.append(first(x.keys()))
 
-    return distinct(source for source in sources)
+    return ldistinct(source for source in sources)
 
 
 def _matches(targets, config_file, plot_id):
@@ -485,10 +508,7 @@ def _collect_definitions(
     result: Dict = defaultdict(dict)
     props = props or {}
 
-    from dvc.fs.dvc import DVCFileSystem
-
-    fs = DVCFileSystem(repo=repo)
-
+    fs = repo.dvcfs
     dpath.merge(
         result,
         _collect_pipeline_files(repo, targets, props, onerror=onerror),
@@ -524,20 +544,25 @@ def unpack_if_dir(fs, path, props: Dict[str, str], onerror: Optional[Callable] =
 
 
 @error_handler
-def parse(fs, path, props=None, **kwargs):
+def parse(fs, path, props=None, **fs_kwargs):
     props = props or {}
     _, extension = os.path.splitext(path)
-    if extension in (".tsv", ".csv"):
-        header = props.get("header", True)
-        if extension == ".csv":
-            return _load_sv(path=path, fs=fs, delimiter=",", header=header)
-        return _load_sv(path=path, fs=fs, delimiter="\t", header=header)
-    if extension in LOADERS or extension in (".yml", ".yaml"):
-        return LOADERS[extension](path=path, fs=fs)
-    if extension in (".jpeg", ".jpg", ".gif", ".png", ".svg"):
-        with fs.open(path, "rb") as fd:
+    if extension in SUPPORTED_IMAGE_EXTENSIONS:
+        with fs.open(path, mode="rb", **fs_kwargs) as fd:
             return fd.read()
-    raise PlotMetricTypeError(path)
+
+    if extension not in PARSERS.keys() | {".yml", ".yaml", ".csv", ".tsv"}:
+        raise PlotMetricTypeError(path)
+
+    with reraise(UnicodeDecodeError, EncodingError(path, "utf8")):
+        with fs.open(path, mode="r", encoding="utf8", **fs_kwargs) as fd:
+            contents = fd.read()
+
+    if extension in (".csv", ".tsv"):
+        header = props.get("header", True)
+        delim = "\t" if extension == ".tsv" else ","
+        return _load_sv(contents, delimiter=delim, header=header)
+    return PARSERS[extension](contents, path)
 
 
 def _plot_props(out: "Output") -> Dict:
@@ -553,10 +578,7 @@ def _plot_props(out: "Output") -> Dict:
     return project(out.plot, PLOT_PROPS)
 
 
-def _load_sv(path, fs, delimiter=",", header=True):
-    with fs.open(path, "r") as fd:
-        content = fd.read()
-
+def _load_sv(content, delimiter=",", header=True):
     if header:
         reader = csv.DictReader(io.StringIO(content), delimiter=delimiter)
     else:
@@ -566,5 +588,4 @@ def _load_sv(path, fs, delimiter=",", header=True):
             delimiter=delimiter,
             fieldnames=[str(i) for i in range(len(first_row))],
         )
-
     return list(reader)

dvc/repo/reproduce.py

@@ -3,6 +3,8 @@ from functools import partial
 from typing import TYPE_CHECKING, Iterator, List
 
 from dvc.exceptions import DvcException, ReproductionError
+from dvc.repo.metrics.show import _collect_top_level_metrics
+from dvc.repo.params.show import _collect_top_level_params
 from dvc.repo.scm_context import scm_context
 from dvc.stage.exceptions import CheckpointKilledError
 
@@ -74,6 +76,17 @@ def _track_stage(stage: "Stage") -> None:
     return context.track_changed_files()
 
 
+def _track_top_level(repo: "Repo") -> None:
+    context = repo.scm_context
+    for metric_file in _collect_top_level_metrics(repo):
+        context.track_file(metric_file)
+    for param_file in _collect_top_level_params(repo):
+        context.track_file(param_file)
+    for plot_file in repo.index._plot_sources:  # pylint: disable=protected-access
+        context.track_file(plot_file)
+    context.track_changed_files()
+
+
 @locked
 @scm_context
 def reproduce(  # noqa: C901
@@ -125,7 +138,9 @@ def reproduce(  # noqa: C901
                 )
             )
 
-    return _reproduce_stages(self.index.graph, list(stages), **kwargs)
+    result = _reproduce_stages(self.index.graph, list(stages), **kwargs)
+    _track_top_level(self)
+    return result
 
 
 def _reproduce_stages(  # noqa: C901

dvc/repo/stage.py

@@ -5,7 +5,11 @@ from contextlib import suppress
 from functools import wraps
 from typing import Iterable, List, NamedTuple, Optional, Set, Tuple, Union
 
-from dvc.exceptions import NoOutputOrStageError, OutputNotFoundError
+from dvc.exceptions import (
+    NoOutputOrStageError,
+    OutputDuplicationError,
+    OutputNotFoundError,
+)
 from dvc.repo import lock_repo
 from dvc.ui import ui
 from dvc.utils import as_posix, parse_target
@@ -176,7 +180,12 @@ class StageLoad:
 
                 check_stage_exists(self.repo, stage, stage.path)
 
-            self.repo.check_graph(stages={stage})
+            try:
+                self.repo.check_graph(stages={stage})
+            except OutputDuplicationError as exc:
+                # Don't include the stage currently being added.
+                exc.stages.remove(stage)
+                raise OutputDuplicationError(exc.output, exc.stages) from None
 
         restore_fields(stage)
         return stage

dvc/repo/status.py

@@ -1,7 +1,5 @@
 import logging
-from itertools import compress
-
-from funcy.py3 import cat
+from itertools import chain, compress
 
 from dvc.exceptions import InvalidArgumentError
 
@@ -29,7 +27,7 @@ def _joint_status(pairs):
 
 def _local_status(self, targets=None, with_deps=False, recursive=False):
     targets = targets or [None]
-    pairs = cat(
+    pairs = chain.from_iterable(
         self.stage.collect_granular(t, with_deps=with_deps, recursive=recursive)
         for t in targets
     )

dvc/schema.py

@@ -3,7 +3,7 @@ from collections.abc import Mapping
 from voluptuous import Any, Optional, Required, Schema
 
 from dvc import dependency, output
-from dvc.annotations import ANNOTATION_SCHEMA
+from dvc.annotations import ANNOTATION_SCHEMA, ARTIFACT_SCHEMA
 from dvc.output import (
     CHECKSUMS_SCHEMA,
     CLOUD_SCHEMA,
@@ -114,6 +114,9 @@ PLOT_DEFINITION = {
     Output.PARAM_PLOT_TEMPLATE: str,
 }
 SINGLE_PLOT_SCHEMA = {str: Any(PLOT_DEFINITION, None)}
+ARTIFACTS = "artifacts"
+SINGLE_ARTIFACT_SCHEMA = Schema({str: ARTIFACT_SCHEMA})
+ARTIFACTS_SCHEMA = Any(str, SINGLE_ARTIFACT_SCHEMA)
 FOREACH_IN = {
     Required(FOREACH_KWD): Any(dict, list, str),
     Required(DO_KWD): STAGE_DEFINITION,
@@ -127,6 +130,7 @@ MULTI_STAGE_SCHEMA = {
     VARS_KWD: VARS_SCHEMA,
     StageParams.PARAM_PARAMS: [str],
     StageParams.PARAM_METRICS: [str],
+    ARTIFACTS: ARTIFACTS_SCHEMA,
 }
 
 COMPILED_SINGLE_STAGE_SCHEMA = Schema(SINGLE_STAGE_SCHEMA)

dvc/stage/monitor.py

@@ -1,7 +1,7 @@
 import functools
 import logging
 import os
-import subprocess
+import subprocess  # nosec B404
 import threading
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Callable, List
@@ -96,7 +96,9 @@ class Monitor:
     def _kill_nt(proc):
         # windows stages are spawned with shell=True, proc is the shell process
         # and not the actual stage process - we have to kill the entire tree
-        subprocess.call(["taskkill", "/F", "/T", "/PID", str(proc.pid)])
+        subprocess.call(  # nosec B607, B603
+            ["taskkill", "/F", "/T", "/PID", str(proc.pid)]
+        )
 
     @staticmethod
     def _loop(tasks: List[MonitorTask], done: threading.Event):

dvc/stage/run.py

@@ -1,7 +1,7 @@
 import logging
 import os
 import signal
-import subprocess
+import subprocess  # nosec B404
 import threading
 from typing import TYPE_CHECKING, List
 
@@ -86,7 +86,7 @@ def _run(stage: "Stage", executable, cmd, checkpoint_func, **kwargs):
     exec_cmd = _make_cmd(executable, cmd)
 
     try:
-        p = subprocess.Popen(exec_cmd, **kwargs)
+        p = subprocess.Popen(exec_cmd, **kwargs)  # nosec B603
         if main_thread:
             old_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
 

dvc/testing/fixtures.py

@@ -1,5 +1,5 @@
 import os
-import subprocess
+import subprocess  # nosec B404
 from typing import Dict, Tuple
 
 import pytest
@@ -233,14 +233,16 @@ def docker_services(
         pytest.skip("disabled for Windows on CI")
 
     try:
-        subprocess.check_output("docker ps", stderr=subprocess.STDOUT, shell=True)
+        subprocess.check_output(  # nosec B607, B602
+            "docker ps", stderr=subprocess.STDOUT, shell=True
+        )
     except subprocess.CalledProcessError as err:
         out = (err.output or b"").decode("utf-8")
         pytest.skip(f"docker is not installed or the daemon is not running: {out}")
 
     try:
         cmd = "docker-compose version"
-        subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+        subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)  # nosec
     except subprocess.CalledProcessError as err:
         out = (err.output or b"").decode("utf-8")
         pytest.skip(f"docker-compose is not installed: {out}")

dvc/ui/__init__.py

@@ -1,4 +1,4 @@
-from contextlib import contextmanager
+from contextlib import contextmanager, nullcontext
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -13,6 +13,8 @@ from typing import (
     Union,
 )
 
+import colorama
+
 from dvc.utils.objects import cached_property
 
 if TYPE_CHECKING:
@@ -27,6 +29,21 @@ if TYPE_CHECKING:
     from dvc.ui.table import Headers, Styles, TableData
 
 
+@contextmanager
+def disable_colorama():
+    import sys
+
+    colorama.deinit()
+    try:
+        yield
+    finally:
+        if sys.stdout:
+            sys.stdout.flush()
+        if sys.stderr:
+            sys.stderr.flush()
+        colorama.reinit()
+
+
 class Formatter:
     def __init__(
         self, theme: Optional[Dict] = None, defaults: Optional[Dict] = None
@@ -121,7 +138,13 @@ class Console:
             sort_keys=sort_keys,
         )
         if not highlight:
-            return self.write(json.text, stderr=stderr)
+            import os
+
+            # we don't need colorama to try to strip ansi codes
+            # when highlighting is disabled
+            ctx = nullcontext() if "DVC_TEST" in os.environ else disable_colorama()
+            with ctx:
+                return self.write(json.text, stderr=stderr)
         return self.rich_print(json, stderr=stderr, soft_wrap=True)
 
     def rich_print(

dvc/ui/pager.py

@@ -59,7 +59,7 @@ def find_pager():
     if not pager:
         pager = os.getenv(PAGER_ENV)
     if not pager:
-        if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") != 0:
+        if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") != 0:  # nosec B605
             logger.warning(
                 "Unable to find `less` in the PATH. Check out %s for more info.",
                 format_link("https://man.dvc.org/pipeline/show"),

dvc/updater.py

@@ -166,7 +166,7 @@ class Updater:
         from dvc.config import Config, to_bool
 
         enabled = to_bool(
-            Config(validate=False).get("core", {}).get("check_update", "true")
+            Config.from_cwd(validate=False).get("core", {}).get("check_update", "true")
         )
         logger.debug("Check for update is %sabled.", "en" if enabled else "dis")
         return enabled

dvc/utils/hydra.py

@@ -36,6 +36,8 @@ def compose_and_dump(
     with initialize_config_dir(config_dir, version_base=None):
         cfg = compose(config_name=config_name, overrides=overrides)
 
+    OmegaConf.resolve(cfg)
+
     suffix = Path(output_file).suffix.lower()
     if suffix not in [".yml", ".yaml"]:
         dumper = DUMPERS[suffix]

dvc/utils/serialize/__init__.py

@@ -12,6 +12,13 @@ LOADERS: DefaultDict[str, LoaderFn] = defaultdict(  # noqa: F405
 )
 LOADERS.update({".toml": load_toml, ".json": load_json, ".py": load_py})  # noqa: F405
 
+PARSERS: DefaultDict[str, ParserFn] = defaultdict(  # noqa: F405
+    lambda: parse_yaml  # noqa: F405
+)
+PARSERS.update(
+    {".toml": parse_toml, ".json": parse_json, ".py": parse_py}  # noqa: F405
+)
+
 
 def load_path(fs_path, fs):
     suffix = fs.path.suffix(fs_path).lower()

dvc/utils/strictyaml.py

@@ -177,8 +177,8 @@ def determine_linecol(
         value = get(data, paths, default=None)
         if value is not None:
             with suppress(AttributeError, TypeError):
-                line = value.lc.line + 1
-                col = value.lc.col + 1
+                line = value.lc.line + 1  # type: ignore[attr-defined]
+                col = value.lc.col + 1  # type: ignore[attr-defined]
                 break
         step += 1
         *paths, _ = paths

dvc/utils/studio.py

@@ -0,0 +1,82 @@
+import logging
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, cast
+from urllib.parse import urljoin
+
+import requests
+from funcy import compact, ignore
+from requests.adapters import HTTPAdapter
+
+if TYPE_CHECKING:
+    from requests import Response
+
+logger = logging.getLogger(__name__)
+
+STUDIO_URL = "https://studio.iterative.ai"
+
+
+def post(
+    url: str,
+    token: str,
+    data: Dict[str, Any],
+    base_url: Optional[str] = STUDIO_URL,
+    max_retries: int = 3,
+    timeout: int = 5,
+) -> "Response":
+    url = urljoin(base_url or STUDIO_URL, url)
+    session = requests.Session()
+    session.mount(url, HTTPAdapter(max_retries=max_retries))
+
+    logger.trace("Sending %s to %s", data, url)  # type: ignore[attr-defined]
+
+    headers = {"Authorization": f"token {token}"}
+    r = session.post(
+        url, json=data, headers=headers, timeout=timeout, allow_redirects=False
+    )
+    r.raise_for_status()
+    return r
+
+
+def notify_refs(
+    repo_url: str,
+    token: str,
+    *,
+    base_url: Optional[str] = STUDIO_URL,
+    **refs: List[str],
+) -> Dict[str, Any]:
+    extra_keys = refs.keys() - {"pushed", "removed"}
+    assert not extra_keys, f"got extra args: {extra_keys}"
+
+    refs = compact(refs)
+    if not refs:
+        return {}
+
+    logger.debug(
+        "notifying Studio%s about updated experiments",
+        f" ({base_url})" if base_url else "",
+    )
+    data = {"repo_url": repo_url, "client": "dvc", "refs": refs}
+
+    try:
+        r = post("/webhook/dvc", token, data, base_url=base_url)
+    except requests.RequestException as e:
+        logger.trace("", exc_info=True)  # type: ignore[attr-defined]
+
+        msg = str(e)
+        if e.response is None:
+            logger.warning("failed to notify Studio: %s", msg.lower())
+            return {}
+
+        r = cast("Response", e.response)
+        d = ignore(Exception, default={})(r.json)()
+        status = r.status_code
+        if detail := d.get("detail"):
+            msg = f"{detail} ({status=})"
+        logger.warning("failed to notify Studio: %s", msg.lower())
+    else:
+        d = r.json()
+
+    if d:
+        logger.trace(  # type: ignore[attr-defined]
+            "received response: %s (status=%r)", d, r.status_code
+        )
+    return d

dvc/version.py

@@ -1,7 +1,7 @@
 # pylint: disable=unused-import
 try:
-    from ._dvc_version import version as __version__
-    from ._dvc_version import version_tuple
+    from ._dvc_version import version as __version__  # type: ignore[import]
+    from ._dvc_version import version_tuple  # type: ignore[import]
 except ImportError:
     __version__ = "UNKNOWN"
     version_tuple = (0, 0, __version__)  # type: ignore[assignment]

pyproject.toml

@@ -34,10 +34,10 @@ dependencies = [
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=0.43.0,<0.44",
+    "dvc-data>=0.47.1,<0.48",
     "dvc-http",
-    "dvc-render>=0.1.2",
-    "dvc-studio-client>=0.5.0,<1",
+    "dvc-render>=0.3.1,<0.4.0",
+    "dvc-studio-client>=0.6.1,<1",
     "dvc-task>=0.2.0,<1",
     "flatten_dict<1,>=0.4.1",
     "flufl.lock>=5",
@@ -56,7 +56,7 @@ dependencies = [
     "requests>=2.22",
     "rich>=12",
     "ruamel.yaml>=0.17.11",
-    "scmrepo>=0.1.15,<1",
+    "scmrepo>=0.2.1,<1",
     "shortuuid>=0.5",
     "shtab<2,>=1.3.4",
     "tabulate>=0.8.7",
@@ -74,7 +74,7 @@ gs = ["dvc-gs==2.22.0"]
 hdfs = ["dvc-hdfs==2.19"]
 lint = [
     "mypy==1.1.1",
-    "pylint==2.17.0",
+    "pylint==2.17.2",
     "types-colorama",
     "types-psutil",
     "types-requests",
@@ -257,9 +257,8 @@ ignored-argument-names = "_.*|args|kwargs"
 # external flake8 codes that should be preserved
 external = ["B301", "C901", "E302", "W601"]
 ignore = ["N818", "S101", "PT004", "PT007", "PT019", "SIM105", "SIM108", "SIM110", "SIM117", "TRY003", "TRY200", "TRY300", "TRY301", "PLR2004", "PLW2901", "RUF005"]
-select = ["F", "E", "W", "C90", "N", "UP", "YTT", "S", "BLE", "B", "A", "C4", "T10", "ISC", "ICN", "G", "INP", "PIE", "T20", "PT", "Q", "RET501", "RET504", "SIM", "TID", "TCH", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY", "RUF"]
+select = ["F", "E", "W", "C90", "N", "UP", "YTT", "S", "BLE", "B", "A", "C4", "T10", "EXE", "ISC", "ICN", "G", "INP", "PIE", "T20", "PT", "Q", "RET501", "RET504", "SIM", "TID", "TCH", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY", "RUF"]
 show-source = true
-target-version = "py38"
 
 [tool.ruff.flake8-pytest-style]
 fixture-parentheses = false
@@ -290,3 +289,7 @@ ignore-names = ["M", "SCM"]
 
 [tool.ruff.pylint]
 max-args = 10
+
+[tool.bandit]
+exclude_dirs = ["tests", "scripts"]
+skips = ["B101"]

scripts/build-requirements.txt

@@ -1,3 +1,3 @@
 # NOTE: using strict version due to
 # https://github.com/iterative/dvc/issues/7949
-PyInstaller==5.8.0
+PyInstaller==5.9.0

tests/func/api/test_data.py

@@ -4,7 +4,7 @@ import pytest
 from funcy import first, get_in
 
 from dvc import api
-from dvc.exceptions import FileMissingError, OutputNotFoundError, PathMissingError
+from dvc.exceptions import OutputNotFoundError, PathMissingError
 from dvc.testing.api_tests import TestAPI  # noqa, pylint: disable=unused-import
 from dvc.testing.tmp_dir import make_subrepo
 from dvc.utils.fs import remove
@@ -75,7 +75,7 @@ def test_missing(tmp_dir, dvc, remote):
 
     remove("foo")
 
-    with pytest.raises(FileMissingError):
+    with pytest.raises(PathMissingError):
         api.read("foo")
 
 
@@ -100,7 +100,7 @@ def test_open_not_cached(dvc):
         assert fd.read() == metric_content
 
     os.remove(metric_file)
-    with pytest.raises(FileMissingError):
+    with pytest.raises(PathMissingError):
         api.read(metric_file)
 
 
@@ -193,10 +193,6 @@ def test_get_url_subrepos(tmp_dir, scm, local_cloud):
     assert api.get_url("subrepo/bar") == expected_url
 
 
-@pytest.mark.xfail(
-    raises=PathMissingError,
-    reason="https://github.com/iterative/dvc/issues/7341",
-)
 def test_open_from_remote(tmp_dir, erepo_dir, cloud, local_cloud):
     erepo_dir.add_remote(config=cloud.config, name="other")
     erepo_dir.add_remote(config=local_cloud.config, default=True)

tests/func/api/test_experiments.py

@@ -0,0 +1,18 @@
+import pytest
+
+from dvc import api
+from dvc.repo.experiments.exceptions import ExperimentExistsError
+
+
+def test_exp_save(tmp_dir, dvc, scm, mocker):
+    tmp_dir.scm_gen({"foo": "foo"}, commit="initial")
+
+    api.exp_save()
+
+    api.exp_save("foo")
+    with pytest.raises(
+        ExperimentExistsError,
+        match="Experiment conflicts with existing experiment 'foo'.",
+    ):
+        api.exp_save("foo")
+    api.exp_save("foo", force=True)

tests/func/artifacts/test_artifacts.py

@@ -0,0 +1,132 @@
+import os
+
+import pytest
+
+from dvc.annotations import Artifact
+from dvc.repo.artifacts import name_is_compatible
+from dvc.utils.strictyaml import YAMLSyntaxError, YAMLValidationError
+
+dvcyaml = {
+    "artifacts": {
+        "myart": {"type": "model", "path": "myart.pkl"},
+        "hello": {"type": "file", "path": "hello.txt"},
+        "world": {
+            "type": "object",
+            "path": "world.txt",
+            "desc": "The world is not enough",
+            "labels": ["but", "this", "is"],
+            "meta": {"such": "a", "perfect": "place to start"},
+        },
+    }
+}
+
+
+def test_reading_artifacts_subdir(tmp_dir, dvc):
+    (tmp_dir / "dvc.yaml").dump(dvcyaml)
+
+    subdir = tmp_dir / "subdir"
+    subdir.mkdir()
+
+    (subdir / "dvc.yaml").dump(dvcyaml)
+
+    artifacts = {
+        name: Artifact(**values) for name, values in dvcyaml["artifacts"].items()
+    }
+    assert tmp_dir.dvc.artifacts.read() == {
+        "dvc.yaml": artifacts,
+        f"subdir{os.path.sep}dvc.yaml": artifacts,
+    }
+
+
+bad_dvcyaml_extra_field = {
+    "artifacts": {
+        "lol": {"kek": "cheburek", "path": "lol"},
+        "hello": {"type": "file", "path": "hello.txt"},
+    }
+}
+
+
+bad_dvcyaml_missing_path = {
+    "artifacts": {
+        "lol": {},
+    }
+}
+
+
+@pytest.mark.parametrize(
+    "bad_dvcyaml", [bad_dvcyaml_extra_field, bad_dvcyaml_missing_path]
+)
+def test_broken_dvcyaml_extra_field(tmp_dir, dvc, bad_dvcyaml):
+    (tmp_dir / "dvc.yaml").dump(bad_dvcyaml)
+
+    with pytest.raises(YAMLValidationError):
+        tmp_dir.dvc.artifacts.read()
+
+
+bad_dvcyaml_id_duplication = """
+artifacts:
+  lol:
+    type: kek
+  lol: {}
+"""
+
+
+def test_broken_dvcyaml_id_duplication(tmp_dir, dvc):
+    with open(tmp_dir / "dvc.yaml", "w") as f:
+        f.write(bad_dvcyaml_id_duplication)
+
+    with pytest.raises(YAMLSyntaxError):
+        tmp_dir.dvc.artifacts.read()
+
+
+dvcyaml_redirecting = {"artifacts": "artifacts.yaml"}
+
+
+def test_read_artifacts_yaml(tmp_dir, dvc):
+    (tmp_dir / "dvc.yaml").dump(dvcyaml_redirecting)
+    (tmp_dir / "artifacts.yaml").dump(dvcyaml["artifacts"])
+
+    artifacts = {
+        name: Artifact(**values) for name, values in dvcyaml["artifacts"].items()
+    }
+    assert tmp_dir.dvc.artifacts.read() == {
+        "dvc.yaml": artifacts,
+    }
+
+
+@pytest.mark.parametrize(
+    "name",
+    [
+        "m",
+        "nn",
+        "m1",
+        "model-prod",
+        "model-prod-v1",
+    ],
+)
+def test_check_name_is_valid(name):
+    assert name_is_compatible(name)
+
+
+@pytest.mark.parametrize(
+    "name",
+    [
+        "",
+        "1",
+        "m/",
+        "/m",
+        "1nn",
+        "###",
+        "@@@",
+        "a model",
+        "a_model",
+        "-model",
+        "model-",
+        "model@1",
+        "model#1",
+        "@namespace/model",
+        "namespace/model",
+    ],
+)
+def test_check_name_is_invalid(name):
+    assert not name_is_compatible(name)

tests/func/experiments/test_apply.py

@@ -1,3 +1,5 @@
+import os
+
 import pytest
 from funcy import first
 
@@ -26,6 +28,11 @@ def test_apply(tmp_dir, scm, dvc, exp_stage):
     assert (tmp_dir / "metrics.yaml").read_text().strip() == "foo: 3"
 
 
+@pytest.mark.xfail(
+    os.name == "nt",
+    strict=False,
+    reason="https://github.com/iterative/dvc/issues/9143",
+)
 def test_apply_failed(tmp_dir, scm, dvc, failed_exp_stage, session_queue):
     from dvc.exceptions import InvalidArgumentError
 
@@ -35,7 +42,7 @@ def test_apply_failed(tmp_dir, scm, dvc, failed_exp_stage, session_queue):
     dvc.experiments.run(
         failed_exp_stage.addressing, params=["foo=3"], queue=True, name="foo"
     )
-    dvc.experiments.run(run_all=True)
+    session_queue.wait(["foo"])
 
     with pytest.raises(InvalidArgumentError):
         dvc.experiments.apply("bar")

tests/func/experiments/test_experiments.py

@@ -5,10 +5,11 @@ import stat
 from textwrap import dedent
 
 import pytest
+from configobj import ConfigObj
 from funcy import first
 
 from dvc.dvcfile import PROJECT_FILE
-from dvc.exceptions import ReproductionError
+from dvc.exceptions import DvcException, ReproductionError
 from dvc.repo.experiments.exceptions import ExperimentExistsError
 from dvc.repo.experiments.queue.base import BaseStashQueue
 from dvc.repo.experiments.refs import CELERY_STASH
@@ -246,11 +247,12 @@ def test_branch(tmp_dir, scm, dvc, exp_stage):
 
     with pytest.raises(InvalidArgumentError):
         dvc.experiments.branch("foo", "branch-exists")
+    dvc.experiments.branch("foo")
     dvc.experiments.branch("foo", "branch-name")
     dvc.experiments.branch(exp_a, "branch-rev")
     dvc.experiments.branch(ref_a, "branch-ref")
 
-    for name in ["branch-name", "branch-rev", "branch-ref"]:
+    for name in ["foo-branch", "branch-name", "branch-rev", "branch-ref"]:
         assert name in scm.list_branches()
         assert scm.resolve_rev(name) == exp_a
 
@@ -620,7 +622,7 @@ def test_run_env(tmp_dir, dvc, scm, mocker):
     )
     dvc.experiments.run()
     assert (tmp_dir / "DVC_EXP_BASELINE_REV").read_text().strip() == baseline
-    assert (tmp_dir / "DVC_EXP_NAME").read_text().strip() != ""
+    assert (tmp_dir / "DVC_EXP_NAME").read_text().strip()
     assert (tmp_dir / "STUDIO_REPO_URL").read_text().strip() == "REPO_URL"
 
     dvc.experiments.run(name="foo")
@@ -663,3 +665,82 @@ def test_experiment_no_commit(tmp_dir):
             repo.experiments.ls()
     finally:
         repo.close()
+
+
+def test_local_config_is_propagated_to_tmp(tmp_dir, scm, dvc):
+    with dvc.config.edit("local") as conf:
+        conf["cache"]["type"] = "hardlink"
+
+    stage = dvc.stage.add(
+        cmd="cat .dvc/config.local > file", name="foo", outs_no_cache=["file"]
+    )
+    scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
+
+    results = dvc.experiments.run(stage.addressing, tmp_dir=True)
+    exp = first(results)
+    fs = scm.get_fs(exp)
+
+    with fs.open("file") as fobj:
+        conf_obj = ConfigObj(fobj)
+        assert conf_obj["cache"]["type"] == "hardlink"
+
+
+@pytest.mark.parametrize("tmp", [True, False])
+def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc, tmp):
+    (tmp_dir / "dvc.yaml").dump(
+        {
+            "metrics": ["metrics.json"],
+            "params": ["params.yaml"],
+            "plots": ["plots.csv"],
+        }
+    )
+    stage = dvc.stage.add(
+        cmd="touch metrics.json && touch params.yaml && touch plots.csv",
+        name="top-level",
+    )
+    scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
+    results = dvc.experiments.run(stage.addressing, tmp_dir=tmp)
+    exp = first(results)
+    fs = scm.get_fs(exp)
+    for file in ["metrics.json", "params.yaml", "plots.csv"]:
+        assert fs.exists(file)
+
+
+@pytest.mark.parametrize("tmp", [True, False])
+def test_copy_paths(tmp_dir, scm, dvc, tmp):
+    stage = dvc.stage.add(
+        cmd="cat file && ls dir",
+        name="foo",
+    )
+    scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
+
+    (tmp_dir / "dir").mkdir()
+    (tmp_dir / "dir" / "file").write_text("dir/file")
+    scm.ignore(tmp_dir / "dir")
+    (tmp_dir / "file").write_text("file")
+    scm.ignore(tmp_dir / "file")
+
+    results = dvc.experiments.run(
+        stage.addressing, tmp_dir=tmp, copy_paths=["dir", "file"]
+    )
+    exp = first(results)
+    fs = scm.get_fs(exp)
+    assert not fs.exists("dir")
+    assert not fs.exists("file")
+
+
+def test_copy_paths_errors(tmp_dir, scm, dvc, mocker):
+    stage = dvc.stage.add(
+        cmd="echo foo",
+        name="foo",
+    )
+    scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
+
+    with pytest.raises(DvcException, match="Unable to copy"):
+        dvc.experiments.run(stage.addressing, tmp_dir=True, copy_paths=["foo"])
+
+    (tmp_dir / "foo").write_text("foo")
+    mocker.patch("shutil.copy", side_effect=OSError)
+
+    with pytest.raises(DvcException, match="Unable to copy"):
+        dvc.experiments.run(stage.addressing, tmp_dir=True, copy_paths=["foo"])

tests/func/experiments/test_init.py

@@ -166,7 +166,7 @@ def test_when_params_is_omitted_in_interactive_mode(tmp_dir, scm, dvc):
         }
     }
     assert not (tmp_dir / "dvc.lock").exists()
-    assert (tmp_dir / "script.py").read_text() == ""
+    assert not (tmp_dir / "script.py").read_text()
     assert (tmp_dir / "data").is_dir()
     scm._reset()
     assert scm.is_tracked("dvc.yaml")
@@ -194,7 +194,7 @@ def test_init_interactive_params_validation(tmp_dir, dvc, capsys):
             }
         }
     }
-    assert (tmp_dir / "script.py").read_text() == ""
+    assert not (tmp_dir / "script.py").read_text()
     assert (tmp_dir / "data").is_dir()
 
     out, err = capsys.readouterr()
@@ -226,7 +226,7 @@ def test_init_with_no_defaults_interactive(tmp_dir, dvc):
             }
         }
     }
-    assert (tmp_dir / "script.py").read_text() == ""
+    assert not (tmp_dir / "script.py").read_text()
     assert (tmp_dir / "data").is_dir()
 
 
@@ -275,7 +275,7 @@ def test_init_default(tmp_dir, scm, dvc, interactive, overrides, inp, capsys):
         }
     }
     assert not (tmp_dir / "dvc.lock").exists()
-    assert (tmp_dir / "script.py").read_text() == ""
+    assert not (tmp_dir / "script.py").read_text()
     assert (tmp_dir / "data").is_dir()
     scm._reset()
     assert scm.is_tracked("dvc.yaml")
@@ -345,7 +345,7 @@ def test_init_interactive_live(tmp_dir, scm, dvc, interactive, overrides, inp, c
         }
     }
     assert not (tmp_dir / "dvc.lock").exists()
-    assert (tmp_dir / "script.py").read_text() == ""
+    assert not (tmp_dir / "script.py").read_text()
     assert (tmp_dir / "data").is_dir()
     scm._reset()
     assert scm.is_tracked("dvc.yaml")

tests/func/experiments/test_queue.py

@@ -22,8 +22,9 @@ def test_celery_logs(
     test_queue,
 ):
     celery_queue = dvc.experiments.celery_queue
-    dvc.experiments.run(failed_exp_stage.addressing, queue=True)
+    dvc.experiments.run(failed_exp_stage.addressing, queue=True, name="foo")
     dvc.experiments.run(run_all=True)
+    test_queue.wait(["foo"])
 
     done_result = first(celery_queue.iter_done())
 
@@ -63,6 +64,7 @@ def test_queue_remove_done(
             name=name_fail,
         )
     dvc.experiments.run(run_all=True)
+    test_queue.wait(success_tasks + failed_tasks)
     assert len(celery_queue.failed_stash) == 3
     status = to_dict(celery_queue.status())
     assert len(status) == 6
@@ -90,3 +92,41 @@ def test_queue_remove_done(
     assert set(celery_queue.clear(success=True)) == set(success_tasks[:2])
 
     assert celery_queue.status() == []
+
+
+def test_queue_doesnt_remove_untracked_params_file(tmp_dir, dvc, scm):
+    """Regression test for https://github.com/iterative/dvc/issues/7842"""
+    tmp_dir.gen("params.yaml", "foo: 1")
+    stage = dvc.run(cmd="echo ${foo}", params=["foo"], name="echo-foo")
+    scm.add(
+        [
+            "dvc.yaml",
+            "dvc.lock",
+            ".gitignore",
+        ]
+    )
+    scm.commit("init")
+    dvc.experiments.run(stage.addressing, params=["foo=2"], queue=True)
+    assert (tmp_dir / "params.yaml").exists()
+
+
+def test_copy_paths_queue(tmp_dir, scm, dvc):
+    stage = dvc.stage.add(
+        cmd="cat file && ls dir",
+        name="foo",
+    )
+    scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
+
+    (tmp_dir / "dir").mkdir()
+    (tmp_dir / "dir" / "file").write_text("dir/file")
+    scm.ignore(tmp_dir / "dir")
+    (tmp_dir / "file").write_text("file")
+    scm.ignore(tmp_dir / "file")
+
+    dvc.experiments.run(stage.addressing, queue=True)
+    results = dvc.experiments.run(run_all=True)
+
+    exp = first(results)
+    fs = scm.get_fs(exp)
+    assert not fs.exists("dir")
+    assert not fs.exists("file")

tests/func/experiments/test_remote.py

@@ -76,7 +76,11 @@ def test_push_diverged(tmp_dir, scm, dvc, git_upstream, exp_stage):
 
     git_upstream.tmp_dir.scm.set_ref(str(ref_info), remote_rev)
 
-    assert dvc.experiments.push(git_upstream.remote, [ref_info.name]) == []
+    assert dvc.experiments.push(git_upstream.remote, [ref_info.name]) == {
+        "diverged": [ref_info.name],
+        "url": None,
+        "uploaded": 0,
+    }
     assert git_upstream.tmp_dir.scm.get_ref(str(ref_info)) == remote_rev
 
     dvc.experiments.push(git_upstream.remote, [ref_info.name], force=True)

tests/func/experiments/test_remove.py

@@ -2,6 +2,7 @@ import pytest
 from funcy import first
 
 from dvc.exceptions import InvalidArgumentError
+from dvc.repo.experiments.exceptions import UnresolvedExpNamesError
 from dvc.repo.experiments.utils import exp_refs_by_rev
 
 
@@ -121,6 +122,11 @@ def test_remove_remote(tmp_dir, scm, dvc, exp_stage, git_upstream, use_url):
     assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_list[1])) is None
     assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_list[2])) == exp_list[2]
 
+    with pytest.raises(
+        UnresolvedExpNamesError, match=f"Experiment 'foo' does not exist in '{remote}'"
+    ):
+        dvc.experiments.remove(git_remote=remote, exp_names=["foo"])
+
 
 def test_remove_experiments_by_rev(tmp_dir, scm, dvc, exp_stage):
     baseline = scm.get_rev()

tests/func/experiments/test_save.py

@@ -1,5 +1,3 @@
-import logging
-
 import pytest
 from funcy import first
 
@@ -8,18 +6,21 @@ from dvc.repo.experiments.utils import exp_refs_by_rev
 from dvc.scm import resolve_rev
 
 
-@pytest.fixture
-def modified_exp_stage(exp_stage, tmp_dir):
-    with open(tmp_dir / "copy.py", "a", encoding="utf-8") as fh:
-        fh.write("\n# dummy change")
+def setup_stage(tmp_dir, dvc, scm):
+    tmp_dir.gen("params.yaml", "foo: 1")
+    dvc.run(name="echo-foo", outs=["bar"], cmd="echo ${foo} > bar")
+    scm.add(["dvc.yaml", "dvc.lock", ".gitignore", "params.yaml"])
+    scm.commit("init")
 
 
-def test_exp_save_unchanged(tmp_dir, dvc, scm, exp_stage):
+def test_exp_save_unchanged(tmp_dir, dvc, scm):
+    setup_stage(tmp_dir, dvc, scm)
     dvc.experiments.save()
 
 
 @pytest.mark.parametrize("name", (None, "test"))
-def test_exp_save(tmp_dir, dvc, scm, exp_stage, name, modified_exp_stage):
+def test_exp_save(tmp_dir, dvc, scm, name):
+    setup_stage(tmp_dir, dvc, scm)
     baseline = scm.get_rev()
 
     exp = dvc.experiments.save(name=name)
@@ -32,12 +33,12 @@ def test_exp_save(tmp_dir, dvc, scm, exp_stage, name, modified_exp_stage):
     assert resolve_rev(scm, exp_name) == exp
 
 
-def test_exp_save_overwrite_experiment(
-    tmp_dir, dvc, scm, exp_stage, modified_exp_stage
-):
+def test_exp_save_overwrite_experiment(tmp_dir, dvc, scm):
+    setup_stage(tmp_dir, dvc, scm)
     name = "dummy"
     dvc.experiments.save(name=name)
 
+    tmp_dir.gen("params.yaml", "foo: 2")
     with pytest.raises(ExperimentExistsError):
         dvc.experiments.save(name=name)
 
@@ -54,12 +55,14 @@ def test_exp_save_overwrite_experiment(
         "invalidname.",
     ),
 )
-def test_exp_save_invalid_name(tmp_dir, dvc, scm, exp_stage, name):
+def test_exp_save_invalid_name(tmp_dir, dvc, scm, name):
+    setup_stage(tmp_dir, dvc, scm)
     with pytest.raises(InvalidArgumentError):
         dvc.experiments.save(name=name, force=True)
 
 
-def test_exp_save_after_commit(tmp_dir, dvc, scm, exp_stage):
+def test_exp_save_after_commit(tmp_dir, dvc, scm):
+    setup_stage(tmp_dir, dvc, scm)
     baseline = scm.get_rev()
     dvc.experiments.save(name="exp-1", force=True)
 
@@ -72,36 +75,81 @@ def test_exp_save_after_commit(tmp_dir, dvc, scm, exp_stage):
 
 
 def test_exp_save_with_staged_changes(tmp_dir, dvc, scm):
+    setup_stage(tmp_dir, dvc, scm)
+    tmp_dir.gen({"deleted": "deleted", "modified": "modified"})
+    scm.add_commit(["deleted", "modified"], "init")
+
+    (tmp_dir / "deleted").unlink()
     tmp_dir.gen({"new_file": "new_file"})
-    scm.add("new_file")
+    (tmp_dir / "modified").write_text("foo")
+    scm.add(["deleted", "new_file", "modified"])
 
-    dvc.experiments.save(name="exp")
-
-    _, _, unstaged = scm.status()
-    assert "new_file" in unstaged
+    exp_rev = dvc.experiments.save(name="exp")
+    scm.checkout(exp_rev, force=True)
+    assert not (tmp_dir / "deleted").exists()
+    assert (tmp_dir / "new_file").exists()
+    assert (tmp_dir / "modified").read_text() == "foo"
 
 
-def test_exp_save_include_untracked(tmp_dir, dvc, scm, exp_stage):
+def test_exp_save_include_untracked(tmp_dir, dvc, scm):
+    setup_stage(tmp_dir, dvc, scm)
+
     new_file = tmp_dir / "new_file"
-    for i in range(2):
-        new_file.write_text(f"exp-{i}")
-        dvc.experiments.save(name=f"exp-{i}", include_untracked=["new_file"])
+    new_file.write_text("new_file")
+    dvc.experiments.save(name="exp", include_untracked=["new_file"])
 
     _, _, unstaged = scm.status()
     assert "new_file" in unstaged
-    assert new_file.read_text() == f"exp-{i}"
-
-    dvc.experiments.apply("exp-0")
-    assert new_file.read_text() == "exp-0"
+    assert new_file.read_text() == "new_file"
 
 
-def test_exp_save_include_untracked_warning(tmp_dir, dvc, scm, caplog, exp_stage):
+def test_exp_save_include_untracked_warning(tmp_dir, dvc, scm, mocker):
     """Regression test for https://github.com/iterative/dvc/issues/9061"""
+    setup_stage(tmp_dir, dvc, scm)
+
     new_dir = tmp_dir / "new_dir"
     new_dir.mkdir()
     (new_dir / "foo").write_text("foo")
     (new_dir / "bar").write_text("bar")
 
-    with caplog.at_level(logging.WARNING, logger="dvc.repo.experiments.save"):
-        dvc.experiments.save(name="exp", include_untracked=["new_dir"])
-        assert not caplog.records
+    logger = mocker.patch("dvc.repo.experiments.executor.base.logger")
+
+    dvc.experiments.save(name="exp", include_untracked=["new_dir"])
+    assert not logger.warning.called
+
+
+def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc):
+    (tmp_dir / "dvc.yaml").dump(
+        {
+            "metrics": ["metrics.json"],
+            "params": ["params.yaml"],
+            "plots": ["plots.csv"],
+        }
+    )
+    stage = dvc.stage.add(
+        cmd="touch metrics.json && touch params.yaml && touch plots.csv",
+        name="top-level",
+    )
+    scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
+    dvc.reproduce(stage.addressing)
+    exp = dvc.experiments.save()
+    fs = scm.get_fs(exp)
+    for file in ["metrics.json", "params.yaml", "plots.csv", "dvc.lock"]:
+        assert fs.exists(file)
+
+
+def test_untracked_dvclock_is_included_in_exp(tmp_dir, scm, dvc):
+    stage = dvc.stage.add(
+        cmd="echo foo",
+        name="foo",
+    )
+    scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
+    dvc.reproduce(stage.addressing)
+
+    # dvc.reproduce automatically stages `dvc.lock`
+    # force it to be untracked
+    scm.reset()
+
+    exp = dvc.experiments.save()
+    fs = scm.get_fs(exp)
+    assert fs.exists("dvc.lock")

tests/func/experiments/test_set_params.py

@@ -116,10 +116,7 @@ def test_hydra_sweep(
     assert patched.call_count == len(expected)
     for e in expected:
         patched.assert_any_call(
-            mocker.ANY,
-            params=e,
-            reset=True,
-            targets=None,
+            mocker.ANY, params=e, reset=True, targets=None, copy_paths=None
         )
 
 

tests/func/experiments/test_show.py

@@ -156,15 +156,18 @@ def test_show_queued(tmp_dir, scm, dvc, exp_stage):
 
 
 @pytest.mark.vscode
-def test_show_failed_experiment(tmp_dir, scm, dvc, failed_exp_stage):
+def test_show_failed_experiment(tmp_dir, scm, dvc, failed_exp_stage, test_queue):
     baseline_rev = scm.get_rev()
     timestamp = datetime.fromtimestamp(
         scm.gitpython.repo.rev_parse(baseline_rev).committed_date
     )
 
-    dvc.experiments.run(failed_exp_stage.addressing, params=["foo=2"], queue=True)
+    dvc.experiments.run(
+        failed_exp_stage.addressing, params=["foo=2"], queue=True, name="show-failed"
+    )
     exp_rev = dvc.experiments.scm.resolve_rev(f"{CELERY_STASH}@{{0}}")
     dvc.experiments.run(run_all=True)
+    test_queue.wait(["show-failed"])
     experiments = dvc.experiments.show()[baseline_rev]
 
     expected_baseline = {
@@ -542,15 +545,20 @@ def test_show_running_celery(tmp_dir, scm, dvc, exp_stage, mocker):
     assert results["workspace"]["baseline"]["data"]["status"] == "Success"
 
 
-def test_show_running_checkpoint(tmp_dir, scm, dvc, checkpoint_stage, mocker):
+def test_show_running_checkpoint(
+    tmp_dir, scm, dvc, checkpoint_stage, mocker, test_queue
+):
     from dvc.repo.experiments.executor.local import TempDirExecutor
 
     baseline_rev = scm.get_rev()
-    dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"], queue=True)
+    dvc.experiments.run(
+        checkpoint_stage.addressing, params=["foo=2"], queue=True, name="foo"
+    )
     queue = dvc.experiments.celery_queue
     entries = list(queue.iter_queued())
 
     run_results = dvc.experiments.run(run_all=True)
+    test_queue.wait(["foo"])
     checkpoint_rev = first(run_results)
     exp_ref = first(exp_refs_by_rev(scm, checkpoint_rev))
 

tests/func/parsing/test_foreach.py

@@ -44,6 +44,23 @@ def test_with_dict_data(tmp_dir, dvc):
     assert not resolver.tracked_vars["build@model2"]
 
 
+def test_with_dict_with_non_str_keys(tmp_dir, dvc):
+    resolver = DataResolver(dvc, tmp_dir.fs_path, {})
+    context = Context()
+
+    foreach_data = {2021: {"thresh": "foo"}, 2022: {"thresh": "bar"}}
+    data = {"foreach": foreach_data, "do": {"cmd": "echo ${key} ${item.thresh}"}}
+    definition = ForeachDefinition(resolver, context, "build", data)
+
+    assert definition.resolve_one("2021") == {"build@2021": {"cmd": "echo 2021 foo"}}
+    assert definition.resolve_one("2022") == {"build@2022": {"cmd": "echo 2022 bar"}}
+
+    # check that `foreach` item-key replacement didnot leave any leftovers.
+    assert not context
+    assert not resolver.tracked_vars["build@2021"]
+    assert not resolver.tracked_vars["build@2022"]
+
+
 def test_with_composite_list(tmp_dir, dvc):
     resolver = DataResolver(dvc, tmp_dir.fs_path, {})
 

tests/func/plots/test_show.py

@@ -418,5 +418,8 @@ def test_show_plots_defined_with_native_os_path(tmp_dir, dvc, scm, capsys):
     assert main(["plots", "show", "--json"]) == 0
     out, _ = capsys.readouterr()
     json_out = json.loads(out)
-    assert json_out[f"dvc.yaml::{top_level_plot}"]
-    assert json_out[stage_plot]
+    assert "errors" not in json_out
+
+    json_data = json_out["data"]
+    assert json_data[f"dvc.yaml::{top_level_plot}"]
+    assert json_data[stage_plot]

tests/func/test_config.py

@@ -245,7 +245,7 @@ def test_merging_two_levels(dvc):
 
 def test_config_loads_without_error_for_non_dvc_repo(tmp_dir):
     # regression testing for https://github.com/iterative/dvc/issues/3328
-    Config(validate=True)
+    Config.from_cwd(validate=True)
 
 
 @pytest.mark.parametrize(
@@ -284,7 +284,7 @@ def test_config_gdrive_fields(tmp_dir, dvc):
             "profile": "myprofile",
         }
 
-    Config(validate=True)
+    Config.from_cwd(validate=True)
 
 
 def test_config_remote(tmp_dir, dvc, capsys):

tests/func/test_data_cloud.py

@@ -226,12 +226,13 @@ def test_pull_git_imports(tmp_dir, dvc, scm, erepo):
 
     assert dvc.pull()["fetched"] == 0
 
-    for item in ["foo", "new_dir", dvc.cache.local.path]:
+    for item in ["foo", "new_dir"]:
         remove(item)
+    dvc.cache.local.clear()
     os.makedirs(dvc.cache.local.path, exist_ok=True)
     clean_repos()
 
-    assert dvc.pull(force=True)["fetched"] == 3
+    assert dvc.pull(force=True)["fetched"] == 2
 
     assert (tmp_dir / "foo").exists()
     assert (tmp_dir / "foo").read_text() == "foo"

tests/func/test_diff.py

@@ -358,11 +358,9 @@ def setup_targets_test(tmp_dir):
 
 
 def test_targets_missing_path(tmp_dir, scm, dvc):
-    from dvc.exceptions import PathMissingError
-
     setup_targets_test(tmp_dir)
 
-    with pytest.raises(PathMissingError):
+    with pytest.raises(FileNotFoundError):
         dvc.diff(targets=["missing"])
 
 

tests/func/test_external_repo.py

@@ -21,11 +21,11 @@ def test_external_repo(erepo_dir, mocker):
     clone_spy = mocker.spy(Git, "clone")
 
     with external_repo(url) as repo:
-        with repo.open_by_relpath("file") as fd:
+        with repo.dvcfs.open("file") as fd:
             assert fd.read() == "master"
 
     with external_repo(url, rev="branch") as repo:
-        with repo.open_by_relpath("file") as fd:
+        with repo.dvcfs.open("file") as fd:
             assert fd.read() == "branch"
 
     assert clone_spy.call_count == 1
@@ -122,7 +122,7 @@ def test_relative_remote(erepo_dir, tmp_dir):
     with external_repo(url) as repo:
         assert os.path.isabs(repo.config["remote"]["upstream"]["url"])
         assert os.path.isdir(repo.config["remote"]["upstream"]["url"])
-        with repo.open_by_relpath("file") as fd:
+        with repo.dvcfs.open("file") as fd:
             assert fd.read() == "contents"
 
 
@@ -136,7 +136,7 @@ def test_shallow_clone_branch(erepo_dir, mocker):
     clone_spy = mocker.spy(Git, "clone")
 
     with external_repo(url, rev="branch") as repo:
-        with repo.open_by_relpath("file") as fd:
+        with repo.dvcfs.open("file") as fd:
             assert fd.read() == "branch"
 
     clone_spy.assert_called_with(url, ANY, shallow_branch="branch", progress=ANY)
@@ -146,7 +146,7 @@ def test_shallow_clone_branch(erepo_dir, mocker):
 
     mock_fetch = mocker.patch.object(Git, "fetch")
     with external_repo(url) as repo:
-        with repo.open_by_relpath("file") as fd:
+        with repo.dvcfs.open("file") as fd:
             assert fd.read() == "master"
     mock_fetch.assert_called_with(unshallow=True)
 
@@ -161,7 +161,7 @@ def test_shallow_clone_tag(erepo_dir, mocker):
 
     clone_spy = mocker.spy(Git, "clone")
     with external_repo(url, rev="v1") as repo:
-        with repo.open_by_relpath("file") as fd:
+        with repo.dvcfs.open("file") as fd:
             assert fd.read() == "foo"
 
     clone_spy.assert_called_with(url, ANY, shallow_branch="v1", progress=ANY)
@@ -171,7 +171,7 @@ def test_shallow_clone_tag(erepo_dir, mocker):
 
     mock_fetch = mocker.patch.object(Git, "fetch")
     with external_repo(url, rev="master") as repo:
-        with repo.open_by_relpath("file") as fd:
+        with repo.dvcfs.open("file") as fd:
             assert fd.read() == "bar"
     mock_fetch.assert_called_with(unshallow=True)
 

tests/func/test_get.py

@@ -217,7 +217,7 @@ def test_get_url_positive(tmp_dir, erepo_dir, caplog, local_cloud):
     caplog.clear()
     with caplog.at_level(logging.ERROR, logger="dvc"):
         assert main(["get", os.fspath(erepo_dir), "foo", "--show-url"]) == 0
-        assert caplog.text == ""
+        assert not caplog.text
 
 
 def test_get_url_not_existing(tmp_dir, erepo_dir, caplog):
@@ -231,17 +231,15 @@ def test_get_url_not_existing(tmp_dir, erepo_dir, caplog):
                     "--show-url",
                 ]
             )
-            == 1
+            != 0
         )
-        assert "failed to show URL" in caplog.text
 
 
 def test_get_url_git_only_repo(tmp_dir, scm, caplog):
     tmp_dir.scm_gen({"foo": "foo"}, commit="initial")
 
     with caplog.at_level(logging.ERROR):
-        assert main(["get", os.fspath(tmp_dir), "foo", "--show-url"]) == 1
-        assert "failed to show URL" in caplog.text
+        assert main(["get", os.fspath(tmp_dir), "foo", "--show-url"]) != 0
 
 
 def test_get_pipeline_tracked_outs(tmp_dir, dvc, scm, git_dir, run_copy, local_remote):

tests/func/test_import.py

@@ -8,7 +8,6 @@ from dvc.annotations import Annotation
 from dvc.cachemgr import CacheManager
 from dvc.config import NoRemoteError
 from dvc.dvcfile import load_file
-from dvc.exceptions import DownloadError, PathMissingError
 from dvc.fs import system
 from dvc.scm import Git
 from dvc.stage.exceptions import StagePathNotFoundError
@@ -240,7 +239,9 @@ def test_pull_import_no_download(tmp_dir, scm, dvc, erepo_dir):
     dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", no_download=True)
 
     dvc.pull(["foo_imported.dvc"])
-    assert os.path.exists("foo_imported")
+    assert (tmp_dir / "foo_imported").exists
+    assert (tmp_dir / "foo_imported" / "bar").read_bytes() == b"bar"
+    assert (tmp_dir / "foo_imported" / "baz").read_bytes() == b"baz contents"
 
     stage = load_file(dvc, "foo_imported.dvc").stage
 
@@ -338,34 +339,7 @@ def test_push_wildcard_from_bare_git_repo(
     with dvc_repo.chdir():
         dvc_repo.dvc.imp(os.fspath(tmp_dir), "dirextra")
 
-        with pytest.raises(PathMissingError):
-            dvc_repo.dvc.imp(os.fspath(tmp_dir), "dir123")
-
-
-def test_download_error_pulling_imported_stage(mocker, tmp_dir, dvc, erepo_dir):
-    with erepo_dir.chdir():
-        erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
-    dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported")
-
-    dst_stage = load_file(dvc, "foo_imported.dvc").stage
-    dst_cache = dst_stage.outs[0].cache_path
-
-    remove("foo_imported")
-    remove(dst_cache)
-
-    def unreliable_download(_from_fs, from_info, _to_fs, to_info, **kwargs):
-        on_error = kwargs["on_error"]
-        assert on_error
-        if isinstance(from_info, str):
-            from_info = [from_info]
-        if isinstance(to_info, str):
-            to_info = [to_info]
-        for from_i, to_i in zip(from_info, to_info):
-            on_error(from_i, to_i, Exception())
-
-    mocker.patch("dvc_objects.fs.generic.transfer", unreliable_download)
-    with pytest.raises(DownloadError):
-        dvc.pull(["foo_imported.dvc"])
+        dvc_repo.dvc.imp(os.fspath(tmp_dir), "dir123")
 
 
 @pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
@@ -405,11 +379,11 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir):
 
 
 def test_import_non_existing(erepo_dir, tmp_dir, dvc):
-    with pytest.raises(PathMissingError):
+    with pytest.raises(FileNotFoundError):
         tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output")
 
     # https://github.com/iterative/dvc/pull/2837#discussion_r352123053
-    with pytest.raises(PathMissingError):
+    with pytest.raises(FileNotFoundError):
         tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root")
 
 
@@ -634,21 +608,6 @@ def test_chained_import(tmp_dir, dvc, make_tmp_dir, erepo_dir, local_cloud):
     assert (dst / "bar").read_text() == "bar"
 
 
-def test_circular_import(tmp_dir, dvc, scm, erepo_dir):
-    from dvc.exceptions import CircularImportError
-
-    with erepo_dir.chdir():
-        erepo_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="init")
-
-    dvc.imp(os.fspath(erepo_dir), "dir", "dir_imported")
-    scm.add("dir_imported.dvc")
-    scm.commit("import")
-
-    with erepo_dir.chdir():
-        with pytest.raises(CircularImportError):
-            erepo_dir.dvc.imp(os.fspath(tmp_dir), "dir_imported", "circular_import")
-
-
 @pytest.mark.parametrize("paths", ([], ["dir"]))
 def test_parameterized_repo(tmp_dir, dvc, scm, erepo_dir, paths):
     path = erepo_dir.joinpath(*paths)

tests/func/test_init.py

@@ -65,7 +65,7 @@ def test_init_quiet_should_not_display_welcome_screen(tmp_dir, scm, caplog):
         ret = main(["init", "--quiet"])
 
         assert ret == 0
-        assert caplog.text == ""
+        assert not caplog.text
 
 
 def test_allow_init_dvc_subdir(tmp_dir, scm, monkeypatch):