logger: init logging config before colorama

Closes #8376

.github/mergify.yml

@@ -1,9 +0,0 @@
-pull_request_rules:
-  - name: backport patches to 1.11
-    conditions:
-      - base=main
-      - label=backport-1.11
-    actions:
-      backport:
-        branches:
-          - "1.11"

.github/workflows/benchmarks.yaml

@@ -1,23 +1,18 @@
 name: benchmarks
 on: [pull_request, workflow_dispatch]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   bench:
     name: run benchmarks
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-18.04]
+    runs-on: ubuntu-latest
     steps:
-      - name: Cancel Previous Runs
-        uses: styfle/[email protected]
-        with:
-          access_token: ${{ github.token }}
       - uses: actions/setup-python@v4
         with:
-            python-version: 3.8
+            python-version: "3.10"
       - uses: iterative/dvc-bench@main
         with:
             pytest_options: "-k 'test_init or test_help'"
-

.github/workflows/tests.yaml

@@ -1,25 +1,27 @@
 name: Tests
 
 on:
-  pull_request: {}
-  push: {}
+  push:
+    branches: [main]
+  pull_request:
   schedule:
     - cron: '5 1 * * *'  # every day at 01:05
+  workflow_dispatch:
 
 env:
   DVC_TEST: "true"
   HOMEBREW_NO_AUTO_UPDATE: 1
   SHELL: /bin/bash
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   lint:
     timeout-minutes: 10
     runs-on: ubuntu-latest
     steps:
-    - name: Cancel Previous Runs
-      uses: styfle/[email protected]
-      with:
-        access_token: ${{ github.token }}
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
@@ -31,7 +33,7 @@ jobs:
       run: |
         pip install --upgrade pip setuptools wheel
         pip install collective.checkdocs==0.2
-        pip install ".[dev]" pre-commit
+        pip install ".[dev]"
     - name: Check README
       run: python setup.py checkdocs
     - uses: pre-commit/[email protected]
@@ -57,25 +59,13 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.pyv }}
-    - name: get pip cache dir
-      id: pip-cache-dir
-      run: |
-        echo "::set-output name=dir::$(pip cache dir)"
-    - name: set pip cache
-      id: pip-cache
-      uses: actions/cache@v3
-      with:
-        path: ${{ steps.pip-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-pip-${{ hashFiles('setup.cfg') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
+        cache: pip
+        cache-dependency-path: setup.cfg
     - name: install
       if: steps.cache.pip-cache-dir.cache-hit != 'true'
       run: |
         pip install --upgrade pip setuptools wheel
         pip install -e ".[dev]"
-      env:
-        AIOHTTP_NO_EXTENSIONS: ${{ matrix.pyv == '3.11-dev' && '1' }}
     - name: run tests
       timeout-minutes: 40
       run: pytest -nauto -ra --durations 100 --cov=dvc --cov-report=xml --cov-report=term -k "${{ matrix.pytest-filter }}"

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       - id: black
         language_version: python3
     repo: https://github.com/ambv/black
-    rev: 22.6.0
+    rev: 22.8.0
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.3.0
     hooks:
@@ -16,9 +16,9 @@ repos:
       - id: codespell
         args: 
           - --ignore-words-list
-          - ba,datas,fo,uptodate,cachable
+          - ba,datas,fo,uptodate,cachable,falsy
     repo: https://github.com/codespell-project/codespell
-    rev: v2.1.0
+    rev: v2.2.1
   - hooks:
       - id: isort
         language_version: python3

README.rst

@@ -82,7 +82,7 @@ To share and back up the *data cache*, DVC supports multiple remote storage plat
 
 |Flowchart|
 
-`DVC pipelines <https://dvc.org/doc/start/data-management/pipelines>`_ (computational graphs) connect code and data together.
+`DVC pipelines <https://dvc.org/doc/start/data-management/data-pipelines>`_ (computational graphs) connect code and data together.
 They specify all steps required to produce a model: input dependencies including code, data, commands to run; and output information to be saved.
 
 Last but not least, `DVC Experiment Versioning <https://dvc.org/doc/start/experiments>`_ lets you prepare and run a large number of experiments.

dvc/analytics.py

@@ -102,6 +102,8 @@ def _runtime_info():
     """
     Gather information from the environment where DVC runs to fill a report.
     """
+    from iterative_telemetry import find_or_create_user_id
+
     from dvc import __version__
     from dvc.utils import is_binary
 
@@ -110,7 +112,7 @@ def _runtime_info():
         "is_binary": is_binary(),
         "scm_class": _scm_in_use(),
         "system_info": _system_info(),
-        "user_id": _find_or_create_user_id(),
+        "user_id": find_or_create_user_id(),
     }
 
 
@@ -146,45 +148,3 @@ def _system_info():
 
     # We don't collect data for any other system.
     raise NotImplementedError
-
-
-def _find_or_create_user_id():
-    """
-    The user's ID is stored on a file under the global config directory.
-
-    The file should contain a JSON with a "user_id" key:
-
-        {"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"}
-
-    IDs are generated randomly with UUID.
-    """
-    import uuid
-
-    from dvc.config import Config
-    from dvc.lock import Lock, LockError
-    from dvc.utils.fs import makedirs
-
-    config_dir = Config.get_dir("global")
-    fname = os.path.join(config_dir, "user_id")
-    lockfile = os.path.join(config_dir, "user_id.lock")
-
-    # Since the `fname` and `lockfile` are under the global config,
-    # we need to make sure such directory exist already.
-    makedirs(config_dir, exist_ok=True)
-
-    try:
-        with Lock(lockfile):
-            try:
-                with open(fname, encoding="utf-8") as fobj:
-                    user_id = json.load(fobj)["user_id"]
-
-            except (FileNotFoundError, ValueError, KeyError):
-                user_id = str(uuid.uuid4())
-
-                with open(fname, "w", encoding="utf-8") as fobj:
-                    json.dump({"user_id": user_id}, fobj)
-
-            return user_id
-
-    except LockError:
-        logger.debug("Failed to acquire '%s'", lockfile)

dvc/api/__init__.py

@@ -1,4 +1,4 @@
-from dvc.fs.dvc import _DvcFileSystem as DvcFileSystem
+from dvc.fs.dvc import _DVCFileSystem as DVCFileSystem
 
 from .data import open  # pylint: disable=redefined-builtin
 from .data import get_url, read
@@ -11,5 +11,5 @@ __all__ = [
     "open",
     "params_show",
     "read",
-    "DvcFileSystem",
+    "DVCFileSystem",
 ]

dvc/cli/parser.py

@@ -29,6 +29,7 @@ from dvc.commands import (
     init,
     install,
     ls,
+    ls_url,
     machine,
     metrics,
     move,
@@ -75,6 +76,7 @@ COMMANDS = [
     install,
     root,
     ls,
+    ls_url,
     freeze,
     dag,
     daemon,

dvc/cli/utils.py

@@ -21,10 +21,13 @@ def append_doc_link(help_message, path):
     )
 
 
-def fix_plumbing_subparsers(subparsers):
-    # metavar needs to be explicitly set in order to hide plumbing subcommands
+def hide_subparsers_from_help(subparsers):
+    # metavar needs to be explicitly set in order to hide subcommands
     # from the 'positional arguments' choices list
     # see: https://bugs.python.org/issue22848
+    # Need to set `add_help=False`, but avoid setting `help`
+    # (not even to `argparse.SUPPPRESS`).
+    # NOTE: The argument is the parent subparser, not the subcommand parser.
     cmds = [
         cmd for cmd, parser in subparsers.choices.items() if parser.add_help
     ]

dvc/commands/checkout.py

@@ -110,7 +110,7 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help=(
             "Limit command scope to these tracked files/directories, "
-            ".dvc files, or stage names."
+            ".dvc files and stage names."
         ),
     ).complete = completion.DVC_FILE
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/commands/commit.py

@@ -70,7 +70,9 @@ def add_parser(subparsers, parent_parser):
     commit_parser.add_argument(
         "targets",
         nargs="*",
-        help="stages or .dvc files to commit. Optional. "
-        "(Finds all DVC files in the workspace by default.)",
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files and stage names."
+        ),
     ).complete = completion.DVCFILES_AND_STAGE
     commit_parser.set_defaults(func=CmdCommit)

dvc/commands/dag.py

@@ -204,7 +204,7 @@ def add_parser(subparsers, parent_parser):
     dag_parser.add_argument(
         "target",
         nargs="?",
-        help="Stage or output to show pipeline for (optional). "
+        help="Stage name or output to show pipeline for. "
         "Finds all stages in the workspace by default.",
     )
     dag_parser.set_defaults(func=CmdDAG)

dvc/commands/data.py

@@ -8,7 +8,11 @@ from funcy import chunks, compact, log_durations
 from dvc.cli import completion
 from dvc.cli.actions import CommaSeparatedArgs
 from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link, fix_subparsers
+from dvc.cli.utils import (
+    append_doc_link,
+    fix_subparsers,
+    hide_subparsers_from_help,
+)
 from dvc.ui import ui
 from dvc.utils import colorize
 
@@ -251,7 +255,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(DATA_LS_HELP, "data/ls"),
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help=DATA_LS_HELP,
+        add_help=False,
     )
     data_ls_parser.add_argument(
         "--md",
@@ -289,5 +293,5 @@ def add_parser(subparsers, parent_parser):
             ".dvc files, or stage names."
         ),
     ).complete = completion.DVCFILES_AND_STAGE
-
     data_ls_parser.set_defaults(func=CmdDataLs)
+    hide_subparsers_from_help(data_subparsers)

dvc/commands/data_sync.py

@@ -120,7 +120,7 @@ def shared_parent_parser():
         nargs="*",
         help=(
             "Limit command scope to these tracked files/directories, "
-            ".dvc files, or stage names."
+            ".dvc files and stage names."
         ),
     ).complete = completion.DVC_FILE
 
@@ -195,7 +195,7 @@ def add_parser(subparsers, _parent_parser):
         "--glob",
         action="store_true",
         default=False,
-        help="Pull cache for targets matching shell-style wildcards.",
+        help=argparse.SUPPRESS,
     )
     pull_parser.set_defaults(func=CmdDataPull)
 

dvc/commands/experiments/__init__.py

@@ -2,8 +2,8 @@ import argparse
 
 from dvc.cli.utils import (
     append_doc_link,
-    fix_plumbing_subparsers,
     fix_subparsers,
+    hide_subparsers_from_help,
 )
 from dvc.commands.experiments import (
     apply,
@@ -59,7 +59,7 @@ def add_parser(subparsers, parent_parser):
     fix_subparsers(experiments_subparsers)
     for cmd in SUB_COMMANDS:
         cmd.add_parser(experiments_subparsers, parent_parser)
-    fix_plumbing_subparsers(experiments_subparsers)
+    hide_subparsers_from_help(experiments_subparsers)
 
 
 def add_rev_selection_flags(

dvc/commands/git_hook.py

@@ -74,7 +74,7 @@ class CmdMergeDriver(CmdHookBase):
             our = Dvcfile(dvc, self.args.our, verify=False)
             their = Dvcfile(dvc, self.args.their, verify=False)
 
-            our.merge(ancestor, their)
+            our.merge(ancestor, their, allowed=["add", "remove", "change"])
 
             return 0
         finally:

dvc/commands/ls_url.py

@@ -0,0 +1,43 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
+from dvc.ui import ui
+
+from .ls import _prettify
+
+logger = logging.getLogger(__name__)
+
+
+class CmdListUrl(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        entries = Repo.ls_url(self.args.url, recursive=self.args.recursive)
+        if entries:
+            entries = _prettify(entries, with_color=True)
+            ui.write("\n".join(entries))
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    LS_HELP = "List directory contents from URL."
+    lsurl_parser = subparsers.add_parser(
+        "list-url",
+        aliases=["ls-url"],
+        parents=[parent_parser],
+        description=append_doc_link(LS_HELP, "list-url"),
+        help=LS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    lsurl_parser.add_argument(
+        "url", help="See `dvc import-url -h` for full list of supported URLs."
+    )
+    lsurl_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        help="Recursively list files.",
+    )
+    lsurl_parser.set_defaults(func=CmdListUrl)

dvc/commands/plots.py

@@ -25,7 +25,7 @@ def _show_json(renderers, split=False):
 
 
 def _adjust_vega_renderers(renderers):
-    from dvc.render import VERSION_FIELD
+    from dvc.render import REVISION_FIELD, VERSION_FIELD
     from dvc_render import VegaRenderer
 
     for r in renderers:
@@ -42,6 +42,7 @@ def _adjust_vega_renderers(renderers):
                         dp["rev"] = "::".join(vi.values())
             else:
                 for dp in r.datapoints:
+                    dp[REVISION_FIELD] = dp[VERSION_FIELD]["revision"]
                     dp.pop(VERSION_FIELD, {})
 
 
@@ -66,18 +67,6 @@ def _data_versions_count(renderer):
     return len(set(x))
 
 
-def _filter_unhandled_renderers(renderers):
-    # filtering out renderers currently unhandled by vscode extension
-    from dvc_render import VegaRenderer
-
-    def _is_json_viable(r):
-        return not (
-            isinstance(r, VegaRenderer) and _data_versions_count(r) > 1
-        )
-
-    return list(filter(_is_json_viable, renderers))
-
-
 class CmdPlots(CmdBase):
     def _func(self, *args, **kwargs):
         raise NotImplementedError
@@ -156,12 +145,10 @@ class CmdPlots(CmdBase):
                 templates_dir=self.repo.plots.templates_dir,
             )
             if self.args.json:
-                renderers = _filter_unhandled_renderers(renderers)
                 _show_json(renderers, self.args.split)
                 return 0
 
             _adjust_vega_renderers(renderers)
-
             if self.args.show_vega:
                 renderer = first(filter(lambda r: r.TYPE == "vega", renderers))
                 if renderer:

dvc/commands/repro.py

@@ -55,7 +55,16 @@ def add_arguments(repro_parser):
     repro_parser.add_argument(
         "targets",
         nargs="*",
-        help="Stages to reproduce. 'dvc.yaml' by default.",
+        help=(
+            """\
+Stages to reproduce. 'dvc.yaml' by default.
+The targets can be path to a dvc.yaml file or `.dvc` file,
+or a stage name from dvc.yaml file from
+current working directory. To run a stage from dvc.yaml
+from other directories, the target must be a path followed by colon `:`
+and then the stage name name.
+"""
+        ),
     ).complete = completion.DVCFILES_AND_STAGE
     repro_parser.add_argument(
         "-f",

dvc/commands/update.py

@@ -60,7 +60,7 @@ def add_parser(subparsers, parent_parser):
         "--no-download",
         action="store_true",
         default=False,
-        help="Update .dvc file hash value(s)"
+        help="Update .dvc file git revision/hash value(s)"
         " but do not download the file(s).",
     )
     update_parser.add_argument(

dvc/config.py

@@ -176,7 +176,10 @@ class Config(dict):
 
         if fs.exists(filename):
             with fs.open(filename) as fobj:
-                conf_obj = ConfigObj(fobj)
+                try:
+                    conf_obj = ConfigObj(fobj)
+                except UnicodeDecodeError as exc:
+                    raise ConfigError(str(exc)) from exc
         else:
             conf_obj = ConfigObj()
         return _parse_named(_lower_keys(conf_obj.dict()))

dvc/dependency/repo.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple
 from voluptuous import Required
 
 from dvc.prompt import confirm
+from dvc_data.hashfile.meta import Meta
 
 from .base import Dependency
 
@@ -32,6 +33,7 @@ class RepoDependency(Dependency):
     def __init__(self, def_repo, stage, *args, **kwargs):
         self.def_repo = def_repo
         self._objs: Dict[str, "HashFile"] = {}
+        self._meta: Dict[str, Meta] = {}
         super().__init__(stage, *args, **kwargs)
 
     def _parse_path(self, fs, fs_path):
@@ -94,12 +96,12 @@ class RepoDependency(Dependency):
     def get_used_objs(
         self, **kwargs
     ) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
-        used, _ = self._get_used_and_obj(**kwargs)
+        used, _, _ = self._get_used_and_obj(**kwargs)
         return used
 
     def _get_used_and_obj(
         self, obj_only=False, **kwargs
-    ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "HashFile"]:
+    ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], Meta, "HashFile"]:
         from dvc.config import NoRemoteError
         from dvc.exceptions import NoOutputOrStageError, PathMissingError
         from dvc.utils import as_posix
@@ -131,7 +133,7 @@ class RepoDependency(Dependency):
                     pass
 
             try:
-                object_store, _, obj = build(
+                object_store, meta, obj = build(
                     local_odb,
                     as_posix(self.def_path),
                     repo.dvcfs,
@@ -145,14 +147,16 @@ class RepoDependency(Dependency):
             object_store.read_only = True
 
             self._objs[rev] = obj
+            self._meta[rev] = meta
+
             used_obj_ids[object_store].add(obj.hash_info)
             if isinstance(obj, Tree):
                 used_obj_ids[object_store].update(oid for _, _, oid in obj)
-            return used_obj_ids, obj
+            return used_obj_ids, meta, obj
 
     def _check_circular_import(self, odb, obj_ids):
         from dvc.exceptions import CircularImportError
-        from dvc.fs.dvc import DvcFileSystem
+        from dvc.fs.dvc import DVCFileSystem
         from dvc_data.hashfile.db.reference import ReferenceHashFileDB
         from dvc_data.hashfile.tree import Tree
 
@@ -169,7 +173,7 @@ class RepoDependency(Dependency):
 
         checked_urls = set()
         for obj in iter_objs():
-            if not isinstance(obj.fs, DvcFileSystem):
+            if not isinstance(obj.fs, DVCFileSystem):
                 continue
             if (
                 obj.fs.repo_url in checked_urls
@@ -190,11 +194,21 @@ class RepoDependency(Dependency):
         rev = self._get_rev(locked=locked)
         if rev in self._objs:
             return self._objs[rev]
-        _, obj = self._get_used_and_obj(
+        _, _, obj = self._get_used_and_obj(
             obj_only=True, filter_info=filter_info, **kwargs
         )
         return obj
 
+    def get_meta(self, filter_info=None, **kwargs):
+        locked = kwargs.get("locked", True)
+        rev = self._get_rev(locked=locked)
+        if rev in self._meta:
+            return self._meta[rev]
+        _, meta, _ = self._get_used_and_obj(
+            obj_only=True, filter_info=filter_info, **kwargs
+        )
+        return meta
+
     def _make_repo(self, locked=True, **kwargs):
         from dvc.external_repo import external_repo
 

dvc/dvcfile.py

@@ -172,7 +172,7 @@ class FileMixin:
     def dump(self, stage, **kwargs):
         raise NotImplementedError
 
-    def merge(self, ancestor, other):
+    def merge(self, ancestor, other, allowed=None):
         raise NotImplementedError
 
 
@@ -204,12 +204,12 @@ class SingleStageFile(FileMixin):
     def remove_stage(self, stage):  # pylint: disable=unused-argument
         self.remove()
 
-    def merge(self, ancestor, other):
+    def merge(self, ancestor, other, allowed=None):
         assert isinstance(ancestor, SingleStageFile)
         assert isinstance(other, SingleStageFile)
 
         stage = self.stage
-        stage.merge(ancestor.stage, other.stage)
+        stage.merge(ancestor.stage, other.stage, allowed=allowed)
         self.dump(stage)
 
 
@@ -237,10 +237,10 @@ class PipelineFile(FileMixin):
             self._dump_pipeline_file(stage)
 
         if update_lock:
-            self._dump_lockfile(stage)
+            self._dump_lockfile(stage, **kwargs)
 
-    def _dump_lockfile(self, stage):
-        self._lockfile.dump(stage)
+    def _dump_lockfile(self, stage, **kwargs):
+        self._lockfile.dump(stage, **kwargs)
 
     @staticmethod
     def _check_if_parametrized(stage, action: str = "dump") -> None:
@@ -310,7 +310,7 @@ class PipelineFile(FileMixin):
         else:
             super().remove()
 
-    def merge(self, ancestor, other):
+    def merge(self, ancestor, other, allowed=None):
         raise NotImplementedError
 
 
@@ -366,7 +366,7 @@ class Lockfile(FileMixin):
         return {SCHEMA_KWD: version}
 
     def dump(self, stage, **kwargs):
-        stage_data = serialize.to_lockfile(stage)
+        stage_data = serialize.to_lockfile(stage, **kwargs)
 
         with modify_yaml(self.path, fs=self.repo.fs) as data:
             version = LOCKFILE_VERSION.from_dict(data)
@@ -411,7 +411,7 @@ class Lockfile(FileMixin):
         else:
             self.remove()
 
-    def merge(self, ancestor, other):
+    def merge(self, ancestor, other, allowed=None):
         raise NotImplementedError
 
 

dvc/exceptions.py

@@ -278,6 +278,11 @@ class PathMissingError(DvcException):
         self.dvc_only = dvc_only
 
 
+class URLMissingError(DvcException):
+    def __init__(self, url):
+        super().__init__(f"The path '{url}' does not exist")
+
+
 class RemoteCacheRequiredError(DvcException):
     def __init__(self, scheme, fs_path):
 

dvc/fs/__init__.py

@@ -2,6 +2,9 @@ from urllib.parse import urlparse
 
 from dvc_http import HTTPFileSystem, HTTPSFileSystem  # noqa: F401
 
+from dvc.config import ConfigError as RepoConfigError
+from dvc.config_schema import SCHEMA, Invalid
+
 # pylint: disable=unused-import
 from dvc_objects.fs import utils  # noqa: F401
 from dvc_objects.fs import (  # noqa: F401
@@ -24,13 +27,13 @@ from dvc_objects.fs.errors import (  # noqa: F401
 from dvc_objects.fs.path import Path  # noqa: F401
 
 from .data import DataFileSystem  # noqa: F401
-from .dvc import DvcFileSystem  # noqa: F401
+from .dvc import DVCFileSystem  # noqa: F401
 from .git import GitFileSystem  # noqa: F401
 
 known_implementations.update(
     {
         "dvc": {
-            "class": "dvc.fs.dvc.DvcFileSystem",
+            "class": "dvc.fs.dvc.DVCFileSystem",
             "err": "dvc is supported, but requires 'dvc' to be installed",
         },
         "git": {
@@ -44,7 +47,7 @@ known_implementations.update(
 # pylint: enable=unused-import
 
 
-def get_fs_config(repo, config, **kwargs):
+def get_fs_config(config, **kwargs):
     name = kwargs.get("name")
     if name:
         try:
@@ -55,10 +58,10 @@ def get_fs_config(repo, config, **kwargs):
             raise RemoteNotFoundError(f"remote '{name}' doesn't exist")
     else:
         remote_conf = kwargs
-    return _resolve_remote_refs(repo, config, remote_conf)
+    return _resolve_remote_refs(config, remote_conf)
 
 
-def _resolve_remote_refs(repo, config, remote_conf):
+def _resolve_remote_refs(config, remote_conf):
     # Support for cross referenced remotes.
     # This will merge the settings, shadowing base ref with remote_conf.
     # For example, having:
@@ -84,21 +87,22 @@ def _resolve_remote_refs(repo, config, remote_conf):
     if parsed.scheme != "remote":
         return remote_conf
 
-    base = get_fs_config(repo, config, name=parsed.netloc)
-    cls, _, _ = get_cloud_fs(repo, **base)
+    base = get_fs_config(config, name=parsed.netloc)
+    cls, _, _ = _get_cloud_fs(config, **base)
     relpath = parsed.path.lstrip("/").replace("/", cls.sep)
     url = cls.sep.join((base["url"], relpath))
     return {**base, **remote_conf, "url": url}
 
 
 def get_cloud_fs(repo, **kwargs):
-    from dvc.config import ConfigError as RepoConfigError
-    from dvc.config_schema import SCHEMA, Invalid
-
     repo_config = repo.config if repo else {}
+    return _get_cloud_fs(repo_config, **kwargs)
+
+
+def _get_cloud_fs(repo_config, **kwargs):
     core_config = repo_config.get("core", {})
 
-    remote_conf = get_fs_config(repo, repo_config, **kwargs)
+    remote_conf = get_fs_config(repo_config, **kwargs)
     try:
         remote_conf = SCHEMA["remote"][str](remote_conf)
     except Invalid as exc:

dvc/fs/callbacks.py

@@ -24,6 +24,7 @@ class RichCallback(Callback):
         bytes: bool = False,  # pylint: disable=redefined-builtin
         unit: str = None,
         disable: bool = False,
+        transient: bool = True,
     ) -> None:
         self._progress = progress
         self.disable = disable
@@ -35,6 +36,7 @@ class RichCallback(Callback):
             "visible": False,
             "progress_type": None if bytes else "summary",
         }
+        self._transient = transient
         self._stack = ExitStack()
         super().__init__(size=size, value=value)
 
@@ -47,7 +49,7 @@ class RichCallback(Callback):
             return self._progress
 
         progress = RichTransferProgress(
-            transient=True,
+            transient=self._transient,
             disable=self.disable,
             console=ui.error_console,
         )
@@ -61,7 +63,8 @@ class RichCallback(Callback):
         return self
 
     def close(self):
-        self.progress.clear_task(self.task)
+        if self._transient:
+            self.progress.clear_task(self.task)
         self._stack.close()
 
     def call(self, hook_name=None, **kwargs):
@@ -74,6 +77,9 @@ class RichCallback(Callback):
 
     def branch(self, path_1, path_2, kwargs, child: Optional[Callback] = None):
         child = child or RichCallback(
-            progress=self.progress, desc=path_1, bytes=True
+            progress=self.progress,
+            desc=path_1,
+            bytes=True,
+            transient=self._transient,
         )
         return super().branch(path_1, path_2, kwargs, child=child)

dvc/fs/dvc.py

@@ -1,3 +1,4 @@
+import errno
 import logging
 import ntpath
 import os
@@ -9,9 +10,7 @@ from typing import TYPE_CHECKING, Any, Callable, Optional, Tuple, Type, Union
 from fsspec.spec import AbstractFileSystem
 from funcy import cached_property, wrap_prop, wrap_with
 
-from dvc.utils.fs import makedirs
 from dvc_objects.fs.base import FileSystem
-from dvc_objects.fs.callbacks import DEFAULT_CALLBACK
 from dvc_objects.fs.path import Path
 
 from .data import DataFileSystem
@@ -64,16 +63,8 @@ def _get_dvc_path(dvc_fs, subkey):
     return dvc_fs.path.join(*subkey) if subkey else ""
 
 
-class _DvcFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
-    """DVC + git-tracked files fs.
-
-    Args:
-        repo: DVC or git repo.
-        subrepos: traverse to subrepos (by default, it ignores subrepos)
-        repo_factory: A function to initialize subrepo with, default is Repo.
-        kwargs: Additional keyword arguments passed to the `DataFileSystem()`.
-    """
-
+class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
+    cachable = False
     root_marker = "/"
 
     def __init__(
@@ -85,6 +76,41 @@ class _DvcFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         repo_factory: Optional[RepoFactory] = None,
         **repo_kwargs: Any,
     ) -> None:
+        """DVC + git-tracked files fs.
+
+        Args:
+            path (str, optional): URL or path to a DVC/Git repository.
+                Defaults to a DVC repository in the current working directory.
+                Both HTTP and SSH protocols are supported for remote Git repos
+                (e.g. [user@]server:project.git).
+            rev (str, optional): Any Git revision such as a branch or tag name,
+                a commit hash or a dvc experiment name.
+                Defaults to the default branch in case of remote repositories.
+                In case of a local repository, if rev is unspecified, it will
+                default to the working directory.
+                If the repo is not a Git repo, this option is ignored.
+            repo (:obj:`Repo`, optional): `Repo` instance.
+            subrepos (bool): traverse to subrepos.
+                By default, it ignores subrepos.
+            repo_factory (callable): A function to initialize subrepo with.
+                The default is `Repo`.
+
+        Examples:
+            - Opening a filesystem from repo in current working directory
+
+            >>> fs = DVCFileSystem()
+
+            - Opening a filesystem from local repository
+
+            >>> fs = DVCFileSystem("path/to/local/repository")
+
+            - Opening a remote repository
+
+            >>> fs = DVCFileSystem(
+            ...    "https://github.com/iterative/example-get-started",
+            ...    rev="main",
+            ... )
+        """
         from pygtrie import Trie
 
         super().__init__()
@@ -222,6 +248,9 @@ class _DvcFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
     def _open(
         self, path, mode="rb", **kwargs
     ):  # pylint: disable=arguments-renamed, arguments-differ
+        if mode != "rb":
+            raise OSError(errno.EROFS, os.strerror(errno.EROFS))
+
         key = self._get_key_from_relative(path)
         fs_path = self._from_key(key)
         try:
@@ -234,7 +263,8 @@ class _DvcFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         dvc_path = _get_dvc_path(dvc_fs, subkey)
         return dvc_fs.open(dvc_path, mode=mode)
 
-    def isdvc(self, path, **kwargs):
+    def isdvc(self, path, **kwargs) -> bool:
+        """Is this entry dvc-tracked?"""
         key = self._get_key_from_relative(path)
         _, dvc_fs, subkey = self._get_subrepo_info(key)
         dvc_path = _get_dvc_path(dvc_fs, subkey)
@@ -291,27 +321,6 @@ class _DvcFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
 
         return infos
 
-    def get_file(  # pylint: disable=arguments-differ
-        self, rpath, lpath, callback=DEFAULT_CALLBACK, **kwargs
-    ):
-        key = self._get_key_from_relative(rpath)
-        fs_path = self._from_key(key)
-        fs = self.repo.fs
-
-        if self.isdir(rpath):
-            makedirs(lpath, exist_ok=True)
-            return None
-
-        try:
-            fs.get_file(fs_path, lpath, callback=callback, **kwargs)
-            return
-        except FileNotFoundError:
-            _, dvc_fs, subkey = self._get_subrepo_info(key)
-            if not dvc_fs:
-                raise
-        dvc_path = _get_dvc_path(dvc_fs, subkey)
-        dvc_fs.get_file(dvc_path, lpath, callback=callback, **kwargs)
-
     def info(self, path, **kwargs):
         key = self._get_key_from_relative(path)
         ignore_subrepos = kwargs.get("ignore_subrepos", True)
@@ -361,7 +370,7 @@ class _DvcFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         return info
 
 
-class DvcFileSystem(FileSystem):
+class DVCFileSystem(FileSystem):
     protocol = "local"
     PARAM_CHECKSUM = "md5"
 
@@ -371,7 +380,7 @@ class DvcFileSystem(FileSystem):
     @wrap_prop(threading.Lock())
     @cached_property
     def fs(self):
-        return _DvcFileSystem(**self.fs_args)
+        return _DVCFileSystem(**self.fs_args)
 
     def isdvc(self, path, **kwargs):
         return self.fs.isdvc(path, **kwargs)

dvc/info.py

@@ -76,7 +76,7 @@ def _get_caches(cache):
 
 def _get_remotes(config):
     schemes = (
-        get_fs_cls(get_fs_config(None, config, name=remote)).protocol
+        get_fs_cls(get_fs_config(config, name=remote)).protocol
         for remote in config["remote"]
     )
 

dvc/logger.py

@@ -213,8 +213,6 @@ def set_loggers_level(level: int = logging.INFO) -> None:
 
 
 def setup(level: int = logging.INFO) -> None:
-    colorama.init()
-
     if level >= logging.DEBUG:
         # Unclosed session errors for asyncio/aiohttp are only available
         # on the tracing mode for extensive debug purposes. They are really
@@ -297,3 +295,8 @@ def setup(level: int = logging.INFO) -> None:
             "disable_existing_loggers": False,
         }
     )
+
+    # NOTE: colorama init needs to go after logging config to avoid potential
+    # conflicts with libs that modify logging (like tensorflow/abseil)
+    # https://github.com/iterative/dvc/issues/8387
+    colorama.init()

dvc/output.py

@@ -765,11 +765,8 @@ class Output:
         if self.remote:
             ret[self.PARAM_REMOTE] = self.remote
 
-        if (
-            self.use_cache
-            and self.is_in_repo
-            and self.hash_info.isdir
-            and (kwargs.get("with_files") or self.files is not None)
+        if self.hash_info.isdir and (
+            kwargs.get("with_files") or self.files is not None
         ):
             if self.obj:
                 obj = self.obj
@@ -1127,7 +1124,7 @@ class Output:
                 "unable to auto-merge outputs that are not directories"
             )
 
-    def merge(self, ancestor, other):
+    def merge(self, ancestor, other, allowed=None):
         from dvc_data.hashfile.tree import MergeError as TreeMergeError
         from dvc_data.hashfile.tree import du, merge
 
@@ -1144,7 +1141,11 @@ class Output:
 
         try:
             merged = merge(
-                self.odb, ancestor_info, self.hash_info, other.hash_info
+                self.odb,
+                ancestor_info,
+                self.hash_info,
+                other.hash_info,
+                allowed=allowed,
             )
         except TreeMergeError as exc:
             raise MergeError(str(exc)) from exc

dvc/render/convert.py

@@ -2,7 +2,13 @@ import json
 from collections import defaultdict
 from typing import Dict, List, Union
 
-from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY
+from dvc.render import (
+    REVISION_FIELD,
+    REVISIONS_KEY,
+    SRC_FIELD,
+    TYPE_KEY,
+    VERSION_FIELD,
+)
 from dvc.render.converter.image import ImageConverter
 from dvc.render.converter.vega import VegaConverter
 
@@ -23,7 +29,7 @@ def _get_converter(
 def _group_by_rev(datapoints):
     grouped = defaultdict(list)
     for datapoint in datapoints:
-        rev = datapoint.pop(REVISION_FIELD)
+        rev = datapoint.get(VERSION_FIELD, {}).get("revision")
         grouped[rev].append(datapoint)
     return dict(grouped)
 

dvc/render/converter/vega.py

@@ -127,13 +127,20 @@ class VegaConverter(Converter):
                 ),
             )
         )
+        self.steps.append(
+            (
+                "infer_y",
+                partial(
+                    self._infer_y,
+                ),
+            )
+        )
 
         self.steps.append(
             (
                 "generate_y",
                 partial(
                     self._generate_y_values,
-                    y_values=self.plot_properties.get("y", None),
                 ),
             )
         )
@@ -156,7 +163,7 @@ class VegaConverter(Converter):
             } - {None}
             self.inferred_properties["fields"] = fields
 
-    def _infer_y(self, datapoints: List[Dict]):
+    def _infer_y(self, datapoints: List[Dict], **kwargs):
         if "y" not in self.plot_properties:
             data_fields = list(first(datapoints))
             skip = (
@@ -178,6 +185,7 @@ class VegaConverter(Converter):
                     )
             else:
                 self.inferred_properties["y"] = inferred_y
+        return datapoints
 
     def convert(
         self,
@@ -197,7 +205,6 @@ class VegaConverter(Converter):
         processed = deepcopy(data)
 
         for step_name, step in self.steps:
-
             if step_name not in skip:
                 processed = step(  # type: ignore
                     processed,
@@ -205,25 +212,28 @@ class VegaConverter(Converter):
                     filename=filename,
                 )
 
-        self._infer_y(processed)  # type: ignore
         return processed, {**self.plot_properties, **self.inferred_properties}
 
     def _generate_y_values(  # noqa: C901
         self,
         datapoints: List[Dict],
-        y_values: Optional[Union[str, List, Dict]],
         revision: str,
         filename: str,
         **kwargs,
     ) -> List[Dict]:
 
+        y_values = self.plot_properties.get(
+            "y", None
+        ) or self.inferred_properties.get("y", None)
+
+        assert y_values is not None
+
         result = []
         properties_update = {}
 
         def _add_version_info(datapoint, version_info):
             tmp = datapoint.copy()
             tmp[VERSION_FIELD] = version_info
-            tmp[REVISION_FIELD] = version_info["revision"]
             return tmp
 
         def _version_info(revision, filename=None, field=None):
@@ -250,12 +260,6 @@ class VegaConverter(Converter):
 
             return tmp
 
-        if not y_values:
-            for dp in datapoints:
-                result.append(
-                    _add_version_info(dp, _version_info(revision, filename))
-                )
-
         if isinstance(y_values, str):
             for datapoint in datapoints:
                 result.append(
@@ -290,13 +294,15 @@ class VegaConverter(Converter):
             ):
                 # if we use the same field from all files,
                 # we dont have to generate it
+                y_field = all_fields.pop()
                 for datapoint in datapoints:
                     result.append(
                         _add_version_info(
-                            datapoint, _version_info(revision, filename)
+                            datapoint,
+                            _version_info(revision, filename, y_field),
                         )
                     )
-                properties_update.update({"y": all_fields.pop()})
+                properties_update.update({"y": y_field})
             else:
                 for def_filename, val in y_values.items():
                     if isinstance(val, str):

dvc/repo/__init__.py

@@ -68,6 +68,7 @@ class Repo:
     from dvc.repo.imp_url import imp_url  # type: ignore[misc]
     from dvc.repo.install import install  # type: ignore[misc]
     from dvc.repo.ls import ls as _ls  # type: ignore[misc]
+    from dvc.repo.ls_url import ls_url as _ls_url  # type: ignore[misc]
     from dvc.repo.move import move  # type: ignore[misc]
     from dvc.repo.pull import pull  # type: ignore[misc]
     from dvc.repo.push import push  # type: ignore[misc]
@@ -80,6 +81,7 @@ class Repo:
     from .data import status as data_status  # type: ignore[misc]
 
     ls = staticmethod(_ls)
+    ls_url = staticmethod(_ls_url)
     get = staticmethod(_get)
     get_url = staticmethod(_get_url)
 
@@ -187,7 +189,7 @@ class Repo:
         self.config = Config(self.dvc_dir, fs=self.fs, config=config)
         self._uninitialized = uninitialized
 
-        # used by DvcFileSystem to determine if it should traverse subrepos
+        # used by DVCFileSystem to determine if it should traverse subrepos
         self.subrepos = subrepos
 
         self.cloud = DataCloud(self)
@@ -526,9 +528,9 @@ class Repo:
 
     @cached_property
     def dvcfs(self):
-        from dvc.fs.dvc import DvcFileSystem
+        from dvc.fs.dvc import DVCFileSystem
 
-        return DvcFileSystem(
+        return DVCFileSystem(
             repo=self, subrepos=self.subrepos, **self._fs_conf
         )
 
@@ -540,13 +542,13 @@ class Repo:
     def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
         from dvc.fs.data import DataFileSystem
-        from dvc.fs.dvc import DvcFileSystem
+        from dvc.fs.dvc import DVCFileSystem
 
         if os.path.isabs(path):
             fs = DataFileSystem(index=self.index.data["local"])
             fs_path = path
         else:
-            fs = DvcFileSystem(repo=self, subrepos=True)
+            fs = DVCFileSystem(repo=self, subrepos=True)
             fs_path = fs.from_os_path(path)
 
         try:

dvc/repo/collect.py

@@ -30,9 +30,9 @@ def _collect_paths(
     recursive: bool = False,
     rev: str = None,
 ) -> StrPaths:
-    from dvc.fs.dvc import DvcFileSystem
+    from dvc.fs.dvc import DVCFileSystem
 
-    fs = DvcFileSystem(repo=repo)
+    fs = DVCFileSystem(repo=repo)
     fs_paths = [fs.from_os_path(target) for target in targets]
 
     target_paths: StrPaths = []

dvc/repo/diff.py

@@ -23,9 +23,9 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
     if self.scm.no_commits:
         return {}
 
-    from dvc.fs.dvc import DvcFileSystem
+    from dvc.fs.dvc import DVCFileSystem
 
-    dvcfs = DvcFileSystem(repo=self)
+    dvcfs = DVCFileSystem(repo=self)
     targets = ensure_list(targets)
     targets = [dvcfs.from_os_path(target) for target in targets]
 

dvc/repo/experiments/__init__.py

@@ -16,7 +16,7 @@ from .exceptions import (
     InvalidExpRefError,
     MultipleBranchError,
 )
-from .executor.base import BaseExecutor, ExecutorInfo
+from .executor.base import BaseExecutor, ExecutorInfo, TaskStatus
 from .queue.base import BaseStashQueue, QueueEntry
 from .queue.celery import LocalCeleryQueue
 from .queue.tempdir import TempDirQueue
@@ -463,6 +463,8 @@ class Experiments:
             if rev == "workspace":
                 # If we are appending to a checkpoint branch in a workspace
                 # run, show the latest checkpoint as running.
+                if info.status > TaskStatus.RUNNING:
+                    return result
                 last_rev = self.scm.get_ref(EXEC_BRANCH)
                 if last_rev:
                     result[last_rev] = info.asdict()

dvc/repo/experiments/executor/base.py

@@ -4,6 +4,7 @@ import pickle
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
 from dataclasses import asdict, dataclass
+from enum import IntEnum
 from functools import partial
 from typing import (
     TYPE_CHECKING,
@@ -35,6 +36,8 @@ from ..refs import (
     EXEC_BASELINE,
     EXEC_BRANCH,
     EXEC_CHECKPOINT,
+    EXEC_HEAD,
+    EXEC_MERGE,
     EXEC_NAMESPACE,
     EXPS_NAMESPACE,
     EXPS_STASH,
@@ -47,10 +50,9 @@ if TYPE_CHECKING:
     from scmrepo.git import Git
 
     from dvc.repo import Repo
+    from dvc.repo.experiments.stash import ExpStashEntry
     from dvc.stage import PipelineStage
 
-    from ..base import ExpStashEntry
-
 logger = logging.getLogger(__name__)
 
 
@@ -64,6 +66,16 @@ class ExecutorResult(NamedTuple):
     force: bool
 
 
+class TaskStatus(IntEnum):
+    PENDING = 0
+    PREPARING = 1
+    RUNNING = 2
+    SUCCESS = 3
+    FAILED = 4
+    CANCELED = 5
+    FINISHED = 6
+
+
 @dataclass
 class ExecutorInfo:
     git_url: str
@@ -73,13 +85,16 @@ class ExecutorInfo:
     dvc_dir: str
     name: Optional[str] = None
     wdir: Optional[str] = None
-    collected: bool = False
     result_hash: Optional[str] = None
     result_ref: Optional[str] = None
     result_force: bool = False
+    status: TaskStatus = TaskStatus.PENDING
 
     @classmethod
     def from_dict(cls, d):
+        if "collected" in d:
+            if d.pop("collected"):
+                d["status"] = TaskStatus.FINISHED
         return cls(**d)
 
     def asdict(self):
@@ -136,6 +151,7 @@ class BaseExecutor(ABC):
         root_dir: str,
         dvc_dir: str,
         baseline_rev: str,
+        status: TaskStatus,
         wdir: Optional[str] = None,
         name: Optional[str] = None,
         location: Optional[str] = None,
@@ -149,9 +165,17 @@ class BaseExecutor(ABC):
         self.baseline_rev = baseline_rev
         self.location: str = location or self.DEFAULT_LOCATION
         self.result = result
+        self.status = status
 
     @abstractmethod
-    def init_git(self, scm: "Git", branch: Optional[str] = None):
+    def init_git(
+        self,
+        scm: "Git",
+        stash_rev: str,
+        entry: "ExpStashEntry",
+        infofile: Optional[str],
+        branch: Optional[str] = None,
+    ):
         """Init git repo and populate it using exp refs from the specified
         SCM instance.
         """
@@ -191,6 +215,7 @@ class BaseExecutor(ABC):
             dvc_dir=self.dvc_dir,
             name=self.name,
             wdir=self.wdir,
+            status=self.status,
             **result_dict,
         )
 
@@ -212,6 +237,7 @@ class BaseExecutor(ABC):
             root_dir=info.root_dir,
             dvc_dir=info.dvc_dir,
             baseline_rev=info.baseline_rev,
+            status=info.status,
             name=info.name,
             wdir=info.wdir,
             result=result,
@@ -222,7 +248,6 @@ class BaseExecutor(ABC):
     def from_stash_entry(
         cls: Type[_T],
         repo: "Repo",
-        stash_rev: str,
         entry: "ExpStashEntry",
         **kwargs,
     ) -> _T:
@@ -232,7 +257,6 @@ class BaseExecutor(ABC):
     def _from_stash_entry(
         cls: Type[_T],
         repo: "Repo",
-        stash_rev: str,
         entry: "ExpStashEntry",
         root_dir: str,
         **kwargs,
@@ -241,12 +265,11 @@ class BaseExecutor(ABC):
             root_dir=root_dir,
             dvc_dir=relpath(repo.dvc_dir, repo.scm.root_dir),
             baseline_rev=entry.baseline_rev,
+            status=TaskStatus.PREPARING,
             name=entry.name,
             wdir=relpath(os.getcwd(), repo.scm.root_dir),
             **kwargs,
         )
-        executor.init_git(repo.scm, branch=entry.branch)
-        executor.init_cache(repo, stash_rev)
         return executor
 
     @staticmethod
@@ -259,8 +282,12 @@ class BaseExecutor(ABC):
                 exp_data.update(to_lockfile(stage))
         return dict_sha256(exp_data)
 
-    def cleanup(self):
-        pass
+    def cleanup(self, infofile: str):
+        if infofile is not None:
+            info = ExecutorInfo.load_json(infofile)
+            if info.status < TaskStatus.FAILED:
+                info.status = TaskStatus.FINISHED
+            info.dump_json(infofile)
 
     # TODO: come up with better way to stash repro arguments
     @staticmethod
@@ -324,11 +351,12 @@ class BaseExecutor(ABC):
                 logger.debug("Replacing existing experiment '%s'", orig_ref)
                 return True
 
+            if on_diverged:
+                return on_diverged(orig_ref, has_checkpoint)
+
             self._raise_ref_conflict(
                 dest_scm, orig_ref, new_rev, has_checkpoint
             )
-            if on_diverged:
-                on_diverged(orig_ref, has_checkpoint)
             logger.debug("Reproduced existing experiment '%s'", orig_ref)
             return False
 
@@ -411,11 +439,9 @@ class BaseExecutor(ABC):
         exp_ref: Optional["ExpRefInfo"] = None
         repro_force: bool = False
 
-        if infofile is not None:
-            info.dump_json(infofile)
-
         with cls._repro_dvc(
             info,
+            infofile,
             log_errors=log_errors,
             **kwargs,
         ) as dvc:
@@ -491,9 +517,6 @@ class BaseExecutor(ABC):
             info.result_ref = ref
             info.result_force = repro_force
 
-        if infofile is not None:
-            info.dump_json(infofile)
-
         # ideally we would return stages here like a normal repro() call, but
         # stages is not currently picklable and cannot be returned across
         # multiprocessing calls
@@ -550,6 +573,7 @@ class BaseExecutor(ABC):
     def _repro_dvc(
         cls,
         info: "ExecutorInfo",
+        infofile: str = None,
         log_errors: bool = True,
         **kwargs,
     ):
@@ -557,6 +581,9 @@ class BaseExecutor(ABC):
         from dvc.stage.monitor import CheckpointKilledError
 
         dvc = Repo(os.path.join(info.root_dir, info.dvc_dir))
+        info.status = TaskStatus.RUNNING
+        if infofile is not None:
+            info.dump_json(infofile)
         if cls.QUIET:
             dvc.scm_context.quiet = cls.QUIET
         old_cwd = os.getcwd()
@@ -568,15 +595,28 @@ class BaseExecutor(ABC):
         try:
             logger.debug("Running repro in '%s'", os.getcwd())
             yield dvc
+            info.status = TaskStatus.SUCCESS
+            if infofile is not None:
+                info.dump_json(infofile)
+
         except CheckpointKilledError:
+            info.status = TaskStatus.FAILED
+            if infofile is not None:
+                info.dump_json(infofile)
             raise
         except DvcException:
             if log_errors:
                 logger.exception("")
+            info.status = TaskStatus.FAILED
+            if infofile is not None:
+                info.dump_json(infofile)
             raise
         except Exception:
             if log_errors:
                 logger.exception("unexpected error")
+            info.status = TaskStatus.FAILED
+            if infofile is not None:
+                info.dump_json(infofile)
             raise
         finally:
             dvc.close()
@@ -714,3 +754,23 @@ class BaseExecutor(ABC):
         disable_other_loggers()
         if level is not None:
             dvc_logger.setLevel(level)
+
+    @contextmanager
+    def set_exec_refs(
+        self, scm: "Git", stash_rev: str, entry: "ExpStashEntry"
+    ):
+        try:
+            # Executor will be initialized with an empty git repo that
+            # we populate by pushing:
+            #   EXEC_HEAD - the base commit for this experiment
+            #   EXEC_MERGE - the unmerged changes (from our stash)
+            #       to be reproduced
+            #   EXEC_BASELINE - the baseline commit for this experiment
+            scm.set_ref(EXEC_HEAD, entry.head_rev)
+            scm.set_ref(EXEC_MERGE, stash_rev)
+            scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
+            yield
+        finally:
+            for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
+                if scm.get_ref(ref):
+                    scm.remove_ref(ref)

dvc/repo/experiments/executor/local.py

@@ -19,7 +19,7 @@ from ..refs import (
     EXEC_MERGE,
     EXEC_NAMESPACE,
 )
-from .base import EXEC_TMP_DIR, BaseExecutor
+from .base import EXEC_TMP_DIR, BaseExecutor, TaskStatus
 
 if TYPE_CHECKING:
     from scmrepo.git import Git
@@ -46,8 +46,8 @@ class BaseLocalExecutor(BaseExecutor):
     def scm(self):
         return SCM(self.root_dir)
 
-    def cleanup(self):
-        super().cleanup()
+    def cleanup(self, infofile: str):
+        super().cleanup(infofile)
         self.scm.close()
         del self.scm
 
@@ -67,15 +67,28 @@ class TempDirExecutor(BaseLocalExecutor):
     QUIET = True
     DEFAULT_LOCATION = "tempdir"
 
-    def init_git(self, scm: "Git", branch: Optional[str] = None):
+    def init_git(
+        self,
+        scm: "Git",
+        stash_rev: str,
+        entry: "ExpStashEntry",
+        infofile: Optional[str],
+        branch: Optional[str] = None,
+    ):
         from dulwich.repo import Repo as DulwichRepo
 
         from ..utils import push_refspec
 
         DulwichRepo.init(os.fspath(self.root_dir))
 
-        refspec = f"{EXEC_NAMESPACE}/"
-        push_refspec(scm, self.git_url, refspec, refspec)
+        self.status = TaskStatus.PREPARING
+        if infofile:
+            self.info.dump_json(infofile)
+
+        with self.set_exec_refs(scm, stash_rev, entry):
+            refspec = f"{EXEC_NAMESPACE}/"
+            push_refspec(scm, self.git_url, refspec, refspec)
+
         if branch:
             push_refspec(scm, self.git_url, branch, branch)
             self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
@@ -84,17 +97,12 @@ class TempDirExecutor(BaseLocalExecutor):
 
         if self.scm.get_ref(EXEC_CHECKPOINT):
             self.scm.remove_ref(EXEC_CHECKPOINT)
-
         # checkout EXEC_HEAD and apply EXEC_MERGE on top of it without
         # committing
         head = EXEC_BRANCH if branch else EXEC_HEAD
         self.scm.checkout(head, detach=True)
         merge_rev = self.scm.get_ref(EXEC_MERGE)
 
-        for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
-            if scm.get_ref(ref):
-                scm.remove_ref(ref)
-
         try:
             self.scm.merge(merge_rev, squash=True, commit=False)
         except _SCMError as exc:
@@ -114,8 +122,8 @@ class TempDirExecutor(BaseLocalExecutor):
         """Initialize DVC cache."""
         self._config(repo.odb.repo.path)
 
-    def cleanup(self):
-        super().cleanup()
+    def cleanup(self, infofile: str):
+        super().cleanup(infofile)
         logger.debug("Removing tmpdir '%s'", self.root_dir)
         remove(self.root_dir)
 
@@ -123,7 +131,6 @@ class TempDirExecutor(BaseLocalExecutor):
     def from_stash_entry(
         cls,
         repo: "Repo",
-        stash_rev: str,
         entry: "ExpStashEntry",
         wdir: Optional[str] = None,
         **kwargs,
@@ -132,9 +139,7 @@ class TempDirExecutor(BaseLocalExecutor):
         makedirs(parent_dir, exist_ok=True)
         tmp_dir = mkdtemp(dir=parent_dir)
         try:
-            executor = cls._from_stash_entry(
-                repo, stash_rev, entry, tmp_dir, **kwargs
-            )
+            executor = cls._from_stash_entry(repo, entry, tmp_dir, **kwargs)
             logger.debug("Init temp dir executor in '%s'", tmp_dir)
             return executor
         except Exception:
@@ -152,18 +157,31 @@ class WorkspaceExecutor(BaseLocalExecutor):
     def from_stash_entry(
         cls,
         repo: "Repo",
-        stash_rev: str,
         entry: "ExpStashEntry",
         **kwargs,
     ):
         root_dir = repo.scm.root_dir
-        executor = cls._from_stash_entry(
-            repo, stash_rev, entry, root_dir, **kwargs
+        executor: "WorkspaceExecutor" = cls._from_stash_entry(
+            repo, entry, root_dir, **kwargs
         )
         logger.debug("Init workspace executor in '%s'", root_dir)
         return executor
 
-    def init_git(self, scm: "Git", branch: Optional[str] = None):
+    def init_git(
+        self,
+        scm: "Git",
+        stash_rev: str,
+        entry: "ExpStashEntry",
+        infofile: Optional[str],
+        branch: Optional[str] = None,
+    ):
+        self.status = TaskStatus.PREPARING
+        if infofile:
+            self.info.dump_json(infofile)
+
+        scm.set_ref(EXEC_HEAD, entry.head_rev)
+        scm.set_ref(EXEC_MERGE, stash_rev)
+        scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
         self._detach_stack.enter_context(
             self.scm.detach_head(
                 self.scm.get_ref(EXEC_HEAD),
@@ -184,7 +202,9 @@ class WorkspaceExecutor(BaseLocalExecutor):
     def init_cache(self, repo: "Repo", rev: str, run_cache: bool = True):
         pass
 
-    def cleanup(self):
+    def cleanup(self, infofile: str):
+        super().cleanup(infofile)
+        remove(os.path.dirname(infofile))
         with self._detach_stack:
             self.scm.remove_ref(EXEC_BASELINE)
             self.scm.remove_ref(EXEC_MERGE)
@@ -193,4 +213,3 @@ class WorkspaceExecutor(BaseLocalExecutor):
             checkpoint = self.scm.get_ref(EXEC_CHECKPOINT)
             if checkpoint and checkpoint != self._orig_checkpoint:
                 self.scm.set_ref(EXEC_APPLY, checkpoint)
-        super().cleanup()

dvc/repo/experiments/executor/manager/base.py

@@ -1,269 +0,0 @@
-import logging
-import os
-from abc import ABC
-from collections import defaultdict, deque
-from collections.abc import Mapping
-from typing import TYPE_CHECKING, Deque, Dict, Generator, Optional, Tuple, Type
-
-from dvc.proc.manager import ProcessManager
-
-from ...exceptions import CheckpointExistsError, ExperimentExistsError
-from ...refs import EXEC_BASELINE, EXEC_HEAD, EXEC_MERGE, ExpRefInfo
-from ...stash import ExpStashEntry
-from ..base import EXEC_PID_DIR, BaseExecutor
-from ..local import TempDirExecutor, WorkspaceExecutor
-
-if TYPE_CHECKING:
-    from scmrepo.git import Git
-
-    from dvc.repo import Repo
-
-logger = logging.getLogger(__name__)
-
-
-class BaseExecutorManager(ABC, Mapping):  # noqa: B024
-    """Manages executors for a collection of experiments to be run."""
-
-    EXECUTOR_CLS: Type = BaseExecutor
-
-    def __init__(
-        self,
-        scm: "Git",
-        wdir: str,
-        **kwargs,
-    ):
-        from dvc.utils.fs import makedirs
-
-        self.scm = scm
-        makedirs(wdir, exist_ok=True)
-        self.wdir = wdir
-        self.proc = ProcessManager(self.pid_dir)
-        self._attached: Dict[str, "BaseExecutor"] = {}
-        self._detached: Dict[str, "BaseExecutor"] = dict(self._load_infos())
-        self._queue: Deque[Tuple[str, "BaseExecutor"]] = deque()
-
-    def __getitem__(self, key: str) -> "BaseExecutor":
-        try:
-            return self._attached[key]
-        except KeyError:
-            pass
-        return self._detached[key]
-
-    def __iter__(self):
-        yield from self._attached
-        yield from self._detached
-
-    def __len__(self):
-        return len(self._attached) + len(self._detached)
-
-    @property
-    def pid_dir(self) -> str:
-        return os.path.join(self.wdir, EXEC_PID_DIR)
-
-    def enqueue(self, rev: str, executor: "BaseExecutor"):
-        assert rev not in self
-        self._queue.append((rev, executor))
-
-    def _load_infos(self) -> Generator[Tuple[str, "BaseExecutor"], None, None]:
-        import json
-        from urllib.parse import urlparse
-
-        from ..base import ExecutorInfo
-        from ..ssh import SSHExecutor
-
-        def make_executor(info: "ExecutorInfo"):
-            if info.git_url:
-                scheme = urlparse(info.git_url).scheme
-                if scheme == "file":
-                    cls: Type = TempDirExecutor
-                elif scheme == "ssh":
-                    cls = SSHExecutor
-                else:
-                    raise NotImplementedError
-            else:
-                cls = WorkspaceExecutor
-            return cls.from_info(info)
-
-        for name in self.proc:
-            infofile = self.get_infofile_path(name)
-            try:
-                with open(infofile, encoding="utf-8") as fobj:
-                    info = ExecutorInfo.from_dict(json.load(fobj))
-                yield name, make_executor(info)
-            except OSError:
-                continue
-
-    def get_infofile_path(self, name: str) -> str:
-        return os.path.join(
-            self.pid_dir,
-            name,
-            f"{name}{BaseExecutor.INFOFILE_EXT}",
-        )
-
-    @classmethod
-    def from_stash_entries(
-        cls,
-        scm: "Git",
-        wdir: str,
-        repo: "Repo",
-        to_run: Dict[str, ExpStashEntry],
-        **kwargs,
-    ):
-        manager = cls(scm, wdir)
-        manager._enqueue_stash_entries(scm, repo, to_run, **kwargs)
-        return manager
-
-    def _enqueue_stash_entries(
-        self,
-        scm: "Git",
-        repo: "Repo",
-        to_run: Dict[str, ExpStashEntry],
-        **kwargs,
-    ):
-        try:
-            for stash_rev, entry in to_run.items():
-                scm.set_ref(EXEC_HEAD, entry.head_rev)
-                scm.set_ref(EXEC_MERGE, stash_rev)
-                scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
-
-                # Executor will be initialized with an empty git repo that
-                # we populate by pushing:
-                #   EXEC_HEAD - the base commit for this experiment
-                #   EXEC_MERGE - the unmerged changes (from our stash)
-                #       to be reproduced
-                #   EXEC_BASELINE - the baseline commit for this experiment
-                executor = self.EXECUTOR_CLS.from_stash_entry(
-                    repo,
-                    stash_rev,
-                    entry,
-                    **kwargs,
-                )
-                self.enqueue(stash_rev, executor)
-        finally:
-            for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
-                scm.remove_ref(ref)
-
-    def exec_queue(
-        self, repo: "Repo", jobs: Optional[int] = 1, detach: bool = False
-    ):
-        """Run dvc repro for queued executors in parallel."""
-        if detach:
-            raise NotImplementedError
-            # TODO use ProcessManager.spawn() to support detached runs
-        return self._exec_attached(repo, jobs=jobs)
-
-    def _exec_attached(self, repo: "Repo", jobs: Optional[int] = 1):
-        import signal
-        from concurrent.futures import (
-            CancelledError,
-            ProcessPoolExecutor,
-            wait,
-        )
-        from multiprocessing import Manager
-
-        from dvc.stage.monitor import CheckpointKilledError
-
-        result: Dict[str, Dict[str, str]] = defaultdict(dict)
-
-        manager = Manager()
-        pid_q = manager.Queue()
-
-        with ProcessPoolExecutor(max_workers=jobs) as workers:
-            futures = {}
-            while self._queue:
-                rev, executor = self._queue.popleft()
-                infofile = self.get_infofile_path(rev)
-                future = workers.submit(
-                    executor.reproduce,
-                    info=executor.info,
-                    rev=rev,
-                    queue=pid_q,
-                    infofile=infofile,
-                    log_level=logger.getEffectiveLevel(),
-                )
-                futures[future] = (rev, executor)
-                self._attached[rev] = executor
-
-            try:
-                wait(futures)
-            except KeyboardInterrupt:
-                # forward SIGINT to any running executor processes and
-                # cancel any remaining futures
-                workers.shutdown(wait=False)
-                pids = {}
-                for future, (rev, _) in futures.items():
-                    if future.running():
-                        # if future has already been started by the scheduler
-                        # we still have to wait until it tells us its PID
-                        while rev not in pids:
-                            rev, pid = pid_q.get()
-                            pids[rev] = pid
-                        os.kill(pids[rev], signal.SIGINT)
-                    elif not future.done():
-                        future.cancel()
-
-            for future, (rev, executor) in futures.items():
-                rev, executor = futures[future]
-
-                try:
-                    exc = future.exception()
-                    if exc is None:
-                        exec_result = future.result()
-                        result[rev].update(
-                            self._collect_executor(repo, executor, exec_result)
-                        )
-                    elif not isinstance(exc, CheckpointKilledError):
-                        logger.error(
-                            "Failed to reproduce experiment '%s'", rev[:7]
-                        )
-                except CancelledError:
-                    logger.error(
-                        "Cancelled before attempting to reproduce experiment "
-                        "'%s'",
-                        rev[:7],
-                    )
-                finally:
-                    self.cleanup_executor(rev, executor)
-
-        return result
-
-    def _collect_executor(self, repo, executor, exec_result) -> Dict[str, str]:
-        # NOTE: GitPython Repo instances cannot be re-used
-        # after process has received SIGINT or SIGTERM, so we
-        # need this hack to re-instantiate git instances after
-        # checkpoint runs. See:
-        # https://github.com/gitpython-developers/GitPython/issues/427
-        # del self.repo.scm
-
-        results = {}
-
-        def on_diverged(ref: str, checkpoint: bool):
-            ref_info = ExpRefInfo.from_ref(ref)
-            if checkpoint:
-                raise CheckpointExistsError(ref_info.name)
-            raise ExperimentExistsError(ref_info.name)
-
-        for ref in executor.fetch_exps(
-            self.scm,
-            force=exec_result.force,
-            on_diverged=on_diverged,
-        ):
-            exp_rev = self.scm.get_ref(ref)
-            if exp_rev:
-                logger.debug("Collected experiment '%s'.", exp_rev[:7])
-                results[exp_rev] = exec_result.exp_hash
-
-        if exec_result.ref_info is not None:
-            executor.collect_cache(repo, exec_result.ref_info)
-
-        return results
-
-    def cleanup_executor(self, rev: str, executor: "BaseExecutor"):
-        from dvc.utils.fs import remove
-
-        executor.cleanup()
-        try:
-            self.proc.remove(rev)
-        except KeyError:
-            pass
-        remove(os.path.join(self.pid_dir, rev))

dvc/repo/experiments/executor/manager/local.py

@@ -1,107 +0,0 @@
-import logging
-from collections import defaultdict
-from typing import TYPE_CHECKING, Dict, Optional
-
-from ...refs import EXEC_BASELINE, EXEC_BRANCH, EXEC_HEAD, EXEC_MERGE
-from ...stash import ExpStashEntry
-from ..local import TempDirExecutor, WorkspaceExecutor
-from .base import BaseExecutorManager
-
-if TYPE_CHECKING:
-    from scmrepo.git import Git
-
-    from dvc.repo import Repo
-
-logger = logging.getLogger(__name__)
-
-
-class TempDirExecutorManager(BaseExecutorManager):
-    EXECUTOR_CLS = TempDirExecutor
-
-
-class WorkspaceExecutorManager(BaseExecutorManager):
-    EXECUTOR_CLS = WorkspaceExecutor
-
-    @classmethod
-    def from_stash_entries(
-        cls,
-        scm: "Git",
-        wdir: str,
-        repo: "Repo",
-        to_run: Dict[str, ExpStashEntry],
-        **kwargs,
-    ):
-        manager = cls(scm, wdir)
-        try:
-            assert len(to_run) == 1
-            for stash_rev, entry in to_run.items():
-                scm.set_ref(EXEC_HEAD, entry.head_rev)
-                scm.set_ref(EXEC_MERGE, stash_rev)
-                scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
-
-                executor = cls.EXECUTOR_CLS.from_stash_entry(
-                    repo,
-                    stash_rev,
-                    entry,
-                    **kwargs,
-                )
-                manager.enqueue(stash_rev, executor)
-        finally:
-            for ref in (EXEC_MERGE,):
-                scm.remove_ref(ref)
-        return manager
-
-    def _collect_executor(self, repo, executor, exec_result) -> Dict[str, str]:
-        results = {}
-        exp_rev = self.scm.get_ref(EXEC_BRANCH)
-        if exp_rev:
-            logger.debug("Collected experiment '%s'.", exp_rev[:7])
-            results[exp_rev] = exec_result.exp_hash
-        return results
-
-    def exec_queue(
-        self, repo: "Repo", jobs: Optional[int] = 1, detach: bool = False
-    ):
-        """Run a single WorkspaceExecutor.
-
-        Workspace execution is done within the main DVC process
-        (rather than in multiprocessing context)
-        """
-        from dvc.exceptions import DvcException
-        from dvc.stage.monitor import CheckpointKilledError
-
-        assert len(self._queue) == 1
-        assert not detach
-        result: Dict[str, Dict[str, str]] = defaultdict(dict)
-        rev, executor = self._queue.popleft()
-
-        exec_name = "workspace"
-        infofile = self.get_infofile_path(exec_name)
-        try:
-            exec_result = executor.reproduce(
-                info=executor.info,
-                rev=rev,
-                infofile=infofile,
-                log_level=logger.getEffectiveLevel(),
-                log_errors=not isinstance(executor, WorkspaceExecutor),
-            )
-            if not exec_result.exp_hash:
-                raise DvcException(
-                    f"Failed to reproduce experiment '{rev[:7]}'"
-                )
-            if exec_result.ref_info:
-                result[rev].update(
-                    self._collect_executor(repo, executor, exec_result)
-                )
-        except CheckpointKilledError:
-            # Checkpoint errors have already been logged
-            return {}
-        except DvcException:
-            raise
-        except Exception as exc:
-            raise DvcException(
-                f"Failed to reproduce experiment '{rev[:7]}'"
-            ) from exc
-        finally:
-            self.cleanup_executor(exec_name, executor)
-        return result

dvc/repo/experiments/executor/manager/ssh.py

@@ -1,109 +0,0 @@
-import logging
-import posixpath
-from collections import defaultdict
-from typing import TYPE_CHECKING, Callable, Dict, Generator, Optional, Tuple
-
-from ...stash import ExpStashEntry
-from ..base import BaseExecutor
-from ..ssh import SSHExecutor, _sshfs
-from .base import BaseExecutorManager
-
-if TYPE_CHECKING:
-    from scmrepo.git import Git
-
-    from dvc.repo import Repo
-
-logger = logging.getLogger(__name__)
-
-
-class SSHExecutorManager(BaseExecutorManager):
-    EXECUTOR_CLS = SSHExecutor
-
-    def __init__(
-        self,
-        scm: "Git",
-        wdir: str,
-        host: Optional[str] = None,
-        port: Optional[int] = None,
-        username: Optional[str] = None,
-        fs_factory: Optional[Callable] = None,
-        **kwargs,
-    ):
-        assert host
-        super().__init__(scm, wdir, **kwargs)
-        self.host = host
-        self.port = port
-        self.username = username
-        self._fs_factory = fs_factory
-
-    def _load_infos(self) -> Generator[Tuple[str, "BaseExecutor"], None, None]:
-        # TODO: load existing infos using sshfs
-        yield from []
-
-    @classmethod
-    def from_stash_entries(
-        cls,
-        scm: "Git",
-        wdir: str,
-        repo: "Repo",
-        to_run: Dict[str, ExpStashEntry],
-        **kwargs,
-    ):
-        machine_name: Optional[str] = kwargs.get("machine_name", None)
-        manager = cls(
-            scm, wdir, **repo.machine.get_executor_kwargs(machine_name)
-        )
-        manager._enqueue_stash_entries(scm, repo, to_run, **kwargs)
-        return manager
-
-    def sshfs(self):
-        return _sshfs(self._fs_factory, host=self.host, port=self.port)
-
-    def get_infofile_path(self, name: str) -> str:
-        return f"{name}{BaseExecutor.INFOFILE_EXT}"
-
-    def _exec_attached(self, repo: "Repo", jobs: Optional[int] = 1):
-        from dvc.exceptions import DvcException
-        from dvc.stage.monitor import CheckpointKilledError
-
-        assert len(self._queue) == 1
-        result: Dict[str, Dict[str, str]] = defaultdict(dict)
-        rev, executor = self._queue.popleft()
-        info = executor.info
-        infofile = posixpath.join(
-            info.root_dir,
-            info.dvc_dir,
-            "tmp",
-            self.get_infofile_path(rev),
-        )
-        try:
-            exec_result = executor.reproduce(
-                info=executor.info,
-                rev=rev,
-                infofile=infofile,
-                log_level=logger.getEffectiveLevel(),
-                fs_factory=self._fs_factory,
-            )
-            if not exec_result.exp_hash:
-                raise DvcException(
-                    f"Failed to reproduce experiment '{rev[:7]}'"
-                )
-            if exec_result.ref_info:
-                result[rev].update(
-                    self._collect_executor(repo, executor, exec_result)
-                )
-        except CheckpointKilledError:
-            # Checkpoint errors have already been logged
-            return {}
-        except DvcException:
-            raise
-        except Exception as exc:
-            raise DvcException(
-                f"Failed to reproduce experiment '{rev[:7]}'"
-            ) from exc
-        finally:
-            self.cleanup_executor(rev, executor)
-        return result
-
-    def cleanup_executor(self, rev: str, executor: "BaseExecutor"):
-        executor.cleanup()

dvc/repo/experiments/executor/ssh.py

@@ -15,7 +15,7 @@ from ..refs import (
     EXEC_MERGE,
     EXEC_NAMESPACE,
 )
-from .base import BaseExecutor, ExecutorInfo, ExecutorResult
+from .base import BaseExecutor, ExecutorInfo, ExecutorResult, TaskStatus
 
 if TYPE_CHECKING:
     from queue import Queue
@@ -23,8 +23,8 @@ if TYPE_CHECKING:
     from scmrepo.git import Git
 
     from dvc.repo import Repo
-
-    from ..base import ExpRefInfo, ExpStashEntry
+    from dvc.repo.experiments.refs import ExpRefInfo
+    from dvc.repo.experiments.stash import ExpStashEntry
 
 logger = logging.getLogger(__name__)
 
@@ -75,14 +75,12 @@ class SSHExecutor(BaseExecutor):
     def from_stash_entry(
         cls,
         repo: "Repo",
-        stash_rev: str,
         entry: "ExpStashEntry",
         **kwargs,
     ):
         machine_name: Optional[str] = kwargs.pop("machine_name", None)
         executor = cls._from_stash_entry(
             repo,
-            stash_rev,
             entry,
             cls.gen_dirname(entry.name),
             location=machine_name,
@@ -119,9 +117,20 @@ class SSHExecutor(BaseExecutor):
         }
         return kwargs
 
-    def init_git(self, scm: "Git", branch: Optional[str] = None):
+    def init_git(
+        self,
+        scm: "Git",
+        stash_rev: str,
+        entry: "ExpStashEntry",
+        infofile: Optional[str],
+        branch: Optional[str] = None,
+    ):
         from ..utils import push_refspec
 
+        self.status = TaskStatus.PREPARING
+        if infofile:
+            self.info.dump_json(infofile)
+
         with self.sshfs() as fs:
             fs.makedirs(self.root_dir)
             self._ssh_cmd(fs, "git init .")
@@ -137,8 +146,11 @@ class SSHExecutor(BaseExecutor):
             # TODO: support multiple client key retries in git backends
             # (see https://github.com/iterative/dvc/issues/6508)
             kwargs = self._git_client_args(fs)
-            refspec = f"{EXEC_NAMESPACE}/"
-            push_refspec(scm, self.git_url, refspec, refspec, **kwargs)
+
+            with self.set_exec_refs(scm, stash_rev, entry):
+                refspec = f"{EXEC_NAMESPACE}/"
+                push_refspec(scm, self.git_url, refspec, refspec, **kwargs)
+
             if branch:
                 push_refspec(scm, self.git_url, branch, branch, **kwargs)
                 self._ssh_cmd(fs, f"git symbolic-ref {EXEC_BRANCH} {branch}")

dvc/repo/experiments/queue/base.py

@@ -33,7 +33,7 @@ from ..executor.base import (
     ExecutorResult,
 )
 from ..executor.local import WorkspaceExecutor
-from ..refs import EXEC_BASELINE, EXEC_HEAD, EXEC_MERGE, ExpRefInfo
+from ..refs import ExpRefInfo
 from ..stash import ExpStash, ExpStashEntry
 from ..utils import exp_refs_by_rev, scm_locked
 
@@ -561,7 +561,7 @@ class BaseStashQueue(ABC):
 
     @staticmethod
     @scm_locked
-    def setup_executor(
+    def init_executor(
         exp: "Experiments",
         queue_entry: QueueEntry,
         executor_cls: Type[BaseExecutor] = WorkspaceExecutor,
@@ -576,21 +576,23 @@ class BaseStashQueue(ABC):
         )
         if stash_entry.stash_index is not None:
             stash.drop(stash_entry.stash_index)
-
-        scm.set_ref(EXEC_HEAD, stash_entry.head_rev)
-        scm.set_ref(EXEC_MERGE, stash_rev)
-        scm.set_ref(EXEC_BASELINE, stash_entry.baseline_rev)
-
-        # Executor will be initialized with an empty git repo that
-        # we populate by pushing:
-        #   EXEC_HEAD - the base commit for this experiment
-        #   EXEC_MERGE - the unmerged changes (from our stash)
-        #       to be reproduced
-        #   EXEC_BASELINE - the baseline commit for this experiment
-        return executor_cls.from_stash_entry(
-            exp.repo, stash_rev, stash_entry, **kwargs
+        executor = executor_cls.from_stash_entry(
+            exp.repo, stash_entry, **kwargs
         )
 
+        infofile = exp.celery_queue.get_infofile_path(stash_rev)
+        executor.init_git(
+            exp.repo.scm,
+            stash_rev,
+            stash_entry,
+            infofile,
+            branch=stash_entry.branch,
+        )
+
+        executor.init_cache(exp.repo, stash_rev)
+
+        return executor
+
     def get_infofile_path(self, name: str) -> str:
         return os.path.join(
             self.pid_dir,

dvc/repo/experiments/queue/celery.py

@@ -23,7 +23,12 @@ from dvc.exceptions import DvcException
 from dvc.ui import ui
 
 from ..exceptions import UnresolvedQueueExpNamesError
-from ..executor.base import EXEC_TMP_DIR, ExecutorInfo, ExecutorResult
+from ..executor.base import (
+    EXEC_TMP_DIR,
+    ExecutorInfo,
+    ExecutorResult,
+    TaskStatus,
+)
 from .base import BaseStashQueue, QueueDoneResult, QueueEntry, QueueGetResult
 from .tasks import run_exp
 
@@ -248,7 +253,7 @@ class LocalCeleryQueue(BaseStashQueue):
 
         def _load_collected(rev: str) -> Optional[ExecutorResult]:
             executor_info = _load_info(rev)
-            if executor_info.collected:
+            if executor_info.status > TaskStatus.SUCCESS:
                 return executor_info.result
             raise FileNotFoundError
 

dvc/repo/experiments/queue/tasks.py

@@ -3,8 +3,6 @@ from typing import Any, Dict
 from celery import shared_task
 from celery.utils.log import get_task_logger
 
-from dvc.utils.fs import remove
-
 from ..executor.base import ExecutorInfo
 from ..executor.local import TempDirExecutor
 from .base import BaseStashQueue, QueueEntry
@@ -13,7 +11,7 @@ logger = get_task_logger(__name__)
 
 
 @shared_task
-def setup_exp(entry_dict: Dict[str, Any]) -> str:
+def setup_exp(entry_dict: Dict[str, Any]) -> TempDirExecutor:
     """Setup an experiment.
 
     Arguments:
@@ -28,7 +26,7 @@ def setup_exp(entry_dict: Dict[str, Any]) -> str:
     repo = Repo(entry.dvc_root)
     # TODO: split executor.init_cache into separate subtask - we can release
     # exp.scm_lock before DVC push
-    executor = BaseStashQueue.setup_executor(
+    executor = BaseStashQueue.init_executor(
         repo.experiments,
         entry,
         TempDirExecutor,
@@ -36,7 +34,7 @@ def setup_exp(entry_dict: Dict[str, Any]) -> str:
     )
     infofile = repo.experiments.celery_queue.get_infofile_path(entry.stash_rev)
     executor.info.dump_json(infofile)
-    return executor.root_dir
+    return executor
 
 
 @shared_task
@@ -79,25 +77,14 @@ def collect_exp(
 
 
 @shared_task
-def cleanup_exp(  # pylint: disable=unused-argument
-    tmp_dir: str, entry_dict: Dict[str, Any]
-) -> None:
+def cleanup_exp(executor: TempDirExecutor, infofile: str) -> None:
     """Cleanup after an experiment.
 
     Arguments:
         tmp_dir: Temp directory to be removed.
         entry_dict: Serialized QueueEntry for this experiment.
     """
-    remove(tmp_dir)
-
-
-def _set_collected(infofile: str):
-    try:
-        executor_info = ExecutorInfo.load_json(infofile)
-        executor_info.collected = True
-        executor_info.dump_json(infofile)
-    except FileNotFoundError:
-        pass
+    executor.cleanup(infofile)
 
 
 @shared_task
@@ -115,11 +102,10 @@ def run_exp(entry_dict: Dict[str, Any]) -> None:
     repo = Repo(entry.dvc_root)
     queue = repo.experiments.celery_queue
     infofile = queue.get_infofile_path(entry.stash_rev)
-    root_dir = setup_exp.s(entry_dict)()
+    executor = setup_exp.s(entry_dict)()
     try:
         cmd = ["dvc", "exp", "exec-run", "--infofile", infofile]
         proc_dict = queue.proc.run_signature(cmd, name=entry.stash_rev)()
         collect_exp.s(proc_dict, entry_dict)()
     finally:
-        cleanup_exp.s(root_dir, entry_dict)()
-        _set_collected(infofile)
+        cleanup_exp.s(executor, infofile)()

dvc/repo/experiments/queue/tempdir.py

@@ -11,6 +11,7 @@ from ..executor.base import (
     BaseExecutor,
     ExecutorInfo,
     ExecutorResult,
+    TaskStatus,
 )
 from ..executor.local import TempDirExecutor
 from .base import BaseStashQueue, QueueEntry, QueueGetResult
@@ -60,7 +61,7 @@ class TempDirQueue(WorkspaceQueue):
             stash_entry.name,
             stash_entry.head_rev,
         )
-        executor = self.setup_executor(
+        executor = self.init_executor(
             self.repo.experiments,
             entry,
             TempDirExecutor,
@@ -75,8 +76,9 @@ class TempDirQueue(WorkspaceQueue):
         for stash_rev in self.proc:
             infofile = self.get_infofile_path(stash_rev)
             executor_info = ExecutorInfo.load_json(infofile)
-            if not executor_info.collected and os.path.exists(
-                executor_info.root_dir
+            if (
+                not executor_info.status <= TaskStatus.SUCCESS
+                and os.path.exists(executor_info.root_dir)
             ):
                 yield QueueEntry(
                     self.repo.root_dir,

dvc/repo/experiments/queue/workspace.py

@@ -1,12 +1,10 @@
 import logging
-import os
 from collections import defaultdict
 from typing import TYPE_CHECKING, Collection, Dict, Generator, Optional
 
 from funcy import first
 
 from dvc.exceptions import DvcException
-from dvc.utils.fs import remove
 
 from ..exceptions import ExpQueueEmptyError
 from ..executor.base import BaseExecutor, ExecutorResult
@@ -41,7 +39,7 @@ class WorkspaceQueue(BaseStashQueue):
             stash_entry.name,
             stash_entry.head_rev,
         )
-        executor = self.setup_executor(self.repo.experiments, entry)
+        executor = self.init_executor(self.repo.experiments, entry)
         return QueueGetResult(entry, executor)
 
     def iter_queued(self) -> Generator[QueueEntry, None, None]:
@@ -118,9 +116,7 @@ class WorkspaceQueue(BaseStashQueue):
                 f"Failed to reproduce experiment '{rev[:7]}'"
             ) from exc
         finally:
-            if self._EXEC_NAME == exec_name:
-                remove(os.path.join(self.pid_dir, exec_name))
-            executor.cleanup()
+            executor.cleanup(infofile)
         return results
 
     @staticmethod
@@ -135,6 +131,7 @@ class WorkspaceQueue(BaseStashQueue):
             assert exec_result.exp_hash
             logger.debug("Collected experiment '%s'.", exp_rev[:7])
             results[exp_rev] = exec_result.exp_hash
+
         return results
 
     def get_result(self, entry: QueueEntry) -> Optional[ExecutorResult]:

dvc/repo/fetch.py

@@ -25,22 +25,6 @@ def _fetch_worktree(repo, remote):
             *key,
         )
     save(index)
-
-    for stage in repo.index.stages:
-        for out in stage.outs:
-            if not out.use_cache:
-                continue
-
-            if not out.is_in_repo:
-                continue
-
-            workspace, key = out.index_key
-            entry = repo.index.data[workspace][key]
-            out.hash_info = entry.hash_info
-            out.meta = entry.meta
-
-        stage.dvcfile.dump(stage)
-
     return len(index)
 
 
@@ -171,9 +155,15 @@ def _fetch_partial_imports(repo, targets, **kwargs):
                 raise DataSourceChanged(f"{stage} ({dep})")
 
             logger.info("Importing '%s' -> '%s'", dep, out)
-            out.transfer(
-                dep.def_path, jobs=kwargs.get("jobs"), odb=repo.odb.local
-            )
+            if stage.is_repo_import:
+                obj = dep.get_obj()
+                meta = dep.get_meta()
+                out.hash_info = obj.hash_info
+                out.meta = meta
+            else:
+                out.transfer(
+                    dep.def_path, jobs=kwargs.get("jobs"), odb=repo.odb.local
+                )
         except DataSourceChanged as exc:
             logger.warning(f"{exc}")
             failed += 1

dvc/repo/index.py

@@ -192,9 +192,12 @@ class Index:
 
             data_index = by_workspace[workspace]
 
+            if out.files:
+                out.obj = out.get_obj()
+
             data_index[key] = DataIndexEntry(
                 meta=out.meta,
-                obj=out.get_obj() if out.files else out.obj,
+                obj=out.obj,
                 hash_info=out.hash_info,
                 odb=out.odb,
                 cache=out.odb,
@@ -253,9 +256,7 @@ class Index:
             collect_targets = ensure_list(targets)
 
         pairs = chain.from_iterable(
-            self.stage_collector.collect_granular(
-                target, recursive=recursive, with_deps=True
-            )
+            self.stage_collector.collect_granular(target, recursive=recursive)
             for target in collect_targets
         )
 

dvc/repo/ls.py

@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Optional
 from dvc.exceptions import PathMissingError
 
 if TYPE_CHECKING:
-    from dvc.fs.dvc import DvcFileSystem
+    from dvc.fs.dvc import DVCFileSystem
 
     from . import Repo
 
@@ -55,7 +55,7 @@ def ls(
 def _ls(
     repo: "Repo", path: str, recursive: bool = None, dvc_only: bool = False
 ):
-    fs: "DvcFileSystem" = repo.dvcfs
+    fs: "DVCFileSystem" = repo.dvcfs
     fs_path = fs.from_os_path(path)
 
     try:

dvc/repo/ls_url.py

@@ -0,0 +1,37 @@
+from dvc.exceptions import URLMissingError
+from dvc.fs import get_cloud_fs
+
+
+def parse_external_url(url, config=None):
+    remote_config = dict(config) if config else {}
+    remote_config["url"] = url
+    fs_cls, fs_config, fs_path = get_cloud_fs(None, **remote_config)
+    fs = fs_cls(**fs_config)
+    return fs, fs_path
+
+
+def ls_url(url, *, config=None, recursive=False):
+    fs, fs_path = parse_external_url(url, config=config)
+    try:
+        info = fs.info(fs_path)
+    except FileNotFoundError as exc:
+        raise URLMissingError(url) from exc
+    if info["type"] != "directory":
+        return [{"path": info["name"], "isdir": False}]
+
+    ret = []
+    for _, dirs, files in fs.walk(fs_path, detail=True):
+        if not recursive:
+            files.update(dirs)
+
+        for info in files.values():
+            ls_info = {
+                "path": fs.path.relpath(info["name"], fs_path),
+                "isdir": info["type"] == "directory",
+            }
+            ret.append(ls_info)
+
+        if not recursive:
+            break
+
+    return ret

dvc/repo/metrics/show.py

@@ -4,7 +4,7 @@ from typing import List
 
 from scmrepo.exceptions import SCMError
 
-from dvc.fs.dvc import DvcFileSystem
+from dvc.fs.dvc import DVCFileSystem
 from dvc.output import Output
 from dvc.repo import locked
 from dvc.repo.collect import StrPaths, collect
@@ -77,7 +77,7 @@ def _read_metric(path, fs, rev, **kwargs):
 
 
 def _read_metrics(repo, metrics, rev, onerror=None):
-    fs = DvcFileSystem(repo=repo)
+    fs = DVCFileSystem(repo=repo)
 
     relpath = ""
     if repo.root_dir != repo.fs.path.getcwd():

dvc/repo/plots/__init__.py

@@ -100,7 +100,7 @@ class Plots:
                             "filename":
                             {
                                 "data_source": callable loading the data,
-                                "props": propreties for the file if it is
+                                "props": properties for the file if it is
                                          plots type output
                             }
                         }
@@ -152,9 +152,9 @@ class Plots:
         props: Optional[Dict] = None,
         onerror: Optional[Callable] = None,
     ):
-        from dvc.fs.dvc import DvcFileSystem
+        from dvc.fs.dvc import DVCFileSystem
 
-        fs = DvcFileSystem(repo=self.repo)
+        fs = DVCFileSystem(repo=self.repo)
 
         props = props or {}
 
@@ -472,9 +472,9 @@ def _collect_definitions(
     result: Dict = defaultdict(dict)
     props = props or {}
 
-    from dvc.fs.dvc import DvcFileSystem
+    from dvc.fs.dvc import DVCFileSystem
 
-    fs = DvcFileSystem(repo=repo)
+    fs = DVCFileSystem(repo=repo)
 
     if not config_files:
         dpath.util.merge(

dvc/repo/push.py

@@ -11,13 +11,10 @@ if TYPE_CHECKING:
 
 def _push_worktree(repo, remote):
     from dvc_data.hashfile.tree import tree_from_index
-    from dvc_data.index import build, checkout, collect
+    from dvc_data.index import checkout
 
     index = repo.index.data["repo"]
-    checkout(index, remote.path, remote.fs, force=True)
-    build(index, remote.path, remote.fs)
-    if any(out.isdir() for out in repo.index.outs):
-        collect(index, remote.path, remote.fs)
+    checkout(index, remote.path, remote.fs)
 
     for stage in repo.index.stages:
         for out in stage.outs:
@@ -50,7 +47,7 @@ def _push_worktree(repo, remote):
             else:
                 out.hash_info = entry.hash_info
                 out.meta = entry.meta
-        stage.dvcfile.dump(stage, with_files=True)
+        stage.dvcfile.dump(stage, with_files=True, update_pipeline=False)
 
     return len(index)
 

dvc/repo/stage.py

@@ -225,10 +225,9 @@ class StageLoad:
         path, name = parse_target(target)
         return self.load_one(path=path, name=name)
 
-    @staticmethod
-    def _get_filepath(path: str = None, name: str = None) -> str:
+    def _get_filepath(self, path: str = None, name: str = None) -> str:
         if path:
-            return path
+            return self.repo.fs.path.realpath(path)
 
         path = PIPELINE_FILE
         logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)

dvc/stage/__init__.py

@@ -568,6 +568,8 @@ class Stage(params.StageParams):
             if kwargs.get("checkpoint_func", None) or no_download:
                 allow_missing = True
             self.save(allow_missing=allow_missing)
+            if no_download:
+                self.ignore_outs()
             if not no_commit:
                 self.commit(allow_missing=allow_missing)
 
@@ -681,7 +683,7 @@ class Stage(params.StageParams):
         self, *args, **kwargs
     ) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
         """Return set of object IDs used by this stage."""
-        if self.is_partial_import:
+        if self.is_partial_import and not self.is_repo_import:
             return {}
 
         used_objs = defaultdict(set)
@@ -706,7 +708,7 @@ class Stage(params.StageParams):
                 "unable to auto-merge DVC files with deleted outputs"
             )
 
-    def merge(self, ancestor, other):
+    def merge(self, ancestor, other, allowed=None):
         assert other
 
         if not other.outs:
@@ -726,7 +728,7 @@ class Stage(params.StageParams):
         self._check_can_merge(self, ancestor_out)
         self._check_can_merge(other, ancestor_out)
 
-        self.outs[0].merge(ancestor_out, other.outs[0])
+        self.outs[0].merge(ancestor_out, other.outs[0], allowed=allowed)
 
     def dump(self, **kwargs):
         self.dvcfile.dump(self, **kwargs)
@@ -768,5 +770,5 @@ class PipelineStage(Stage):
     def _changed_stage_entry(self):
         return f"'cmd' of {self} has changed."
 
-    def merge(self, ancestor, other):
+    def merge(self, ancestor, other, allowed=None):
         raise NotImplementedError

dvc/stage/imports.py

@@ -56,7 +56,10 @@ def sync_import(
         stage.outs[0].checkout()
     else:
         stage.save_deps()
-        if not no_download:
+        if no_download:
+            if stage.is_repo_import:
+                stage.deps[0].update()
+        else:
             stage.deps[0].download(
                 stage.outs[0],
                 jobs=jobs,

dvc/stage/loader.py

@@ -73,6 +73,7 @@ class StageLoader(Mapping):
             item.hash_info = HashInfo(item.hash_name, hash_value)
             if item.hash_info and item.hash_info.isdir:
                 item.meta.isdir = True
+            item.files = get_in(checksums, [key, path, item.PARAM_FILES])
 
     @classmethod
     def load_stage(cls, dvcfile, name, stage_data, lock_data=None):

dvc/stage/serialize.py

@@ -152,7 +152,7 @@ def to_pipeline_file(stage: "PipelineStage"):
     }
 
 
-def to_single_stage_lockfile(stage: "Stage") -> dict:
+def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
     assert stage.cmd
 
     def _dumpd(item):
@@ -164,6 +164,13 @@ def to_single_stage_lockfile(stage: "Stage") -> dict:
             *meta_d.items(),
         ]
 
+        if item.hash_info.isdir and kwargs.get("with_files"):
+            if item.obj:
+                obj = item.obj
+            else:
+                obj = item.get_obj()
+            ret.append((item.PARAM_FILES, obj.as_list(with_meta=True)))
+
         return OrderedDict(ret)
 
     res = OrderedDict([("cmd", stage.cmd)])
@@ -183,9 +190,9 @@ def to_single_stage_lockfile(stage: "Stage") -> dict:
     return res
 
 
-def to_lockfile(stage: "PipelineStage") -> dict:
+def to_lockfile(stage: "PipelineStage", **kwargs) -> dict:
     assert stage.name
-    return {stage.name: to_single_stage_lockfile(stage)}
+    return {stage.name: to_single_stage_lockfile(stage, **kwargs)}
 
 
 def to_single_stage_file(stage: "Stage", **kwargs):

dvc/testing/api_tests.py

@@ -1,7 +1,7 @@
 import pytest
 
 from dvc import api
-from dvc.api import DvcFileSystem
+from dvc.api import DVCFileSystem
 from dvc.utils.fs import remove
 
 
@@ -38,7 +38,7 @@ class TestAPI:
     )
     def test_filesystem(
         self,
-        M,
+        M,  # from `pytest-test-utils`
         tmp_dir,
         make_tmp_dir,
         scm,
@@ -47,6 +47,8 @@ class TestAPI:
         fs_kwargs,
         clear_cache,
     ):
+        fs_kwargs = fs_kwargs.copy()
+
         tmp_dir.scm_gen({"scripts": {"script1": "script1"}}, commit="scripts")
         tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}}, commit="data")
         dvc.push()
@@ -59,7 +61,7 @@ class TestAPI:
                 path=tmp_dir, posixpath=tmp_dir.as_posix()
             )
 
-        fs = DvcFileSystem(**fs_kwargs)
+        fs = DVCFileSystem(**fs_kwargs)
 
         assert fs.ls("/", detail=False) == M.unordered(
             "/.gitignore", "/scripts", "/data"

dvc/testing/remote_tests.py

@@ -71,7 +71,7 @@ class TestRemote:
         _check_status(status_dir, ok=dir_hashes)
 
         # Remove and check status
-        remove(dvc.odb.local.path)
+        dvc.odb.local.clear()
 
         status = dvc.cloud.status(foo_hashes)
         _check_status(status, deleted={foo_hash})

dvc/testing/workspace_tests.py

@@ -2,6 +2,9 @@ import os
 
 import pytest
 
+from dvc.exceptions import URLMissingError
+from dvc.repo.ls_url import ls_url, parse_external_url
+
 
 class TestImport:
     def test_import(self, tmp_dir, dvc, workspace):
@@ -134,3 +137,62 @@ class TestAdd:
         assert (
             workspace / "cache" / dir_hash_value[:2] / dir_hash_value[2:]
         ).is_file()
+
+
+def match_files(fs, entries, expected):
+    entries_content = {
+        (fs.path.normpath(d["path"]), d["isdir"]) for d in entries
+    }
+    expected_content = {
+        (fs.path.normpath(d["path"]), d["isdir"]) for d in expected
+    }
+    assert entries_content == expected_content
+
+
+class TestLsUrl:
+    @pytest.mark.parametrize("fname", ["foo", "foo.dvc", "dir/foo"])
+    def test_file(self, cloud, fname):
+        cloud.gen({fname: "foo contents"})
+        fs, fs_path = parse_external_url(cloud.url, cloud.config)
+        result = ls_url(str(cloud / fname), config=cloud.config)
+        match_files(
+            fs,
+            result,
+            [{"path": fs.path.join(fs_path, fname), "isdir": False}],
+        )
+
+    def test_dir(self, cloud):
+        cloud.gen(
+            {"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"}
+        )
+        fs, _ = parse_external_url(cloud.url, cloud.config)
+        result = ls_url(str(cloud / "dir"), config=cloud.config)
+        match_files(
+            fs,
+            result,
+            [
+                {"path": "foo", "isdir": False},
+                {"path": "subdir", "isdir": True},
+            ],
+        )
+
+    def test_recursive(self, cloud):
+        cloud.gen(
+            {"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"}
+        )
+        fs, _ = parse_external_url(cloud.url, cloud.config)
+        result = ls_url(
+            str(cloud / "dir"), config=cloud.config, recursive=True
+        )
+        match_files(
+            fs,
+            result,
+            [
+                {"path": "foo", "isdir": False},
+                {"path": "subdir/bar", "isdir": False},
+            ],
+        )
+
+    def test_nonexistent(self, cloud):
+        with pytest.raises(URLMissingError):
+            ls_url(str(cloud / "dir"), config=cloud.config)

dvc/utils/hydra.py

@@ -1,4 +1,3 @@
-from collections import defaultdict
 from pathlib import Path
 from typing import TYPE_CHECKING, List
 
@@ -83,27 +82,21 @@ def to_hydra_overrides(path_overrides):
     return parser.parse_overrides(overrides=path_overrides)
 
 
+def dict_product(dicts):
+    import itertools
+
+    return [dict(zip(dicts, x)) for x in itertools.product(*dicts.values())]
+
+
 def get_hydra_sweeps(path_overrides):
-    merged_overrides = []
+    path_sweeps = {}
     for path, overrides in path_overrides.items():
-        merged_overrides.extend(
-            [f"{path}___{override}" for override in overrides]
-        )
-
-    hydra_overrides = to_hydra_overrides(merged_overrides)
-    for hydra_override in hydra_overrides:
-        if hydra_override.value_type == ValueType.GLOB_CHOICE_SWEEP:
-            raise InvalidArgumentError(
-                f"Glob override '{hydra_override.input_line}' "
-                "is not supported."
-            )
-
-    splits = BasicSweeper.split_arguments(hydra_overrides, None)[0]
-    sweeps = []
-    for split in splits:
-        sweep_overrides = defaultdict(list)
-        for merged_override in split:
-            path, override = merged_override.split("___")
-            sweep_overrides[path].append(override)
-        sweeps.append(dict(sweep_overrides))
-    return sweeps
+        overrides = to_hydra_overrides(overrides)
+        for override in overrides:
+            if override.value_type == ValueType.GLOB_CHOICE_SWEEP:
+                raise InvalidArgumentError(
+                    f"Glob override '{override.input_line}' "
+                    "is not supported."
+                )
+        path_sweeps[path] = BasicSweeper.split_arguments(overrides, None)[0]
+    return dict_product(path_sweeps)

setup.cfg

@@ -22,6 +22,7 @@ classifiers =
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
 
 [options]
 setup_requires =
@@ -63,12 +64,13 @@ install_requires =
     pyparsing>=2.4.7
     typing-extensions>=3.7.4
     scmrepo==0.1.1
-    dvc-render==0.0.10
-    dvc-task==0.1.2
+    dvc-render==0.0.11
+    dvc-task==0.1.3
     dvclive>=0.10.0
-    dvc-data==0.8.0
-    dvc-http==2.19.1
+    dvc-data==0.14.0
+    dvc-http==2.27.2
     hydra-core>=1.1.0
+    iterative-telemetry==0.0.5
 
 [options.extras_require]
 all =
@@ -86,13 +88,13 @@ dev =
     %(tests)s
     dvc-data[cli]
 # Extra dependencies for remote integrations
-azure = dvc-azure==2.20.2
+azure = dvc-azure==2.20.4
 gdrive = dvc-gdrive==2.19.0
 gs = dvc-gs==2.19.1
 hdfs =
     dvc-hdfs==2.19.0; python_version < '3.11'
 oss = dvc-oss==2.19.0
-s3 = dvc-s3==2.20.0
+s3 = dvc-s3==2.20.1
 ssh = dvc-ssh==2.19.0
 # gssapi should not be included in all_remotes, because it doesn't have wheels
 # for Linux, so it will fail to compile if user doesn't have all the
@@ -103,16 +105,18 @@ webhdfs = dvc-webhdfs==2.19.0
 # requests-kerberos requires krb5 & gssapi, which does not provide wheels Linux
 webhdfs_kerberos = dvc-webhdfs[kerberos]==2.19.0
 terraform = tpi[ssh]>=2.1.0
+testing =
+    pytest-test-utils==0.0.8
 tests =
     %(terraform)s
     dvc-ssh==2.19.0
     # Test requirements
     pytest==7.1.3
-    pytest-cov==3.0.0
+    pytest-cov==4.0.0
     pytest-xdist==2.5.0
-    pytest-mock==3.8.2
+    pytest-mock==3.9.0
     pytest-lazy-fixture==0.6.3
-    pytest-test-utils==0.0.8
+    %(testing)s
     # https://github.com/docker/docker-py/issues/2902
     pytest-docker==0.11.0; python_version < '3.10' or sys_platform != 'win32'
     flaky==3.7.0
@@ -122,11 +126,13 @@ tests =
     pylint==2.15.2
     # we use this to suppress pytest-related false positives in our tests.
     pylint-pytest==1.1.2
+    # see https://github.com/PyCQA/pylint/issues/7488
+    astroid==2.12.9
     # we use this to suppress some messages in tests, eg: foo/bar naming,
     # and, protected method calls in our tests
     pylint-plugin-utils==0.7
     # type-checking
-    mypy==0.971
+    mypy==0.982
     types-requests>=2.27.15
     types-tabulate>=0.8.6
     types-toml>=0.10.4
@@ -146,7 +152,7 @@ exclude =
 console_scripts =
     dvc = dvc.cli:main
 fsspec.specs =
-    dvc = dvc.api:DvcFileSystem
+    dvc = dvc.api:DVCFileSystem
 
 [flake8]
 ignore=

tests/func/api/test_data.py

@@ -9,7 +9,9 @@ from dvc.exceptions import (
     OutputNotFoundError,
     PathMissingError,
 )
-from dvc.testing.test_api import TestAPI  # noqa, pylint: disable=unused-import
+from dvc.testing.api_tests import (  # noqa, pylint: disable=unused-import
+    TestAPI,
+)
 from dvc.testing.tmp_dir import make_subrepo
 from dvc.utils.fs import remove
 

tests/func/experiments/conftest.py

@@ -1,122 +1,12 @@
-from textwrap import dedent
-
 import pytest
 
-from tests.func.test_repro_multistage import COPY_SCRIPT
-from tests.unit.repo.experiments.conftest import (  # noqa pylint disable=unused-argument
+from tests.unit.repo.experiments.conftest import (  # noqa, pylint disable=unused-argument
+    checkpoint_stage,
+    exp_stage,
+    failed_exp_stage,
     test_queue,
 )
 
-DEFAULT_ITERATIONS = 2
-CHECKPOINT_SCRIPT_FORMAT = dedent(
-    """\
-    import os
-    import sys
-    import shutil
-
-    from dvc.api import make_checkpoint
-
-    checkpoint_file = {}
-    checkpoint_iterations = int({})
-    if os.path.exists(checkpoint_file):
-        with open(checkpoint_file) as fobj:
-            try:
-                value = int(fobj.read())
-            except ValueError:
-                value = 0
-    else:
-        with open(checkpoint_file, "w"):
-            pass
-        value = 0
-
-    shutil.copyfile({}, {})
-
-    if os.getenv("DVC_CHECKPOINT"):
-        for _ in range(checkpoint_iterations):
-            value += 1
-            with open(checkpoint_file, "w") as fobj:
-                fobj.write(str(value))
-            make_checkpoint()
-"""
-)
-CHECKPOINT_SCRIPT = CHECKPOINT_SCRIPT_FORMAT.format(
-    "sys.argv[1]", "sys.argv[2]", "sys.argv[3]", "sys.argv[4]"
-)
-
-
-@pytest.fixture
-def exp_stage(tmp_dir, scm, dvc):
-    tmp_dir.gen("copy.py", COPY_SCRIPT)
-    tmp_dir.gen("params.yaml", "foo: 1")
-    stage = dvc.run(
-        cmd="python copy.py params.yaml metrics.yaml",
-        metrics_no_cache=["metrics.yaml"],
-        params=["foo"],
-        name="copy-file",
-        deps=["copy.py"],
-    )
-    scm.add(
-        [
-            "dvc.yaml",
-            "dvc.lock",
-            "copy.py",
-            "params.yaml",
-            "metrics.yaml",
-            ".gitignore",
-        ]
-    )
-    scm.commit("init")
-    return stage
-
-
-@pytest.fixture
-def checkpoint_stage(tmp_dir, scm, dvc, mocker):
-    mocker.patch("dvc.stage.run.Monitor.AWAIT", 0.01)
-
-    tmp_dir.gen("checkpoint.py", CHECKPOINT_SCRIPT)
-    tmp_dir.gen("params.yaml", "foo: 1")
-    stage = dvc.run(
-        cmd=(
-            f"python checkpoint.py foo {DEFAULT_ITERATIONS} "
-            "params.yaml metrics.yaml"
-        ),
-        metrics_no_cache=["metrics.yaml"],
-        params=["foo"],
-        checkpoints=["foo"],
-        deps=["checkpoint.py"],
-        no_exec=True,
-        name="checkpoint-file",
-    )
-    scm.add(["dvc.yaml", "checkpoint.py", "params.yaml", ".gitignore"])
-    scm.commit("init")
-    stage.iterations = DEFAULT_ITERATIONS
-    return stage
-
-
-@pytest.fixture
-def failed_exp_stage(tmp_dir, scm, dvc):
-    tmp_dir.gen("copy.py", COPY_SCRIPT)
-    tmp_dir.gen("params.yaml", "foo: 1")
-    stage = dvc.stage.add(
-        cmd="python -c 'import sys; sys.exit(1)'",
-        metrics_no_cache=["failed-metrics.yaml"],
-        params=["foo"],
-        name="failed-copy-file",
-        deps=["copy.py"],
-    )
-    scm.add(
-        [
-            "dvc.yaml",
-            "dvc.lock",
-            "copy.py",
-            "params.yaml",
-            "failed-metrics.yaml",
-            ".gitignore",
-        ]
-    )
-    scm.commit("init")
-    return stage
-
 
 @pytest.fixture
 def http_auth_patch(mocker):

tests/func/experiments/executor/test_ssh.py

@@ -29,7 +29,6 @@ def test_init_from_stash(tmp_dir, scm, dvc, machine_instance, mocker):
     mock_entry.name = ""
     SSHExecutor.from_stash_entry(
         dvc,
-        "abc123",
         mock_entry,
         machine_name="foo",
     )
@@ -39,14 +38,14 @@ def test_init_from_stash(tmp_dir, scm, dvc, machine_instance, mocker):
 
 @pytest.mark.needs_internet
 @pytest.mark.parametrize("cloud", [pytest.lazy_fixture("git_ssh")])
-def test_init_git(tmp_dir, scm, cloud):
+def test_init_git(tmp_dir, scm, cloud, mocker):
     tmp_dir.scm_gen({"foo": "foo", "dir": {"bar": "bar"}}, commit="init")
     baseline_rev = scm.get_rev()
-    scm.set_ref(EXEC_HEAD, baseline_rev)
     tmp_dir.gen("foo", "stashed")
     scm.gitpython.git.stash()
     rev = scm.resolve_rev("stash@{0}")
-    scm.set_ref(EXEC_MERGE, rev)
+
+    mock = mocker.Mock(baseline_rev=baseline_rev, head_rev=baseline_rev)
 
     root_url = cloud / SSHExecutor.gen_dirname()
 
@@ -59,7 +58,8 @@ def test_init_git(tmp_dir, scm, cloud):
         username=TEST_SSH_USER,
         fs_factory=partial(_ssh_factory, cloud),
     )
-    executor.init_git(scm)
+    infofile = str((root_url / "foo.run").path)
+    executor.init_git(scm, rev, mock, infofile=infofile)
     assert root_url.path == executor._repo_abspath
 
     fs = cloud._ssh
@@ -117,7 +117,6 @@ def test_reproduce(tmp_dir, scm, dvc, cloud, exp_stage, mocker):
     SSHExecutor.reproduce(
         info,
         rev,
-        infofile=infofile,
         fs_factory=partial(_ssh_factory, cloud),
     )
     assert mock_execute.called_once()

tests/func/experiments/test_show.py

@@ -12,6 +12,7 @@ from dvc.repo.experiments.executor.base import (
     EXEC_TMP_DIR,
     BaseExecutor,
     ExecutorInfo,
+    TaskStatus,
 )
 from dvc.repo.experiments.queue.base import QueueEntry
 from dvc.repo.experiments.refs import CELERY_STASH, ExpRefInfo
@@ -338,9 +339,16 @@ def test_show_sort(tmp_dir, scm, dvc, exp_stage, caplog):
 
 
 @pytest.mark.vscode
-def test_show_running_workspace(tmp_dir, scm, dvc, exp_stage, capsys):
+@pytest.mark.parametrize(
+    "status, running", [(TaskStatus.RUNNING, True), (TaskStatus.FAILED, False)]
+)
+def test_show_running_workspace(
+    tmp_dir, scm, dvc, exp_stage, capsys, status, running
+):
     pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR)
-    info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION)
+    info = make_executor_info(
+        location=BaseExecutor.DEFAULT_LOCATION, status=status
+    )
     pidfile = os.path.join(
         pid_dir,
         "workspace",
@@ -349,7 +357,8 @@ def test_show_running_workspace(tmp_dir, scm, dvc, exp_stage, capsys):
     makedirs(os.path.dirname(pidfile), True)
     (tmp_dir / pidfile).dump_json(info.asdict())
 
-    assert dvc.experiments.show()["workspace"] == {
+    print(dvc.experiments.show())
+    assert dvc.experiments.show().get("workspace") == {
         "baseline": {
             "data": {
                 "deps": {
@@ -363,18 +372,18 @@ def test_show_running_workspace(tmp_dir, scm, dvc, exp_stage, capsys):
                 "params": {"params.yaml": {"data": {"foo": 1}}},
                 "outs": {},
                 "queued": False,
-                "running": True,
-                "executor": info.location,
+                "running": True if running else False,
+                "executor": info.location if running else None,
                 "timestamp": None,
             }
         }
     }
-
     capsys.readouterr()
     assert main(["exp", "show", "--csv"]) == 0
     cap = capsys.readouterr()
-    assert "Running" in cap.out
-    assert info.location in cap.out
+    if running:
+        assert "Running" in cap.out
+        assert info.location in cap.out
 
 
 def test_show_running_tempdir(tmp_dir, scm, dvc, exp_stage, mocker):

tests/func/test_add.py

@@ -34,7 +34,7 @@ from dvc.stage.exceptions import (
     StageExternalOutputsError,
     StagePathNotFoundError,
 )
-from dvc.testing.test_workspace import TestAdd
+from dvc.testing.workspace_tests import TestAdd
 from dvc.utils import LARGE_DIR_SIZE, relpath
 from dvc.utils.fs import path_isin
 from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml
@@ -453,7 +453,9 @@ class TestAddCommit(TestDvc):
         ret = main(["commit", self.FOO + ".dvc"])
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(self.FOO))
-        self.assertEqual(len(os.listdir(self.dvc.odb.local.path)), 1)
+        self.assertTrue(
+            self.dvc.odb.local.exists("acbd18db4cc2f85cedef654fccc4a4d8")
+        )
 
 
 def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
@@ -832,9 +834,9 @@ def test_add_symlink_file(tmp_dir, dvc):
     assert (tmp_dir / "dir" / "foo").read_text() == "bar"
     assert (tmp_dir / "dir" / "bar").read_text() == "bar"
 
-    assert (tmp_dir / ".dvc" / "cache").read_text() == {
-        "37": {"b51d194a7513e45b56f6524f2d51f2": "bar"}
-    }
+    assert (
+        tmp_dir / ".dvc" / "cache" / "37" / "b51d194a7513e45b56f6524f2d51f2"
+    ).read_text() == "bar"
     assert not (
         tmp_dir / ".dvc" / "cache" / "37" / "b51d194a7513e45b56f6524f2d51f2"
     ).is_symlink()
@@ -891,9 +893,9 @@ def test_add_with_cache_link_error(tmp_dir, dvc, mocker, capsys):
 
     assert (tmp_dir / "foo").exists()
     assert (tmp_dir / "foo.dvc").exists()
-    assert (tmp_dir / ".dvc" / "cache").read_text() == {
-        "ac": {"bd18db4cc2f85cedef654fccc4a4d8": "foo"}
-    }
+    assert (
+        tmp_dir / ".dvc" / "cache" / "ac" / "bd18db4cc2f85cedef654fccc4a4d8"
+    ).read_text() == "foo"
 
 
 def test_add_preserve_fields(tmp_dir, dvc):

tests/func/test_data_cloud.py

@@ -9,7 +9,7 @@ import dvc_data
 from dvc.cli import main
 from dvc.external_repo import clean_repos
 from dvc.stage.exceptions import StageNotFound
-from dvc.testing.test_remote import (  # noqa, pylint: disable=unused-import
+from dvc.testing.remote_tests import (  # noqa, pylint: disable=unused-import
     TestRemote,
 )
 from dvc.utils.fs import remove
@@ -49,7 +49,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker):
         for (_args, _kwargs) in oids_exist.call_args_list
     )
 
-    remove(dvc.odb.local.path)
+    dvc.odb.local.clear()
     oids_exist.reset_mock()
 
     assert main(["fetch"] + args) == 0
@@ -164,7 +164,7 @@ def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
     tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
 
     # purge cache
-    remove(dvc.odb.local.path)
+    dvc.odb.local.clear()
 
     header = (
         "Some of the cache files do not exist "
@@ -205,7 +205,7 @@ def test_verify_hashes(
     # remove artifacts and cache to trigger fetching
     remove("file")
     remove("dir")
-    remove(dvc.odb.local.path)
+    dvc.odb.local.clear()
 
     hash_spy = mocker.spy(dvc_data.hashfile.hash, "file_md5")
 
@@ -213,7 +213,7 @@ def test_verify_hashes(
     assert hash_spy.call_count == 0
 
     # Removing cache will invalidate existing state entries
-    remove(dvc.odb.local.path)
+    dvc.odb.local.clear()
 
     dvc.config["remote"]["upstream"]["verify"] = True
 
@@ -310,18 +310,19 @@ def test_pull_external_dvc_imports_mixed(
 
 def clean(outs, dvc=None):
     if dvc:
-        outs = outs + [dvc.odb.local.path]
+        dvc.odb.local.clear()
     for path in outs:
         print(path)
         remove(path)
     if dvc:
-        os.makedirs(dvc.odb.local.path, exist_ok=True)
         clean_repos()
 
 
 def recurse_list_dir(d):
     return [
-        os.path.join(d, f) for _, _, filenames in os.walk(d) for f in filenames
+        os.path.join(root, f)
+        for root, _, filenames in os.walk(d)
+        for f in filenames
     ]
 
 
@@ -488,11 +489,11 @@ def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
 
     dvc.pull("copy-foo-bar")
     assert (tmp_dir / "bar").exists()
-    assert len(recurse_list_dir(dvc.odb.local.path)) == 1
+    assert len(recurse_list_dir(dvc.odb.local.path)) == 2
     clean(["bar"], dvc)
 
     dvc.fetch("copy-foo-bar")
-    assert len(recurse_list_dir(dvc.odb.local.path)) == 1
+    assert len(recurse_list_dir(dvc.odb.local.path)) == 2
 
 
 def test_pull_partial(tmp_dir, dvc, local_remote):

tests/func/test_fs.py

@@ -1,7 +1,6 @@
 import os
 from operator import itemgetter
 
-from dvc.fs.callbacks import Callback
 from dvc.repo import Repo
 
 
@@ -42,57 +41,3 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
 
     assert set(get_dirs(next(dvc_fs.walk(path)))) == {".dvc", "subdir", ".git"}
     assert set(get_dirs(next(scm_fs.walk("/")))) == {".dvc", "subdir"}
-
-
-def test_callback_on_dvcfs(tmp_dir, dvc, scm, mocker):
-    tmp_dir.dvc_gen({"dir": {"bar": "bar"}}, commit="dvc")
-    tmp_dir.scm_gen({"dir": {"foo": "foo"}}, commit="git")
-
-    fs = dvc.dvcfs
-
-    callback = Callback()
-    fs.get(
-        "dir",
-        (tmp_dir / "dir2").fs_path,
-        callback=callback,
-    )
-
-    assert (tmp_dir / "dir2").read_text() == {"foo": "foo", "bar": "bar"}
-    assert callback.size == 2
-    assert callback.value == 2
-
-    callback = Callback()
-    branch = mocker.spy(callback, "branch")
-    fs.get(
-        os.path.join("dir", "foo"),
-        (tmp_dir / "foo").fs_path,
-        callback=callback,
-    )
-
-    size = os.path.getsize(tmp_dir / "dir" / "foo")
-    assert (tmp_dir / "foo").read_text() == "foo"
-    assert callback.size == 1
-    assert callback.value == 1
-
-    assert branch.call_count == 1
-    assert branch.spy_return.size == size
-    assert branch.spy_return.value == size
-
-    branch.reset_mock()
-
-    callback = Callback()
-    branch = mocker.spy(callback, "branch")
-    fs.get(
-        os.path.join("dir", "bar"),
-        (tmp_dir / "bar").fs_path,
-        callback=callback,
-    )
-
-    size = os.path.getsize(tmp_dir / "dir" / "bar")
-    assert (tmp_dir / "bar").read_text() == "bar"
-    assert callback.size == 1
-    assert callback.value == 1
-
-    assert branch.call_count == 1
-    assert branch.spy_return.size == size
-    assert branch.spy_return.value == size

tests/func/test_ignore.py

@@ -421,8 +421,6 @@ def test_run_dvcignored_dep(tmp_dir, dvc, run_copy):
 
 
 def test_pull_ignore(tmp_dir, dvc, local_cloud):
-    from dvc.utils.fs import remove
-
     tmp_dir.dvc_gen(
         {
             ".dvcignore": "data/processed/",
@@ -437,7 +435,7 @@ def test_pull_ignore(tmp_dir, dvc, local_cloud):
     foo_path.unlink()
     assert not foo_path.exists()
 
-    remove(tmp_dir / ".dvc/cache")
+    dvc.odb.local.clear()
     dvc.pull()
 
     assert foo_path.exists()

tests/func/test_import.py

@@ -217,6 +217,68 @@ def test_pull_imported_stage(tmp_dir, dvc, erepo_dir):
     assert os.path.isfile(dst_cache)
 
 
+def test_import_no_download(tmp_dir, scm, dvc, erepo_dir):
+    with erepo_dir.chdir():
+        erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
+
+    dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", no_download=True)
+
+    assert not os.path.exists("foo_imported")
+
+    dst_stage = Dvcfile(dvc, "foo_imported.dvc").stage
+
+    assert dst_stage.deps[0].def_repo == {
+        "url": os.fspath(erepo_dir),
+        "rev_lock": erepo_dir.scm.get_rev(),
+    }
+    assert scm.is_ignored("foo_imported")
+
+
+def test_pull_import_no_download(tmp_dir, scm, dvc, erepo_dir):
+    with erepo_dir.chdir():
+        erepo_dir.scm_gen(os.path.join("foo", "bar"), b"bar", commit="add bar")
+        erepo_dir.dvc_gen(
+            os.path.join("foo", "baz"), b"baz contents", commit="add baz"
+        )
+        size = (
+            len(b"bar")
+            + len(b"baz contents")
+            + len((erepo_dir / "foo" / ".gitignore").read_bytes())
+        )
+
+    dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", no_download=True)
+
+    dvc.pull(["foo_imported.dvc"])
+    assert os.path.exists("foo_imported")
+
+    stage = Dvcfile(dvc, "foo_imported.dvc").stage
+
+    assert (
+        stage.outs[0].hash_info.value == "bdb8641831d8fcb03939637e09011c21.dir"
+    )
+
+    assert stage.outs[0].meta.size == size
+    assert stage.outs[0].meta.nfiles == 3
+    assert stage.outs[0].meta.isdir
+
+
+def test_pull_import_no_download_rev_lock(
+    tmp_dir,
+    dvc,
+    erepo_dir,
+):
+    with erepo_dir.chdir():
+        erepo_dir.dvc_gen("foo", "foo content", commit="add")
+
+    dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", no_download=True)
+
+    with erepo_dir.chdir():
+        erepo_dir.dvc_gen("foo", "modified foo content", commit="modify foo")
+
+    dvc.pull(["foo_imported.dvc"])
+    assert (tmp_dir / "foo_imported").read_text() == "foo content"
+
+
 def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir):
     with erepo_dir.chdir():
         with erepo_dir.dvc.config.edit() as conf:

tests/func/test_import_url.py

@@ -8,9 +8,10 @@ import pytest
 from dvc.annotations import Annotation
 from dvc.cli import main
 from dvc.dependency.base import Dependency, DependencyDoesNotExistError
+from dvc.dvcfile import Dvcfile
 from dvc.exceptions import InvalidArgumentError
 from dvc.stage import Stage
-from dvc.testing.test_workspace import TestImport as _TestImport
+from dvc.testing.workspace_tests import TestImport as _TestImport
 from dvc.utils.fs import makedirs
 from tests.basic_env import TestDvc
 
@@ -282,7 +283,7 @@ def test_import_url_to_remote_status(tmp_dir, dvc, local_cloud, local_remote):
     assert len(status) == 0
 
 
-def test_import_url_no_download(tmp_dir, dvc, local_workspace):
+def test_import_url_no_download(tmp_dir, scm, dvc, local_workspace):
     local_workspace.gen("file", "file content")
     dst = tmp_dir / "file"
     stage = dvc.imp_url(
@@ -292,6 +293,7 @@ def test_import_url_no_download(tmp_dir, dvc, local_workspace):
     assert stage.deps[0].hash_info.value == "d10b4c3ff123b26dc068d43a8bef2d23"
 
     assert not dst.exists()
+    assert scm.is_ignored(dst)
 
     out = stage.outs[0]
     assert not out.hash_info
@@ -303,6 +305,21 @@ def test_import_url_no_download(tmp_dir, dvc, local_workspace):
     ]
 
 
+def test_partial_import_pull(tmp_dir, scm, dvc, local_workspace):
+    local_workspace.gen("file", "file content")
+    dst = tmp_dir / "file"
+    dvc.imp_url("remote://workspace/file", os.fspath(dst), no_download=True)
+
+    dvc.pull(["file.dvc"])
+
+    assert dst.exists()
+
+    stage = Dvcfile(dvc, "file.dvc").stage
+
+    assert stage.outs[0].hash_info.value == "d10b4c3ff123b26dc068d43a8bef2d23"
+    assert stage.outs[0].meta.size == 12
+
+
 def test_imp_url_with_annotations(M, tmp_dir, dvc, local_workspace):
     local_workspace.gen("foo", "foo")
     annot = {

tests/func/test_ls_url.py

@@ -0,0 +1,5 @@
+from dvc.testing.workspace_tests import TestLsUrl as _TestLsUrl
+
+
+class TestLsUrl(_TestLsUrl):
+    pass

tests/func/test_merge_driver.py

@@ -39,6 +39,22 @@ def _gen(tmp_dir, struct, name):
         (None, {"foo": "foo"}, {"bar": "bar"}, {"foo": "foo", "bar": "bar"}),
         (None, None, {"bar": "bar"}, {"bar": "bar"}),
         (None, {"foo": "foo"}, None, {"foo": "foo"}),
+        (
+            {"foo": "foo"},
+            {"foo": "bar"},
+            {"foo": "foo", "baz": "baz"},
+            {"foo": "bar", "baz": "baz"},
+        ),
+        ({"foo": "foo"}, {}, {"foo": "foo", "bar": "bar"}, {"bar": "bar"}),
+        (
+            {"common": "common", "subdir": {"foo": "foo", "bar": "bar"}},
+            {"common": "common", "subdir": {"foo": "foo", "bar": "baz"}},
+            {"common": "common", "subdir": {"bar": "bar", "bizz": "bizz"}},
+            {
+                "common": "common",
+                "subdir": {"bar": "baz", "bizz": "bizz"},
+            },
+        ),
     ],
 )
 def test_merge(tmp_dir, dvc, ancestor, our, their, merged):
@@ -74,18 +90,15 @@ def test_merge(tmp_dir, dvc, ancestor, our, their, merged):
             {"foo": "foo"},
             {"foo": "bar"},
             {"foo": "baz"},
-            (
-                "unable to auto-merge directories with "
-                "diff that contains 'change'ed files"
-            ),
+            ("unable to auto-merge the following paths:\nfoo"),
         ),
         (
             {"common": "common", "foo": "foo"},
             {"common": "common", "bar": "bar"},
             {"baz": "baz"},
             (
-                "unable to auto-merge directories with "
-                "diff that contains 'remove'ed files"
+                "unable to auto-merge the following paths:\n"
+                "both deleted: ('foo',)"
             ),
         ),
     ],

tests/func/test_odb.py

@@ -112,18 +112,15 @@ def test_shared_cache_dir(tmp_dir):
 
     assert not os.path.exists(os.path.join("dir1", ".dvc", "cache"))
     assert not os.path.exists(os.path.join("dir2", ".dvc", "cache"))
-
-    subdirs = list(
-        filter(
-            lambda x: os.path.isdir(os.path.join(cache_dir, x)),
-            os.listdir(cache_dir),
-        )
+    assert os.path.exists(
+        os.path.join(cache_dir, "dc", "f6c2fa538b445a3a095255c3641dfc")
+    )
+    assert os.path.exists(
+        os.path.join(cache_dir, "b4", "333c8cfa2ebba7ef20ec6c3265902b")
+    )
+    assert os.path.exists(
+        os.path.join(cache_dir, "9e", "fab2399c7c560b34de477b9aa0a465")
     )
-    assert len(subdirs) == 3
-    assert len(os.listdir(os.path.join(cache_dir, subdirs[0]))) == 1
-
-    assert len(os.listdir(os.path.join(cache_dir, subdirs[1]))) == 1
-    assert len(os.listdir(os.path.join(cache_dir, subdirs[2]))) == 1
 
 
 def test_cache_link_type(tmp_dir, scm, dvc):
@@ -167,10 +164,9 @@ def test_cmd_cache_relative_path(tmp_dir, scm, dvc, make_tmp_dir):
 
     tmp_dir.dvc_gen({"foo": "foo"})
 
-    subdirs = os.listdir(cache_dir)
-    assert len(subdirs) == 1
-    files = os.listdir(os.path.join(cache_dir, subdirs[0]))
-    assert len(files) == 1
+    assert os.path.exists(
+        os.path.join(cache_dir, "ac", "bd18db4cc2f85cedef654fccc4a4d8")
+    )
 
 
 def test_default_cache_type(dvc):
@@ -194,28 +190,13 @@ def test_shared_cache(tmp_dir, dvc, group):
         {"file": "file content", "dir": {"file2": "file 2 " "content"}}
     )
 
-    actual = {}
-    for root, dnames, fnames in os.walk(cache_dir):
-        for name in dnames + fnames:
-            path = os.path.join(root, name)
-            actual[path] = oct(stat.S_IMODE(os.stat(path).st_mode))
-
     file_mode = oct(0o444)
     dir_mode = oct(0o2775 if group else (0o777 & ~system.umask))
+    for root, dnames, fnames in os.walk(cache_dir):
+        for dname in dnames:
+            path = os.path.join(root, dname)
+            assert oct(stat.S_IMODE(os.stat(path).st_mode)) == dir_mode
 
-    expected = {
-        os.path.join(cache_dir, "17"): dir_mode,
-        os.path.join(
-            cache_dir, "17", "4eaa1dd94050255b7b98a7e1924b31.dir"
-        ): file_mode,
-        os.path.join(cache_dir, "97"): dir_mode,
-        os.path.join(
-            cache_dir, "97", "e17781c198500e2766ea56bd697c03"
-        ): file_mode,
-        os.path.join(cache_dir, "d1"): dir_mode,
-        os.path.join(
-            cache_dir, "d1", "0b4c3ff123b26dc068d43a8bef2d23"
-        ): file_mode,
-    }
-
-    assert expected == actual
+        for fname in fnames:
+            path = os.path.join(root, fname)
+            assert oct(stat.S_IMODE(os.stat(path).st_mode)) == file_mode

tests/func/test_remote.py

@@ -201,7 +201,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
 
     # Push everything and delete local cache
     dvc.push()
-    remove(dvc.odb.local.path)
+    dvc.odb.local.clear()
 
     baz._collect_used_dir_cache()
     with patch.object(generic, "transfer", side_effect=Exception):

tests/func/test_run_single_stage.py

@@ -726,7 +726,7 @@ class TestRunCommit(TestDvc):
         ret = main(["commit", fname + ".dvc"])
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(fname))
-        self.assertEqual(len(os.listdir(self.dvc.odb.local.path)), 1)
+        self.assertEqual(len(list(self.dvc.odb.local.all())), 1)
 
 
 class TestRunPersist(TestDvc):

tests/func/test_stage.py

@@ -265,6 +265,23 @@ def test_collect_repo_ignored_dir_unignored_pattern(tmp_dir, dvc, scm):
     assert dvc.stage.collect_repo() == [stage]
 
 
+@pytest.mark.parametrize("with_deps", (False, True))
+def test_collect_symlink(tmp_dir, dvc, with_deps):
+    tmp_dir.gen({"data": {"foo": "foo contents"}})
+    foo_path = os.path.join("data", "foo")
+    dvc.add(foo_path)
+
+    data_link = tmp_dir / "data_link"
+    data_link.symlink_to("data")
+    stage = list(
+        dvc.stage.collect(
+            target=str(data_link / "foo.dvc"), with_deps=with_deps
+        )
+    )[0]
+
+    assert stage.addressing == f"{foo_path}.dvc"
+
+
 def test_stage_strings_representation(tmp_dir, dvc, run_copy):
     tmp_dir.dvc_gen("foo", "foo")
     stage1 = run_copy("foo", "bar", single_stage=True)

tests/func/utils/test_hydra.py

@@ -183,6 +183,22 @@ def test_compose_and_dump(tmp_dir, suffix, overrides, expected):
 @pytest.mark.parametrize(
     "overrides, expected",
     [
+        (
+            {"params.yaml": ["defaults/foo=1,2"]},
+            [
+                {"params.yaml": ["defaults/foo=1"]},
+                {"params.yaml": ["defaults/foo=2"]},
+            ],
+        ),
+        (
+            {"params.yaml": ["+foo=1,2", "~bar", "++foobar=5,6"]},
+            [
+                {"params.yaml": ["+foo=1", "~bar=null", "++foobar=5"]},
+                {"params.yaml": ["+foo=1", "~bar=null", "++foobar=6"]},
+                {"params.yaml": ["+foo=2", "~bar=null", "++foobar=5"]},
+                {"params.yaml": ["+foo=2", "~bar=null", "++foobar=6"]},
+            ],
+        ),
         (
             {"params.yaml": ["foo=1,2", "bar=3,4"]},
             [

tests/integration/plots/test_plots.py

@@ -11,6 +11,7 @@ from bs4 import BeautifulSoup
 from funcy import first
 
 from dvc.cli import main
+from dvc.render import REVISION_FIELD, VERSION_FIELD
 
 JSON_OUT = "vis_data"
 
@@ -60,17 +61,6 @@ def extract_vega_specs(html_path, plots_ids):
     return result
 
 
-def filter_fields(datapoints: List[Dict], fields: List[str]):
-
-    tmp = deepcopy(datapoints)
-    for datapoint in tmp:
-        keys = set(datapoint.keys())
-        for key in keys:
-            if key not in fields:
-                datapoint.pop(key)
-    return tmp
-
-
 def drop_fields(datapoints: List[Dict], fields: List[str]):
     tmp = deepcopy(datapoints)
     for datapoint in tmp:
@@ -219,10 +209,35 @@ def test_repo_with_plots(
     html_result = extract_vega_specs(
         html_path, ["linear.json", "confusion.json"]
     )
+
+    assert json_result["linear.json"][0]["content"]["data"][
+        "values"
+    ] == _update_datapoints(
+        linear_v1,
+        {
+            VERSION_FIELD: {
+                "revision": "workspace",
+                "filename": "linear.json",
+                "field": "y",
+            },
+        },
+    )
     assert html_result["linear.json"]["data"]["values"] == _update_datapoints(
         linear_v1,
         {
-            "rev": "workspace",
+            REVISION_FIELD: "workspace",
+        },
+    )
+    assert json_result["confusion.json"][0]["content"]["data"][
+        "values"
+    ] == _update_datapoints(
+        confusion_v1,
+        {
+            VERSION_FIELD: {
+                "revision": "workspace",
+                "filename": "confusion.json",
+                "field": "actual",
+            },
         },
     )
     assert html_result["confusion.json"]["data"][
@@ -230,10 +245,9 @@ def test_repo_with_plots(
     ] == _update_datapoints(
         confusion_v1,
         {
-            "rev": "workspace",
+            REVISION_FIELD: "workspace",
         },
     )
-
     verify_image(
         tmp_dir, "workspace", "image.png", image_v1, html_path, json_result
     )
@@ -280,17 +294,59 @@ def test_repo_with_plots(
             html_path,
             ["../linear.json", "../confusion.json"],
         )
+        assert json_result["../linear.json"][0]["content"]["data"][
+            "values"
+        ] == _update_datapoints(
+            linear_v2,
+            {
+                VERSION_FIELD: {
+                    "revision": "workspace",
+                    "filename": "../linear.json",
+                    "field": "y",
+                },
+            },
+        ) + _update_datapoints(
+            linear_v1,
+            {
+                VERSION_FIELD: {
+                    "revision": "HEAD",
+                    "filename": "../linear.json",
+                    "field": "y",
+                },
+            },
+        )
         assert html_result["../linear.json"]["data"][
             "values"
         ] == _update_datapoints(
             linear_v2,
             {
-                "rev": "workspace",
+                REVISION_FIELD: "workspace",
             },
         ) + _update_datapoints(
             linear_v1,
             {
-                "rev": "HEAD",
+                REVISION_FIELD: "HEAD",
+            },
+        )
+        assert json_result["../confusion.json"][0]["content"]["data"][
+            "values"
+        ] == _update_datapoints(
+            confusion_v2,
+            {
+                VERSION_FIELD: {
+                    "revision": "workspace",
+                    "filename": "../confusion.json",
+                    "field": "actual",
+                },
+            },
+        ) + _update_datapoints(
+            confusion_v1,
+            {
+                VERSION_FIELD: {
+                    "revision": "HEAD",
+                    "filename": "../confusion.json",
+                    "field": "actual",
+                },
             },
         )
         assert html_result["../confusion.json"]["data"][
@@ -298,12 +354,12 @@ def test_repo_with_plots(
         ] == _update_datapoints(
             confusion_v2,
             {
-                "rev": "workspace",
+                REVISION_FIELD: "workspace",
             },
         ) + _update_datapoints(
             confusion_v1,
             {
-                "rev": "HEAD",
+                REVISION_FIELD: "HEAD",
             },
         )
 
@@ -390,11 +446,9 @@ def test_repo_with_config_plots(tmp_dir, capsys, repo_with_config_plots):
     repo_state = repo_with_config_plots()
     plots = next(repo_state)
 
-    html_path, json_result, split_json_result = call(capsys)
+    html_path, _, __ = call(capsys)
 
     assert os.path.exists(html_path)
-    assert json_result == {}
-    assert split_json_result == {}
     html_result = extract_vega_specs(
         html_path,
         [
@@ -405,9 +459,15 @@ def test_repo_with_config_plots(tmp_dir, capsys, repo_with_config_plots):
     assert html_result["dvc.yaml::linear_train_vs_test"]["data"][
         "values"
     ] == _update_datapoints(
-        plots["data"]["linear_train.json"], {"rev": "linear_train.json"}
+        plots["data"]["linear_train.json"],
+        {
+            REVISION_FIELD: "linear_train.json",
+        },
     ) + _update_datapoints(
-        plots["data"]["linear_test.json"], {"rev": "linear_test.json"}
+        plots["data"]["linear_test.json"],
+        {
+            REVISION_FIELD: "linear_test.json",
+        },
     )
 
     # TODO check json results once vscode is able to handle flexible plots

tests/unit/command/test_ls_url.py

@@ -0,0 +1,24 @@
+from dvc.cli import parse_args
+from dvc.commands.ls_url import CmdListUrl
+
+
+def test_ls_url(mocker):
+    cli_args = parse_args(["ls-url", "src"])
+    assert cli_args.func == CmdListUrl
+    cmd = cli_args.func(cli_args)
+    m = mocker.patch("dvc.repo.Repo.ls_url", autospec=True)
+
+    assert cmd.run() == 0
+
+    m.assert_called_once_with("src", recursive=False)
+
+
+def test_recursive(mocker):
+    cli_args = parse_args(["ls-url", "-R", "src"])
+    assert cli_args.func == CmdListUrl
+    cmd = cli_args.func(cli_args)
+    m = mocker.patch("dvc.repo.Repo.ls_url", autospec=True)
+
+    assert cmd.run() == 0
+
+    m.assert_called_once_with("src", recursive=True)

tests/unit/command/test_plots.py

@@ -334,10 +334,6 @@ def test_plots_diff_json(dvc, mocker, capsys):
     )
     render_mock = mocker.patch("dvc_render.render_html")
 
-    show_json_mock = mocker.patch(
-        "dvc.commands.plots._filter_unhandled_renderers",
-        return_value=renderers,
-    )
     show_json_mock = mocker.patch("dvc.commands.plots._show_json")
 
     assert cmd.run() == 0

tests/unit/command/test_run.py

@@ -8,7 +8,7 @@ def test_run(mocker, dvc):
         [
             "run",
             "--name",
-            "nam",
+            "name",
             "--deps",
             "deps",
             "--outs",
@@ -86,7 +86,7 @@ def test_run(mocker, dvc):
         no_commit=True,
         always_changed=True,
         cmd="command",
-        name="nam",
+        name="name",
         single_stage=False,
         external=True,
         desc="description",

tests/unit/fs/test_data.py

@@ -50,7 +50,7 @@ def test_open_dirty_hash(tmp_dir, dvc):
 
     fs = DataFileSystem(index=dvc.index.data["repo"])
     with fs.open("file", "r") as fobj:
-        # NOTE: Unlike DvcFileSystem, DataFileSystem should not
+        # NOTE: Unlike DVCFileSystem, DataFileSystem should not
         # be affected by a dirty workspace.
         assert fobj.read() == "file"
 
@@ -71,7 +71,7 @@ def test_open_dirty_no_hash(tmp_dir, dvc):
     (tmp_dir / "file.dvc").write_text("outs:\n- path: file\n")
 
     fs = DataFileSystem(index=dvc.index.data["repo"])
-    # NOTE: Unlike DvcFileSystem, DataFileSystem should not
+    # NOTE: Unlike DVCFileSystem, DataFileSystem should not
     # be affected by a dirty workspace.
     with pytest.raises(FileNotFoundError):
         with fs.open("file", "r"):

tests/unit/fs/test_dvc.py

@@ -5,7 +5,7 @@ from unittest import mock
 
 import pytest
 
-from dvc.fs.dvc import DvcFileSystem
+from dvc.fs.dvc import DVCFileSystem
 from dvc.testing.tmp_dir import make_subrepo
 from dvc_data.hashfile.build import build
 from dvc_data.hashfile.hash_info import HashInfo
@@ -16,7 +16,7 @@ def test_exists(tmp_dir, dvc):
     dvc.add("foo")
     (tmp_dir / "foo").unlink()
 
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     assert fs.exists("foo")
 
 
@@ -25,7 +25,7 @@ def test_open(tmp_dir, dvc):
     dvc.add("foo")
     (tmp_dir / "foo").unlink()
 
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     with fs.open("foo", "r") as fobj:
         assert fobj.read() == "foo"
 
@@ -34,7 +34,7 @@ def test_open_dirty_hash(tmp_dir, dvc):
     tmp_dir.dvc_gen("file", "file")
     (tmp_dir / "file").write_text("something")
 
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     with fs.open("file", "r") as fobj:
         assert fobj.read() == "something"
 
@@ -43,7 +43,7 @@ def test_open_dirty_no_hash(tmp_dir, dvc):
     tmp_dir.gen("file", "file")
     (tmp_dir / "file.dvc").write_text("outs:\n- path: file\n")
 
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     with fs.open("file", "r") as fobj:
         assert fobj.read() == "file"
 
@@ -63,7 +63,7 @@ def test_open_in_history(tmp_dir, scm, dvc):
         if rev == "workspace":
             continue
 
-        fs = DvcFileSystem(repo=dvc)
+        fs = DVCFileSystem(repo=dvc)
         with fs.open("foo", "r") as fobj:
             assert fobj.read() == "foo"
 
@@ -86,7 +86,7 @@ def test_isdir_isfile(tmp_dir, dvc):
         },
     )
 
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     assert fs.isdir("datadir")
     assert not fs.isfile("datadir")
     assert not fs.isdvc("datadir")
@@ -107,7 +107,7 @@ def test_isdir_isfile(tmp_dir, dvc):
     (tmp_dir / "datafile").unlink()
     (tmp_dir / "subdir" / "baz").unlink()
 
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     assert fs.isdir("datadir")
     assert not fs.isfile("datadir")
     assert fs.isdvc("datadir")
@@ -127,7 +127,7 @@ def test_exists_isdir_isfile_dirty(tmp_dir, dvc):
         {"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}}
     )
 
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     shutil.rmtree(tmp_dir / "datadir")
     (tmp_dir / "datafile").unlink()
 
@@ -162,7 +162,7 @@ def test_isdir_mixed(tmp_dir, dvc):
 
     dvc.add(str(tmp_dir / "dir" / "foo"))
 
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     assert fs.isdir("dir")
     assert not fs.isfile("dir")
 
@@ -192,7 +192,7 @@ def test_walk(tmp_dir, dvc, dvcfiles, extra_expected):
     )
     dvc.add(str(tmp_dir / "dir"), recursive=True)
     tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
 
     expected = [
         "dir/subdir1",
@@ -227,7 +227,7 @@ def test_walk_dirty(tmp_dir, dvc):
     tmp_dir.gen({"dir": {"bar": "bar", "subdir3": {"foo3": "foo3"}}})
     (tmp_dir / "dir" / "foo").unlink()
 
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     expected = [
         "dir/subdir1",
         "dir/subdir2",
@@ -253,7 +253,7 @@ def test_walk_dirty_cached_dir(tmp_dir, scm, dvc):
     tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}}, commit="add data")
     (tmp_dir / "data" / "foo").unlink()
 
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
 
     actual = []
     for root, dirs, files in fs.walk("data"):
@@ -277,7 +277,7 @@ def test_walk_mixed_dir(tmp_dir, scm, dvc):
     )
     tmp_dir.scm.commit("add dir")
 
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
 
     expected = [
         "dir/foo",
@@ -294,7 +294,7 @@ def test_walk_mixed_dir(tmp_dir, scm, dvc):
 
 
 def test_walk_missing(tmp_dir, dvc):
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
 
     for _ in fs.walk("dir"):
         pass
@@ -302,7 +302,7 @@ def test_walk_missing(tmp_dir, dvc):
 
 def test_walk_not_a_dir(tmp_dir, dvc):
     tmp_dir.dvc_gen("foo", "foo")
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
 
     for _ in fs.walk("foo"):
         pass
@@ -312,7 +312,7 @@ def test_isdvc(tmp_dir, dvc):
     tmp_dir.gen({"foo": "foo", "bar": "bar", "dir": {"baz": "baz"}})
     dvc.add("foo")
     dvc.add("dir")
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     assert fs.isdvc("foo")
     assert not fs.isdvc("bar")
     assert fs.isdvc("dir")
@@ -322,7 +322,7 @@ def test_isdvc(tmp_dir, dvc):
 
 def test_subrepos(tmp_dir, scm, dvc, mocker):
     tmp_dir.scm_gen(
-        {"dir": {"repo.txt": "file to confuse DvcFileSystem"}},
+        {"dir": {"repo.txt": "file to confuse DVCFileSystem"}},
         commit="dir/repo.txt",
     )
 
@@ -340,7 +340,7 @@ def test_subrepos(tmp_dir, scm, dvc, mocker):
         )
 
     dvc._reset()
-    fs = DvcFileSystem(repo=dvc, subrepos=True)
+    fs = DVCFileSystem(repo=dvc, subrepos=True)
 
     def assert_fs_belongs_to_repo(ret_val):
         method = fs.fs._get_repo
@@ -400,7 +400,7 @@ def test_subrepos(tmp_dir, scm, dvc, mocker):
 )
 def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
     tmp_dir.scm_gen(
-        {"dir": {"repo.txt": "file to confuse DvcFileSystem"}},
+        {"dir": {"repo.txt": "file to confuse DVCFileSystem"}},
         commit="dir/repo.txt",
     )
 
@@ -420,7 +420,7 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
 
     # using fs that does not have dvcignore
     dvc._reset()
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     expected = [
         "dir/repo",
         "dir/repo.txt",
@@ -451,7 +451,7 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
 
 def test_dvcfs_no_subrepos(tmp_dir, dvc, scm):
     tmp_dir.scm_gen(
-        {"dir": {"repo.txt": "file to confuse DvcFileSystem"}},
+        {"dir": {"repo.txt": "file to confuse DVCFileSystem"}},
         commit="dir/repo.txt",
     )
     tmp_dir.dvc_gen({"lorem": "lorem"}, commit="add foo")
@@ -464,7 +464,7 @@ def test_dvcfs_no_subrepos(tmp_dir, dvc, scm):
 
     # using fs that does not have dvcignore
     dvc._reset()
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     expected = [
         "/.dvcignore",
         "/.gitignore",
@@ -496,7 +496,7 @@ def test_dvcfs_no_subrepos(tmp_dir, dvc, scm):
 
 def test_get_hash_cached_file(tmp_dir, dvc, mocker):
     tmp_dir.dvc_gen({"foo": "foo"})
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     expected = "acbd18db4cc2f85cedef654fccc4a4d8"
     assert fs.info("foo").get("md5") is None
     _, _, obj = build(dvc.odb.local, "foo", fs, "md5")
@@ -509,7 +509,7 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker):
     tmp_dir.dvc_gen(
         {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
     )
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     expected = "8761c4e9acad696bee718615e23e22db.dir"
     assert fs.info("dir").get("md5") is None
     _, _, obj = build(dvc.odb.local, "dir", fs, "md5")
@@ -529,7 +529,7 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
     tmp_dir.dvc_gen(
         {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
     )
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     subdir = "dir/subdir"
     assert fs.info(subdir).get("md5") is None
     _, _, obj = build(dvc.odb.local, subdir, fs, "md5")
@@ -558,7 +558,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc):
     )
     tmp_dir.scm.commit("add dir")
 
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     _, _, obj = build(dvc.odb.local, "dir", fs, "md5")
     assert obj.hash_info == HashInfo(
         "md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir"
@@ -577,7 +577,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
 
     # file is modified in workspace
     # hash_file(file) should return workspace hash, not DVC cached hash
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     assert fs.info("file").get("md5") is None
     staging, _, obj = build(dvc.odb.local, "file", fs, "md5")
     assert obj.hash_info == something_hash_info
@@ -599,7 +599,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
     tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
     (tmp_dir / "dir" / "baz").write_text("baz")
 
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     _, meta, obj = build(dvc.odb.local, "dir", fs, "md5")
     assert obj.hash_info == HashInfo(
         "md5", "ba75a2162ca9c29acecb7957105a0bc2.dir"
@@ -661,7 +661,7 @@ def test_walk_nested_subrepos(tmp_dir, dvc, scm, traverse_subrepos):
         expected["/subrepo1"].add("subrepo3")
 
     actual = {}
-    fs = DvcFileSystem(repo=dvc)
+    fs = DVCFileSystem(repo=dvc)
     for root, dirs, files in fs.walk(
         "/", ignore_subrepos=not traverse_subrepos
     ):

tests/unit/fs/test_dvc_info.py

@@ -2,7 +2,7 @@ import os
 
 import pytest
 
-from dvc.fs.dvc import DvcFileSystem
+from dvc.fs.dvc import DVCFileSystem
 from dvc.testing.tmp_dir import make_subrepo
 
 
@@ -38,7 +38,7 @@ def dvcfs(tmp_dir, dvc, scm):
     tmp_dir.scm_gen(fs_structure, commit="repo init")
     tmp_dir.dvc_gen(dvc_structure, commit="use dvc")
 
-    yield DvcFileSystem(repo=dvc, subrepos=True)
+    yield DVCFileSystem(repo=dvc, subrepos=True)
 
 
 def test_info_not_existing(dvcfs):

tests/unit/fs/test_fs.py

@@ -1,10 +1,12 @@
+from dataclasses import dataclass
+
 import pytest
 from dvc_http import HTTPFileSystem, HTTPSFileSystem
 from dvc_s3 import S3FileSystem
 from dvc_ssh import SSHFileSystem
 
 from dvc.config import RemoteNotFoundError
-from dvc.fs import LocalFileSystem, get_fs_cls, get_fs_config
+from dvc.fs import LocalFileSystem, get_cloud_fs, get_fs_cls, get_fs_config
 
 url_cls_pairs = [
     ("s3://bucket/path", S3FileSystem),
@@ -36,5 +38,39 @@ def test_get_fs_cls(url, cls):
 
 
 def test_get_fs_config():
+    result = get_fs_config({}, url="ssh://example.com:/dir/path")
+    assert result == {"url": "ssh://example.com:/dir/path"}
+
+
+def test_get_fs_config_error():
     with pytest.raises(RemoteNotFoundError):
-        get_fs_config(None, {"remote": {}}, name="myremote")
+        get_fs_config({"remote": {}}, name="myremote")
+
+
+def test_remote_url():
+    config = {
+        "remote": {
+            "base": {"url": "http://example.com"},
+            "r1": {"url": "remote://base/r1", "user": "user"},
+            "r2": {"url": "remote://r1/r2", "password": "123"},
+        }
+    }
+    result = get_fs_config(config, url="remote://r2/foo")
+    assert result == {
+        "password": "123",
+        "user": "user",
+        "url": "http://example.com/r1/r2/foo",
+    }
+
+
+@dataclass
+class FakeRepo:
+    config: dict
+
+
+def test_get_cloud_fs():
+    repo = FakeRepo({})
+    cls, config, path = get_cloud_fs(repo, url="ssh://example.com:/dir/path")
+    assert cls is SSHFileSystem
+    assert config == {"host": "example.com", "verify": False}
+    assert path == "/dir/path"

tests/unit/render/test_convert.py

@@ -1,4 +1,10 @@
-from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY
+from dvc.render import (
+    REVISION_FIELD,
+    REVISIONS_KEY,
+    SRC_FIELD,
+    TYPE_KEY,
+    VERSION_FIELD,
+)
 from dvc.render.convert import to_json
 
 
@@ -7,8 +13,18 @@ def test_to_json_vega(mocker):
     vega_renderer.TYPE = "vega"
     vega_renderer.get_filled_template.return_value = '{"this": "is vega"}'
     vega_renderer.datapoints = [
-        {"x": 1, "y": 2, REVISION_FIELD: "foo", "filename": "foo.json"},
-        {"x": 2, "y": 1, REVISION_FIELD: "bar", "filename": "foo.json"},
+        {
+            "x": 1,
+            "y": 2,
+            VERSION_FIELD: {"revision": "foo"},
+            "filename": "foo.json",
+        },
+        {
+            "x": 2,
+            "y": 1,
+            VERSION_FIELD: {"revision": "bar"},
+            "filename": "foo.json",
+        },
     ]
     result = to_json(vega_renderer)
     assert result[0] == {
@@ -17,10 +33,20 @@ def test_to_json_vega(mocker):
         "content": {"this": "is vega"},
         "datapoints": {
             "foo": [
-                {"x": 1, "y": 2, "filename": "foo.json"},
+                {
+                    "x": 1,
+                    "y": 2,
+                    "filename": "foo.json",
+                    VERSION_FIELD: {"revision": "foo"},
+                },
             ],
             "bar": [
-                {"x": 2, "y": 1, "filename": "foo.json"},
+                {
+                    "x": 2,
+                    "y": 1,
+                    "filename": "foo.json",
+                    VERSION_FIELD: {"revision": "bar"},
+                },
             ],
         },
     }
@@ -34,8 +60,18 @@ def test_to_json_vega_split(mocker):
         '{"this": "is split vega"}'
     )
     vega_renderer.datapoints = [
-        {"x": 1, "y": 2, REVISION_FIELD: "foo", "filename": "foo.json"},
-        {"x": 2, "y": 1, REVISION_FIELD: "bar", "filename": "foo.json"},
+        {
+            "x": 1,
+            "y": 2,
+            VERSION_FIELD: {"revision": "foo"},
+            "filename": "foo.json",
+        },
+        {
+            "x": 2,
+            "y": 1,
+            VERSION_FIELD: {"revision": "bar"},
+            "filename": "foo.json",
+        },
     ]
     result = to_json(vega_renderer, split=True)
     assert result[0] == {
@@ -43,8 +79,22 @@ def test_to_json_vega_split(mocker):
         REVISIONS_KEY: ["bar", "foo"],
         "content": {"this": "is split vega"},
         "datapoints": {
-            "foo": [{"x": 1, "y": 2, "filename": "foo.json"}],
-            "bar": [{"x": 2, "y": 1, "filename": "foo.json"}],
+            "foo": [
+                {
+                    "x": 1,
+                    "y": 2,
+                    "filename": "foo.json",
+                    VERSION_FIELD: {"revision": "foo"},
+                }
+            ],
+            "bar": [
+                {
+                    "x": 2,
+                    "y": 1,
+                    "filename": "foo.json",
+                    VERSION_FIELD: {"revision": "bar"},
+                }
+            ],
         },
     }
     vega_renderer.get_filled_template.assert_called_with(skip_anchors=["data"])

tests/unit/render/test_match.py

@@ -89,14 +89,20 @@ def test_match_renderers(mocker):
     assert len(renderers) == 1
     assert renderers[0].datapoints == [
         {
-            VERSION_FIELD: {"revision": "v1", "filename": "file.json"},
-            "rev": "v1",
+            VERSION_FIELD: {
+                "revision": "v1",
+                "filename": "file.json",
+                "field": "y",
+            },
             "x": 1,
             "y": 1,
         },
         {
-            VERSION_FIELD: {"revision": "v1", "filename": "file.json"},
-            "rev": "v1",
+            VERSION_FIELD: {
+                "revision": "v1",
+                "filename": "file.json",
+                "field": "y",
+            },
             "x": 2,
             "y": 2,
         },

tests/unit/render/test_vega_converter.py

@@ -57,42 +57,53 @@ def test_finding_lists(dictionary, expected_result):
 @pytest.mark.parametrize(
     "input_data,properties,expected_datapoints,expected_properties",
     [
-        (
-            # default x and y
+        pytest.param(
             {"metric": [{"v": 1}, {"v": 2}]},
             {},
             [
                 {
                     "v": 1,
                     "step": 0,
-                    VERSION_FIELD: {"revision": "r", "filename": "f"},
-                    "rev": "r",
+                    VERSION_FIELD: {
+                        "revision": "r",
+                        "filename": "f",
+                        "field": "v",
+                    },
                 },
                 {
                     "v": 2,
                     "step": 1,
-                    VERSION_FIELD: {"revision": "r", "filename": "f"},
-                    "rev": "r",
+                    VERSION_FIELD: {
+                        "revision": "r",
+                        "filename": "f",
+                        "field": "v",
+                    },
                 },
             ],
             {"x": "step", "y": "v"},
+            id="default_x_y",
         ),
-        (
-            # filter fields
+        pytest.param(
             {"metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]},
             {"fields": {"v"}},
             [
                 {
                     "v": 1,
                     "step": 0,
-                    VERSION_FIELD: {"revision": "r", "filename": "f"},
-                    "rev": "r",
+                    VERSION_FIELD: {
+                        "revision": "r",
+                        "filename": "f",
+                        "field": "v",
+                    },
                 },
                 {
                     "v": 2,
                     "step": 1,
-                    VERSION_FIELD: {"revision": "r", "filename": "f"},
-                    "rev": "r",
+                    VERSION_FIELD: {
+                        "revision": "r",
+                        "filename": "f",
+                        "field": "v",
+                    },
                 },
             ],
             {
@@ -100,9 +111,9 @@ def test_finding_lists(dictionary, expected_result):
                 "y": "v",
                 "fields": {"v", "step"},
             },
+            id="filter_fields",
         ),
-        (
-            # choose x and y
+        pytest.param(
             {"metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]},
             {"x": "v", "y": "v2"},
             [
@@ -114,7 +125,6 @@ def test_finding_lists(dictionary, expected_result):
                         "filename": "f",
                         "field": "v2",
                     },
-                    "rev": "r",
                 },
                 {
                     "v": 2,
@@ -124,13 +134,12 @@ def test_finding_lists(dictionary, expected_result):
                         "filename": "f",
                         "field": "v2",
                     },
-                    "rev": "r",
                 },
             ],
             {"x": "v", "y": "v2"},
+            id="choose_x_y",
         ),
-        (
-            # append x and y to filtered fields
+        pytest.param(
             {
                 "metric": [
                     {"v": 1, "v2": 0.1, "v3": 0.01, "v4": 0.001},
@@ -148,7 +157,6 @@ def test_finding_lists(dictionary, expected_result):
                         "filename": "f",
                         "field": "v4",
                     },
-                    "rev": "r",
                 },
                 {
                     "v": 2,
@@ -159,13 +167,12 @@ def test_finding_lists(dictionary, expected_result):
                         "filename": "f",
                         "field": "v4",
                     },
-                    "rev": "r",
                 },
             ],
             {"x": "v3", "y": "v4", "fields": {"v", "v3", "v4"}},
+            id="append_x_y_to_fields",
         ),
-        (
-            # find metric in nested structure
+        pytest.param(
             {
                 "some": "noise",
                 "very": {
@@ -184,7 +191,6 @@ def test_finding_lists(dictionary, expected_result):
                         "filename": "f",
                         "field": "v2",
                     },
-                    "rev": "r",
                 },
                 {
                     "v": 2,
@@ -194,13 +200,12 @@ def test_finding_lists(dictionary, expected_result):
                         "filename": "f",
                         "field": "v2",
                     },
-                    "rev": "r",
                 },
             ],
             {"x": "v", "y": "v2"},
+            id="find_in_nested_structure",
         ),
-        (
-            # provide list of fields in y def
+        pytest.param(
             {"metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]},
             {"y": {"f": ["v", "v2"]}},
             [
@@ -212,7 +217,6 @@ def test_finding_lists(dictionary, expected_result):
                         "filename": "f",
                         "field": "v",
                     },
-                    "rev": "r",
                     "dvc_inferred_y_value": 1,
                     "step": 0,
                 },
@@ -224,7 +228,6 @@ def test_finding_lists(dictionary, expected_result):
                         "filename": "f",
                         "field": "v2",
                     },
-                    "rev": "r",
                     "dvc_inferred_y_value": 0.1,
                     "step": 0,
                 },
@@ -236,7 +239,6 @@ def test_finding_lists(dictionary, expected_result):
                         "filename": "f",
                         "field": "v",
                     },
-                    "rev": "r",
                     "dvc_inferred_y_value": 2,
                     "step": 1,
                 },
@@ -248,12 +250,12 @@ def test_finding_lists(dictionary, expected_result):
                         "filename": "f",
                         "field": "v2",
                     },
-                    "rev": "r",
                     "dvc_inferred_y_value": 0.2,
                     "step": 1,
                 },
             ],
             {"x": "step", "y": "dvc_inferred_y_value", "y_label": "y"},
+            id="y_def_list",
         ),
     ],
 )
@@ -275,6 +277,7 @@ def test_convert(
 def test_convert_skip_step():
     converter = VegaConverter()
     converter.skip_step("append_index")
+    converter.skip_step("generate_y")
 
     datapoints, resolved_properties = converter.convert(
         data={"a": "b", "metric": [{"v": 1}, {"v": 2}]},
@@ -285,13 +288,9 @@ def test_convert_skip_step():
     assert datapoints == [
         {
             "v": 1,
-            "rev": "r",
-            VERSION_FIELD: {"revision": "r", "filename": "f"},
         },
         {
             "v": 2,
-            "rev": "r",
-            VERSION_FIELD: {"revision": "r", "filename": "f"},
         },
     ]
     assert resolved_properties == {"x": "step", "y": "v"}

tests/unit/repo/experiments/conftest.py

@@ -1,8 +1,120 @@
 from contextlib import contextmanager
+from textwrap import dedent
 
 import pytest
 
 from dvc.repo.experiments.queue.celery import LocalCeleryQueue
+from tests.func.test_repro_multistage import COPY_SCRIPT
+
+DEFAULT_ITERATIONS = 2
+CHECKPOINT_SCRIPT_FORMAT = dedent(
+    """\
+    import os
+    import sys
+    import shutil
+
+    from dvc.api import make_checkpoint
+
+    checkpoint_file = {}
+    checkpoint_iterations = int({})
+    if os.path.exists(checkpoint_file):
+        with open(checkpoint_file) as fobj:
+            try:
+                value = int(fobj.read())
+            except ValueError:
+                value = 0
+    else:
+        with open(checkpoint_file, "w"):
+            pass
+        value = 0
+
+    shutil.copyfile({}, {})
+
+    if os.getenv("DVC_CHECKPOINT"):
+        for _ in range(checkpoint_iterations):
+            value += 1
+            with open(checkpoint_file, "w") as fobj:
+                fobj.write(str(value))
+            make_checkpoint()
+"""
+)
+CHECKPOINT_SCRIPT = CHECKPOINT_SCRIPT_FORMAT.format(
+    "sys.argv[1]", "sys.argv[2]", "sys.argv[3]", "sys.argv[4]"
+)
+
+
+@pytest.fixture
+def exp_stage(tmp_dir, scm, dvc):
+    tmp_dir.gen("copy.py", COPY_SCRIPT)
+    tmp_dir.gen("params.yaml", "foo: 1")
+    stage = dvc.run(
+        cmd="python copy.py params.yaml metrics.yaml",
+        metrics_no_cache=["metrics.yaml"],
+        params=["foo"],
+        name="copy-file",
+        deps=["copy.py"],
+    )
+    scm.add(
+        [
+            "dvc.yaml",
+            "dvc.lock",
+            "copy.py",
+            "params.yaml",
+            "metrics.yaml",
+            ".gitignore",
+        ]
+    )
+    scm.commit("init")
+    return stage
+
+
+@pytest.fixture
+def checkpoint_stage(tmp_dir, scm, dvc, mocker):
+    mocker.patch("dvc.stage.run.Monitor.AWAIT", 0.01)
+
+    tmp_dir.gen("checkpoint.py", CHECKPOINT_SCRIPT)
+    tmp_dir.gen("params.yaml", "foo: 1")
+    stage = dvc.run(
+        cmd=(
+            f"python checkpoint.py foo {DEFAULT_ITERATIONS} "
+            "params.yaml metrics.yaml"
+        ),
+        metrics_no_cache=["metrics.yaml"],
+        params=["foo"],
+        checkpoints=["foo"],
+        deps=["checkpoint.py"],
+        no_exec=True,
+        name="checkpoint-file",
+    )
+    scm.add(["dvc.yaml", "checkpoint.py", "params.yaml", ".gitignore"])
+    scm.commit("init")
+    stage.iterations = DEFAULT_ITERATIONS
+    return stage
+
+
+@pytest.fixture
+def failed_exp_stage(tmp_dir, scm, dvc):
+    tmp_dir.gen("copy.py", COPY_SCRIPT)
+    tmp_dir.gen("params.yaml", "foo: 1")
+    stage = dvc.stage.add(
+        cmd="python -c 'import sys; sys.exit(1)'",
+        metrics_no_cache=["failed-metrics.yaml"],
+        params=["foo"],
+        name="failed-copy-file",
+        deps=["copy.py"],
+    )
+    scm.add(
+        [
+            "dvc.yaml",
+            "dvc.lock",
+            "copy.py",
+            "params.yaml",
+            "failed-metrics.yaml",
+            ".gitignore",
+        ]
+    )
+    scm.commit("init")
+    return stage
 
 
 @contextmanager

tests/unit/repo/experiments/queue/test_local.py

@@ -6,9 +6,7 @@ from flaky.flaky_decorator import flaky
 
 from dvc.exceptions import DvcException
 from dvc.repo.experiments.exceptions import UnresolvedExpNamesError
-from dvc.repo.experiments.executor.local import TempDirExecutor
 from dvc.repo.experiments.queue.base import QueueDoneResult
-from dvc.repo.experiments.refs import EXEC_BASELINE, EXEC_HEAD, EXEC_MERGE
 
 
 def test_shutdown_no_tasks(test_queue, mocker):
@@ -110,20 +108,6 @@ def test_celery_queue_kill(test_queue, mocker):
     assert spy.called_once_with(mock_entry.stash_rev)
 
 
-def test_queue_clean_workspace_refs(git_dir, tmp_dir):
-
-    rev = git_dir.scm.get_rev()
-    exec_heads = (EXEC_MERGE, EXEC_BASELINE, EXEC_HEAD)
-    for ref in exec_heads:
-        git_dir.scm.set_ref(ref, rev)
-
-    temp_dir_executor = TempDirExecutor(str(tmp_dir), str(tmp_dir), rev)
-    temp_dir_executor.init_git(git_dir.scm)
-
-    for ref in exec_heads:
-        assert git_dir.scm.get_ref(ref) is None
-
-
 @pytest.mark.parametrize("status", ["FAILURE", "SUCCESS"])
 def test_queue_iter_done_task(test_queue, mocker, status):
 

tests/unit/repo/experiments/test_executor_status.py

@@ -0,0 +1,117 @@
+import os
+
+import pytest
+
+from dvc.exceptions import DvcException
+from dvc.repo.experiments.executor.base import ExecutorInfo, TaskStatus
+from dvc.repo.experiments.queue.tasks import (
+    cleanup_exp,
+    collect_exp,
+    setup_exp,
+)
+
+
+def test_celery_queue_success_status(dvc, scm, test_queue, exp_stage):
+    queue_entry = test_queue._stash_exp(
+        params={"params.yaml": ["foo=1"]},
+        targets=exp_stage.addressing,
+        name="success",
+    )
+    infofile = test_queue.get_infofile_path(queue_entry.stash_rev)
+    executor = setup_exp.s(queue_entry.asdict())()
+    executor_info = ExecutorInfo.load_json(infofile)
+    assert executor_info.status == TaskStatus.PREPARING
+
+    cmd = ["dvc", "exp", "exec-run", "--infofile", infofile]
+    proc_dict = test_queue.proc.run_signature(
+        cmd, name=queue_entry.stash_rev
+    )()
+
+    executor_info = ExecutorInfo.load_json(infofile)
+    assert executor_info.status == TaskStatus.SUCCESS
+
+    collect_exp.s(proc_dict, queue_entry.asdict())()
+    cleanup_exp.s(executor, infofile)()
+    executor_info = ExecutorInfo.load_json(infofile)
+    assert executor_info.status == TaskStatus.FINISHED
+
+
+def test_celery_queue_failure_status(dvc, scm, test_queue, failed_exp_stage):
+    queue_entry = test_queue._stash_exp(
+        params={"params.yaml": ["foo=1"]},
+        targets=failed_exp_stage.addressing,
+        name="failed",
+    )
+    infofile = test_queue.get_infofile_path(queue_entry.stash_rev)
+    setup_exp.s(queue_entry.asdict())()
+    cmd = ["dvc", "exp", "exec-run", "--infofile", infofile]
+    test_queue.proc.run_signature(cmd, name=queue_entry.stash_rev)()
+    executor_info = ExecutorInfo.load_json(infofile)
+    assert executor_info.status == TaskStatus.FAILED
+
+
+@pytest.mark.parametrize("queue_type", ["workspace_queue", "tempdir_queue"])
+def test_workspace_executor_success_status(dvc, scm, exp_stage, queue_type):
+    workspace_queue = getattr(dvc.experiments, queue_type)
+    queue_entry = workspace_queue.put(
+        params={"params.yaml": ["foo=1"]},
+        targets=exp_stage.addressing,
+        name="success",
+    )
+    name = workspace_queue._EXEC_NAME or queue_entry.stash_rev
+    infofile = workspace_queue.get_infofile_path(name)
+    entry, executor = workspace_queue.get()
+    rev = entry.stash_rev
+    exec_result = executor.reproduce(
+        info=executor.info,
+        rev=rev,
+        infofile=infofile,
+    )
+    executor_info = ExecutorInfo.load_json(infofile)
+    assert executor_info.status == TaskStatus.SUCCESS
+    if exec_result.ref_info:
+        workspace_queue.collect_executor(
+            dvc.experiments, executor, exec_result
+        )
+    executor.cleanup(infofile)
+
+    if queue_type == "tempdir_queue":
+        executor_info = ExecutorInfo.load_json(infofile)
+        assert executor_info.status == TaskStatus.FINISHED
+    else:
+        assert not os.path.exists(infofile)
+
+
+@pytest.mark.parametrize("queue_type", ["workspace_queue", "tempdir_queue"])
+def test_workspace_executor_failed_status(
+    dvc, scm, failed_exp_stage, queue_type
+):
+    workspace_queue = getattr(dvc.experiments, queue_type)
+    queue_entry = workspace_queue.put(
+        params={"params.yaml": ["foo=1"]},
+        targets=failed_exp_stage.addressing,
+        name="failed",
+    )
+    name = workspace_queue._EXEC_NAME or queue_entry.stash_rev
+
+    infofile = workspace_queue.get_infofile_path(name)
+    with pytest.raises(DvcException):
+        workspace_queue.reproduce()
+    if queue_type == "workspace_queue":
+        assert not os.path.exists(infofile)
+    else:
+        executor_info = ExecutorInfo.load_json(infofile)
+        assert executor_info.status == TaskStatus.FAILED
+
+
+def test_executor_status_compatiblity():
+    data = {
+        "git_url": "file:///Users/home",
+        "baseline_rev": "123",
+        "location": "dvc-task",
+        "root_dir": "/Users/home/8088/.dvc/tmp/exps/tmpx85892cx",
+        "dvc_dir": ".dvc",
+        "collected": True,
+    }
+    result = ExecutorInfo.from_dict(data)
+    assert result.status == TaskStatus.FINISHED

tests/unit/test_analytics.py

@@ -14,7 +14,11 @@ def tmp_global_dir(mocker, tmp_path):
     """
     Fixture to prevent modifying the actual global config
     """
-    mocker.patch("dvc.config.Config.get_dir", return_value=str(tmp_path))
+
+    def _user_config_dir(appname, *_args, **_kwargs):
+        return str(tmp_path / appname)
+
+    mocker.patch("iterative_telemetry.user_config_dir", _user_config_dir)
 
 
 def test_collect_and_send_report(mocker, tmp_global_dir):
@@ -155,10 +159,3 @@ def test_system_info():
         )
 
     assert schema(analytics._system_info())
-
-
-def test_find_or_create_user_id(tmp_global_dir):
-    created = analytics._find_or_create_user_id()
-    found = analytics._find_or_create_user_id()
-
-    assert created == found

tests/unit/test_config.py

@@ -3,7 +3,7 @@ import textwrap
 
 import pytest
 
-from dvc.config import Config
+from dvc.config import Config, ConfigError
 from dvc.fs import LocalFileSystem
 
 
@@ -70,3 +70,14 @@ def test_s3_ssl_verify(tmp_dir, dvc):
             ssl_verify = /path/to/custom/cabundle.pem
         """
     )
+
+
+def test_load_unicode_error(tmp_dir, dvc, mocker):
+    config = Config(validate=False)
+    mocker.patch(
+        "configobj.ConfigObj",
+        side_effect=UnicodeDecodeError("", b"", 0, 0, ""),
+    )
+    with pytest.raises(ConfigError):
+        with config.edit():
+            pass