deps: explicitly specify pyarrow for hdfs

Bumps [mypy](https://github.com/python/mypy) from 0.920 to 0.921.
- [Release notes](https://github.com/python/mypy/releases)
- [Commits](https://github.com/python/mypy/compare/v0.920...v0.921)

---
updated-dependencies:
- dependency-name: mypy
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>

.github/mergify.yml

@@ -1,7 +1,7 @@
 pull_request_rules:
   - name: backport patches to 1.11
     conditions:
-      - base=master
+      - base=main
       - label=backport-1.11
     actions:
       backport:

.github/workflows/benchmarks.yaml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/setup-python@v2
         with:
             python-version: 3.7
-      - uses: iterative/dvc-bench@master
+      - uses: iterative/dvc-bench@main
         with:
             pytest_options: "-k 'test_init or test_help'"
 

.github/workflows/packages.yaml

@@ -2,7 +2,7 @@ name: Build packages
 on:
   push:
     branches:
-    - master
+    - main
   release:
     types: [released, prereleased]
   workflow_dispatch:

.github/workflows/tests.yaml

@@ -83,7 +83,7 @@ jobs:
         file: ./coverage.xml
         fail_ci_if_error: false
   notify:
-    if: github.ref == 'refs/heads/master' && failure()
+    if: github.ref == 'refs/heads/main' && failure()
     needs:
       - lint
       - tests
@@ -94,6 +94,6 @@ jobs:
       env:
         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         SLACK_COLOR: ${{ job.status }}
-        SLACK_MESSAGE: 'CI Failed on master :boom:'
+        SLACK_MESSAGE: 'CI Failed on main :boom:'
         SLACK_TITLE: CI Status
         SLACK_USERNAME: DVC-CI

README.rst

@@ -103,7 +103,7 @@ Snap (Snapcraft/Linux)
 
 This corresponds to the latest tagged release.
 Add ``--beta`` for the latest tagged release candidate,
-or ``--edge`` for the latest ``master`` version.
+or ``--edge`` for the latest ``main`` version.
 
 Choco (Chocolatey/Windows)
 --------------------------
@@ -254,7 +254,7 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://dvc.org
    :alt: DVC logo
 
-.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=master
+.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=main
    :target: https://github.com/iterative/dvc/actions
    :alt: GHA Tests
 
@@ -262,7 +262,7 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://codeclimate.com/github/iterative/dvc
    :alt: Code Climate
 
-.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
+.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/main/graph/badge.svg
    :target: https://codecov.io/gh/iterative/dvc
    :alt: Codecov
 

dvc/command/experiments/__init__.py

@@ -0,0 +1,53 @@
+import argparse
+
+from dvc.command.base import append_doc_link, fix_subparsers
+from dvc.command.experiments import (
+    apply,
+    branch,
+    diff,
+    gc,
+    init,
+    ls,
+    pull,
+    push,
+    remove,
+    run,
+    show,
+)
+
+SUB_COMMANDS = [
+    apply,
+    branch,
+    diff,
+    gc,
+    init,
+    ls,
+    pull,
+    push,
+    remove,
+    run,
+    show,
+]
+
+
+def add_parser(subparsers, parent_parser):
+    EXPERIMENTS_HELP = "Commands to run and compare experiments."
+
+    experiments_parser = subparsers.add_parser(
+        "experiments",
+        parents=[parent_parser],
+        aliases=["exp"],
+        description=append_doc_link(EXPERIMENTS_HELP, "exp"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        help=EXPERIMENTS_HELP,
+    )
+
+    experiments_subparsers = experiments_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc experiments CMD --help` to display "
+        "command-specific help.",
+    )
+
+    fix_subparsers(experiments_subparsers)
+    for cmd in SUB_COMMANDS:
+        cmd.add_parser(experiments_subparsers, parent_parser)

dvc/command/experiments/apply.py

@@ -0,0 +1,41 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsApply(CmdBase):
+    def run(self):
+
+        self.repo.experiments.apply(
+            self.args.experiment, force=self.args.force
+        )
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_APPLY_HELP = (
+        "Apply the changes from an experiment to your workspace."
+    )
+    experiments_apply_parser = experiments_subparsers.add_parser(
+        "apply",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_APPLY_HELP, "exp/apply"),
+        help=EXPERIMENTS_APPLY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_apply_parser.add_argument(
+        "--no-force",
+        action="store_false",
+        dest="force",
+        help="Fail if this command would overwrite conflicting changes.",
+    )
+    experiments_apply_parser.add_argument(
+        "experiment", help="Experiment to be applied."
+    ).complete = completion.EXPERIMENT
+    experiments_apply_parser.set_defaults(func=CmdExperimentsApply)

dvc/command/experiments/branch.py

@@ -0,0 +1,33 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsBranch(CmdBase):
+    def run(self):
+
+        self.repo.experiments.branch(self.args.experiment, self.args.branch)
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_BRANCH_HELP = "Promote an experiment to a Git branch."
+    experiments_branch_parser = experiments_subparsers.add_parser(
+        "branch",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_BRANCH_HELP, "exp/branch"),
+        help=EXPERIMENTS_BRANCH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_branch_parser.add_argument(
+        "experiment", help="Experiment to be promoted."
+    )
+    experiments_branch_parser.add_argument(
+        "branch", help="Git branch name to use."
+    )
+    experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)

dvc/command/experiments/diff.py

@@ -0,0 +1,117 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.metrics import DEFAULT_PRECISION
+from dvc.exceptions import DvcException
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsDiff(CmdBase):
+    def run(self):
+
+        try:
+            diff = self.repo.experiments.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                all=self.args.all,
+                param_deps=self.args.param_deps,
+            )
+        except DvcException:
+            logger.exception("failed to show experiments diff")
+            return 1
+
+        if self.args.json:
+            ui.write_json(diff)
+        else:
+            from dvc.compare import show_diff
+
+            precision = self.args.precision or DEFAULT_PRECISION
+            diffs = [("metrics", "Metric"), ("params", "Param")]
+            for idx, (key, title) in enumerate(diffs):
+                if idx:
+                    # we are printing tables even in `--quiet` mode
+                    # so we should also be printing the "table" separator
+                    ui.write(force=True)
+
+                show_diff(
+                    diff[key],
+                    title=title,
+                    markdown=self.args.markdown,
+                    no_path=self.args.no_path,
+                    on_empty_diff="diff not supported",
+                    precision=precision if key == "metrics" else None,
+                    a_rev=self.args.a_rev,
+                    b_rev=self.args.b_rev,
+                )
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_DIFF_HELP = (
+        "Show changes between experiments in the DVC repository."
+    )
+
+    experiments_diff_parser = experiments_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_DIFF_HELP, "exp/diff"),
+        help=EXPERIMENTS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_diff_parser.add_argument(
+        "a_rev", nargs="?", help="Old experiment to compare (defaults to HEAD)"
+    ).complete = completion.EXPERIMENT
+    experiments_diff_parser.add_argument(
+        "b_rev",
+        nargs="?",
+        help="New experiment to compare (defaults to the current workspace)",
+    ).complete = completion.EXPERIMENT
+    experiments_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged metrics/params as well.",
+    )
+    experiments_diff_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
+    experiments_diff_parser.add_argument(
+        "--json",
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    experiments_diff_parser.add_argument(
+        "--md",
+        "--show-md",
+        action="store_true",
+        default=False,
+        dest="markdown",
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    experiments_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show metric/param path.",
+    )
+    experiments_diff_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics/params to `n` digits precision after the decimal "
+            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
+    experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)

dvc/command/experiments/gc.py

@@ -0,0 +1,128 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+def _raise_error_if_all_disabled(**kwargs):
+    if not any(kwargs.values()):
+        raise InvalidArgumentError(
+            "Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags` "
+            "or `--all-commits` needs to be set."
+        )
+
+
+class CmdExperimentsGC(CmdBase):
+    def run(self):
+        _raise_error_if_all_disabled(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+        )
+
+        msg = "This will remove all experiments except those derived from "
+
+        msg += "the workspace"
+        if self.args.all_commits:
+            msg += " and all git commits"
+        elif self.args.all_branches and self.args.all_tags:
+            msg += " and all git branches and tags"
+        elif self.args.all_branches:
+            msg += " and all git branches"
+        elif self.args.all_tags:
+            msg += " and all git tags"
+        msg += " of the current repo."
+        if self.args.queued:
+            msg += " Run queued experiments will be preserved."
+        else:
+            msg += " Run queued experiments will be removed."
+
+        logger.warning(msg)
+
+        msg = "Are you sure you want to proceed?"
+        if not self.args.force and not ui.confirm(msg):
+            return 1
+
+        removed = self.repo.experiments.gc(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+            queued=self.args.queued,
+        )
+
+        if removed:
+            ui.write(
+                f"Removed {removed} experiments.",
+                "To remove unused cache files",
+                "use 'dvc gc'.",
+            )
+        else:
+            ui.write("No experiments to remove.")
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments."
+    EXPERIMENTS_GC_DESCRIPTION = (
+        "Removes all experiments which are not derived from the specified"
+        "Git revisions."
+    )
+    experiments_gc_parser = experiments_subparsers.add_parser(
+        "gc",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_GC_DESCRIPTION, "exp/gc"),
+        help=EXPERIMENTS_GC_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_gc_parser.add_argument(
+        "-w",
+        "--workspace",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from the current workspace.",
+    )
+    experiments_gc_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from the tips of all Git branches.",
+    )
+    experiments_gc_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from all Git tags.",
+    )
+    experiments_gc_parser.add_argument(
+        "-A",
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from all Git commits.",
+    )
+    experiments_gc_parser.add_argument(
+        "--queued",
+        action="store_true",
+        default=False,
+        help=(
+            "Keep queued experiments (experiments run queue will be cleared "
+            "by default)."
+        ),
+    )
+    experiments_gc_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force garbage collection - automatically agree to all prompts.",
+    )
+    experiments_gc_parser.set_defaults(func=CmdExperimentsGC)

dvc/command/experiments/init.py

@@ -0,0 +1,187 @@
+import argparse
+import logging
+
+from funcy import compact
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsInit(CmdBase):
+    DEFAULT_NAME = "train"
+    CODE = "src"
+    DATA = "data"
+    MODELS = "models"
+    DEFAULT_METRICS = "metrics.json"
+    DEFAULT_PARAMS = "params.yaml"
+    PLOTS = "plots"
+    DVCLIVE = "dvclive"
+    DEFAULTS = {
+        "code": CODE,
+        "data": DATA,
+        "models": MODELS,
+        "metrics": DEFAULT_METRICS,
+        "params": DEFAULT_PARAMS,
+        "plots": PLOTS,
+        "live": DVCLIVE,
+    }
+    EXP_LINK = "https://s.dvc.org/g/exp/run"
+
+    def run(self):
+        from dvc.command.stage import parse_cmd
+
+        cmd = parse_cmd(self.args.command)
+        if not self.args.interactive and not cmd:
+            raise InvalidArgumentError("command is not specified")
+
+        from dvc.repo.experiments.init import init
+
+        defaults = {}
+        if not self.args.explicit:
+            config = self.repo.config["exp"]
+            defaults.update({**self.DEFAULTS, **config})
+
+        cli_args = compact(
+            {
+                "cmd": cmd,
+                "code": self.args.code,
+                "data": self.args.data,
+                "models": self.args.models,
+                "metrics": self.args.metrics,
+                "params": self.args.params,
+                "plots": self.args.plots,
+                "live": self.args.live,
+            }
+        )
+
+        initialized_stage = init(
+            self.repo,
+            name=self.args.name,
+            type=self.args.type,
+            defaults=defaults,
+            overrides=cli_args,
+            interactive=self.args.interactive,
+            force=self.args.force,
+        )
+
+        text = ui.rich_text.assemble(
+            "\n" if self.args.interactive else "",
+            "Created ",
+            (self.args.name, "bright_blue"),
+            " stage in ",
+            ("dvc.yaml", "green"),
+            ".",
+        )
+        if not self.args.run:
+            text.append_text(
+                ui.rich_text.assemble(
+                    " To run, use ",
+                    ('"dvc exp run"', "green"),
+                    ".\nSee ",
+                    (self.EXP_LINK, "repr.url"),
+                    ".",
+                )
+            )
+
+        ui.write(text, styled=True)
+        if self.args.run:
+            return self.repo.experiments.run(
+                targets=[initialized_stage.addressing]
+            )
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_INIT_HELP = "Initialize experiments."
+    experiments_init_parser = experiments_subparsers.add_parser(
+        "init",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_INIT_HELP, "exp/init"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        help=EXPERIMENTS_INIT_HELP,
+    )
+    experiments_init_parser.add_argument(
+        "command",
+        nargs=argparse.REMAINDER,
+        help="Command to execute.",
+        metavar="command",
+    )
+    experiments_init_parser.add_argument(
+        "--run",
+        action="store_true",
+        help="Run the experiment after initializing it",
+    )
+    experiments_init_parser.add_argument(
+        "--interactive",
+        "-i",
+        action="store_true",
+        help="Prompt for values that are not provided",
+    )
+    experiments_init_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Overwrite existing stage",
+    )
+    experiments_init_parser.add_argument(
+        "--explicit",
+        action="store_true",
+        default=False,
+        help="Only use the path values explicitly provided",
+    )
+    experiments_init_parser.add_argument(
+        "--name",
+        "-n",
+        help="Name of the stage to create (default: %(default)s)",
+        default=CmdExperimentsInit.DEFAULT_NAME,
+    )
+    experiments_init_parser.add_argument(
+        "--code",
+        help="Path to the source file or directory "
+        "which your experiments depend"
+        f" (default: {CmdExperimentsInit.CODE})",
+    )
+    experiments_init_parser.add_argument(
+        "--data",
+        help="Path to the data file or directory "
+        "which your experiments depend"
+        f" (default: {CmdExperimentsInit.DATA})",
+    )
+    experiments_init_parser.add_argument(
+        "--models",
+        help="Path to the model file or directory for your experiments"
+        f" (default: {CmdExperimentsInit.MODELS})",
+    )
+    experiments_init_parser.add_argument(
+        "--params",
+        help="Path to the parameters file for your experiments"
+        f" (default: {CmdExperimentsInit.DEFAULT_PARAMS})",
+    )
+    experiments_init_parser.add_argument(
+        "--metrics",
+        help="Path to the metrics file for your experiments"
+        f" (default: {CmdExperimentsInit.DEFAULT_METRICS})",
+    )
+    experiments_init_parser.add_argument(
+        "--plots",
+        help="Path to the plots file or directory for your experiments"
+        f" (default: {CmdExperimentsInit.PLOTS})",
+    )
+    experiments_init_parser.add_argument(
+        "--live",
+        help="Path to log dvclive outputs for your experiments"
+        f" (default: {CmdExperimentsInit.DVCLIVE})",
+    )
+    experiments_init_parser.add_argument(
+        "--type",
+        choices=["default", "dl"],
+        default="default",
+        help="Select type of stage to create (default: %(default)s)",
+    )
+    experiments_init_parser.set_defaults(func=CmdExperimentsInit)

dvc/command/experiments/ls.py

@@ -0,0 +1,72 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsList(CmdBase):
+    def run(self):
+        names_only = self.args.names_only
+        exps = self.repo.experiments.ls(
+            rev=self.args.rev,
+            git_remote=self.args.git_remote,
+            all_=self.args.all,
+        )
+        for baseline in exps:
+            tag = self.repo.scm.describe(baseline)
+            if not tag:
+                branch = self.repo.scm.describe(baseline, base="refs/heads")
+                if branch:
+                    tag = branch.split("/")[-1]
+            name = tag if tag else baseline[:7]
+            if not names_only:
+                print(f"{name}:")
+            for exp_name in exps[baseline]:
+                indent = "" if names_only else "\t"
+                print(f"{indent}{exp_name}")
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_LIST_HELP = "List local and remote experiments."
+    experiments_list_parser = experiments_subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_LIST_HELP, "exp/list"),
+        help=EXPERIMENTS_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_list_parser.add_argument(
+        "--rev",
+        type=str,
+        default=None,
+        help=(
+            "List experiments derived from the specified revision. "
+            "Defaults to HEAD if neither `--rev` nor `--all` are specified."
+        ),
+        metavar="<rev>",
+    )
+    experiments_list_parser.add_argument(
+        "--all", action="store_true", help="List all experiments."
+    )
+    experiments_list_parser.add_argument(
+        "--names-only",
+        action="store_true",
+        help="Only output experiment names (without parent commits).",
+    )
+    experiments_list_parser.add_argument(
+        "git_remote",
+        nargs="?",
+        default=None,
+        help=(
+            "Optional Git remote name or Git URL. "
+            "If provided, experiments from the specified Git repository "
+            " will be listed instead of local ones."
+        ),
+        metavar="[<git_remote>]",
+    )
+    experiments_list_parser.set_defaults(func=CmdExperimentsList)

dvc/command/experiments/pull.py

@@ -0,0 +1,92 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsPull(CmdBase):
+    def run(self):
+
+        self.repo.experiments.pull(
+            self.args.git_remote,
+            self.args.experiment,
+            force=self.args.force,
+            pull_cache=self.args.pull_cache,
+            dvc_remote=self.args.dvc_remote,
+            jobs=self.args.jobs,
+            run_cache=self.args.run_cache,
+        )
+
+        ui.write(
+            f"Pulled experiment '{self.args.experiment}'",
+            f"from Git remote '{self.args.git_remote}'.",
+        )
+        if not self.args.pull_cache:
+            ui.write(
+                "To pull cached outputs for this experiment"
+                "from DVC remote storage,"
+                "re-run this command without '--no-cache'."
+            )
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    EXPERIMENTS_PULL_HELP = "Pull an experiment from a Git remote."
+    experiments_pull_parser = experiments_subparsers.add_parser(
+        "pull",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_PULL_HELP, "exp/pull"),
+        help=EXPERIMENTS_PULL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_pull_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="Replace local experiment already exists.",
+    )
+    experiments_pull_parser.add_argument(
+        "--no-cache",
+        action="store_false",
+        dest="pull_cache",
+        help=(
+            "Do not pull cached outputs for this experiment from DVC remote "
+            "storage."
+        ),
+    )
+    experiments_pull_parser.add_argument(
+        "-r",
+        "--remote",
+        dest="dvc_remote",
+        metavar="<name>",
+        help="Name of the DVC remote to use when pulling cached outputs.",
+    )
+    experiments_pull_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        metavar="<number>",
+        help=(
+            "Number of jobs to run simultaneously when pulling from DVC "
+            "remote storage."
+        ),
+    )
+    experiments_pull_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help="Pull run history for all stages.",
+    )
+    experiments_pull_parser.add_argument(
+        "git_remote",
+        help="Git remote name or Git URL.",
+        metavar="<git_remote>",
+    )
+    experiments_pull_parser.add_argument(
+        "experiment", help="Experiment to pull.", metavar="<experiment>"
+    )
+    experiments_pull_parser.set_defaults(func=CmdExperimentsPull)

dvc/command/experiments/push.py

@@ -0,0 +1,93 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsPush(CmdBase):
+    def run(self):
+
+        self.repo.experiments.push(
+            self.args.git_remote,
+            self.args.experiment,
+            force=self.args.force,
+            push_cache=self.args.push_cache,
+            dvc_remote=self.args.dvc_remote,
+            jobs=self.args.jobs,
+            run_cache=self.args.run_cache,
+        )
+
+        ui.write(
+            f"Pushed experiment '{self.args.experiment}'"
+            f"to Git remote '{self.args.git_remote}'."
+        )
+        if not self.args.push_cache:
+            ui.write(
+                "To push cached outputs",
+                "for this experiment to DVC remote storage,"
+                "re-run this command without '--no-cache'.",
+            )
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    EXPERIMENTS_PUSH_HELP = "Push a local experiment to a Git remote."
+    experiments_push_parser = experiments_subparsers.add_parser(
+        "push",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_PUSH_HELP, "exp/push"),
+        help=EXPERIMENTS_PUSH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_push_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="Replace experiment in the Git remote if it already exists.",
+    )
+    experiments_push_parser.add_argument(
+        "--no-cache",
+        action="store_false",
+        dest="push_cache",
+        help=(
+            "Do not push cached outputs for this experiment to DVC remote "
+            "storage."
+        ),
+    )
+    experiments_push_parser.add_argument(
+        "-r",
+        "--remote",
+        dest="dvc_remote",
+        metavar="<name>",
+        help="Name of the DVC remote to use when pushing cached outputs.",
+    )
+    experiments_push_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        metavar="<number>",
+        help=(
+            "Number of jobs to run simultaneously when pushing to DVC remote "
+            "storage."
+        ),
+    )
+    experiments_push_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help="Push run history for all stages.",
+    )
+    experiments_push_parser.add_argument(
+        "git_remote",
+        help="Git remote name or Git URL.",
+        metavar="<git_remote>",
+    )
+    experiments_push_parser.add_argument(
+        "experiment", help="Experiment to push.", metavar="<experiment>"
+    ).complete = completion.EXPERIMENT
+    experiments_push_parser.set_defaults(func=CmdExperimentsPush)

dvc/command/experiments/remove.py

@@ -0,0 +1,54 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsRemove(CmdBase):
+    def run(self):
+
+        self.repo.experiments.remove(
+            exp_names=self.args.experiment,
+            queue=self.args.queue,
+            clear_all=self.args.all,
+            remote=self.args.git_remote,
+        )
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_REMOVE_HELP = "Remove experiments."
+    experiments_remove_parser = experiments_subparsers.add_parser(
+        "remove",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_REMOVE_HELP, "exp/remove"),
+        help=EXPERIMENTS_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remove_group = experiments_remove_parser.add_mutually_exclusive_group()
+    remove_group.add_argument(
+        "--queue", action="store_true", help="Remove all queued experiments."
+    )
+    remove_group.add_argument(
+        "-A",
+        "--all",
+        action="store_true",
+        help="Remove all committed experiments.",
+    )
+    remove_group.add_argument(
+        "-g",
+        "--git-remote",
+        metavar="<git_remote>",
+        help="Name or URL of the Git remote to remove the experiment from",
+    )
+    experiments_remove_parser.add_argument(
+        "experiment",
+        nargs="*",
+        help="Experiments to remove.",
+        metavar="<experiment>",
+    )
+    experiments_remove_parser.set_defaults(func=CmdExperimentsRemove)

dvc/command/experiments/run.py

@@ -0,0 +1,132 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import append_doc_link
+from dvc.command.repro import CmdRepro
+from dvc.command.repro import add_arguments as add_repro_arguments
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsRun(CmdRepro):
+    def run(self):
+        from dvc.compare import show_metrics
+
+        if self.args.checkpoint_resume:
+            if self.args.reset:
+                raise InvalidArgumentError(
+                    "--reset and --rev are mutually exclusive."
+                )
+            if not (self.args.queue or self.args.tmp_dir):
+                raise InvalidArgumentError(
+                    "--rev can only be used in conjunction with "
+                    "--queue or --temp."
+                )
+
+        if self.args.reset:
+            ui.write("Any existing checkpoints will be reset and re-run.")
+
+        results = self.repo.experiments.run(
+            name=self.args.name,
+            queue=self.args.queue,
+            run_all=self.args.run_all,
+            jobs=self.args.jobs,
+            params=self.args.set_param,
+            checkpoint_resume=self.args.checkpoint_resume,
+            reset=self.args.reset,
+            tmp_dir=self.args.tmp_dir,
+            **self._repro_kwargs,
+        )
+
+        if self.args.metrics and results:
+            metrics = self.repo.metrics.show(revs=list(results))
+            metrics.pop("workspace", None)
+            show_metrics(metrics)
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_RUN_HELP = "Run or resume an experiment."
+    experiments_run_parser = experiments_subparsers.add_parser(
+        "run",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_RUN_HELP, "exp/run"),
+        help=EXPERIMENTS_RUN_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    _add_run_common(experiments_run_parser)
+    experiments_run_parser.add_argument(
+        "-r",
+        "--rev",
+        type=str,
+        dest="checkpoint_resume",
+        help=(
+            "Continue the specified checkpoint experiment. Can only be used "
+            "in conjunction with --queue or --temp."
+        ),
+        metavar="<experiment_rev>",
+    ).complete = completion.EXPERIMENT
+    experiments_run_parser.add_argument(
+        "--reset",
+        action="store_true",
+        help="Reset existing checkpoints and restart the experiment.",
+    )
+    experiments_run_parser.set_defaults(func=CmdExperimentsRun)
+
+
+def _add_run_common(parser):
+    """Add common args for 'exp run' and 'exp resume'."""
+    # inherit arguments from `dvc repro`
+    add_repro_arguments(parser)
+    parser.add_argument(
+        "-n",
+        "--name",
+        default=None,
+        help=(
+            "Human-readable experiment name. If not specified, a name will "
+            "be auto-generated."
+        ),
+        metavar="<name>",
+    )
+    parser.add_argument(
+        "-S",
+        "--set-param",
+        action="append",
+        default=[],
+        help="Use the specified param value when reproducing pipelines.",
+        metavar="[<filename>:]<param_name>=<param_value>",
+    )
+    parser.add_argument(
+        "--queue",
+        action="store_true",
+        default=False,
+        help="Stage this experiment in the run queue for future execution.",
+    )
+    parser.add_argument(
+        "--run-all",
+        action="store_true",
+        default=False,
+        help="Execute all experiments in the run queue. Implies --temp.",
+    )
+    parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        default=1,
+        help="Run the specified number of experiments at a time in parallel.",
+        metavar="<number>",
+    )
+    parser.add_argument(
+        "--temp",
+        action="store_true",
+        dest="tmp_dir",
+        help=(
+            "Run this experiment in a separate temporary directory instead of "
+            "your workspace."
+        ),
+    )

dvc/command/experiments/show.py

@@ -0,0 +1,741 @@
+import argparse
+import logging
+from collections import Counter, OrderedDict, defaultdict
+from datetime import date, datetime
+from fnmatch import fnmatch
+from pathlib import Path
+from typing import TYPE_CHECKING, Dict, Iterable, Optional
+
+from funcy import lmap
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.metrics import DEFAULT_PRECISION
+from dvc.exceptions import DvcException, InvalidArgumentError
+from dvc.ui import ui
+from dvc.utils.flatten import flatten
+from dvc.utils.serialize import encode_exception
+
+if TYPE_CHECKING:
+    from dvc.compare import TabularData
+    from dvc.ui import RichText
+
+FILL_VALUE = "-"
+FILL_VALUE_ERRORED = "!"
+
+
+logger = logging.getLogger(__name__)
+
+
+def _filter_name(names, label, filter_strs):
+    ret = defaultdict(dict)
+    path_filters = defaultdict(list)
+
+    for filter_s in filter_strs:
+        path, _, name = filter_s.rpartition(":")
+        path_filters[path].append(name)
+
+    for path, filters in path_filters.items():
+        if path:
+            match_paths = [path]
+        else:
+            match_paths = names.keys()
+        for match_path in match_paths:
+            for f in filters:
+                matches = [
+                    name for name in names[match_path] if fnmatch(name, f)
+                ]
+                if not matches:
+                    raise InvalidArgumentError(
+                        f"'{f}' does not match any known {label}"
+                    )
+                ret[match_path].update({match: None for match in matches})
+
+    return ret
+
+
+def _filter_names(
+    names: Dict[str, Dict[str, None]],
+    label: str,
+    include: Optional[Iterable],
+    exclude: Optional[Iterable],
+):
+    if include and exclude:
+        intersection = set(include) & set(exclude)
+        if intersection:
+            values = ", ".join(intersection)
+            raise InvalidArgumentError(
+                f"'{values}' specified in both --include-{label} and"
+                f" --exclude-{label}"
+            )
+
+    if include:
+        ret = _filter_name(names, label, include)
+    else:
+        ret = names
+
+    if exclude:
+        to_remove = _filter_name(names, label, exclude)
+        for path in to_remove:
+            if path in ret:
+                for key in to_remove[path]:
+                    if key in ret[path]:
+                        del ret[path][key]
+
+    return ret
+
+
+def _update_names(names, items):
+    for name, item in items:
+        item = item.get("data", {})
+        if isinstance(item, dict):
+            item = flatten(item)
+            names[name].update({key: None for key in item})
+
+
+def _collect_names(all_experiments, **kwargs):
+    metric_names = defaultdict(dict)
+    param_names = defaultdict(dict)
+
+    for _, experiments in all_experiments.items():
+        for exp_data in experiments.values():
+            exp = exp_data.get("data", {})
+            _update_names(metric_names, exp.get("metrics", {}).items())
+            _update_names(param_names, exp.get("params", {}).items())
+    metric_names = _filter_names(
+        metric_names,
+        "metrics",
+        kwargs.get("include_metrics"),
+        kwargs.get("exclude_metrics"),
+    )
+    param_names = _filter_names(
+        param_names,
+        "params",
+        kwargs.get("include_params"),
+        kwargs.get("exclude_params"),
+    )
+
+    return metric_names, param_names
+
+
+experiment_types = {
+    "checkpoint_tip": "│ ╓",
+    "checkpoint_commit": "│ ╟",
+    "checkpoint_base": "├─╨",
+    "branch_commit": "├──",
+    "branch_base": "└──",
+    "baseline": "",
+}
+
+
+def _collect_rows(
+    base_rev,
+    experiments,
+    metric_names,
+    param_names,
+    precision=DEFAULT_PRECISION,
+    sort_by=None,
+    sort_order=None,
+    fill_value=FILL_VALUE,
+    iso=False,
+):
+    from scmrepo.git import Git
+
+    if sort_by:
+        sort_path, sort_name, sort_type = _sort_column(
+            sort_by, metric_names, param_names
+        )
+        reverse = sort_order == "desc"
+        experiments = _sort_exp(
+            experiments, sort_path, sort_name, sort_type, reverse
+        )
+
+    new_checkpoint = True
+    for i, (rev, results) in enumerate(experiments.items()):
+        exp = results.get("data", {})
+        if exp.get("running"):
+            state = "Running"
+        elif exp.get("queued"):
+            state = "Queued"
+        else:
+            state = fill_value
+        executor = exp.get("executor", fill_value)
+        is_baseline = rev == "baseline"
+
+        if is_baseline:
+            name_rev = base_rev[:7] if Git.is_sha(base_rev) else base_rev
+        else:
+            name_rev = rev[:7]
+
+        exp_name = exp.get("name", "")
+        tip = exp.get("checkpoint_tip")
+
+        parent_rev = exp.get("checkpoint_parent", "")
+        parent_exp = experiments.get(parent_rev, {}).get("data", {})
+        parent_tip = parent_exp.get("checkpoint_tip")
+
+        parent = ""
+        if is_baseline:
+            typ = "baseline"
+        elif tip:
+            if tip == parent_tip:
+                typ = (
+                    "checkpoint_tip" if new_checkpoint else "checkpoint_commit"
+                )
+            elif parent_rev == base_rev:
+                typ = "checkpoint_base"
+            else:
+                typ = "checkpoint_commit"
+                parent = parent_rev[:7]
+        elif i < len(experiments) - 1:
+            typ = "branch_commit"
+        else:
+            typ = "branch_base"
+
+        if not is_baseline:
+            new_checkpoint = not (tip and tip == parent_tip)
+
+        row = [
+            exp_name,
+            name_rev,
+            typ,
+            _format_time(exp.get("timestamp"), fill_value, iso),
+            parent,
+            state,
+            executor,
+        ]
+        fill_value = FILL_VALUE_ERRORED if results.get("error") else fill_value
+        _extend_row(
+            row,
+            metric_names,
+            exp.get("metrics", {}).items(),
+            precision,
+            fill_value=fill_value,
+        )
+        _extend_row(
+            row,
+            param_names,
+            exp.get("params", {}).items(),
+            precision,
+            fill_value=fill_value,
+        )
+
+        yield row
+
+
+def _sort_column(sort_by, metric_names, param_names):
+    path, _, sort_name = sort_by.rpartition(":")
+    matches = set()
+
+    if path:
+        if path in metric_names and sort_name in metric_names[path]:
+            matches.add((path, sort_name, "metrics"))
+        if path in param_names and sort_name in param_names[path]:
+            matches.add((path, sort_name, "params"))
+    else:
+        for path in metric_names:
+            if sort_name in metric_names[path]:
+                matches.add((path, sort_name, "metrics"))
+        for path in param_names:
+            if sort_name in param_names[path]:
+                matches.add((path, sort_name, "params"))
+
+    if len(matches) == 1:
+        return matches.pop()
+    if len(matches) > 1:
+        raise InvalidArgumentError(
+            "Ambiguous sort column '{}' matched '{}'".format(
+                sort_by,
+                ", ".join([f"{path}:{name}" for path, name, _ in matches]),
+            )
+        )
+    raise InvalidArgumentError(f"Unknown sort column '{sort_by}'")
+
+
+def _sort_exp(experiments, sort_path, sort_name, typ, reverse):
+    def _sort(item):
+        rev, exp = item
+        exp_data = exp.get("data", {})
+        tip = exp_data.get("checkpoint_tip")
+        if tip and tip != rev:
+            # Sort checkpoint experiments by tip commit
+            return _sort((tip, experiments[tip]))
+        data = exp_data.get(typ, {}).get(sort_path, {}).get("data", {})
+        val = flatten(data).get(sort_name)
+        return val is None, val
+
+    ret = OrderedDict()
+    if "baseline" in experiments:
+        ret["baseline"] = experiments.pop("baseline")
+
+    ret.update(sorted(experiments.items(), key=_sort, reverse=reverse))
+    return ret
+
+
+def _format_time(datetime_obj, fill_value=FILL_VALUE, iso=False):
+    if datetime_obj is None:
+        return fill_value
+
+    if iso:
+        return datetime_obj.isoformat()
+
+    if datetime_obj.date() == date.today():
+        fmt = "%I:%M %p"
+    else:
+        fmt = "%b %d, %Y"
+    return datetime_obj.strftime(fmt)
+
+
+def _extend_row(row, names, items, precision, fill_value=FILL_VALUE):
+    from dvc.compare import _format_field, with_value
+
+    if not items:
+        row.extend(fill_value for keys in names.values() for _ in keys)
+        return
+
+    for fname, data in items:
+        item = data.get("data", {})
+        item = flatten(item) if isinstance(item, dict) else {fname: item}
+        for name in names[fname]:
+            value = with_value(
+                item.get(name),
+                FILL_VALUE_ERRORED if data.get("error", None) else fill_value,
+            )
+            # wrap field data in ui.rich_text, otherwise rich may
+            # interpret unescaped braces from list/dict types as rich
+            # markup tags
+            row.append(ui.rich_text(str(_format_field(value, precision))))
+
+
+def _parse_filter_list(param_list):
+    ret = []
+    for param_str in param_list:
+        path, _, param_str = param_str.rpartition(":")
+        if path:
+            ret.extend(f"{path}:{param}" for param in param_str.split(","))
+        else:
+            ret.extend(param_str.split(","))
+    return ret
+
+
+def experiments_table(
+    all_experiments,
+    headers,
+    metric_headers,
+    metric_names,
+    param_headers,
+    param_names,
+    sort_by=None,
+    sort_order=None,
+    precision=DEFAULT_PRECISION,
+    fill_value=FILL_VALUE,
+    iso=False,
+) -> "TabularData":
+    from funcy import lconcat
+
+    from dvc.compare import TabularData
+
+    td = TabularData(
+        lconcat(headers, metric_headers, param_headers), fill_value=fill_value
+    )
+    for base_rev, experiments in all_experiments.items():
+        rows = _collect_rows(
+            base_rev,
+            experiments,
+            metric_names,
+            param_names,
+            sort_by=sort_by,
+            sort_order=sort_order,
+            precision=precision,
+            fill_value=fill_value,
+            iso=iso,
+        )
+        td.extend(rows)
+
+    return td
+
+
+def prepare_exp_id(kwargs) -> "RichText":
+    exp_name = kwargs["Experiment"]
+    rev = kwargs["rev"]
+    typ = kwargs.get("typ", "baseline")
+
+    if typ == "baseline" or not exp_name:
+        text = ui.rich_text(exp_name or rev)
+    else:
+        text = ui.rich_text.assemble(rev, " [", (exp_name, "bold"), "]")
+
+    parent = kwargs.get("parent")
+    suff = f" ({parent})" if parent else ""
+    text.append(suff)
+
+    tree = experiment_types[typ]
+    pref = f"{tree} " if tree else ""
+    return ui.rich_text(pref) + text
+
+
+def baseline_styler(typ):
+    return {"style": "bold"} if typ == "baseline" else {"style": "default"}
+
+
+def show_experiments(
+    all_experiments,
+    pager=True,
+    no_timestamp=False,
+    csv=False,
+    markdown=False,
+    html=False,
+    **kwargs,
+):
+    from funcy.seqs import flatten as flatten_list
+
+    include_metrics = _parse_filter_list(kwargs.pop("include_metrics", []))
+    exclude_metrics = _parse_filter_list(kwargs.pop("exclude_metrics", []))
+    include_params = _parse_filter_list(kwargs.pop("include_params", []))
+    exclude_params = _parse_filter_list(kwargs.pop("exclude_params", []))
+
+    metric_names, param_names = _collect_names(
+        all_experiments,
+        include_metrics=include_metrics,
+        exclude_metrics=exclude_metrics,
+        include_params=include_params,
+        exclude_params=exclude_params,
+    )
+
+    headers = [
+        "Experiment",
+        "rev",
+        "typ",
+        "Created",
+        "parent",
+        "State",
+        "Executor",
+    ]
+
+    names = {**metric_names, **param_names}
+    counter = Counter(flatten_list([list(a.keys()) for a in names.values()]))
+    counter.update(headers)
+    metric_headers = _normalize_headers(metric_names, counter)
+    param_headers = _normalize_headers(param_names, counter)
+
+    td = experiments_table(
+        all_experiments,
+        headers,
+        metric_headers,
+        metric_names,
+        param_headers,
+        param_names,
+        kwargs.get("sort_by"),
+        kwargs.get("sort_order"),
+        kwargs.get("precision"),
+        kwargs.get("fill_value"),
+        kwargs.get("iso"),
+    )
+
+    if no_timestamp or html:
+        td.drop("Created")
+
+    for col in ("State", "Executor"):
+        if td.is_empty(col):
+            td.drop(col)
+
+    row_styles = lmap(baseline_styler, td.column("typ"))
+    for n, row_style in enumerate(row_styles):
+        if n % 2 != 0:
+            row_style["style"] += " on grey23"
+
+    if not csv:
+        merge_headers = ["Experiment", "rev", "typ", "parent"]
+        td.column("Experiment")[:] = map(
+            prepare_exp_id, td.as_dict(merge_headers)
+        )
+        td.drop(*merge_headers[1:])
+
+    headers = {"metrics": metric_headers, "params": param_headers}
+    styles = {
+        "Experiment": {"no_wrap": True, "header_style": "black on grey93"},
+        "Created": {"header_style": "black on grey93"},
+        "State": {"header_style": "black on grey93"},
+        "Executor": {"header_style": "black on grey93"},
+    }
+    header_bg_colors = {"metrics": "cornsilk1", "params": "light_cyan1"}
+    styles.update(
+        {
+            header: {
+                "justify": "right" if typ == "metrics" else "left",
+                "header_style": f"black on {header_bg_colors[typ]}",
+                "collapse": idx != 0,
+                "no_wrap": typ == "metrics",
+            }
+            for typ, hs in headers.items()
+            for idx, header in enumerate(hs)
+        }
+    )
+
+    if kwargs.get("only_changed", False) or html:
+        td.drop_duplicates("cols")
+
+    html_args = {}
+    if html:
+        td.dropna("rows", how="all")
+        td.column("Experiment")[:] = [
+            # remove tree characters
+            str(x).encode("ascii", "ignore").strip().decode()
+            for x in td.column("Experiment")
+        ]
+        out = kwargs.get("out") or "dvc_plots"
+        html_args["output_path"] = (Path.cwd() / out).resolve()
+        html_args["color_by"] = kwargs.get("sort_by") or "Experiment"
+
+    td.render(
+        pager=pager,
+        borders="horizontals",
+        rich_table=True,
+        header_styles=styles,
+        row_styles=row_styles,
+        csv=csv,
+        markdown=markdown,
+        html=html,
+        **html_args,
+    )
+
+    if html and kwargs.get("open"):
+        return ui.open_browser(Path(out) / "index.html")
+
+
+def _normalize_headers(names, count):
+    return [
+        name if count[name] == 1 else f"{path}:{name}"
+        for path in names
+        for name in names[path]
+    ]
+
+
+def _format_json(item):
+    if isinstance(item, (date, datetime)):
+        return item.isoformat()
+    return encode_exception(item)
+
+
+def _raise_error_if_all_disabled(**kwargs):
+    if not any(kwargs.values()):
+        raise InvalidArgumentError(
+            "Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags` "
+            "or `--all-commits` needs to be set."
+        )
+
+
+class CmdExperimentsShow(CmdBase):
+    def run(self):
+        try:
+            all_experiments = self.repo.experiments.show(
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+                sha_only=self.args.sha,
+                num=self.args.num,
+                param_deps=self.args.param_deps,
+            )
+        except DvcException:
+            logger.exception("failed to show experiments")
+            return 1
+
+        if self.args.json:
+            ui.write_json(all_experiments, default=_format_json)
+        else:
+            precision = (
+                self.args.precision or None
+                if self.args.csv
+                else DEFAULT_PRECISION
+            )
+            fill_value = "" if self.args.csv else FILL_VALUE
+            iso = True if self.args.csv else False
+
+            show_experiments(
+                all_experiments,
+                include_metrics=self.args.include_metrics,
+                exclude_metrics=self.args.exclude_metrics,
+                include_params=self.args.include_params,
+                exclude_params=self.args.exclude_params,
+                no_timestamp=self.args.no_timestamp,
+                sort_by=self.args.sort_by,
+                sort_order=self.args.sort_order,
+                precision=precision,
+                fill_value=fill_value,
+                iso=iso,
+                pager=not self.args.no_pager,
+                csv=self.args.csv,
+                markdown=self.args.markdown,
+                only_changed=self.args.only_changed,
+                html=self.args.html,
+                out=self.args.out,
+                open=self.args.open,
+            )
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    EXPERIMENTS_SHOW_HELP = "Print experiments."
+    experiments_show_parser = experiments_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_SHOW_HELP, "exp/show"),
+        help=EXPERIMENTS_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_show_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Show experiments derived from the tip of all Git branches.",
+    )
+    experiments_show_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Show experiments derived from all Git tags.",
+    )
+    experiments_show_parser.add_argument(
+        "-A",
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show experiments derived from all Git commits.",
+    )
+    experiments_show_parser.add_argument(
+        "-n",
+        "--num",
+        type=int,
+        default=1,
+        dest="num",
+        metavar="<num>",
+        help="Show the last `num` commits from HEAD.",
+    )
+    experiments_show_parser.add_argument(
+        "--no-pager",
+        action="store_true",
+        default=False,
+        help="Do not pipe output into a pager.",
+    )
+    experiments_show_parser.add_argument(
+        "--include-metrics",
+        action="append",
+        default=[],
+        help="Include the specified metrics in output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-metrics",
+        action="append",
+        default=[],
+        help="Exclude the specified metrics from output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--include-params",
+        action="append",
+        default=[],
+        help="Include the specified params in output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-params",
+        action="append",
+        default=[],
+        help="Exclude the specified params from output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
+    experiments_show_parser.add_argument(
+        "--sort-by",
+        help="Sort related experiments by the specified metric or param.",
+        metavar="<metric/param>",
+    )
+    experiments_show_parser.add_argument(
+        "--sort-order",
+        help=(
+            "Sort order to use with --sort-by."
+            " Defaults to ascending ('asc')."
+        ),
+        choices=("asc", "desc"),
+        default="asc",
+    )
+    experiments_show_parser.add_argument(
+        "--no-timestamp",
+        action="store_true",
+        default=False,
+        help="Do not show experiment timestamps.",
+    )
+    experiments_show_parser.add_argument(
+        "--sha",
+        action="store_true",
+        default=False,
+        help="Always show git commit SHAs instead of branch/tag names.",
+    )
+    experiments_show_parser.add_argument(
+        "--json",
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Print output in JSON format instead of a human-readable table.",
+    )
+    experiments_show_parser.add_argument(
+        "--csv",
+        "--show-csv",
+        action="store_true",
+        default=False,
+        help="Print output in csv format instead of a human-readable table.",
+    )
+    experiments_show_parser.add_argument(
+        "--md",
+        "--show-md",
+        action="store_true",
+        default=False,
+        dest="markdown",
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    experiments_show_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics/params to `n` digits precision after the decimal "
+            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
+    experiments_show_parser.add_argument(
+        "--only-changed",
+        action="store_true",
+        default=False,
+        help=(
+            "Only show metrics/params with values varying "
+            "across the selected experiments."
+        ),
+    )
+    experiments_show_parser.add_argument(
+        "--html",
+        action="store_true",
+        default=False,
+        help="Generate a parallel coordinates plot from the tabulated output.",
+    )
+    experiments_show_parser.add_argument(
+        "-o",
+        "--out",
+        default=None,
+        help="Destination folder to save the HTML to",
+        metavar="<path>",
+    ).complete = completion.DIR
+    experiments_show_parser.add_argument(
+        "--open",
+        action="store_true",
+        default=False,
+        help="Open the HTML directly in the browser.",
+    )
+    experiments_show_parser.set_defaults(func=CmdExperimentsShow)

dvc/command/live.py

@@ -3,6 +3,7 @@ from pathlib import Path
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, fix_subparsers
+from dvc.render.utils import match_renderers
 from dvc.ui import ui
 
 
@@ -16,7 +17,8 @@ class CmdLive(CmdBase):
             html_path = Path.cwd() / (self.args.target + "_html")
             from dvc.render.utils import render
 
-            index_path = render(self.repo, plots, metrics, html_path)
+            renderers = match_renderers(plots, self.repo.plots.templates)
+            index_path = render(self.repo, renderers, metrics, html_path)
             ui.write(index_path.as_uri())
             return 0
         return 1

dvc/command/plots.py

@@ -1,16 +1,32 @@
 import argparse
+import json
 import logging
 
+from funcy import first
+
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.render.utils import find_vega, render
+from dvc.render.utils import match_renderers, render
+from dvc.render.vega import VegaRenderer
 from dvc.ui import ui
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
 
+def _show_json(renderers, path: None):
+    if any(lambda r: r.needs_output_path for r in renderers) and not path:
+        raise DvcException("Output path ('-o') is required!")
+
+    result = {
+        renderer.filename: json.loads(renderer.as_json(path=path))
+        for renderer in renderers
+    }
+    if result:
+        ui.write_json(result)
+
+
 class CmdPlots(CmdBase):
     def _func(self, *args, **kwargs):
         raise NotImplementedError
@@ -34,8 +50,15 @@ class CmdPlots(CmdBase):
                     "you can only specify one target for `--show-vega`"
                 )
                 return 1
+            if self.args.json:
+                logger.error(
+                    "'--show-vega' and '--json' are mutually exclusive "
+                    "options."
+                )
+                return 1
 
         try:
+
             plots_data = self._func(
                 targets=self.args.targets, props=self._props()
             )
@@ -46,18 +69,26 @@ class CmdPlots(CmdBase):
                     "visualization file will not be created."
                 )
 
+            renderers = match_renderers(
+                plots_data=plots_data, templates=self.repo.plots.templates
+            )
+
             if self.args.show_vega:
-                target = self.args.targets[0]
-                plot_json = find_vega(self.repo, plots_data, target)
-                if plot_json:
-                    ui.write(plot_json)
+                renderer = first(
+                    filter(lambda r: isinstance(r, VegaRenderer), renderers)
+                )
+                if renderer:
+                    ui.write_json(renderer.asdict())
+                return 0
+            if self.args.json:
+                _show_json(renderers, self.args.out)
                 return 0
 
             rel: str = self.args.out or "dvc_plots"
             path: Path = (Path.cwd() / rel).resolve()
             index_path = render(
                 self.repo,
-                plots_data,
+                renderers,
                 path=path,
                 html_template_path=self.args.html_template,
             )
@@ -251,6 +282,13 @@ def _add_output_arguments(parser):
         default=False,
         help="Show output in Vega format.",
     )
+    parser.add_argument(
+        "--json",
+        "--show-json",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     parser.add_argument(
         "--open",
         action="store_true",

dvc/compare.py

@@ -153,6 +153,18 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             writer.writerow(row)
         return buff.getvalue()
 
+    def to_parallel_coordinates(self, output_path, color_by):
+        from dvc.render.html import write
+        from dvc.render.plotly import ParallelCoordinatesRenderer
+
+        index_path = write(
+            output_path,
+            renderers=[
+                ParallelCoordinatesRenderer(self, color_by, self._fill_value)
+            ],
+        )
+        return index_path.as_uri()
+
     def add_column(self, name: str) -> None:
         self._columns[name] = Column([self._fill_value] * len(self))
         self._keys.append(name)
@@ -173,6 +185,14 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
 
         if kwargs.pop("csv", False):
             ui.write(self.to_csv(), end="")
+
+        elif kwargs.pop("html", False):
+            ui.write(
+                self.to_parallel_coordinates(
+                    kwargs["output_path"], kwargs.get("color_by")
+                )
+            )
+
         else:
             ui.table(self, headers=self.keys(), **kwargs)
 

dvc/fs/fsspec_wrapper.py

@@ -141,7 +141,9 @@ class FSSpecWrapper(FileSystem):
         self.makedirs(self.path.parent(to_info))
         with self.open(to_info, "wb") as fdest:
             shutil.copyfileobj(
-                fobj, fdest, length=getattr(fdest, "blocksize", None)
+                fobj,
+                fdest,
+                length=getattr(fdest, "blocksize", None),  # type: ignore
             )
 
 

dvc/fs/hdfs.py

@@ -27,9 +27,9 @@ class HDFSFileSystem(CallbackMixin, FSSpecWrapper):
 
     def unstrip_protocol(self, path: str) -> str:
         host = self.fs_args["host"]
-        port = self.fs_args["port"]
-        path = path.lstrip("/")
-        return f"hdfs://{host}:{port}/{path}"
+        port = self.fs_args.get("port")
+        netloc = host + (f":{port}" if port else "")
+        return "hdfs://" + netloc + "/" + path.lstrip("/")
 
     @staticmethod
     def _get_kwargs_from_urls(urlpath):

dvc/fs/path.py

@@ -80,12 +80,9 @@ class Path:
         # pylint: disable=arguments-out-of-order
         return self.isin_or_eq(left, right) or self.isin(right, left)
 
-    def relpath(self, path, base):
-        assert len(path) > len(base)
-        assert path.startswith(base)
-        normpath = path.rstrip(self.flavour.sep)
-        normbase = base.rstrip(self.flavour.sep)
-        return normpath[len(normbase) + 1 :]
+    def relpath(self, path, start):
+        assert start
+        return self.flavour.relpath(path, start=start)
 
     def relparts(self, path, base):
         return self.parts(self.relpath(path, base))

dvc/fs/repo.py

@@ -1,3 +1,4 @@
+import errno
 import logging
 import os
 import threading
@@ -475,7 +476,9 @@ class RepoFileSystem(FileSystem):  # pylint:disable=abstract-method
             info_result = fs.info(fs_path)
 
         if not info_result and not dvc_meta:
-            raise FileNotFoundError
+            raise FileNotFoundError(
+                errno.ENOENT, os.strerror(errno.ENOENT), fs_path
+            )
 
         from ._metadata import Metadata
 

dvc/render/base.py

@@ -16,6 +16,9 @@ class BadTemplateError(DvcException):
 
 
 class Renderer(abc.ABC):
+    REVISIONS_KEY = "revisions"
+    TYPE_KEY = "type"
+
     def __init__(self, data: Dict, templates=None):
         self.data = data
         self.templates = templates
@@ -28,7 +31,17 @@ class Renderer(abc.ABC):
         assert len(files) == 1
         self.filename = files.pop()
 
-    def _convert(self, path: "StrPath"):
+    def partial_html(self, **kwargs):
+        """
+        Us this method to generate partial HTML content,
+        that will fill self.DIV
+        """
+
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def TYPE(self):
         raise NotImplementedError
 
     @property
@@ -42,7 +55,7 @@ class Renderer(abc.ABC):
         raise NotImplementedError
 
     @abc.abstractmethod
-    def as_json(self):
+    def as_json(self, **kwargs):
         raise NotImplementedError
 
     @staticmethod
@@ -51,9 +64,13 @@ class Renderer(abc.ABC):
             {ord(c): "_" for c in r"!@#$%^&*()[]{};,<>?\/:.|`~=_+"}
         )
 
+    @property
+    def needs_output_path(self):
+        return False
+
     def generate_html(self, path: "StrPath"):
         """this method might edit content of path"""
-        partial = self._convert(path)
+        partial = self.partial_html(path=path)
 
         div_id = self._remove_special_chars(self.filename)
         div_id = f"plot_{div_id}"

dvc/render/image.py

@@ -1,6 +1,10 @@
+import json
 import os
 from typing import TYPE_CHECKING
 
+from funcy import reraise
+
+from dvc.exceptions import DvcException
 from dvc.render.base import Renderer
 from dvc.render.utils import get_files
 from dvc.utils import relpath
@@ -10,6 +14,7 @@ if TYPE_CHECKING:
 
 
 class ImageRenderer(Renderer):
+    TYPE = "image"
     DIV = """
         <div
             id="{id}"
@@ -20,49 +25,83 @@ class ImageRenderer(Renderer):
 
     SCRIPTS = ""
 
+    @property
+    def needs_output_path(self):
+        return True
+
     def _write_image(
         self,
-        page_dir_path: "StrPath",
+        path: "StrPath",
         revision: str,
         filename: str,
         image_data: bytes,
     ):
-        static = os.path.join(page_dir_path, "static")
-        os.makedirs(static, exist_ok=True)
-
         img_path = os.path.join(
-            static, f"{revision}_{filename.replace(os.sep, '_')}"
+            path, f"{revision}_{filename.replace(os.sep, '_')}"
         )
         with open(img_path, "wb") as fd:
             fd.write(image_data)
-        return """
+
+        return img_path
+
+    def _save_images(self, path: "StrPath"):
+
+        for rev, rev_data in self.data.items():
+            if "data" in rev_data:
+                for file, file_data in rev_data.get("data", {}).items():
+                    if "data" in file_data:
+                        if not os.path.isdir(path):
+                            os.makedirs(path, exist_ok=True)
+                        yield rev, file, self._write_image(
+                            os.path.abspath(path), rev, file, file_data["data"]
+                        )
+
+    def partial_html(self, **kwargs):
+        path = kwargs.get("path", None)
+        if not path:
+            raise DvcException("Can't save here")
+        static = os.path.join(path, "static")
+
+        div_content = []
+        for rev, _, img_path in self._save_images(static):
+            div_content.append(
+                """
         <div
             style="border:1px dotted black;margin:2px;display: inline-block;
             overflow:hidden;margin-left:8px;">
             <p>{title}</p>
             <img src="{src}">
         </div>""".format(
-            title=revision, src=(relpath(img_path, page_dir_path))
-        )
-
-    def _convert(self, path: "StrPath"):
-        div_content = []
-        for rev, rev_data in self.data.items():
-            if "data" in rev_data:
-                for file, file_data in rev_data.get("data", {}).items():
-                    if "data" in file_data:
-                        div_content.append(
-                            self._write_image(
-                                path, rev, file, file_data["data"]
-                            )
-                        )
+                    title=rev, src=(relpath(img_path, path))
+                )
+            )
         if div_content:
             div_content.insert(0, f"<p>{self.filename}</p>")
             return "\n".join(div_content)
         return ""
 
-    def as_json(self):
-        raise NotImplementedError
+    def as_json(self, **kwargs):
+
+        with reraise(
+            KeyError,
+            DvcException(
+                f"{type(self).__name__} needs 'path' to store images."
+            ),
+        ):
+            path = kwargs["path"]
+
+        results = []
+
+        for revision, _, img_path in self._save_images(path):
+            results.append(
+                {
+                    self.TYPE_KEY: self.TYPE,
+                    self.REVISIONS_KEY: [revision],
+                    "url": img_path,
+                }
+            )
+
+        return json.dumps(results)
 
     @staticmethod
     def matches(data):

dvc/render/plotly.py

@@ -0,0 +1,92 @@
+import json
+from collections import defaultdict
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+from dvc.render.base import Renderer
+
+if TYPE_CHECKING:
+    from dvc.compare import TabularData
+
+
+class ParallelCoordinatesRenderer(Renderer):
+    TYPE = "plotly"
+
+    DIV = """
+    <div id = "{id}">
+        <script type = "text/javascript">
+            var plotly_data = {partial};
+            Plotly.newPlot("{id}", plotly_data.data, plotly_data.layout);
+        </script>
+    </div>
+    """
+
+    SCRIPTS = """
+    <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
+    """
+
+    # pylint: disable=W0231
+    def __init__(
+        self,
+        tabular_data: "TabularData",
+        color_by: Optional[str] = None,
+        fill_value: str = "",
+    ):
+        self.tabular_data = tabular_data
+        self.color_by = color_by
+        self.filename = "experiments"
+        self.fill_value = fill_value
+
+    def partial_html(self, **kwargs):
+        return self.as_json()
+
+    def as_json(self, **kwargs) -> str:
+        tabular_dict = defaultdict(list)
+        for row in self.tabular_data.as_dict():
+            for col_name, value in row.items():
+                tabular_dict[col_name].append(str(value))
+
+        trace: Dict[str, Any] = {"type": "parcoords", "dimensions": []}
+        for label, values in tabular_dict.items():
+            is_categorical = False
+            try:
+                float_values = [
+                    float(x) if x != self.fill_value else None for x in values
+                ]
+            except ValueError:
+                is_categorical = True
+
+            if is_categorical:
+                non_missing = [x for x in values if x != self.fill_value]
+                unique_values = sorted(set(non_missing))
+                unique_values.append(self.fill_value)
+
+                dummy_values = [unique_values.index(x) for x in values]
+
+                values = [
+                    x if x != self.fill_value else "Missing" for x in values
+                ]
+                trace["dimensions"].append(
+                    {
+                        "label": label,
+                        "values": dummy_values,
+                        "tickvals": dummy_values,
+                        "ticktext": values,
+                    }
+                )
+            else:
+                trace["dimensions"].append(
+                    {"label": label, "values": float_values}
+                )
+
+            if label == self.color_by:
+                trace["line"] = {
+                    "color": dummy_values if is_categorical else float_values,
+                    "showscale": True,
+                    "colorbar": {"title": self.color_by},
+                }
+                if is_categorical:
+                    trace["line"]["colorbar"]["tickmode"] = "array"
+                    trace["line"]["colorbar"]["tickvals"] = dummy_values
+                    trace["line"]["colorbar"]["ticktext"] = values
+
+        return json.dumps({"data": [trace], "layout": {}})

dvc/render/utils.py

@@ -21,15 +21,6 @@ def group_by_filename(plots_data: Dict) -> List[Dict]:
     return grouped
 
 
-def find_vega(repo, plots_data, target):
-    found = dpath.util.search(plots_data, ["*", "*", target])
-    from dvc.render.vega import VegaRenderer
-
-    if found and VegaRenderer.matches(found):
-        return VegaRenderer(found, repo.plots.templates).as_json()
-    return ""
-
-
 def match_renderers(plots_data, templates):
     from dvc.render import RENDERERS
 
@@ -43,14 +34,12 @@ def match_renderers(plots_data, templates):
 
 def render(
     repo,
-    plots_data,
+    renderers,
     metrics=None,
     path=None,
     html_template_path=None,
     refresh_seconds=None,
 ):
-    # TODO we could probably remove repo usages (here and in VegaRenderer)
-    renderers = match_renderers(plots_data, repo.plots.templates)
     if not html_template_path:
         html_template_path = repo.config.get("plots", {}).get(
             "html_template", None

dvc/render/vega.py

@@ -1,6 +1,7 @@
+import json
 import os
 from copy import copy, deepcopy
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from typing import Dict, List, Optional, Union
 
 from funcy import first
 
@@ -13,9 +14,6 @@ from dvc.render.base import (
 )
 from dvc.render.utils import get_files
 
-if TYPE_CHECKING:
-    from dvc.types import StrPath
-
 
 class PlotDataStructureError(DvcException):
     def __init__(self):
@@ -92,6 +90,8 @@ def _append_revision(datapoints: List[Dict], revision) -> List[Dict]:
 
 
 class VegaRenderer(Renderer):
+    TYPE = "vega"
+
     DIV = """
     <div id = "{id}">
         <script type = "text/javascript">
@@ -115,6 +115,9 @@ class VegaRenderer(Renderer):
                 resolved = {**resolved, **props}
         return resolved
 
+    def _revisions(self):
+        return list(self.data.keys())
+
     def _datapoints(self, props: Dict):
         fields = props.get("fields", set())
         if fields:
@@ -139,7 +142,7 @@ class VegaRenderer(Renderer):
                     datapoints.extend(tmp)
         return datapoints
 
-    def fill_template(self, template, datapoints, props=None):
+    def _fill_template(self, template, datapoints, props=None):
         props = props or {}
 
         content = deepcopy(template.content)
@@ -168,7 +171,7 @@ class VegaRenderer(Renderer):
 
         return content
 
-    def as_json(self) -> Optional[str]:
+    def get_filled_template(self):
         props = self._squash_props()
 
         template = self.templates.load(props.get("template", None))
@@ -186,14 +189,33 @@ class VegaRenderer(Renderer):
                 props["y"] = first(
                     f for f in reversed(fields) if f not in skip
                 )
-            filled_template = self.fill_template(template, datapoints, props)
+            filled_template = self._fill_template(template, datapoints, props)
 
             return filled_template
-
         return None
 
-    def _convert(self, path: "StrPath"):
-        return self.as_json()
+    def asdict(self):
+        filled_template = self.get_filled_template()
+        if filled_template:
+            return json.loads(filled_template)
+        return {}
+
+    def as_json(self, **kwargs) -> Optional[str]:
+
+        content = self.asdict()
+
+        return json.dumps(
+            [
+                {
+                    self.TYPE_KEY: self.TYPE,
+                    self.REVISIONS_KEY: self._revisions(),
+                    "content": content,
+                }
+            ],
+        )
+
+    def partial_html(self, **kwargs):
+        return self.get_filled_template() or ""
 
     @staticmethod
     def matches(data):

dvc/repo/diff.py

@@ -170,13 +170,20 @@ def _output_paths(repo, targets):
 
 
 def _dir_output_paths(fs, fs_path, obj, targets=None):
+    if fs.scheme == "local":
+        # NOTE: workaround for filesystems that are based on full local paths
+        # (e.g. gitfs, dvcfs, repofs). Proper solution is to use upcoming
+        # fsspec's prefixfs to use relpaths as fs_paths.
+        base = fs.path.relpath(fs_path, os.getcwd())
+    else:
+        base = fs_path
     for key, _, oid in obj:
         fname = fs.path.join(fs_path, *key)
         if targets is None or any(
             fs.path.isin_or_eq(fname, target) for target in targets
         ):
             # pylint: disable=no-member
-            yield fs.path.join(fs.path.name(fs_path), *key), oid.value
+            yield fs.path.join(base, *key), oid.value
 
 
 def _filter_missing(repo_fs, paths):

dvc/repo/experiments/__init__.py

@@ -1,12 +1,8 @@
 import logging
 import os
 import re
-import signal
-from collections import defaultdict, namedtuple
-from concurrent.futures import CancelledError, ProcessPoolExecutor, wait
 from functools import wraps
-from multiprocessing import Manager
-from typing import Dict, Iterable, Mapping, Optional
+from typing import Dict, Iterable, List, Mapping, Optional, Type
 
 from funcy import cached_property, first
 
@@ -17,20 +13,28 @@ from dvc.utils import relpath
 
 from .base import (
     EXEC_APPLY,
-    EXEC_BASELINE,
     EXEC_BRANCH,
     EXEC_CHECKPOINT,
-    EXEC_HEAD,
-    EXEC_MERGE,
     EXEC_NAMESPACE,
     EXPS_NAMESPACE,
     EXPS_STASH,
     BaselineMismatchError,
-    CheckpointExistsError,
     ExperimentExistsError,
     ExpRefInfo,
+    ExpStashEntry,
     MultipleBranchError,
 )
+from .executor.base import (
+    EXEC_PID_DIR,
+    EXEC_TMP_DIR,
+    BaseExecutor,
+    ExecutorInfo,
+)
+from .executor.manager import (
+    BaseExecutorManager,
+    TempDirExecutorManager,
+    WorkspaceExecutorManager,
+)
 from .utils import exp_refs_by_rev
 
 logger = logging.getLogger(__name__)
@@ -81,12 +85,6 @@ class Experiments:
         r"^(?P<baseline_rev>[a-f0-9]{7})-(?P<exp_sha>[a-f0-9]+)"
         r"(?P<checkpoint>-checkpoint)?$"
     )
-    EXEC_TMP_DIR = "exps"
-    EXEC_PID_DIR = os.path.join(EXEC_TMP_DIR, "run")
-
-    StashEntry = namedtuple(
-        "StashEntry", ["index", "rev", "baseline_rev", "branch", "name"]
-    )
 
     def __init__(self, repo):
         from dvc.lock import make_lock
@@ -112,8 +110,6 @@ class Experiments:
 
     @cached_property
     def args_file(self):
-        from .executor.base import BaseExecutor
-
         return os.path.join(self.repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE)
 
     @cached_property
@@ -123,13 +119,13 @@ class Experiments:
         return Stash(self.scm, EXPS_STASH)
 
     @property
-    def stash_revs(self):
+    def stash_revs(self) -> Dict[str, ExpStashEntry]:
         revs = {}
         for i, entry in enumerate(self.stash):
             msg = entry.message.decode("utf-8").strip()
             m = self.STASH_EXPERIMENT_RE.match(msg)
             if m:
-                revs[entry.new_sha.decode("utf-8")] = self.StashEntry(
+                revs[entry.new_sha.decode("utf-8")] = ExpStashEntry(
                     i,
                     m.group("rev"),
                     m.group("baseline_rev"),
@@ -324,8 +320,6 @@ class Experiments:
     def _pack_args(self, *args, **kwargs):
         import pickle
 
-        from .executor.base import BaseExecutor
-
         if os.path.exists(self.args_file) and self.scm.is_tracked(
             self.args_file
         ):
@@ -433,9 +427,14 @@ class Experiments:
             )
             return [stash_rev]
         if tmp_dir or queue:
-            results = self._reproduce_revs(revs=[stash_rev], keep_stash=False)
+            manager_cls: Type = TempDirExecutorManager
         else:
-            results = self._workspace_repro()
+            manager_cls = WorkspaceExecutorManager
+        results = self._reproduce_revs(
+            revs=[stash_rev],
+            keep_stash=False,
+            manager_cls=manager_cls,
+        )
         exp_rev = first(results)
         if exp_rev is not None:
             self._log_reproduced(results, tmp_dir=tmp_dir)
@@ -589,6 +588,7 @@ class Experiments:
         self,
         revs: Optional[Iterable] = None,
         keep_stash: Optional[bool] = True,
+        manager_cls: Type = TempDirExecutorManager,
         **kwargs,
     ) -> Mapping[str, str]:
         """Reproduce the specified experiments.
@@ -616,7 +616,7 @@ class Experiments:
             to_run = {
                 rev: stash_revs[rev]
                 if rev in stash_revs
-                else self.StashEntry(None, rev, rev, None, None)
+                else ExpStashEntry(None, rev, rev, None, None)
                 for rev in revs
             }
 
@@ -625,17 +625,22 @@ class Experiments:
             ", ".join(rev[:7] for rev in to_run),
         )
 
-        executors = self._init_executors(to_run)
+        manager = manager_cls.from_stash_entries(
+            self.scm,
+            os.path.join(self.repo.tmp_dir, EXEC_TMP_DIR),
+            self.repo,
+            to_run,
+        )
         try:
             exec_results = {}
-            exec_results.update(self._executors_repro(executors, **kwargs))
+            exec_results.update(self._executors_repro(manager, **kwargs))
         finally:
             # only drop successfully run stashed experiments
-            to_drop = [
-                entry.index
+            to_drop: List[int] = [
+                entry.stash_index
                 for rev, entry in to_run.items()
                 if (
-                    entry.index is not None
+                    entry.stash_index is not None
                     and (not keep_stash or rev in exec_results)
                 )
             ]
@@ -647,47 +652,11 @@ class Experiments:
             result.update(exp_result)
         return result
 
-    def _init_executors(self, to_run):
-        from dvc.utils.fs import makedirs
-
-        from .executor.local import TempDirExecutor
-
-        executors = {}
-        base_tmp_dir = os.path.join(self.repo.tmp_dir, self.EXEC_TMP_DIR)
-        if not os.path.exists(base_tmp_dir):
-            makedirs(base_tmp_dir)
-        pid_dir = os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR)
-        if not os.path.exists(pid_dir):
-            makedirs(pid_dir)
-        for stash_rev, item in to_run.items():
-            self.scm.set_ref(EXEC_HEAD, item.rev)
-            self.scm.set_ref(EXEC_MERGE, stash_rev)
-            self.scm.set_ref(EXEC_BASELINE, item.baseline_rev)
-
-            # Executor will be initialized with an empty git repo that
-            # we populate by pushing:
-            #   EXEC_HEAD - the base commit for this experiment
-            #   EXEC_MERGE - the unmerged changes (from our stash)
-            #       to be reproduced
-            #   EXEC_BASELINE - the baseline commit for this experiment
-            executor = TempDirExecutor(
-                self.scm,
-                self.dvc_dir,
-                name=item.name,
-                branch=item.branch,
-                tmp_dir=base_tmp_dir,
-            )
-            executor.init_cache(self.repo, stash_rev)
-            executors[stash_rev] = executor
-
-        for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
-            self.scm.remove_ref(ref)
-
-        return executors
-
     @unlocked_repo
     def _executors_repro(
-        self, executors: dict, jobs: Optional[int] = 1
+        self,
+        manager: "BaseExecutorManager",
+        **kwargs,
     ) -> Dict[str, Dict[str, str]]:
         """Run dvc repro for the specified BaseExecutors in parallel.
 
@@ -695,174 +664,7 @@ class Experiments:
             dict mapping stash revs to the successfully executed experiments
             for each stash rev.
         """
-        from dvc.stage.monitor import CheckpointKilledError
-
-        result: Dict[str, Dict[str, str]] = defaultdict(dict)
-
-        manager = Manager()
-        pid_q = manager.Queue()
-
-        rel_cwd = relpath(os.getcwd(), self.repo.root_dir)
-        with ProcessPoolExecutor(max_workers=jobs) as workers:
-            futures = {}
-            for rev, executor in executors.items():
-                pidfile = os.path.join(
-                    self.repo.tmp_dir,
-                    self.EXEC_PID_DIR,
-                    f"{rev}{executor.PIDFILE_EXT}",
-                )
-                future = workers.submit(
-                    executor.reproduce,
-                    executor.dvc_dir,
-                    rev,
-                    queue=pid_q,
-                    name=executor.name,
-                    rel_cwd=rel_cwd,
-                    log_level=logger.getEffectiveLevel(),
-                    pidfile=pidfile,
-                    git_url=executor.git_url,
-                )
-                futures[future] = (rev, executor)
-
-            try:
-                wait(futures)
-            except KeyboardInterrupt:
-                # forward SIGINT to any running executor processes and
-                # cancel any remaining futures
-                workers.shutdown(wait=False)
-                pids = {}
-                for future, (rev, _) in futures.items():
-                    if future.running():
-                        # if future has already been started by the scheduler
-                        # we still have to wait until it tells us its PID
-                        while rev not in pids:
-                            rev, pid = pid_q.get()
-                            pids[rev] = pid
-                        os.kill(pids[rev], signal.SIGINT)
-                    elif not future.done():
-                        future.cancel()
-
-            for future, (rev, executor) in futures.items():
-                rev, executor = futures[future]
-
-                try:
-                    exc = future.exception()
-                    if exc is None:
-                        exec_result = future.result()
-                        result[rev].update(
-                            self._collect_executor(executor, exec_result)
-                        )
-                    elif not isinstance(exc, CheckpointKilledError):
-                        logger.error(
-                            "Failed to reproduce experiment '%s'", rev[:7]
-                        )
-                except CancelledError:
-                    logger.error(
-                        "Cancelled before attempting to reproduce experiment "
-                        "'%s'",
-                        rev[:7],
-                    )
-                finally:
-                    executor.cleanup()
-
-        return result
-
-    def _collect_executor(self, executor, exec_result) -> Mapping[str, str]:
-        # NOTE: GitPython Repo instances cannot be re-used
-        # after process has received SIGINT or SIGTERM, so we
-        # need this hack to re-instantiate git instances after
-        # checkpoint runs. See:
-        # https://github.com/gitpython-developers/GitPython/issues/427
-        del self.repo.scm
-
-        results = {}
-
-        def on_diverged(ref: str, checkpoint: bool):
-            ref_info = ExpRefInfo.from_ref(ref)
-            if checkpoint:
-                raise CheckpointExistsError(ref_info.name)
-            raise ExperimentExistsError(ref_info.name)
-
-        for ref in executor.fetch_exps(
-            self.scm,
-            executor.git_url,
-            force=exec_result.force,
-            on_diverged=on_diverged,
-        ):
-            exp_rev = self.scm.get_ref(ref)
-            if exp_rev:
-                logger.debug("Collected experiment '%s'.", exp_rev[:7])
-                results[exp_rev] = exec_result.exp_hash
-
-        return results
-
-    @unlocked_repo
-    def _workspace_repro(self) -> Mapping[str, str]:
-        """Run the most recently stashed experiment in the workspace."""
-        from dvc.stage.monitor import CheckpointKilledError
-        from dvc.utils.fs import makedirs
-
-        from .executor.base import BaseExecutor
-
-        entry = first(self.stash_revs.values())
-        assert entry.index == 0
-
-        # NOTE: the stash commit to be popped already contains all the current
-        # workspace changes plus CLI modified --params changes.
-        # `checkout --force` here will not lose any data (popping stash commit
-        # will result in conflict between workspace params and stashed CLI
-        # params, but we always want the stashed version).
-        with self.scm.detach_head(entry.rev, force=True, client="dvc"):
-            rev = self.stash.pop()
-            self.scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
-            if entry.branch:
-                self.scm.set_ref(EXEC_BRANCH, entry.branch, symbolic=True)
-            elif self.scm.get_ref(EXEC_BRANCH):
-                self.scm.remove_ref(EXEC_BRANCH)
-            try:
-                orig_checkpoint = self.scm.get_ref(EXEC_CHECKPOINT)
-                pid_dir = os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR)
-                if not os.path.exists(pid_dir):
-                    makedirs(pid_dir)
-                pidfile = os.path.join(
-                    pid_dir, f"workspace{BaseExecutor.PIDFILE_EXT}"
-                )
-                exec_result = BaseExecutor.reproduce(
-                    None,
-                    rev,
-                    name=entry.name,
-                    rel_cwd=relpath(os.getcwd(), self.scm.root_dir),
-                    log_errors=False,
-                    pidfile=pidfile,
-                )
-
-                if not exec_result.exp_hash:
-                    raise DvcException(
-                        f"Failed to reproduce experiment '{rev[:7]}'"
-                    )
-                if not exec_result.ref_info:
-                    # repro succeeded but result matches baseline
-                    # (no experiment generated or applied)
-                    return {}
-                exp_rev = self.scm.get_ref(str(exec_result.ref_info))
-                self.scm.set_ref(EXEC_APPLY, exp_rev)
-                return {exp_rev: exec_result.exp_hash}
-            except CheckpointKilledError:
-                # Checkpoint errors have already been logged
-                return {}
-            except DvcException:
-                raise
-            except Exception as exc:
-                raise DvcException(
-                    f"Failed to reproduce experiment '{rev[:7]}'"
-                ) from exc
-            finally:
-                self.scm.remove_ref(EXEC_BASELINE)
-                if entry.branch:
-                    self.scm.remove_ref(EXEC_BRANCH)
-                checkpoint = self.scm.get_ref(EXEC_CHECKPOINT)
-                if checkpoint and checkpoint != orig_checkpoint:
-                    self.scm.set_ref(EXEC_APPLY, checkpoint)
+        return manager.exec_queue(**kwargs)
 
     def check_baseline(self, exp_rev):
         baseline_sha = self.repo.scm.get_rev()
@@ -931,45 +733,55 @@ class Experiments:
 
     def get_running_exps(self) -> Dict[str, int]:
         """Return info for running experiments."""
-        from dvc.utils.serialize import load_yaml
-
-        from .executor.base import BaseExecutor, ExecutorInfo
+        from dvc.scm import InvalidRemoteSCMRepo
+        from dvc.utils.serialize import load_json
 
         result = {}
-        for pidfile in self.repo.fs.find(
-            os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR)
-        ):
-            rev, _ = os.path.splitext(os.path.basename(pidfile))
+        pid_dir = os.path.join(
+            self.repo.tmp_dir,
+            EXEC_TMP_DIR,
+            EXEC_PID_DIR,
+        )
+        for fname in self.repo.fs.find(pid_dir):
+            rev, ext = os.path.splitext(os.path.basename(fname))
+            if ext != BaseExecutor.INFOFILE_EXT:
+                continue
 
             try:
-                info = ExecutorInfo.from_dict(load_yaml(pidfile))
+                info = ExecutorInfo.from_dict(load_json(fname))
+                if info.result is not None:
+                    continue
                 if rev == "workspace":
                     # If we are appending to a checkpoint branch in a workspace
                     # run, show the latest checkpoint as running.
                     last_rev = self.scm.get_ref(EXEC_BRANCH)
                     if last_rev:
-                        result[last_rev] = info.to_dict()
+                        result[last_rev] = info.asdict()
                     else:
-                        result[rev] = info.to_dict()
+                        result[rev] = info.asdict()
                 else:
-                    result[rev] = info.to_dict()
+                    result[rev] = info.asdict()
                     if info.git_url:
 
                         def on_diverged(_ref: str, _checkpoint: bool):
                             return False
 
-                        for ref in BaseExecutor.fetch_exps(
-                            self.scm,
-                            info.git_url,
-                            on_diverged=on_diverged,
-                        ):
-                            logger.debug(
-                                "Updated running experiment '%s'.", ref
-                            )
-                            last_rev = self.scm.get_ref(ref)
-                            result[rev]["last"] = last_rev
-                            if last_rev:
-                                result[last_rev] = info.to_dict()
+                        try:
+                            for ref in BaseExecutor.fetch_exps(
+                                self.scm,
+                                info.git_url,
+                                on_diverged=on_diverged,
+                            ):
+                                logger.debug(
+                                    "Updated running experiment '%s'.", ref
+                                )
+                                last_rev = self.scm.get_ref(ref)
+                                result[rev]["last"] = last_rev
+                                if last_rev:
+                                    result[last_rev] = info.asdict()
+                        except InvalidRemoteSCMRepo:
+                            # ignore stale info files
+                            del result[rev]
             except OSError:
                 pass
         return result

dvc/repo/experiments/base.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import NamedTuple, Optional
 
 from dvc.exceptions import DvcException, InvalidArgumentError
 
@@ -141,3 +141,20 @@ class ExpRefInfo:
         baseline_sha = parts[2] + parts[3] if len(parts) >= 4 else None
         name = parts[4] if len(parts) == 5 else None
         return cls(baseline_sha, name)
+
+
+class ExpStashEntry(NamedTuple):
+    """Experiment stash entry.
+
+    stash_index: EXPS_STASH index for this entry. Can be None if this commit
+        is not pushed onto the stash ref.
+    head_rev: HEAD Git commit to be checked out for this experiment.
+    baseline_rev: Experiment baseline commit.
+    branch: Optional exp (checkpoint) branch name for this experiment.
+    """
+
+    stash_index: Optional[int]
+    head_rev: str
+    baseline_rev: str
+    branch: Optional[str]
+    name: Optional[str]

dvc/repo/experiments/executor/base.py

@@ -3,21 +3,28 @@ import os
 import pickle
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
 from functools import partial
 from typing import (
     TYPE_CHECKING,
+    Any,
     Callable,
+    Dict,
     Iterable,
     NamedTuple,
     Optional,
+    Type,
+    TypeVar,
     Union,
 )
 
 from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE
 from dvc.exceptions import DvcException
-from dvc.repo import Repo
-from dvc.repo.experiments.base import (
+from dvc.stage.serialize import to_lockfile
+from dvc.utils import dict_sha256, env2bool, relpath
+from dvc.utils.fs import remove
+
+from ..base import (
     EXEC_BASELINE,
     EXEC_BRANCH,
     EXEC_CHECKPOINT,
@@ -29,20 +36,24 @@ from dvc.repo.experiments.base import (
     ExpRefInfo,
     UnchangedExperimentError,
 )
-from dvc.stage import PipelineStage
-from dvc.stage.monitor import CheckpointKilledError
-from dvc.stage.serialize import to_lockfile
-from dvc.utils import dict_sha256, env2bool
-from dvc.utils.fs import remove
 
 if TYPE_CHECKING:
     from multiprocessing import Queue
 
     from scmrepo.git import Git
 
+    from dvc.repo import Repo
+    from dvc.stage import PipelineStage
+
+    from ..base import ExpStashEntry
+
 logger = logging.getLogger(__name__)
 
 
+EXEC_TMP_DIR = "exps"
+EXEC_PID_DIR = "run"
+
+
 class ExecutorResult(NamedTuple):
     exp_hash: Optional[str]
     ref_info: Optional["ExpRefInfo"]
@@ -51,68 +62,77 @@ class ExecutorResult(NamedTuple):
 
 @dataclass
 class ExecutorInfo:
-    PARAM_PID = "pid"
-    PARAM_GIT_URL = "git"
-    PARAM_BASELINE_REV = "baseline"
-    PARAM_LOCATION = "location"
-
-    pid: Optional[int]
-    git_url: Optional[str]
-    baseline_rev: Optional[str]
-    location: Optional[str]
+    git_url: str
+    baseline_rev: str
+    location: str
+    root_dir: str
+    dvc_dir: str
+    name: Optional[str] = None
+    wdir: Optional[str] = None
+    result_hash: Optional[str] = None
+    result_ref: Optional[str] = None
+    result_force: bool = False
 
     @classmethod
     def from_dict(cls, d):
-        return cls(
-            d.get(cls.PARAM_PID),
-            d.get(cls.PARAM_GIT_URL),
-            d.get(cls.PARAM_BASELINE_REV),
-            d.get(cls.PARAM_LOCATION),
+        return cls(**d)
+
+    def asdict(self):
+        return asdict(self)
+
+    @property
+    def result(self) -> Optional["ExecutorResult"]:
+        if self.result_hash is None:
+            return None
+        return ExecutorResult(
+            self.result_hash,
+            ExpRefInfo.from_ref(self.result_ref) if self.result_ref else None,
+            self.result_force,
         )
 
-    def to_dict(self):
-        return {
-            self.PARAM_PID: self.pid,
-            self.PARAM_GIT_URL: self.git_url,
-            self.PARAM_BASELINE_REV: self.baseline_rev,
-            self.PARAM_LOCATION: self.location,
-        }
+
+_T = TypeVar("_T", bound="BaseExecutor")
 
 
 class BaseExecutor(ABC):
     """Base class for executing experiments in parallel.
 
-    Args:
-        src: source Git SCM instance.
-        dvc_dir: relpath to DVC root from SCM root.
-
-    Optional keyword args:
-        branch: Existing git branch for this experiment.
+    Parameters:
+        root_dir: Path to SCM root.
+        dvc_dir: Path to .dvc dir relative to SCM root.
+        baseline_rev: Experiment baseline revision.
+        wdir: Path to exec working directory relative to SCM root.
+        name: Executor (experiment) name.
+        result: Completed executor result.
     """
 
     PACKED_ARGS_FILE = "repro.dat"
     WARN_UNTRACKED = False
     QUIET = False
-    PIDFILE_EXT = ".run"
-    DEFAULT_LOCATION: Optional[str] = "workspace"
+    INFOFILE_EXT = ".run"
+    DEFAULT_LOCATION: str = "workspace"
 
     def __init__(
         self,
-        src: "Git",
+        root_dir: str,
         dvc_dir: str,
-        root_dir: Optional[str] = None,
-        branch: Optional[str] = None,
+        baseline_rev: str,
+        wdir: Optional[str] = None,
         name: Optional[str] = None,
+        location: Optional[str] = None,
+        result: Optional["ExecutorResult"] = None,
         **kwargs,
     ):
-        assert root_dir is not None
         self._dvc_dir = dvc_dir
         self.root_dir = root_dir
-        self._init_git(src, branch)
+        self.wdir = wdir
         self.name = name
+        self.baseline_rev = baseline_rev
+        self.location: str = location or self.DEFAULT_LOCATION
+        self.result = result
 
     @abstractmethod
-    def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs):
+    def init_git(self, scm: "Git", branch: Optional[str] = None):
         """Init git repo and populate it using exp refs from the specified
         SCM instance.
         """
@@ -126,12 +146,92 @@ class BaseExecutor(ABC):
     def init_cache(self, dvc: "Repo", rev: str, run_cache: bool = True):
         """Initialize DVC (cache)."""
 
+    @property
+    def info(self) -> "ExecutorInfo":
+        if self.result is not None:
+            result_dict: Dict[str, Any] = {
+                "result_hash": self.result.exp_hash,
+                "result_ref": (
+                    str(self.result.ref_info) if self.result.ref_info else None
+                ),
+                "result_force": self.result.force,
+            }
+        else:
+            result_dict = {}
+        return ExecutorInfo(
+            git_url=self.git_url,
+            baseline_rev=self.baseline_rev,
+            location=self.location,
+            root_dir=self.root_dir,
+            dvc_dir=self.dvc_dir,
+            name=self.name,
+            wdir=self.wdir,
+            **result_dict,
+        )
+
+    @classmethod
+    def from_info(cls: Type[_T], info: "ExecutorInfo") -> _T:
+        if info.result_hash:
+            result: Optional["ExecutorResult"] = ExecutorResult(
+                info.result_hash,
+                (
+                    ExpRefInfo.from_ref(info.result_ref)
+                    if info.result_ref
+                    else None
+                ),
+                info.result_force,
+            )
+        else:
+            result = None
+        return cls(
+            root_dir=info.root_dir,
+            dvc_dir=info.dvc_dir,
+            baseline_rev=info.baseline_rev,
+            name=info.name,
+            wdir=info.wdir,
+            result=result,
+        )
+
+    @classmethod
+    @abstractmethod
+    def from_stash_entry(
+        cls: Type[_T],
+        repo: "Repo",
+        stash_rev: str,
+        entry: "ExpStashEntry",
+        **kwargs,
+    ) -> _T:
+        pass
+
+    @classmethod
+    def _from_stash_entry(
+        cls: Type[_T],
+        repo: "Repo",
+        stash_rev: str,
+        entry: "ExpStashEntry",
+        root_dir: str,
+        **kwargs,
+    ) -> _T:
+        executor = cls(
+            root_dir=root_dir,
+            dvc_dir=relpath(repo.dvc_dir, repo.scm.root_dir),
+            baseline_rev=entry.baseline_rev,
+            name=entry.name,
+            wdir=relpath(os.getcwd(), repo.scm.root_dir),
+            **kwargs,
+        )
+        executor.init_git(repo.scm, branch=entry.branch)
+        executor.init_cache(repo, stash_rev)
+        return executor
+
     @property
     def dvc_dir(self) -> str:
         return os.path.join(self.root_dir, self._dvc_dir)
 
     @staticmethod
     def hash_exp(stages: Iterable["PipelineStage"]) -> str:
+        from dvc.stage import PipelineStage
+
         exp_data = {}
         for stage in stages:
             if isinstance(stage, PipelineStage):
@@ -244,11 +344,10 @@ class BaseExecutor(ABC):
     @classmethod
     def reproduce(
         cls,
-        dvc_dir: Optional[str],
+        info: "ExecutorInfo",
         rev: str,
         queue: Optional["Queue"] = None,
-        rel_cwd: Optional[str] = None,
-        name: Optional[str] = None,
+        infofile: Optional[str] = None,
         log_errors: bool = True,
         log_level: Optional[int] = None,
         **kwargs,
@@ -262,6 +361,7 @@ class BaseExecutor(ABC):
         """
         from dvc.repo.checkout import checkout as dvc_checkout
         from dvc.repo.reproduce import reproduce as dvc_reproduce
+        from dvc.stage import PipelineStage
 
         auto_push = env2bool(DVC_EXP_AUTO_PUSH)
         git_remote = os.getenv(DVC_EXP_GIT_REMOTE, None)
@@ -283,9 +383,9 @@ class BaseExecutor(ABC):
         repro_force: bool = False
 
         with cls._repro_dvc(
-            dvc_dir,
-            rel_cwd,
-            log_errors,
+            info,
+            log_errors=log_errors,
+            infofile=infofile,
             **kwargs,
         ) as dvc:
             if auto_push:
@@ -333,7 +433,7 @@ class BaseExecutor(ABC):
                 cls.checkpoint_callback,
                 dvc,
                 dvc.scm,
-                name,
+                info.name,
                 repro_force or checkpoint_reset,
             )
             stages = dvc_reproduce(
@@ -358,7 +458,7 @@ class BaseExecutor(ABC):
                     cls.commit(
                         dvc.scm,
                         exp_hash,
-                        exp_name=name,
+                        exp_name=info.name,
                         force=repro_force,
                         checkpoint=is_checkpoint,
                     )
@@ -379,6 +479,9 @@ class BaseExecutor(ABC):
                             "\t%s",
                             ", ".join(untracked),
                         )
+            info.result_hash = exp_hash
+            info.result_ref = ref
+            info.result_force = repro_force
 
         # ideally we would return stages here like a normal repro() call, but
         # stages is not currently picklable and cannot be returned across
@@ -389,38 +492,32 @@ class BaseExecutor(ABC):
     @contextmanager
     def _repro_dvc(
         cls,
-        dvc_dir: Optional[str],
-        rel_cwd: Optional[str],
-        log_errors: bool,
-        pidfile: Optional[str] = None,
-        git_url: Optional[str] = None,
+        info: "ExecutorInfo",
+        log_errors: bool = True,
+        infofile: Optional[str] = None,
         **kwargs,
     ):
-        from dvc.utils.serialize import modify_yaml
+        from dvc.repo import Repo
+        from dvc.stage.monitor import CheckpointKilledError
+        from dvc.utils.fs import makedirs
+        from dvc.utils.serialize import modify_json
 
-        dvc = Repo(dvc_dir)
+        dvc = Repo(os.path.join(info.root_dir, info.dvc_dir))
         if cls.QUIET:
             dvc.scm_context.quiet = cls.QUIET
-        if dvc_dir is not None:
-            old_cwd: Optional[str] = os.getcwd()
-            if rel_cwd:
-                os.chdir(os.path.join(dvc.root_dir, rel_cwd))
-            else:
-                os.chdir(dvc.root_dir)
+        old_cwd = os.getcwd()
+        if info.wdir:
+            os.chdir(os.path.join(dvc.scm.root_dir, info.wdir))
         else:
-            old_cwd = None
-        if pidfile is not None:
-            info = ExecutorInfo(
-                os.getpid(),
-                git_url,
-                dvc.scm.get_rev(),
-                cls.DEFAULT_LOCATION,
-            )
-            with modify_yaml(pidfile) as d:
-                d.update(info.to_dict())
-        logger.debug("Running repro in '%s'", os.getcwd())
+            os.chdir(dvc.root_dir)
+
+        if infofile is not None:
+            makedirs(os.path.dirname(infofile), exist_ok=True)
+            with modify_json(infofile) as d:
+                d.update(info.asdict())
 
         try:
+            logger.debug("Running repro in '%s'", os.getcwd())
             yield dvc
         except CheckpointKilledError:
             raise
@@ -433,11 +530,11 @@ class BaseExecutor(ABC):
                 logger.exception("unexpected error")
             raise
         finally:
-            if pidfile is not None:
-                remove(pidfile)
+            if infofile is not None:
+                with modify_json(infofile) as d:
+                    d.update(info.asdict())
             dvc.close()
-            if old_cwd:
-                os.chdir(old_cwd)
+            os.chdir(old_cwd)
 
     @classmethod
     def _repro_args(cls, dvc):

dvc/repo/experiments/executor/local.py

@@ -1,47 +1,35 @@
 import logging
 import os
-from tempfile import TemporaryDirectory
+from contextlib import ExitStack
+from tempfile import mkdtemp
 from typing import TYPE_CHECKING, Optional
 
 from funcy import cached_property
 
-from dvc.repo.experiments.base import (
+from dvc.scm import SCM
+from dvc.utils.fs import remove
+
+from ..base import (
+    EXEC_APPLY,
+    EXEC_BASELINE,
     EXEC_BRANCH,
     EXEC_CHECKPOINT,
     EXEC_HEAD,
     EXEC_MERGE,
     EXEC_NAMESPACE,
 )
-from dvc.scm import SCM
-
-from .base import BaseExecutor
+from .base import EXEC_TMP_DIR, BaseExecutor
 
 if TYPE_CHECKING:
     from scmrepo.git import Git
 
     from dvc.repo import Repo
 
+    from ..base import ExpStashEntry
+
 logger = logging.getLogger(__name__)
 
 
-class ExpTemporaryDirectory(TemporaryDirectory):
-    # Python's TemporaryDirectory cleanup shutil.rmtree wrapper does not handle
-    # git read-only dirs cleanly in Windows on Python <3.8, so we use our own
-    # remove(). See:
-    # https://github.com/iterative/dvc/pull/5425
-    # https://bugs.python.org/issue26660
-
-    @classmethod
-    def _rmtree(cls, name):
-        from dvc.utils.fs import remove
-
-        remove(name)
-
-    def cleanup(self):
-        if self._finalizer.detach():
-            self._rmtree(self.name)
-
-
 class BaseLocalExecutor(BaseExecutor):
     """Base local machine executor."""
 
@@ -70,20 +58,9 @@ class TempDirExecutor(BaseLocalExecutor):
     # suggestions) that are not applicable outside of workspace runs
     WARN_UNTRACKED = True
     QUIET = True
-    DEFAULT_LOCATION: Optional[str] = "temp"
+    DEFAULT_LOCATION = "temp"
 
-    def __init__(
-        self,
-        *args,
-        tmp_dir: Optional[str] = None,
-        **kwargs,
-    ):
-        self._tmp_dir = ExpTemporaryDirectory(dir=tmp_dir)
-        kwargs["root_dir"] = self._tmp_dir.name
-        super().__init__(*args, **kwargs)
-        logger.debug("Init temp dir executor in dir '%s'", self._tmp_dir)
-
-    def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs):
+    def init_git(self, scm: "Git", branch: Optional[str] = None):
         from dulwich.repo import Repo as DulwichRepo
 
         from ..utils import push_refspec
@@ -91,9 +68,9 @@ class TempDirExecutor(BaseLocalExecutor):
         DulwichRepo.init(os.fspath(self.root_dir))
 
         refspec = f"{EXEC_NAMESPACE}/"
-        push_refspec(scm, self.git_url, refspec, refspec, **kwargs)
+        push_refspec(scm, self.git_url, refspec, refspec)
         if branch:
-            push_refspec(scm, self.git_url, branch, branch, **kwargs)
+            push_refspec(scm, self.git_url, branch, branch)
             self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
         elif self.scm.get_ref(EXEC_BRANCH):
             self.scm.remove_ref(EXEC_BRANCH)
@@ -120,5 +97,70 @@ class TempDirExecutor(BaseLocalExecutor):
 
     def cleanup(self):
         super().cleanup()
-        logger.debug("Removing tmpdir '%s'", self._tmp_dir)
-        self._tmp_dir.cleanup()
+        logger.debug("Removing tmpdir '%s'", self.root_dir)
+        remove(self.root_dir)
+
+    @classmethod
+    def from_stash_entry(
+        cls,
+        repo: "Repo",
+        stash_rev: str,
+        entry: "ExpStashEntry",
+        **kwargs,
+    ):
+        tmp_dir = mkdtemp(dir=os.path.join(repo.tmp_dir, EXEC_TMP_DIR))
+        try:
+            executor = cls._from_stash_entry(repo, stash_rev, entry, tmp_dir)
+            logger.debug("Init temp dir executor in '%s'", tmp_dir)
+            return executor
+        except Exception:
+            remove(tmp_dir)
+            raise
+
+
+class WorkspaceExecutor(BaseLocalExecutor):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._detach_stack = ExitStack()
+        self._orig_checkpoint = self.scm.get_ref(EXEC_CHECKPOINT)
+
+    @classmethod
+    def from_stash_entry(
+        cls,
+        repo: "Repo",
+        stash_rev: str,
+        entry: "ExpStashEntry",
+        **kwargs,
+    ):
+        root_dir = repo.scm.root_dir
+        executor = cls._from_stash_entry(repo, stash_rev, entry, root_dir)
+        logger.debug("Init workspace executor in '%s'", root_dir)
+        return executor
+
+    def init_git(self, scm: "Git", branch: Optional[str] = None):
+        self._detach_stack.enter_context(
+            self.scm.detach_head(
+                self.scm.get_ref(EXEC_HEAD),
+                force=True,
+                client="dvc",
+            )
+        )
+        merge_rev = self.scm.get_ref(EXEC_MERGE)
+        self.scm.merge(merge_rev, squash=True, commit=False)
+        if branch:
+            self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
+        elif scm.get_ref(EXEC_BRANCH):
+            self.scm.remove_ref(EXEC_BRANCH)
+
+    def init_cache(self, dvc: "Repo", rev: str, run_cache: bool = True):
+        pass
+
+    def cleanup(self):
+        with self._detach_stack:
+            self.scm.remove_ref(EXEC_BASELINE)
+            if self.scm.get_ref(EXEC_BRANCH):
+                self.scm.remove_ref(EXEC_BRANCH)
+            checkpoint = self.scm.get_ref(EXEC_CHECKPOINT)
+            if checkpoint and checkpoint != self._orig_checkpoint:
+                self.scm.set_ref(EXEC_APPLY, checkpoint)
+        super().cleanup()

dvc/repo/experiments/executor/manager.py

@@ -0,0 +1,351 @@
+import logging
+import os
+from abc import ABC
+from collections import defaultdict, deque
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Deque, Dict, Generator, Optional, Tuple, Type
+
+from dvc.proc.manager import ProcessManager
+
+from ..base import (
+    EXEC_BASELINE,
+    EXEC_BRANCH,
+    EXEC_HEAD,
+    EXEC_MERGE,
+    CheckpointExistsError,
+    ExperimentExistsError,
+    ExpRefInfo,
+    ExpStashEntry,
+)
+from .base import EXEC_PID_DIR, BaseExecutor
+from .local import TempDirExecutor, WorkspaceExecutor
+
+if TYPE_CHECKING:
+    from scmrepo.git import Git
+
+    from dvc.repo import Repo
+
+logger = logging.getLogger(__name__)
+
+
+class BaseExecutorManager(ABC, Mapping):
+    """Manages executors for a collection of experiments to be run."""
+
+    EXECUTOR_CLS: Type = BaseExecutor
+
+    def __init__(
+        self,
+        scm: "Git",
+        wdir: str,
+    ):
+        from dvc.utils.fs import makedirs
+
+        self.scm = scm
+        makedirs(wdir, exist_ok=True)
+        self.wdir = wdir
+        self.proc = ProcessManager(self.pid_dir)
+        self._attached: Dict[str, "BaseExecutor"] = {}
+        self._detached: Dict[str, "BaseExecutor"] = dict(self._load_infos())
+        self._queue: Deque[Tuple[str, "BaseExecutor"]] = deque()
+
+    def __getitem__(self, key: str) -> "BaseExecutor":
+        try:
+            return self._attached[key]
+        except KeyError:
+            pass
+        return self._detached[key]
+
+    def __iter__(self):
+        yield from self._attached
+        yield from self._detached
+
+    def __len__(self):
+        return len(self._attached) + len(self._detached)
+
+    @property
+    def pid_dir(self) -> str:
+        return os.path.join(self.wdir, EXEC_PID_DIR)
+
+    def enqueue(self, rev: str, executor: "BaseExecutor"):
+        assert rev not in self
+        self._queue.append((rev, executor))
+
+    def _load_infos(self) -> Generator[Tuple[str, "BaseExecutor"], None, None]:
+        import json
+        from urllib.parse import urlparse
+
+        from .base import ExecutorInfo
+        from .ssh import SSHExecutor
+
+        def make_executor(info: "ExecutorInfo"):
+            if info.git_url:
+                scheme = urlparse(info.git_url).scheme
+                if scheme == "file":
+                    cls: Type = TempDirExecutor
+                elif scheme == "ssh":
+                    cls = SSHExecutor
+                else:
+                    raise NotImplementedError
+            else:
+                cls = WorkspaceExecutor
+            return cls.from_info(info)
+
+        for name in self.proc:
+            infofile = self.get_infofile_path(name)
+            try:
+                with open(infofile, encoding="utf-8") as fobj:
+                    info = ExecutorInfo.from_dict(json.load(fobj))
+                yield name, make_executor(info)
+            except OSError:
+                continue
+
+    def get_infofile_path(self, name: str) -> str:
+        return os.path.join(
+            self.pid_dir,
+            name,
+            f"{name}{BaseExecutor.INFOFILE_EXT}",
+        )
+
+    @classmethod
+    def from_stash_entries(
+        cls,
+        scm: "Git",
+        wdir: str,
+        repo: "Repo",
+        to_run: Dict[str, ExpStashEntry],
+        **kwargs,
+    ):
+        manager = cls(scm, wdir)
+        try:
+            for stash_rev, entry in to_run.items():
+                scm.set_ref(EXEC_HEAD, entry.head_rev)
+                scm.set_ref(EXEC_MERGE, stash_rev)
+                scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
+
+                # Executor will be initialized with an empty git repo that
+                # we populate by pushing:
+                #   EXEC_HEAD - the base commit for this experiment
+                #   EXEC_MERGE - the unmerged changes (from our stash)
+                #       to be reproduced
+                #   EXEC_BASELINE - the baseline commit for this experiment
+                executor = cls.EXECUTOR_CLS.from_stash_entry(
+                    repo,
+                    stash_rev,
+                    entry,
+                    **kwargs,
+                )
+                manager.enqueue(stash_rev, executor)
+        finally:
+            for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
+                scm.remove_ref(ref)
+        return manager
+
+    def exec_queue(self, jobs: Optional[int] = 1, detach: bool = False):
+        """Run dvc repro for queued executors in parallel."""
+        if detach:
+            raise NotImplementedError
+            # TODO use ProcessManager.spawn() to support detached runs
+        return self._exec_attached(jobs=jobs)
+
+    def _exec_attached(self, jobs: Optional[int] = 1):
+        import signal
+        from concurrent.futures import (
+            CancelledError,
+            ProcessPoolExecutor,
+            wait,
+        )
+        from multiprocessing import Manager
+
+        from dvc.stage.monitor import CheckpointKilledError
+
+        result: Dict[str, Dict[str, str]] = defaultdict(dict)
+
+        manager = Manager()
+        pid_q = manager.Queue()
+
+        with ProcessPoolExecutor(max_workers=jobs) as workers:
+            futures = {}
+            while self._queue:
+                rev, executor = self._queue.popleft()
+                infofile = self.get_infofile_path(rev)
+                future = workers.submit(
+                    executor.reproduce,
+                    info=executor.info,
+                    rev=rev,
+                    queue=pid_q,
+                    infofile=infofile,
+                    log_level=logger.getEffectiveLevel(),
+                )
+                futures[future] = (rev, executor)
+                self._attached[rev] = executor
+
+            try:
+                wait(futures)
+            except KeyboardInterrupt:
+                # forward SIGINT to any running executor processes and
+                # cancel any remaining futures
+                workers.shutdown(wait=False)
+                pids = {}
+                for future, (rev, _) in futures.items():
+                    if future.running():
+                        # if future has already been started by the scheduler
+                        # we still have to wait until it tells us its PID
+                        while rev not in pids:
+                            rev, pid = pid_q.get()
+                            pids[rev] = pid
+                        os.kill(pids[rev], signal.SIGINT)
+                    elif not future.done():
+                        future.cancel()
+
+            for future, (rev, executor) in futures.items():
+                rev, executor = futures[future]
+
+                try:
+                    exc = future.exception()
+                    if exc is None:
+                        exec_result = future.result()
+                        result[rev].update(
+                            self._collect_executor(executor, exec_result)
+                        )
+                    elif not isinstance(exc, CheckpointKilledError):
+                        logger.error(
+                            "Failed to reproduce experiment '%s'", rev[:7]
+                        )
+                except CancelledError:
+                    logger.error(
+                        "Cancelled before attempting to reproduce experiment "
+                        "'%s'",
+                        rev[:7],
+                    )
+                finally:
+                    self.cleanup_executor(rev, executor)
+
+        return result
+
+    def _collect_executor(self, executor, exec_result) -> Dict[str, str]:
+        # NOTE: GitPython Repo instances cannot be re-used
+        # after process has received SIGINT or SIGTERM, so we
+        # need this hack to re-instantiate git instances after
+        # checkpoint runs. See:
+        # https://github.com/gitpython-developers/GitPython/issues/427
+        # del self.repo.scm
+
+        results = {}
+
+        def on_diverged(ref: str, checkpoint: bool):
+            ref_info = ExpRefInfo.from_ref(ref)
+            if checkpoint:
+                raise CheckpointExistsError(ref_info.name)
+            raise ExperimentExistsError(ref_info.name)
+
+        for ref in executor.fetch_exps(
+            self.scm,
+            executor.git_url,
+            force=exec_result.force,
+            on_diverged=on_diverged,
+        ):
+            exp_rev = self.scm.get_ref(ref)
+            if exp_rev:
+                logger.debug("Collected experiment '%s'.", exp_rev[:7])
+                results[exp_rev] = exec_result.exp_hash
+
+        return results
+
+    def cleanup_executor(self, rev: str, executor: "BaseExecutor"):
+        from dvc.utils.fs import remove
+
+        executor.cleanup()
+        try:
+            self.proc.remove(rev)
+        except KeyError:
+            pass
+        remove(os.path.join(self.pid_dir, rev))
+
+
+class TempDirExecutorManager(BaseExecutorManager):
+    EXECUTOR_CLS = TempDirExecutor
+
+
+class WorkspaceExecutorManager(BaseExecutorManager):
+    EXECUTOR_CLS = WorkspaceExecutor
+
+    @classmethod
+    def from_stash_entries(
+        cls,
+        scm: "Git",
+        wdir: str,
+        repo: "Repo",
+        to_run: Dict[str, ExpStashEntry],
+        **kwargs,
+    ):
+        manager = cls(scm, wdir)
+        try:
+            assert len(to_run) == 1
+            for stash_rev, entry in to_run.items():
+                scm.set_ref(EXEC_HEAD, entry.head_rev)
+                scm.set_ref(EXEC_MERGE, stash_rev)
+                scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
+
+                executor = cls.EXECUTOR_CLS.from_stash_entry(
+                    repo,
+                    stash_rev,
+                    entry,
+                    **kwargs,
+                )
+                manager.enqueue(stash_rev, executor)
+        finally:
+            for ref in (EXEC_MERGE,):
+                scm.remove_ref(ref)
+        return manager
+
+    def _collect_executor(self, executor, exec_result) -> Dict[str, str]:
+        results = {}
+        exp_rev = self.scm.get_ref(EXEC_BRANCH)
+        if exp_rev:
+            logger.debug("Collected experiment '%s'.", exp_rev[:7])
+            results[exp_rev] = exec_result.exp_hash
+        return results
+
+    def exec_queue(self, jobs: Optional[int] = 1, detach: bool = False):
+        """Run a single WorkspaceExecutor.
+
+        Workspace execution is done within the main DVC process
+        (rather than in multiprocessing context)
+        """
+        from dvc.exceptions import DvcException
+        from dvc.stage.monitor import CheckpointKilledError
+
+        assert len(self._queue) == 1
+        assert not detach
+        result: Dict[str, Dict[str, str]] = defaultdict(dict)
+        rev, executor = self._queue.popleft()
+
+        exec_name = "workspace"
+        infofile = self.get_infofile_path(exec_name)
+        try:
+            exec_result = executor.reproduce(
+                info=executor.info,
+                rev=rev,
+                infofile=infofile,
+                log_level=logger.getEffectiveLevel(),
+            )
+            if not exec_result.exp_hash:
+                raise DvcException(
+                    f"Failed to reproduce experiment '{rev[:7]}'"
+                )
+            if exec_result.ref_info:
+                result[rev].update(
+                    self._collect_executor(executor, exec_result)
+                )
+        except CheckpointKilledError:
+            # Checkpoint errors have already been logged
+            return {}
+        except DvcException:
+            raise
+        except Exception as exc:
+            raise DvcException(
+                f"Failed to reproduce experiment '{rev[:7]}'"
+            ) from exc
+        finally:
+            self.cleanup_executor(exec_name, executor)
+        return result

dvc/repo/experiments/executor/ssh.py

@@ -22,6 +22,8 @@ if TYPE_CHECKING:
     from dvc.machine import MachineManager
     from dvc.repo import Repo
 
+    from ..base import ExpStashEntry
+
 logger = logging.getLogger(__name__)
 
 
@@ -51,13 +53,12 @@ class SSHExecutor(BaseExecutor):
     ):
         assert host
 
+        super().__init__(*args, **kwargs)
         self.host: str = host
         self.port = port
         self.username = username
         self._fs_factory = fs_factory
-
-        super().__init__(*args, **kwargs)
-        logger.debug("Init SSH executor for host '%s'", self.host)
+        self._repo_abspath = ""
 
     @classmethod
     def gen_dirname(cls, name: Optional[str] = None):
@@ -66,16 +67,24 @@ class SSHExecutor(BaseExecutor):
         return "-".join([name or "dvc-exp", "executor", uuid()])
 
     @classmethod
-    def from_machine(
+    def from_stash_entry(
         cls,
-        manager: "MachineManager",
-        machine_name: Optional[str],
-        *args,
+        repo: "Repo",
+        stash_rev: str,
+        entry: "ExpStashEntry",
         **kwargs,
     ):
-        kwargs["root_dir"] = cls.gen_dirname(kwargs.get("name"))
-        kwargs.update(manager.get_executor_kwargs(machine_name))
-        return cls(*args, **kwargs)
+        manager: "MachineManager" = kwargs.pop("manager")
+        machine_name: Optional[str] = kwargs.pop("machine_name", None)
+        executor = cls._from_stash_entry(
+            repo,
+            stash_rev,
+            entry,
+            cls.gen_dirname(entry.name),
+            **manager.get_executor_kwargs(machine_name),
+        )
+        logger.debug("Init SSH executor for host '%s'", executor.host)
+        return executor
 
     def sshfs(self):
         return _sshfs(self._fs_factory, host=self.host, port=self.port)
@@ -104,7 +113,7 @@ class SSHExecutor(BaseExecutor):
         }
         return kwargs
 
-    def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs):
+    def init_git(self, scm: "Git", branch: Optional[str] = None):
         from ..utils import push_refspec
 
         with self.sshfs() as fs:
@@ -121,7 +130,7 @@ class SSHExecutor(BaseExecutor):
 
             # TODO: support multiple client key retries in git backends
             # (see https://github.com/iterative/dvc/issues/6508)
-            kwargs.update(self._git_client_args(fs))
+            kwargs = self._git_client_args(fs)
             refspec = f"{EXEC_NAMESPACE}/"
             push_refspec(scm, self.git_url, refspec, refspec, **kwargs)
             if branch:

dvc/repo/experiments/gc.py

@@ -42,7 +42,7 @@ def gc(
     delete_stashes = []
     for _, entry in repo.experiments.stash_revs.items():
         if not queued or entry.baseline_rev not in keep_revs:
-            delete_stashes.append(entry.index)
+            delete_stashes.append(entry.stash_index)
     for index in sorted(delete_stashes, reverse=True):
         repo.experiments.stash.drop(index)
     removed += len(delete_stashes)

dvc/repo/experiments/remove.py

@@ -49,16 +49,16 @@ def _clear_all(repo):
 
 def _get_exp_stash_index(repo, ref_or_rev: str) -> Optional[int]:
     stash_revs = repo.experiments.stash_revs
-    for _, ref_info in stash_revs.items():
-        if ref_info.name == ref_or_rev:
-            return ref_info.index
+    for _, entry in stash_revs.items():
+        if entry.name == ref_or_rev:
+            return entry.stash_index
 
     from dvc.scm import resolve_rev
 
     try:
         rev = resolve_rev(repo.scm, ref_or_rev)
         if rev in stash_revs:
-            return stash_revs.get(rev).index
+            return stash_revs.get(rev).stash_index
     except RevError:
         pass
     return None

dvc/repo/experiments/show.py

@@ -6,7 +6,6 @@ from typing import Any, Callable, Dict, Optional
 from dvc.exceptions import InvalidArgumentError
 from dvc.repo import locked
 from dvc.repo.experiments.base import ExpRefInfo
-from dvc.repo.experiments.executor.base import ExecutorInfo
 from dvc.repo.experiments.utils import fix_exp_head
 from dvc.repo.metrics.show import _gather_metrics
 from dvc.repo.params.show import _gather_params
@@ -44,7 +43,7 @@ def _collect_experiment_commit(
         res["queued"] = stash
         if running is not None and exp_rev in running:
             res["running"] = True
-            res["executor"] = running[exp_rev].get(ExecutorInfo.PARAM_LOCATION)
+            res["executor"] = running[exp_rev].get("location")
         else:
             res["running"] = False
             res["executor"] = None

dvc/repo/live.py

@@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, List, Optional
 
 from funcy import once_per_args
 
-from dvc.render.utils import render
+from dvc.render.utils import match_renderers, render
 
 logger = logging.getLogger(__name__)
 
@@ -27,8 +27,9 @@ def create_summary(out):
 
     html_path = out.fs_path + "_dvc_plots"
 
+    renderers = match_renderers(plots, out.repo.plots.templates)
     index_path = render(
-        out.repo, plots, metrics=metrics, path=html_path, refresh_seconds=5
+        out.repo, renderers, metrics=metrics, path=html_path, refresh_seconds=5
     )
 
     webbrowser_open(index_path)

dvc/ui/table.py

@@ -91,6 +91,7 @@ def rich_table(
         False: None,
         "simple": box.SIMPLE,
         "minimal": box.MINIMAL,
+        "horizontals": box.HORIZONTALS,
     }
 
     table = Table(box=border_style[borders])

setup.cfg

@@ -75,13 +75,9 @@ install_requires =
     aiohttp-retry>=2.4.5
     diskcache>=5.2.1
     jaraco.windows>=5.7.0; python_version < '3.8' and sys_platform == 'win32'
-    scmrepo==0.0.4
+    scmrepo==0.0.7
 
 [options.extras_require]
-# gssapi should not be included in all_remotes, because it doesn't have wheels
-# for linux and mac, so it will fail to compile if user doesn't have all the
-# requirements, including kerberos itself. Once all the wheels are available,
-# we can start shipping it by default.
 all =
     %(azure)s
     %(gdrive)s
@@ -102,17 +98,26 @@ azure =
 gdrive = pydrive2[fsspec]>=1.9.4
 gs = gcsfs>=2021.11.1
 hdfs =
+    # due to https://github.com/python-poetry/poetry/issues/4683, we have to
+    # explicitly specify pyarrow here
+    pyarrow>=1
     fsspec[arrow]
 oss = ossfs>=2021.8.0
-s3 =
-    s3fs>=2021.11.1
-    aiobotocore[boto3]>1.0.1
-ssh = sshfs[bcrypt]>=2021.11.2
+s3 = s3fs[boto3]>=2021.11.1
+ssh =
+    # due to https://github.com/python-poetry/poetry/issues/4683, we have to
+    # explicitly specify bcrypt here
+    bcrypt
+    sshfs[bcrypt]>=2021.11.2
+# gssapi should not be included in all_remotes, because it doesn't have wheels
+# for Linux, so it will fail to compile if user doesn't have all the
+# requirements, including kerberos itself.
 ssh_gssapi = sshfs[gssapi]>=2021.11.2
 webdav = webdav4>=0.9.3
 # not to break `dvc[webhdfs]`
 webhdfs =
-    requests-kerberos==0.14.0
+# requests-kerberos requires krb5 & gssapi, which does not provide wheels Linux
+webdhfs_kerberos = requests-kerberos==0.14.0
 terraform = tpi[ssh]>=2.1.0
 tests =
     %(terraform)s
@@ -121,14 +126,14 @@ tests =
     # Test requirements
     pytest==6.2.5
     pytest-cov==3.0.0
-    pytest-xdist==2.4.0
+    pytest-xdist==2.5.0
     pytest-mock==3.6.1
     pytest-lazy-fixture==0.6.3
     # https://github.com/docker/docker-py/issues/2902
     pytest-docker==0.10.3; python_version < '3.10' or sys_platform != 'win32'
     flaky==3.7.0
     mock==4.0.3
-    pytest-timeout==2.0.1
+    pytest-timeout==2.0.2
     rangehttpserver==1.2.0
     mock-ssh-server==0.9.1
     wget==3.2
@@ -152,8 +157,8 @@ tests =
     # and, protected method calls in our tests
     pylint-plugin-utils==0.6
     # type-checking
-    mypy==0.910
-    types-requests==2.26.1
+    mypy==0.921
+    types-requests==2.26.2
     types-tabulate==0.8.3
     types-toml==0.10.1
     # optional dependencies

tests/basic_env.py

@@ -140,15 +140,6 @@ class TestGitFixture(TestDirFixture):
         super().tearDown()
 
 
-class TestGitSubmoduleFixture(TestGitFixture):
-    def setUp(self):
-        super().setUp()
-        subrepo = Repo.init()
-        subrepo_path = "subrepo"
-        self.git.create_submodule(subrepo_path, subrepo_path, subrepo.git_dir)
-        self._pushd(subrepo_path)
-
-
 class TestDvcFixture(TestDirFixture):
     def setUp(self):
         super().setUp()
@@ -185,12 +176,6 @@ class TestGit(TestGitFixture, TestCase):
         TestCase.__init__(self, methodName)
 
 
-class TestGitSubmodule(TestGitSubmoduleFixture, TestCase):
-    def __init__(self, methodName):
-        TestGitSubmoduleFixture.__init__(self)
-        TestCase.__init__(self, methodName)
-
-
 class TestDvc(TestDvcFixture, TestCase):
     def __init__(self, methodName):
         TestDvcFixture.__init__(self)

tests/func/experiments/executor/test_ssh.py

@@ -21,10 +21,19 @@ def _ssh_factory(cloud):
     )
 
 
-def test_from_machine(tmp_dir, scm, dvc, machine_instance, mocker):
-    mocker.patch.object(SSHExecutor, "_init_git")
-    executor = SSHExecutor.from_machine(dvc.machine, "foo", scm, "")
-    assert executor.host == machine_instance["instance_ip"]
+def test_init_from_stash(tmp_dir, scm, dvc, machine_instance, mocker):
+    mock = mocker.patch.object(SSHExecutor, "_from_stash_entry")
+    mock_entry = mocker.Mock()
+    mock_entry.name = ""
+    SSHExecutor.from_stash_entry(
+        None,
+        "",
+        mock_entry,
+        manager=dvc.machine,
+        machine_name="foo",
+    )
+    _args, kwargs = mock.call_args
+    assert kwargs["host"] == machine_instance["instance_ip"]
 
 
 @pytest.mark.needs_internet

tests/func/experiments/test_experiments.py

@@ -629,7 +629,8 @@ def test_fix_exp_head(tmp_dir, scm, tail):
     head = "HEAD" + tail
     assert head == fix_exp_head(scm, head)
 
-    scm.set_ref(EXEC_BASELINE, "refs/heads/master")
+    rev = "1" * 40
+    scm.set_ref(EXEC_BASELINE, rev)
     assert EXEC_BASELINE + tail == fix_exp_head(scm, head)
     assert "foo" + tail == fix_exp_head(scm, "foo" + tail)
 

tests/func/experiments/test_init.py

@@ -3,7 +3,7 @@ import os
 
 import pytest
 
-from dvc.command.experiments import CmdExperimentsInit
+from dvc.command.experiments.init import CmdExperimentsInit
 from dvc.exceptions import DvcException
 from dvc.main import main
 from dvc.repo.experiments.init import init

tests/func/experiments/test_show.py

@@ -8,7 +8,12 @@ from funcy import first, get_in
 from dvc.exceptions import InvalidArgumentError
 from dvc.main import main
 from dvc.repo.experiments.base import EXPS_STASH, ExpRefInfo
-from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorInfo
+from dvc.repo.experiments.executor.base import (
+    EXEC_PID_DIR,
+    EXEC_TMP_DIR,
+    BaseExecutor,
+    ExecutorInfo,
+)
 from dvc.repo.experiments.utils import exp_refs_by_rev
 from dvc.utils.fs import makedirs
 from dvc.utils.serialize import YAMLFileCorruptedError
@@ -16,6 +21,20 @@ from tests.func.test_repro_multistage import COPY_SCRIPT
 from tests.utils import console_width
 
 
+def make_executor_info(**kwargs):
+    # set default values for required info fields
+    for key in (
+        "git_url",
+        "baseline_rev",
+        "location",
+        "root_dir",
+        "dvc_dir",
+    ):
+        if key not in kwargs:
+            kwargs[key] = ""
+    return ExecutorInfo(**kwargs)
+
+
 def test_show_simple(tmp_dir, scm, dvc, exp_stage):
     assert dvc.experiments.show()["workspace"] == {
         "baseline": {
@@ -274,7 +293,6 @@ def test_show_filter(
     from dvc.ui import ui
 
     capsys.readouterr()
-    div = "│" if os.name == "nt" else "┃"
 
     tmp_dir.gen("copy.py", COPY_SCRIPT)
     params_file = tmp_dir / "params.yaml"
@@ -321,11 +339,11 @@ def test_show_filter(
     cap = capsys.readouterr()
 
     for i in included:
-        assert f"{div} params.yaml:{i} {div}" in cap.out
-        assert f"{div} metrics.yaml:{i} {div}" in cap.out
+        assert f"params.yaml:{i}" in cap.out
+        assert f"metrics.yaml:{i}" in cap.out
     for e in excluded:
-        assert f"{div} params.yaml:{e} {div}" not in cap.out
-        assert f"{div} metrics.yaml:{e} {div}" not in cap.out
+        assert f"params.yaml:{e}" not in cap.out
+        assert f"metrics.yaml:{e}" not in cap.out
 
 
 def test_show_multiple_commits(tmp_dir, scm, dvc, exp_stage):
@@ -367,11 +385,15 @@ def test_show_sort(tmp_dir, scm, dvc, exp_stage, caplog):
 
 
 def test_show_running_workspace(tmp_dir, scm, dvc, exp_stage, capsys):
-    pid_dir = os.path.join(dvc.tmp_dir, dvc.experiments.EXEC_PID_DIR)
-    makedirs(pid_dir, True)
-    info = ExecutorInfo(None, None, None, BaseExecutor.DEFAULT_LOCATION)
-    pidfile = os.path.join(pid_dir, f"workspace{BaseExecutor.PIDFILE_EXT}")
-    (tmp_dir / pidfile).dump(info.to_dict())
+    pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR)
+    info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION)
+    pidfile = os.path.join(
+        pid_dir,
+        "workspace",
+        f"workspace{BaseExecutor.INFOFILE_EXT}",
+    )
+    makedirs(os.path.dirname(pidfile), True)
+    (tmp_dir / pidfile).dump_json(info.asdict())
 
     assert dvc.experiments.show()["workspace"] == {
         "baseline": {
@@ -398,11 +420,15 @@ def test_show_running_executor(tmp_dir, scm, dvc, exp_stage):
     dvc.experiments.run(exp_stage.addressing, params=["foo=2"], queue=True)
     exp_rev = dvc.experiments.scm.resolve_rev(f"{EXPS_STASH}@{{0}}")
 
-    pid_dir = os.path.join(dvc.tmp_dir, dvc.experiments.EXEC_PID_DIR)
-    makedirs(pid_dir, True)
-    info = ExecutorInfo(None, None, None, BaseExecutor.DEFAULT_LOCATION)
-    pidfile = os.path.join(pid_dir, f"{exp_rev}{BaseExecutor.PIDFILE_EXT}")
-    (tmp_dir / pidfile).dump(info.to_dict())
+    pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR)
+    info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION)
+    pidfile = os.path.join(
+        pid_dir,
+        exp_rev,
+        f"{exp_rev}{BaseExecutor.INFOFILE_EXT}",
+    )
+    makedirs(os.path.dirname(pidfile), True)
+    (tmp_dir / pidfile).dump_json(info.asdict())
 
     results = dvc.experiments.show()
     exp_data = get_in(results, [baseline_rev, exp_rev, "data"])
@@ -430,17 +456,21 @@ def test_show_running_checkpoint(
     checkpoint_rev = first(run_results)
     exp_ref = first(exp_refs_by_rev(scm, checkpoint_rev))
 
-    pid_dir = os.path.join(dvc.tmp_dir, dvc.experiments.EXEC_PID_DIR)
-    makedirs(pid_dir, True)
+    pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR)
     executor = (
         BaseExecutor.DEFAULT_LOCATION
         if workspace
         else TempDirExecutor.DEFAULT_LOCATION
     )
-    info = ExecutorInfo(123, "foo.git", baseline_rev, executor)
+    info = make_executor_info(
+        git_url="foo.git",
+        baseline_rev=baseline_rev,
+        location=executor,
+    )
     rev = "workspace" if workspace else stash_rev
-    pidfile = os.path.join(pid_dir, f"{rev}{BaseExecutor.PIDFILE_EXT}")
-    (tmp_dir / pidfile).dump(info.to_dict())
+    pidfile = os.path.join(pid_dir, f"{rev}{BaseExecutor.INFOFILE_EXT}")
+    makedirs(os.path.dirname(pidfile), True)
+    (tmp_dir / pidfile).dump_json(info.asdict())
 
     mocker.patch.object(
         BaseExecutor, "fetch_exps", return_value=[str(exp_ref)]
@@ -559,7 +589,6 @@ def test_show_only_changed(tmp_dir, dvc, scm, capsys):
     assert main(["exp", "show"]) == 0
     cap = capsys.readouterr()
 
-    print(cap)
     assert "bar" in cap.out
 
     capsys.readouterr()
@@ -567,3 +596,78 @@ def test_show_only_changed(tmp_dir, dvc, scm, capsys):
     cap = capsys.readouterr()
 
     assert "bar" not in cap.out
+
+
+def test_show_parallel_coordinates(tmp_dir, dvc, scm, mocker):
+    from dvc.command.experiments import show
+
+    webbroser_open = mocker.patch("webbrowser.open")
+    show_experiments = mocker.spy(show, "show_experiments")
+
+    tmp_dir.gen("copy.py", COPY_SCRIPT)
+    params_file = tmp_dir / "params.yaml"
+    params_data = {
+        "foo": 1,
+        "bar": 1,
+    }
+    (tmp_dir / params_file).dump(params_data)
+
+    dvc.run(
+        cmd="python copy.py params.yaml metrics.yaml",
+        metrics_no_cache=["metrics.yaml"],
+        params=["foo", "bar"],
+        name="copy-file",
+        deps=["copy.py"],
+    )
+    scm.add(
+        [
+            "dvc.yaml",
+            "dvc.lock",
+            "copy.py",
+            "params.yaml",
+            "metrics.yaml",
+            ".gitignore",
+        ]
+    )
+    scm.commit("init")
+
+    dvc.experiments.run(params=["foo=2"])
+
+    assert main(["exp", "show", "--html"]) == 0
+    kwargs = show_experiments.call_args[1]
+
+    html_text = (tmp_dir / "dvc_plots" / "index.html").read_text()
+    assert all(rev in html_text for rev in ["workspace", "master", "[exp-"])
+
+    assert (
+        '{"label": "metrics.yaml:foo", "values": [2.0, 1.0, 2.0]}' in html_text
+    )
+    assert (
+        '{"label": "params.yaml:foo", "values": [2.0, 1.0, 2.0]}' in html_text
+    )
+    assert '"line": {"color": [2, 1, 0]' in html_text
+    assert '"label": "metrics.yaml:bar"' not in html_text
+
+    assert (
+        main(["exp", "show", "--html", "--sort-by", "metrics.yaml:foo"]) == 0
+    )
+    kwargs = show_experiments.call_args[1]
+
+    html_text = (tmp_dir / "dvc_plots" / "index.html").read_text()
+    assert '"line": {"color": [2.0, 1.0, 2.0]' in html_text
+
+    assert main(["exp", "show", "--html", "--out", "experiments"]) == 0
+    kwargs = show_experiments.call_args[1]
+
+    assert kwargs["out"] == "experiments"
+    assert (tmp_dir / "experiments" / "index.html").exists()
+
+    assert main(["exp", "show", "--html", "--open"]) == 0
+
+    webbroser_open.assert_called()
+
+    params_data = {"foo": 1, "bar": 1, "foobar": 2}
+    (tmp_dir / params_file).dump(params_data)
+    assert main(["exp", "show", "--html"]) == 0
+    html_text = (tmp_dir / "dvc_plots" / "index.html").read_text()
+    assert '{"label": "foobar", "values": [2.0, null, null]}' in html_text

tests/func/test_diff.py

@@ -33,6 +33,28 @@ def test_added(tmp_dir, scm, dvc):
     }
 
 
+def test_added_deep(tmp_dir, scm, dvc):
+    tmp_dir.gen({"datas": {"data": {"file": "text"}}})
+    dvc.add(os.path.join("datas", "data"))
+
+    assert dvc.diff() == {
+        "added": [
+            {
+                "path": os.path.join("datas", "data" + os.sep),
+                "hash": "0dab3fae569586d4c33272e5011605bf.dir",
+            },
+            {
+                "path": os.path.join("datas", "data", "file"),
+                "hash": "1cb251ec0d568de6a929b520c4aed8d1",
+            },
+        ],
+        "deleted": [],
+        "modified": [],
+        "not in cache": [],
+        "renamed": [],
+    }
+
+
 def test_no_cache_entry(tmp_dir, scm, dvc):
     tmp_dir.dvc_gen("file", "first", commit="add a file")
 

tests/func/test_fs.py

@@ -9,187 +9,157 @@ import pytest
 from dvc.fs import get_cloud_fs
 from dvc.fs.local import LocalFileSystem
 from dvc.repo import Repo
-from dvc.scm import SCM
-from tests.basic_env import TestDir, TestGit, TestGitSubmodule
 
 
-class TestLocalFileSystem(TestDir):
-    def setUp(self):
-        super().setUp()
-        self.fs = LocalFileSystem()
+def test_local_fs_open(tmp_dir):
+    tmp_dir.gen(
+        {
+            "foo": "foo",
+            "bar": "bar",
+            "тест": "проверка",
+            "code.py": "import sys\nimport shutil\n"
+            "shutil.copyfile(sys.argv[1], sys.argv[2])",
+            "data_dir": {
+                "data": "data",
+                "data_sub_dir": {"data_sub": "data_sub"},
+            },
+        }
+    )
+    fs = LocalFileSystem()
 
-    def test_open(self):
-        with self.fs.open(self.FOO) as fd:
-            self.assertEqual(fd.read(), self.FOO_CONTENTS)
-        with self.fs.open(self.UNICODE, encoding="utf-8") as fd:
-            self.assertEqual(fd.read(), self.UNICODE_CONTENTS)
-
-    def test_exists(self):
-        self.assertTrue(self.fs.exists(self.FOO))
-        self.assertTrue(self.fs.exists(self.UNICODE))
-        self.assertFalse(self.fs.exists("not-existing-file"))
-
-    def test_isdir(self):
-        self.assertTrue(self.fs.isdir(self.DATA_DIR))
-        self.assertFalse(self.fs.isdir(self.FOO))
-        self.assertFalse(self.fs.isdir("not-existing-file"))
-
-    def test_isfile(self):
-        self.assertTrue(self.fs.isfile(self.FOO))
-        self.assertFalse(self.fs.isfile(self.DATA_DIR))
-        self.assertFalse(self.fs.isfile("not-existing-file"))
+    with fs.open("foo", encoding="utf-8") as fobj:
+        assert fobj.read() == "foo"
+    with fs.open("тест", encoding="utf-8") as fobj:
+        assert fobj.read() == "проверка"
 
 
-class GitFileSystemTests:
-    # pylint: disable=no-member
-    def test_open(self):
-        self.scm.add([self.FOO, self.UNICODE, self.DATA_DIR])
-        self.scm.commit("add")
+def test_local_fs_exists(tmp_dir):
+    tmp_dir.gen(
+        {
+            "foo": "foo",
+            "bar": "bar",
+            "тест": "проверка",
+            "code.py": "import sys\nimport shutil\n"
+            "shutil.copyfile(sys.argv[1], sys.argv[2])",
+            "data_dir": {
+                "data": "data",
+                "data_sub_dir": {"data_sub": "data_sub"},
+            },
+        }
+    )
+    fs = LocalFileSystem()
 
-        fs = self.scm.get_fs("master")
-        with fs.open(self.FOO, mode="r", encoding="utf-8") as fd:
-            self.assertEqual(fd.read(), self.FOO_CONTENTS)
-        with fs.open(self.UNICODE, mode="r", encoding="utf-8") as fd:
-            self.assertEqual(fd.read(), self.UNICODE_CONTENTS)
-        with self.assertRaises(IOError):
-            fs.open("not-existing-file")
-        with self.assertRaises(IOError):
-            fs.open(self.DATA_DIR)
-
-    def test_exists(self):
-        fs = self.scm.get_fs("master")
-        self.assertFalse(fs.exists(self.FOO))
-        self.assertFalse(fs.exists(self.UNICODE))
-        self.assertFalse(fs.exists(self.DATA_DIR))
-        self.scm.add([self.FOO, self.UNICODE, self.DATA])
-        self.scm.commit("add")
-
-        fs = self.scm.get_fs("master")
-        self.assertTrue(fs.exists(self.FOO))
-        self.assertTrue(fs.exists(self.UNICODE))
-        self.assertTrue(fs.exists(self.DATA_DIR))
-        self.assertFalse(fs.exists("non-existing-file"))
-
-    def test_isdir(self):
-        self.scm.add([self.FOO, self.DATA_DIR])
-        self.scm.commit("add")
-
-        fs = self.scm.get_fs("master")
-        self.assertTrue(fs.isdir(self.DATA_DIR))
-        self.assertFalse(fs.isdir(self.FOO))
-        self.assertFalse(fs.isdir("non-existing-file"))
-
-    def test_isfile(self):
-        self.scm.add([self.FOO, self.DATA_DIR])
-        self.scm.commit("add")
-
-        fs = self.scm.get_fs("master")
-        self.assertTrue(fs.isfile(self.FOO))
-        self.assertFalse(fs.isfile(self.DATA_DIR))
-        self.assertFalse(fs.isfile("not-existing-file"))
+    assert fs.exists("foo")
+    assert fs.exists("тест")
+    assert not fs.exists("not-existing-file")
 
 
-class TestGitFileSystem(TestGit, GitFileSystemTests):
-    def setUp(self):
-        super().setUp()
-        self.scm = SCM(self._root_dir)
+def test_local_fs_isdir(tmp_dir):
+    tmp_dir.gen(
+        {
+            "foo": "foo",
+            "bar": "bar",
+            "тест": "проверка",
+            "code.py": "import sys\nimport shutil\n"
+            "shutil.copyfile(sys.argv[1], sys.argv[2])",
+            "data_dir": {
+                "data": "data",
+                "data_sub_dir": {"data_sub": "data_sub"},
+            },
+        }
+    )
+    fs = LocalFileSystem()
+
+    assert fs.isdir("data_dir")
+    assert not fs.isdir("foo")
+    assert not fs.isdir("not-existing-file")
 
 
-class TestGitSubmoduleFileSystem(TestGitSubmodule, GitFileSystemTests):
-    def setUp(self):
-        super().setUp()
-        self.scm = SCM(self._root_dir)
-        self._pushd(self._root_dir)
+def test_local_fs_isfile(tmp_dir):
+    tmp_dir.gen(
+        {
+            "foo": "foo",
+            "bar": "bar",
+            "тест": "проверка",
+            "code.py": "import sys\nimport shutil\n"
+            "shutil.copyfile(sys.argv[1], sys.argv[2])",
+            "data_dir": {
+                "data": "data",
+                "data_sub_dir": {"data_sub": "data_sub"},
+            },
+        }
+    )
+    fs = LocalFileSystem()
+
+    assert fs.isfile("foo")
+    assert not fs.isfile("data_dir")
+    assert not fs.isfile("not-existing-file")
 
 
-class AssertWalkEqualMixin:
-    def assertWalkEqual(self, actual, expected, msg=None):
-        def convert_to_sets(walk_results):
-            return [
-                (root, set(dirs), set(nondirs))
-                for root, dirs, nondirs in walk_results
-            ]
-
-        self.assertEqual(
-            convert_to_sets(actual), convert_to_sets(expected), msg=msg
-        )
+def convert_to_sets(walk_results):
+    return [
+        (root, set(dirs), set(nondirs)) for root, dirs, nondirs in walk_results
+    ]
 
 
-class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir):
-    def test(self):
-        fs = LocalFileSystem()
-        self.assertWalkEqual(
-            fs.walk(self._root_dir),
-            [
-                (
-                    self._root_dir,
-                    ["data_dir"],
-                    ["code.py", "bar", "тест", "foo"],
-                ),
-                (join(self._root_dir, "data_dir"), ["data_sub_dir"], ["data"]),
-                (
-                    join(self._root_dir, "data_dir", "data_sub_dir"),
-                    [],
-                    ["data_sub"],
-                ),
-            ],
-        )
+def test_walk_no_scm(tmp_dir):
+    tmp_dir.gen(
+        {
+            "foo": "foo",
+            "bar": "bar",
+            "тест": "проверка",
+            "code.py": "import sys\nimport shutil\n"
+            "shutil.copyfile(sys.argv[1], sys.argv[2])",
+            "data_dir": {
+                "data": "data",
+                "data_sub_dir": {"data_sub": "data_sub"},
+            },
+        }
+    )
+    fs = LocalFileSystem()
+    walk_results = fs.walk(str(tmp_dir))
+    assert convert_to_sets(walk_results) == [
+        (str(tmp_dir), {"data_dir"}, {"code.py", "bar", "тест", "foo"}),
+        (str(tmp_dir / "data_dir"), {"data_sub_dir"}, {"data"}),
+        (str(tmp_dir / "data_dir" / "data_sub_dir"), set(), {"data_sub"}),
+    ]
 
-    def test_subdir(self):
-        fs = LocalFileSystem()
-        self.assertWalkEqual(
-            fs.walk(join("data_dir", "data_sub_dir")),
-            [(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
-        )
+    walk_results = fs.walk(join("data_dir", "data_sub_dir"))
+    assert convert_to_sets(walk_results) == [
+        (join("data_dir", "data_sub_dir"), set(), {"data_sub"}),
+    ]
 
 
-class TestWalkInGit(AssertWalkEqualMixin, TestGit):
-    def test_nobranch(self):
-        fs = LocalFileSystem(url=self._root_dir)
-        walk_result = []
-        for root, dirs, files in fs.walk("."):
-            dirs[:] = [i for i in dirs if i != ".git"]
-            walk_result.append((root, dirs, files))
-        self.assertWalkEqual(
-            walk_result,
-            [
-                (".", ["data_dir"], ["bar", "тест", "code.py", "foo"]),
-                (join("data_dir"), ["data_sub_dir"], ["data"]),
-                (join("data_dir", "data_sub_dir"), [], ["data_sub"]),
-            ],
-        )
-        self.assertWalkEqual(
-            fs.walk(join("data_dir", "data_sub_dir")),
-            [(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
-        )
+def test_walk_fs_with_git(tmp_dir, scm):
+    tmp_dir.gen(
+        {
+            "foo": "foo",
+            "bar": "bar",
+            "тест": "проверка",
+            "code.py": "import sys\nimport shutil\n"
+            "shutil.copyfile(sys.argv[1], sys.argv[2])",
+            "data_dir": {
+                "data": "data",
+                "data_sub_dir": {"data_sub": "data_sub"},
+            },
+        }
+    )
+    fs = LocalFileSystem(url=str(tmp_dir))
+    walk_result = []
+    for root, dirs, files in fs.walk("."):
+        dirs[:] = [i for i in dirs if i != ".git"]
+        walk_result.append((root, dirs, files))
 
-    def test_branch(self):
-        scm = SCM(self._root_dir)
-        scm.add([self.DATA_SUB_DIR])
-        scm.commit("add data_dir/data_sub_dir/data_sub")
-        fs = scm.get_fs("master")
-        self.assertWalkEqual(
-            fs.walk("."),
-            [
-                (self._root_dir, ["data_dir"], ["code.py"]),
-                (join(self._root_dir, "data_dir"), ["data_sub_dir"], []),
-                (
-                    join(self._root_dir, "data_dir", "data_sub_dir"),
-                    [],
-                    ["data_sub"],
-                ),
-            ],
-        )
-        self.assertWalkEqual(
-            fs.walk(join("data_dir", "data_sub_dir")),
-            [
-                (
-                    join(self._root_dir, "data_dir", "data_sub_dir"),
-                    [],
-                    ["data_sub"],
-                )
-            ],
-        )
+    assert convert_to_sets(walk_result) == [
+        (".", {"data_dir"}, {"bar", "тест", "code.py", "foo"}),
+        (join("data_dir"), {"data_sub_dir"}, {"data"}),
+        (join("data_dir", "data_sub_dir"), set(), {"data_sub"}),
+    ]
+
+    walk_result = fs.walk(join("data_dir", "data_sub_dir"))
+    assert convert_to_sets(walk_result) == [
+        (join("data_dir", "data_sub_dir"), set(), {"data_sub"})
+    ]
 
 
 def test_cleanfs_subrepo(tmp_dir, dvc, scm, monkeypatch):

tests/func/test_scm.py

@@ -1,13 +1,9 @@
 import os
-import sys
 
 import pytest
 from git import Repo
 
 from dvc.scm import SCM, Git, NoSCM, SCMError
-from dvc.system import System
-from tests.basic_env import TestGitSubmodule
-from tests.utils import get_gitignore_content
 
 
 def test_init_none(tmp_dir):
@@ -31,351 +27,3 @@ def test_init_sub_dir(tmp_dir):
 
     scm = SCM(os.fspath(subdir))
     assert scm.root_dir == os.fspath(tmp_dir)
-
-
-def test_commit(tmp_dir, scm):
-    tmp_dir.gen({"foo": "foo"})
-    scm.add(["foo"])
-    scm.commit("add")
-    assert "foo" in scm.gitpython.git.ls_files()
-
-
-def test_is_tracked(tmp_dir, scm):
-    tmp_dir.gen({"foo": "foo", "тест": "проверка"})
-    scm.add(["foo", "тест"])
-    abs_foo = os.path.abspath("foo")
-    assert scm.is_tracked(abs_foo)
-    assert scm.is_tracked("foo")
-    assert scm.is_tracked("тест")
-    scm.commit("add")
-    assert scm.is_tracked(abs_foo)
-    assert scm.is_tracked("foo")
-    scm.gitpython.repo.index.remove(["foo"], working_tree=True)
-    assert not scm.is_tracked(abs_foo)
-    assert not scm.is_tracked("foo")
-    assert not scm.is_tracked("not-existing-file")
-
-
-class TestSCMGitSubmodule(TestGitSubmodule):
-    def test_git_submodule(self):
-        self.assertIsInstance(SCM(os.curdir), Git)
-
-    def test_commit_in_submodule(self):
-        G = Git(self._root_dir)
-        G.add(["foo"])
-        G.commit("add")
-        self.assertTrue("foo" in self.git.git.ls_files())
-
-
-def _count_gitignore_entries(line):
-    lines = get_gitignore_content()
-    return lines.count(line)
-
-
-def test_ignore(tmp_dir, scm):
-    foo = os.fspath(tmp_dir / "foo")
-    target = "/foo"
-
-    scm.ignore(foo)
-    assert (tmp_dir / ".gitignore").is_file()
-    scm._reset()
-    assert _count_gitignore_entries(target) == 1
-
-    scm.ignore(foo)
-    assert (tmp_dir / ".gitignore").is_file()
-    scm._reset()
-    assert _count_gitignore_entries(target) == 1
-
-    scm.ignore_remove(foo)
-    assert not (tmp_dir / ".gitignore").exists()
-
-
-def test_ignored(tmp_dir, scm):
-    tmp_dir.gen({"dir1": {"file1.jpg": "cont", "file2.txt": "cont"}})
-    tmp_dir.gen({".gitignore": "dir1/*.jpg"})
-
-    assert scm.is_ignored(tmp_dir / "dir1" / "file1.jpg")
-    assert not scm.is_ignored(tmp_dir / "dir1" / "file2.txt")
-
-
-def test_ignored_dir_unignored_subdirs(tmp_dir, scm):
-    tmp_dir.gen({".gitignore": "data/**\n!data/**/\n!data/**/*.csv"})
-    scm.add([".gitignore"])
-    tmp_dir.gen(
-        {
-            os.path.join("data", "raw", "tracked.csv"): "cont",
-            os.path.join("data", "raw", "not_tracked.json"): "cont",
-        }
-    )
-
-    assert not scm.is_ignored(tmp_dir / "data" / "raw" / "tracked.csv")
-    assert scm.is_ignored(tmp_dir / "data" / "raw" / "not_tracked.json")
-    assert not scm.is_ignored(tmp_dir / "data" / "raw" / "non_existent.csv")
-    assert scm.is_ignored(tmp_dir / "data" / "raw" / "non_existent.json")
-    assert not scm.is_ignored(tmp_dir / "data" / "non_existent.csv")
-    assert scm.is_ignored(tmp_dir / "data" / "non_existent.json")
-
-    assert not scm.is_ignored(f"data{os.sep}")
-    # git check-ignore would now mark "data/raw" as ignored
-    # after detecting it's a directory in the file system;
-    # instead, we rely on the trailing separator to determine if handling a
-    # a directory - for consistency between existent and non-existent paths
-    assert scm.is_ignored(os.path.join("data", "raw"))
-    assert not scm.is_ignored(os.path.join("data", f"raw{os.sep}"))
-
-    assert scm.is_ignored(os.path.join("data", "non_existent"))
-    assert not scm.is_ignored(os.path.join("data", f"non_existent{os.sep}"))
-
-
-def test_get_gitignore(tmp_dir, scm):
-    tmp_dir.gen({"file1": "contents", "dir": {}})
-
-    data_dir = os.fspath(tmp_dir / "file1")
-    entry, gitignore = scm._get_gitignore(data_dir)
-    assert entry == "/file1"
-    assert gitignore == os.fspath(tmp_dir / ".gitignore")
-
-    data_dir = os.fspath(tmp_dir / "dir")
-    entry, gitignore = scm._get_gitignore(data_dir)
-
-    assert entry == "/dir"
-    assert gitignore == os.fspath(tmp_dir / ".gitignore")
-
-
-def test_get_gitignore_symlink(tmp_dir, scm):
-    tmp_dir.gen({"dir": {"subdir": {"data": "contents"}}})
-    link = os.fspath(tmp_dir / "link")
-    target = os.fspath(tmp_dir / "dir" / "subdir" / "data")
-    System.symlink(target, link)
-    entry, gitignore = scm._get_gitignore(link)
-    assert entry == "/link"
-    assert gitignore == os.fspath(tmp_dir / ".gitignore")
-
-
-def test_get_gitignore_subdir(tmp_dir, scm):
-    tmp_dir.gen({"dir1": {"file1": "cont", "dir2": {}}})
-
-    data_dir = os.fspath(tmp_dir / "dir1" / "file1")
-    entry, gitignore = scm._get_gitignore(data_dir)
-    assert entry == "/file1"
-    assert gitignore == os.fspath(tmp_dir / "dir1" / ".gitignore")
-
-    data_dir = os.fspath(tmp_dir / "dir1" / "dir2")
-    entry, gitignore = scm._get_gitignore(data_dir)
-    assert entry == "/dir2"
-    assert gitignore == os.fspath(tmp_dir / "dir1" / ".gitignore")
-
-
-def test_gitignore_should_end_with_newline(tmp_dir, scm):
-    tmp_dir.gen({"foo": "foo", "bar": "bar"})
-
-    foo = os.fspath(tmp_dir / "foo")
-    bar = os.fspath(tmp_dir / "bar")
-    gitignore = tmp_dir / ".gitignore"
-
-    scm.ignore(foo)
-    assert gitignore.read_text().endswith("\n")
-
-    scm.ignore(bar)
-    assert gitignore.read_text().endswith("\n")
-
-
-def test_gitignore_should_append_newline_to_gitignore(tmp_dir, scm):
-    tmp_dir.gen({"foo": "foo", "bar": "bar"})
-
-    bar_path = os.fspath(tmp_dir / "bar")
-    gitignore = tmp_dir / ".gitignore"
-
-    gitignore.write_text("/foo")
-    assert not gitignore.read_text().endswith("\n")
-
-    scm.ignore(bar_path)
-    contents = gitignore.read_text()
-    assert gitignore.read_text().endswith("\n")
-
-    assert contents.splitlines() == ["/foo", "/bar"]
-
-
-def test_git_detach_head(tmp_dir, scm):
-    tmp_dir.scm_gen({"file": "0"}, commit="init")
-    init_rev = scm.get_rev()
-
-    with scm.detach_head() as rev:
-        assert init_rev == rev
-        assert init_rev == (tmp_dir / ".git" / "HEAD").read_text().strip()
-    assert (
-        "ref: refs/heads/master"
-        == (tmp_dir / ".git" / "HEAD").read_text().strip()
-    )
-
-
-def test_git_stash_workspace(tmp_dir, scm):
-    tmp_dir.scm_gen({"file": "0"}, commit="init")
-    tmp_dir.gen("file", "1")
-
-    with scm.stash_workspace():
-        assert not scm.is_dirty()
-        assert "0" == (tmp_dir / "file").read_text()
-    assert scm.is_dirty()
-    assert "1" == (tmp_dir / "file").read_text()
-
-
-@pytest.mark.parametrize(
-    "ref, include_untracked",
-    [
-        (None, True),
-        (None, False),
-        ("refs/foo/stash", True),
-        ("refs/foo/stash", False),
-    ],
-)
-def test_git_stash_push(tmp_dir, scm, ref, include_untracked):
-    from scmrepo.git import Stash
-
-    tmp_dir.scm_gen({"file": "0"}, commit="init")
-    tmp_dir.gen({"file": "1", "untracked": "0"})
-
-    stash = Stash(scm, ref=ref)
-    rev = stash.push(include_untracked=include_untracked)
-    assert rev == scm.get_ref(stash.ref)
-    assert "0" == (tmp_dir / "file").read_text()
-    assert include_untracked != (tmp_dir / "untracked").exists()
-    assert len(stash) == 1
-
-    stash.apply(rev)
-    assert "1" == (tmp_dir / "file").read_text()
-    assert "0" == (tmp_dir / "untracked").read_text()
-
-    parts = list(stash.ref.split("/"))
-    assert os.path.exists(os.path.join(os.fspath(tmp_dir), ".git", *parts))
-    assert os.path.exists(
-        os.path.join(os.fspath(tmp_dir), ".git", "logs", *parts)
-    )
-
-
-@pytest.mark.parametrize("ref", [None, "refs/foo/stash"])
-def test_git_stash_drop(tmp_dir, scm, ref):
-    from scmrepo.git import Stash
-
-    tmp_dir.scm_gen({"file": "0"}, commit="init")
-    tmp_dir.gen("file", "1")
-
-    stash = Stash(scm, ref=ref)
-    stash.push()
-
-    tmp_dir.gen("file", "2")
-    expected = stash.push()
-
-    stash.drop(1)
-    assert expected == scm.get_ref(stash.ref)
-    assert len(stash) == 1
-
-
-reason = """libgit2 stash_save() is flaky on linux when run inside pytest
-    https://github.com/iterative/dvc/pull/5286#issuecomment-792574294"""
-
-
-@pytest.mark.parametrize(
-    "ref",
-    [
-        pytest.param(
-            None,
-            marks=pytest.mark.xfail(
-                sys.platform == "linux", raises=AssertionError, reason=reason
-            ),
-        ),
-        "refs/foo/stash",
-    ],
-)
-def test_git_stash_pop(tmp_dir, scm, ref):
-    from scmrepo.git import Stash
-
-    tmp_dir.scm_gen({"file": "0"}, commit="init")
-    tmp_dir.gen("file", "1")
-
-    stash = Stash(scm, ref=ref)
-    first = stash.push()
-
-    tmp_dir.gen("file", "2")
-    second = stash.push()
-
-    assert second == stash.pop()
-    assert len(stash) == 1
-    assert first == scm.get_ref(stash.ref)
-    assert "2" == (tmp_dir / "file").read_text()
-
-
-@pytest.mark.parametrize("ref", [None, "refs/foo/stash"])
-def test_git_stash_clear(tmp_dir, scm, ref):
-    from scmrepo.git import Stash
-
-    tmp_dir.scm_gen({"file": "0"}, commit="init")
-    tmp_dir.gen("file", "1")
-
-    stash = Stash(scm, ref=ref)
-    stash.push()
-
-    tmp_dir.gen("file", "2")
-    stash.push()
-
-    stash.clear()
-    assert len(stash) == 0
-
-    parts = list(stash.ref.split("/"))
-    assert not os.path.exists(os.path.join(os.fspath(tmp_dir), ".git", *parts))
-    # NOTE: some backends will completely remove reflog file on clear, some
-    # will only truncate it, either case means an empty stash
-    log_path = os.path.join(os.fspath(tmp_dir), ".git", "logs", *parts)
-    assert (
-        not os.path.exists(log_path)
-        or not open(log_path, encoding="utf-8").read()
-    )
-
-
-@pytest.mark.needs_internet
-@pytest.mark.parametrize("server", [pytest.lazy_fixture("git_ssh")])
-def test_git_ssh(tmp_dir, scm, server):
-    from dulwich.repo import Repo as DulwichRepo
-    from dvc_ssh.tests.cloud import TEST_SSH_KEY_PATH, TEST_SSH_USER
-    from sshfs import SSHFileSystem
-
-    from dvc.utils.fs import remove
-
-    fs = SSHFileSystem(
-        host=server.host,
-        port=server.port,
-        username=TEST_SSH_USER,
-        client_keys=[TEST_SSH_KEY_PATH],
-    )
-    server._ssh.execute("git init --bare test-repo.git")
-    url = f"ssh://{TEST_SSH_USER}@{server.host}:{server.port}/~/test-repo.git"
-
-    tmp_dir.scm_gen("foo", "foo", commit="init")
-    rev = scm.get_rev()
-
-    scm.push_refspec(
-        url,
-        "refs/heads/master",
-        "refs/heads/master",
-        force=True,
-        key_filename=TEST_SSH_KEY_PATH,
-    )
-
-    assert (
-        rev.encode("ascii")
-        == fs.open("test-repo.git/refs/heads/master").read().strip()
-    )
-
-    remove(tmp_dir / ".git")
-    remove(tmp_dir / "foo")
-    DulwichRepo.init(str(tmp_dir))
-
-    scm.fetch_refspecs(
-        url,
-        ["refs/heads/master"],
-        force=True,
-        key_filename=TEST_SSH_KEY_PATH,
-    )
-    assert rev == scm.get_ref("refs/heads/master")
-    scm.checkout("master")
-    assert "foo" == (tmp_dir / "foo").read_text()

tests/unit/command/test_compat_flag.py

@@ -29,6 +29,8 @@ def _id_gen(val) -> str:
         (["params", "diff", "--show-json"], "json"),
         (["params", "diff", "--show-md"], "markdown"),
         (["status", "--show-json"], "json"),
+        (["plots", "show", "--show-json"], "json"),
+        (["plots", "diff", "--show-json"], "json"),
     ],
     ids=_id_gen,
 )

tests/unit/command/test_experiments.py

@@ -5,20 +5,17 @@ from datetime import datetime
 import pytest
 
 from dvc.cli import parse_args
-from dvc.command.experiments import (
-    CmdExperimentsApply,
-    CmdExperimentsBranch,
-    CmdExperimentsDiff,
-    CmdExperimentsGC,
-    CmdExperimentsInit,
-    CmdExperimentsList,
-    CmdExperimentsPull,
-    CmdExperimentsPush,
-    CmdExperimentsRemove,
-    CmdExperimentsRun,
-    CmdExperimentsShow,
-    show_experiments,
-)
+from dvc.command.experiments.apply import CmdExperimentsApply
+from dvc.command.experiments.branch import CmdExperimentsBranch
+from dvc.command.experiments.diff import CmdExperimentsDiff
+from dvc.command.experiments.gc import CmdExperimentsGC
+from dvc.command.experiments.init import CmdExperimentsInit
+from dvc.command.experiments.ls import CmdExperimentsList
+from dvc.command.experiments.pull import CmdExperimentsPull
+from dvc.command.experiments.push import CmdExperimentsPush
+from dvc.command.experiments.remove import CmdExperimentsRemove
+from dvc.command.experiments.run import CmdExperimentsRun
+from dvc.command.experiments.show import CmdExperimentsShow, show_experiments
 from dvc.exceptions import DvcParserError, InvalidArgumentError
 from dvc.repo import Repo
 from dvc.stage import PipelineStage
@@ -649,6 +646,7 @@ def test_experiments_init_config(dvc, scm, mocker):
 
     assert isinstance(cmd, CmdExperimentsInit)
     assert cmd.run() == 0
+
     m.assert_called_once_with(
         ANY(Repo),
         name="train",
@@ -761,3 +759,37 @@ def test_experiments_init_extra_args(extra_args, expected_kw, mocker):
 def test_experiments_init_type_invalid_choice():
     with pytest.raises(DvcParserError):
         parse_args(["exp", "init", "--type=invalid", "cmd"])
+
+
+def test_show_experiments_html(tmp_dir, mocker):
+    all_experiments = {
+        "workspace": {
+            "baseline": {
+                "data": {
+                    "timestamp": None,
+                    "params": {"params.yaml": {"data": {"foo": 1}}},
+                    "queued": False,
+                    "running": False,
+                    "executor": None,
+                    "metrics": {
+                        "scores.json": {"data": {"bar": 0.9544670443829399}}
+                    },
+                }
+            }
+        },
+    }
+    experiments_table = mocker.patch(
+        "dvc.command.experiments.show.experiments_table"
+    )
+    td = experiments_table.return_value
+
+    show_experiments(all_experiments, html=True)
+
+    td.dropna.assert_called_with("rows", how="all")
+
+    render_kwargs = td.render.call_args[1]
+
+    for arg in ["html", "output_path", "color_by"]:
+        assert arg in render_kwargs
+    assert render_kwargs["output_path"] == tmp_dir / "dvc_plots"
+    assert render_kwargs["color_by"] == "Experiment"

tests/unit/command/test_plots.py

@@ -1,3 +1,4 @@
+import json
 import os
 import posixpath
 from pathlib import Path
@@ -125,16 +126,15 @@ def test_plots_diff_vega(dvc, mocker, capsys, plots_data):
     cmd = cli_args.func(cli_args)
     mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data)
     mocker.patch(
-        "dvc.command.plots.find_vega", return_value="vega_json_content"
-    )
-    render_mock = mocker.patch(
-        "dvc.command.plots.render", return_value="html_path"
+        "dvc.command.plots.VegaRenderer.asdict",
+        return_value={"this": "is vega json"},
     )
+    render_mock = mocker.patch("dvc.command.plots.render")
     assert cmd.run() == 0
 
     out, _ = capsys.readouterr()
 
-    assert "vega_json_content" in out
+    assert json.dumps({"this": "is vega json"}) in out
     render_mock.assert_not_called()
 
 
@@ -243,6 +243,8 @@ def test_should_call_render(tmp_dir, mocker, capsys, plots_data, output):
 
     output = output or "dvc_plots"
     index_path = tmp_dir / output / "index.html"
+    renderers = mocker.MagicMock()
+    mocker.patch("dvc.command.plots.match_renderers", return_value=renderers)
     render_mock = mocker.patch(
         "dvc.command.plots.render", return_value=index_path
     )
@@ -253,5 +255,37 @@ def test_should_call_render(tmp_dir, mocker, capsys, plots_data, output):
     assert index_path.as_uri() in out
 
     render_mock.assert_called_once_with(
-        cmd.repo, plots_data, path=tmp_dir / output, html_template_path=None
+        cmd.repo, renderers, path=tmp_dir / output, html_template_path=None
     )
+
+
+def test_plots_diff_json(dvc, mocker, capsys):
+    cli_args = parse_args(
+        [
+            "plots",
+            "diff",
+            "HEAD~10",
+            "HEAD~1",
+            "--json",
+            "--targets",
+            "plot.csv",
+            "-o",
+            "out",
+        ]
+    )
+    cmd = cli_args.func(cli_args)
+
+    data = mocker.MagicMock()
+    mocker.patch("dvc.repo.plots.diff.diff", return_value=data)
+
+    renderers = mocker.MagicMock()
+    mocker.patch("dvc.command.plots.match_renderers", return_value=renderers)
+    render_mock = mocker.patch("dvc.command.plots.render")
+
+    show_json_mock = mocker.patch("dvc.command.plots._show_json")
+
+    assert cmd.run() == 0
+
+    show_json_mock.assert_called_once_with(renderers, "out")
+
+    render_mock.assert_not_called()

tests/unit/fs/test_hdfs.py

@@ -0,0 +1,16 @@
+import pytest
+
+from dvc.fs import HDFSFileSystem
+
+
+@pytest.mark.parametrize(
+    "fs_args, expected_url",
+    [
+        ({"host": "example.com"}, "hdfs://example.com"),
+        ({"host": "example.com", "port": None}, "hdfs://example.com"),
+        ({"host": "example.com", "port": 8020}, "hdfs://example.com:8020"),
+    ],
+)
+def test_hdfs_unstrip_protocol(fs_args, expected_url):
+    fs = HDFSFileSystem(**fs_args)
+    assert fs.unstrip_protocol("/path") == expected_url + "/path"

tests/unit/render/test_image.py

@@ -1,3 +1,4 @@
+import json
 import os
 
 import pytest
@@ -44,3 +45,35 @@ def test_render(tmp_dir):
     assert (
         f'<img src="{os.path.join("static", "workspace_file.jpg")}">' in html
     )
+
+
+def test_as_json(tmp_dir):
+    data = {
+        "workspace": {"data": {"file.jpg": {"data": b"content"}}},
+        "HEAD": {"data": {"file.jpg": {"data": b"content2"}}},
+    }
+
+    page_dir = os.path.join("some", "path")
+    json_content = json.loads(ImageRenderer(data).as_json(path=page_dir))
+
+    img1 = os.path.join(page_dir, "workspace_file.jpg")
+    img2 = os.path.join(page_dir, "HEAD_file.jpg")
+
+    assert {
+        "type": "image",
+        "revisions": ["workspace"],
+        "url": os.path.abspath(img1),
+    } in json_content
+    assert {
+        "type": "image",
+        "revisions": ["HEAD"],
+        "url": os.path.abspath(img2),
+    } in json_content
+
+    def assert_file_with_content(path, content):
+        assert os.path.isfile(path)
+        with open(path, "rb") as fd:
+            assert fd.read() == content
+
+    assert_file_with_content(img1, b"content")
+    assert_file_with_content(img2, b"content2")

tests/unit/render/test_parallel_coordinates.py

@@ -0,0 +1,156 @@
+import json
+
+from dvc.compare import TabularData
+from dvc.render.html import write
+from dvc.render.plotly import ParallelCoordinatesRenderer
+
+# pylint: disable=W1514
+
+
+def expected_format(result):
+    assert "data" in result
+    assert "layout" in result
+    assert isinstance(result["data"], list)
+    assert result["data"][0]["type"] == "parcoords"
+    assert isinstance(result["data"][0]["dimensions"], list)
+    return True
+
+
+def test_scalar_columns():
+    td = TabularData(["col-1", "col-2", "col-3"])
+    td.extend([["0.1", "1", ""], ["2", "0.2", "0"]])
+    renderer = ParallelCoordinatesRenderer(td)
+
+    result = json.loads(renderer.as_json())
+
+    assert expected_format(result)
+
+    assert result["data"][0]["dimensions"][0] == {
+        "label": "col-1",
+        "values": [0.1, 2.0],
+    }
+    assert result["data"][0]["dimensions"][1] == {
+        "label": "col-2",
+        "values": [1.0, 0.2],
+    }
+    assert result["data"][0]["dimensions"][2] == {
+        "label": "col-3",
+        "values": [None, 0],
+    }
+
+
+def test_categorical_columns():
+    td = TabularData(["col-1", "col-2"])
+    td.extend([["foo", ""], ["bar", "foobar"], ["foo", ""]])
+    renderer = ParallelCoordinatesRenderer(td)
+
+    result = json.loads(renderer.as_json())
+
+    assert expected_format(result)
+
+    assert result["data"][0]["dimensions"][0] == {
+        "label": "col-1",
+        "values": [1, 0, 1],
+        "tickvals": [1, 0, 1],
+        "ticktext": ["foo", "bar", "foo"],
+    }
+    assert result["data"][0]["dimensions"][1] == {
+        "label": "col-2",
+        "values": [1, 0, 1],
+        "tickvals": [1, 0, 1],
+        "ticktext": ["Missing", "foobar", "Missing"],
+    }
+
+
+def test_mixed_columns():
+    td = TabularData(["categorical", "scalar"])
+    td.extend([["foo", "0.1"], ["bar", "2"]])
+    renderer = ParallelCoordinatesRenderer(td)
+
+    result = json.loads(renderer.as_json())
+
+    assert expected_format(result)
+
+    assert result["data"][0]["dimensions"][0] == {
+        "label": "categorical",
+        "values": [1, 0],
+        "tickvals": [1, 0],
+        "ticktext": ["foo", "bar"],
+    }
+    assert result["data"][0]["dimensions"][1] == {
+        "label": "scalar",
+        "values": [0.1, 2.0],
+    }
+
+
+def test_color_by_scalar():
+    td = TabularData(["categorical", "scalar"])
+    td.extend([["foo", "0.1"], ["bar", "2"]])
+    renderer = ParallelCoordinatesRenderer(td, color_by="scalar")
+
+    result = json.loads(renderer.as_json())
+
+    assert expected_format(result)
+    assert result["data"][0]["line"] == {
+        "color": [0.1, 2.0],
+        "showscale": True,
+        "colorbar": {"title": "scalar"},
+    }
+
+
+def test_color_by_categorical():
+    td = TabularData(["categorical", "scalar"])
+    td.extend([["foo", "0.1"], ["bar", "2"]])
+    renderer = ParallelCoordinatesRenderer(td, color_by="categorical")
+
+    result = json.loads(renderer.as_json())
+
+    assert expected_format(result)
+    assert result["data"][0]["line"] == {
+        "color": [1, 0],
+        "showscale": True,
+        "colorbar": {
+            "title": "categorical",
+            "tickmode": "array",
+            "tickvals": [1, 0],
+            "ticktext": ["foo", "bar"],
+        },
+    }
+
+
+def test_write_parallel_coordinates(tmp_dir):
+    td = TabularData(["categorical", "scalar"])
+    td.extend([["foo", "0.1"], ["bar", "2"]])
+
+    renderer = ParallelCoordinatesRenderer(td)
+    html_path = write(tmp_dir, renderers=[renderer])
+
+    html_text = html_path.read_text()
+
+    assert ParallelCoordinatesRenderer.SCRIPTS in html_text
+
+    div = ParallelCoordinatesRenderer.DIV.format(
+        id="plot_experiments", partial=renderer.as_json()
+    )
+    assert div in html_text
+
+
+def test_fill_value():
+    td = TabularData(["categorical", "scalar"])
+    td.extend([["foo", "-"], ["-", "2"]])
+    renderer = ParallelCoordinatesRenderer(td, fill_value="-")
+
+    result = json.loads(renderer.as_json())
+
+    assert expected_format(result)
+
+    assert result["data"][0]["dimensions"][0] == {
+        "label": "categorical",
+        "values": [0, 1],
+        "tickvals": [0, 1],
+        "ticktext": ["foo", "Missing"],
+    }
+    assert result["data"][0]["dimensions"][1] == {
+        "label": "scalar",
+        "values": [None, 2.0],
+    }

tests/unit/render/test_render.py

@@ -1,6 +1,9 @@
 import os
 
-from dvc.render.utils import find_vega, render
+import dpath
+
+from dvc.render.utils import match_renderers, render
+from dvc.render.vega import VegaRenderer
 
 
 def assert_website_has_image(page_path, revision, filename, image_content):
@@ -48,8 +51,9 @@ def test_render(tmp_dir, dvc):
             }
         },
     }
+    renderers = match_renderers(data, dvc.plots.templates)
 
-    render(dvc, data, path=os.path.join("results", "dir"))
+    render(dvc, renderers, path=os.path.join("results", "dir"))
     page_path = tmp_dir / "results" / "dir"
     index_path = page_path / "index.html"
 
@@ -60,12 +64,13 @@ def test_render(tmp_dir, dvc):
     assert_website_has_image(page_path, "v2", "other_file.jpg", b"content2")
     assert_website_has_image(page_path, "v1", "another.gif", b"content3")
 
-    index_content = index_path.read_text()
-    file_vega = find_vega(dvc, data, "file.json")
-    some_vega = find_vega(dvc, data, "some.csv")
-
     def clean(txt: str) -> str:
         return txt.replace("\n", "").replace("\r", "").replace(" ", "")
 
-    assert clean(file_vega) in clean(index_content)
-    assert clean(some_vega) in clean(index_content)
+    def get_vega_string(data, filename):
+        file_data = dpath.util.search(data, ["*", "*", filename])
+        return VegaRenderer(file_data, dvc.plots.templates).partial_html()
+
+    index_content = index_path.read_text()
+    assert clean(get_vega_string(data, "file.json")) in clean(index_content)
+    assert clean(get_vega_string(data, "some.csv")) in clean(index_content)

tests/unit/render/test_vega.py

@@ -5,7 +5,7 @@ from collections import OrderedDict
 import pytest
 from funcy import first
 
-from dvc.render.utils import find_vega, group_by_filename
+from dvc.render.utils import group_by_filename
 from dvc.render.vega import (
     INDEX_FIELD,
     REVISION_FIELD,
@@ -107,9 +107,8 @@ def test_one_column(tmp_dir, scm, dvc):
         }
     }
 
-    plot_string = VegaRenderer(data, dvc.plots.templates).as_json()
+    plot_content = VegaRenderer(data, dvc.plots.templates).asdict()
 
-    plot_content = json.loads(plot_string)
     assert plot_content["title"] == "mytitle"
     assert plot_content["data"]["values"] == [
         {"val": 2, INDEX_FIELD: 0, REVISION_FIELD: "workspace"},
@@ -133,9 +132,8 @@ def test_multiple_columns(tmp_dir, scm, dvc):
         "workspace": {"data": {"file.json": {"data": metric, "props": {}}}}
     }
 
-    plot_string = VegaRenderer(data, dvc.plots.templates).as_json()
+    plot_content = VegaRenderer(data, dvc.plots.templates).asdict()
 
-    plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
         {
             "val": 2,
@@ -169,9 +167,8 @@ def test_choose_axes(tmp_dir, scm, dvc):
     data = {
         "workspace": {"data": {"file.json": {"data": metric, "props": props}}}
     }
-    plot_string = VegaRenderer(data, dvc.plots.templates).as_json()
+    plot_content = VegaRenderer(data, dvc.plots.templates).asdict()
 
-    plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
         {
             "val": 2,
@@ -207,9 +204,8 @@ def test_confusion(tmp_dir, dvc):
         }
     }
 
-    plot_string = VegaRenderer(data, dvc.plots.templates).as_json()
+    plot_content = VegaRenderer(data, dvc.plots.templates).asdict()
 
-    plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
         {"predicted": "B", "actual": "A", REVISION_FIELD: "workspace"},
         {"predicted": "A", "actual": "A", REVISION_FIELD: "workspace"},
@@ -245,9 +241,8 @@ def test_multiple_revs_default(tmp_dir, scm, dvc):
         },
     }
 
-    plot_string = VegaRenderer(data, dvc.plots.templates).as_json()
+    plot_content = VegaRenderer(data, dvc.plots.templates).asdict()
 
-    plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
         {"y": 5, INDEX_FIELD: 0, REVISION_FIELD: "HEAD"},
         {"y": 6, INDEX_FIELD: 1, REVISION_FIELD: "HEAD"},
@@ -271,9 +266,8 @@ def test_metric_missing(tmp_dir, scm, dvc, caplog):
             "data": {"file.json": {"error": FileNotFoundError(), "props": {}}}
         },
     }
-    plot_string = VegaRenderer(data, dvc.plots.templates).as_json()
+    plot_content = VegaRenderer(data, dvc.plots.templates).asdict()
 
-    plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
         {"y": 2, INDEX_FIELD: 0, REVISION_FIELD: "v2"},
         {"y": 3, INDEX_FIELD: 1, REVISION_FIELD: "v2"},
@@ -291,9 +285,8 @@ def test_custom_template(tmp_dir, scm, dvc, custom_template):
         "workspace": {"data": {"file.json": {"data": metric, "props": props}}}
     }
 
-    plot_string = VegaRenderer(data, dvc.plots.templates).as_json()
+    plot_content = VegaRenderer(data, dvc.plots.templates).asdict()
 
-    plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
         {"a": 1, "b": 2, REVISION_FIELD: "workspace"},
         {"a": 2, "b": 3, REVISION_FIELD: "workspace"},
@@ -310,7 +303,7 @@ def test_raise_on_no_template(tmp_dir, dvc):
     }
 
     with pytest.raises(TemplateNotFoundError):
-        VegaRenderer(data, dvc.plots.templates).as_json()
+        VegaRenderer(data, dvc.plots.templates).asdict()
 
 
 def test_bad_template(tmp_dir, dvc):
@@ -322,7 +315,7 @@ def test_bad_template(tmp_dir, dvc):
     }
 
     with pytest.raises(BadTemplateError):
-        VegaRenderer(data, dvc.plots.templates).as_json()
+        VegaRenderer(data, dvc.plots.templates).asdict()
 
 
 def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template):
@@ -337,9 +330,8 @@ def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template):
         "workspace": {"data": {"file.json": {"data": metric, "props": props}}}
     }
 
-    plot_string = VegaRenderer(data, dvc.plots.templates).as_json()
+    plot_content = VegaRenderer(data, dvc.plots.templates).asdict()
 
-    plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
         {"b": 2, "c": 3, REVISION_FIELD: "workspace"},
         {"b": 3, "c": 4, REVISION_FIELD: "workspace"},
@@ -356,8 +348,7 @@ def test_plot_default_choose_column(tmp_dir, scm, dvc):
         }
     }
 
-    plot_string = VegaRenderer(data, dvc.plots.templates).as_json()
-    plot_content = json.loads(plot_string)
+    plot_content = VegaRenderer(data, dvc.plots.templates).asdict()
 
     assert plot_content["data"]["values"] == [
         {INDEX_FIELD: 0, "b": 2, REVISION_FIELD: "workspace"},
@@ -378,7 +369,7 @@ def test_raise_on_wrong_field(tmp_dir, scm, dvc):
     }
 
     with pytest.raises(NoFieldInDataError):
-        VegaRenderer(data, dvc.plots.templates).as_json()
+        VegaRenderer(data, dvc.plots.templates).asdict()
 
 
 @pytest.mark.parametrize(
@@ -403,53 +394,6 @@ def test_matches(extension, matches):
     assert VegaRenderer.matches(data) == matches
 
 
-def test_find_vega(tmp_dir, dvc):
-    data = {
-        "HEAD": {
-            "data": {
-                "file.json": {
-                    "data": [{"y": 5}, {"y": 6}],
-                    "props": {"fields": {"y"}},
-                },
-                "other_file.jpg": {"data": b"content"},
-            }
-        },
-        "v2": {
-            "data": {
-                "file.json": {
-                    "data": [{"y": 3}, {"y": 5}],
-                    "props": {"fields": {"y"}},
-                },
-                "other_file.jpg": {"data": b"content2"},
-            }
-        },
-        "v1": {
-            "data": {
-                "file.json": {
-                    "data": [{"y": 2}, {"y": 3}],
-                    "props": {"fields": {"y"}},
-                },
-                "another.gif": {"data": b"content2"},
-            }
-        },
-    }
-
-    plot_content = json.loads(find_vega(dvc, data, "file.json"))
-
-    assert plot_content["data"]["values"] == [
-        {"y": 5, INDEX_FIELD: 0, REVISION_FIELD: "HEAD"},
-        {"y": 6, INDEX_FIELD: 1, REVISION_FIELD: "HEAD"},
-        {"y": 3, INDEX_FIELD: 0, REVISION_FIELD: "v2"},
-        {"y": 5, INDEX_FIELD: 1, REVISION_FIELD: "v2"},
-        {"y": 2, INDEX_FIELD: 0, REVISION_FIELD: "v1"},
-        {"y": 3, INDEX_FIELD: 1, REVISION_FIELD: "v1"},
-    ]
-    assert (
-        first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD
-    )
-    assert first(plot_content["layer"])["encoding"]["y"]["field"] == "y"
-
-
 @pytest.mark.parametrize(
     "template_path, target_name",
     [
@@ -474,3 +418,20 @@ def test_should_resolve_template(tmp_dir, dvc, template_path, target_name):
     assert dvc.plots.templates.get_template(target_name) == os.path.abspath(
         template_path
     )
+
+
+def test_as_json(tmp_dir, scm, dvc):
+    metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
+    data = {
+        "workspace": {
+            "data": {"file.json": {"data": metric, "props": {"fields": {"b"}}}}
+        }
+    }
+
+    renderer = VegaRenderer(data, dvc.plots.templates)
+    plot_content = renderer.asdict()
+    plot_as_json = first(json.loads(renderer.as_json()))
+
+    assert plot_as_json["type"] == "vega"
+    assert plot_as_json["revisions"] == ["workspace"]
+    assert plot_as_json["content"] == plot_content

tests/unit/test_tabular_data.py

@@ -264,3 +264,21 @@ def test_drop_duplicates_invalid_axis():
 
     with pytest.raises(ValueError, match="Invalid 'axis' value foo."):
         td.drop_duplicates("foo")
+
+
+def test_to_parallel_coordinates(tmp_dir, mocker):
+    (tmp_dir / "foo").mkdir()
+    td = TabularData(["categorical", "scalar"])
+    td.extend([["foo", "0.1"], ["bar", "2"]])
+
+    write = mocker.patch("dvc.render.html.write")
+    renderer_class = mocker.patch(
+        "dvc.render.plotly.ParallelCoordinatesRenderer"
+    )
+    renderer = renderer_class.return_value
+
+    td.render(html=True, output_path="foo")
+
+    renderer_class.assert_called_with(td, None, td._fill_value)
+
+    write.assert_called_with("foo", renderers=[renderer])