dvc: bump to 2.7.0

* extract transfer related logic from odb to fs.utils.transfer

* don't makedirs on dirinfo

As local odb needs to set a mode for dir, we cannot create a directory
in the transfer

.github/workflows/tests.yaml

@@ -1,12 +1,8 @@
 name: Tests
 
 on:
-  pull_request:
-    branches:
-      - master
-  push:
-    branches:
-      - master
+  pull_request: {}
+  push: {}
   schedule:
     - cron: '5 1 * * *'  # every day at 01:05
 
@@ -27,7 +23,7 @@ jobs:
     - name: Install requirements
       run: |
         pip install wheel
-        pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+        pip install ".[all,tests,terraform]" Pygments collective.checkdocs pre-commit
     - name: install temporary dependencies
       run: |
         pip install git+https://github.com/isidentical/gdrivefs@service-account
@@ -77,7 +73,7 @@ jobs:
       if: steps.cache.pip-cache-dir.cache-hit != 'true'
       run: |
         pip install --upgrade pip setuptools wheel
-        pip install -e ".[all,tests]"
+        pip install -e ".[all,tests,terraform]"
     - name: install temporary dependencies
       run: |
         pip install git+https://github.com/isidentical/gdrivefs@service-account
@@ -113,7 +109,7 @@ jobs:
         --cov-report=xml --cov-report=term
         --all ${{ env.extra_test_args }}
     - name: upload coverage report
-      uses: codecov/[email protected].2
+      uses: codecov/[email protected].3
       with:
         file: ./coverage.xml
         fail_ci_if_error: false

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       - id: black
         language_version: python3
     repo: https://github.com/ambv/black
-    rev: 21.7b0
+    rev: 21.8b0
   - hooks:
       - id: codespell
         args: 

README.rst

@@ -8,7 +8,7 @@
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
-|Release| |CI| |Maintainability| |Coverage| |Donate| |DOI|
+|CI| |Maintainability| |Coverage| |Donate| |DOI|
 
 |PyPI| |Packages| |Brew| |Conda| |Choco| |Snap|
 
@@ -20,8 +20,9 @@ learning projects. Key features:
 #. Simple **command line** Git-like experience. Does not require installing and maintaining
    any databases. Does not depend on any proprietary online services.
 
-#. Management and versioning of **datasets** and **machine learning models**. Data is saved in
-   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
+#. Management and versioning of **datasets** and **machine learning
+   models**. Data can be saved in S3, Google cloud, Azure, Alibaba 
+   cloud, SSH server, HDFS, or even local HDD RAID.
 
 #. Makes projects **reproducible** and **shareable**; helping to answer questions about how
    a model was built.
@@ -65,25 +66,25 @@ Quick start
 
 Please read `Get Started <https://dvc.org/doc/get-started>`_ guide for a full version. Common workflow commands include:
 
-+-----------------------------------+-------------------------------------------------------------------+
-| Step                              | Command                                                           |
-+===================================+===================================================================+
-| Track data                        | | ``$ git add train.py``                                          |
-|                                   | | ``$ dvc add images.zip``                                        |
-+-----------------------------------+-------------------------------------------------------------------+
-| Connect code and data by commands | | ``$ dvc run -d images.zip -o images/ unzip -q images.zip``      |
-|                                   | | ``$ dvc run -d images/ -d train.py -o model.p python train.py`` |
-+-----------------------------------+-------------------------------------------------------------------+
-| Make changes and reproduce        | | ``$ vi train.py``                                               |
-|                                   | | ``$ dvc repro model.p.dvc``                                     |
-+-----------------------------------+-------------------------------------------------------------------+
-| Share code                        | | ``$ git add .``                                                 |
-|                                   | | ``$ git commit -m 'The baseline model'``                        |
-|                                   | | ``$ git push``                                                  |
-+-----------------------------------+-------------------------------------------------------------------+
-| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``        |
-|                                   | | ``$ dvc push``                                                  |
-+-----------------------------------+-------------------------------------------------------------------+
++-----------------------------------+----------------------------------------------------------------------------+
+| Step                              | Command                                                                    |
++===================================+============================================================================+
+| Track data                        | | ``$ git add train.py``                                                   |
+|                                   | | ``$ dvc add images.zip``                                                 |
++-----------------------------------+----------------------------------------------------------------------------+
+| Connect code and data by commands | | ``$ dvc run -n prepare -d images.zip -o images/ unzip -q images.zip``    |
+|                                   | | ``$ dvc run -n train -d images/ -d train.py -o model.p python train.py`` |
++-----------------------------------+----------------------------------------------------------------------------+
+| Make changes and reproduce        | | ``$ vi train.py``                                                        |
+|                                   | | ``$ dvc repro model.p.dvc``                                              |
++-----------------------------------+----------------------------------------------------------------------------+
+| Share code                        | | ``$ git add .``                                                          |
+|                                   | | ``$ git commit -m 'The baseline model'``                                 |
+|                                   | | ``$ git push``                                                           |
++-----------------------------------+----------------------------------------------------------------------------+
+| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``                 |
+|                                   | | ``$ dvc push``                                                           |
++-----------------------------------+----------------------------------------------------------------------------+
 
 Installation
 ============
@@ -182,30 +183,37 @@ Fedora / CentOS (rpm)
 Comparison to related technologies
 ==================================
 
-#. *Data Engineering* tools such as `AirFlow <https://airflow.apache.org/>`_, `Luigi <https://github.com/spotify/luigi>`_ and
-   others - in DVC data, model and ML pipelines represent a single ML project and focuses on data scientist' experience while
-   data engineering pipeline orchestrates multiple data projects and focuses on efficient execution. DVC project can be used
-   from data pipeline as a single execution step. `AirFlow DVC <https://github.com/covid-genomics/airflow-dvc>`_ is an example
-   of such integration.
+#. Data Engineering tools such as `AirFlow <https://airflow.apache.org/>`,
+   `Luigi <https://github.com/spotify/luigi>`, and others - in DVC data,
+   model and ML pipelines represent a single ML project focused on data
+   scientists' experience.  Data engineering tools orchestrate multiple data
+   projects and focus on efficient execution. A DVC project can be used from
+   existing data pipelines as a single execution step.
 
 #. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (which should
    not be in a Git repository) in a local key-value store, and uses file hardlinks/symlinks instead of
    copying/duplicating files.
 
-#. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with any remote storage (S3, Google Cloud, Azure, SSH,
-   etc). DVC also uses reflinks or hardlinks to avoid copy operations on checkouts; thus handling large data files
-   much more efficiently.
+#. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with many
+   remote storage services (S3, Google Cloud, Azure, SSH, etc). DVC also
+   uses reflinks or hardlinks to avoid copy operations on checkouts; thus
+   handling large data files much more efficiently.
 
-#. *Makefile* (and analogues including ad-hoc scripts) - DVC tracks dependencies (in a directed acyclic graph).
+#. Makefile (and analogues including ad-hoc scripts) - DVC tracks
+   dependencies (in a directed acyclic graph).
 
 #. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_ - DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-#. `DAGsHub <https://dagshub.com/>`_ - online service of Git+DVC repositories with pipeline and metrics visualization and
-   DVC-specific cloud storage.
+#. `DAGsHub <https://dagshub.com/>`_ - online service to host DVC
+   projects.  It provides a useful UI around DVC repositories and integrates
+   other tools.
 
-#. `DVC Studio <https://studio.iterative.ai/>`_ - online service of DVC repositories visualisation from DVC team. Also,
-   integrated with `CML (CI/CD for ML) <https://cml.dev/>`_ for training models in clouds and Kubernetes.
+#. `DVC Studio <https://studio.iterative.ai/>`_ - official online
+   platform for DVC projects.  It can be used to manage data and models, run
+   and track experiments, and visualize and share results.  Also, it
+   integrates with `CML (CI/CD for ML) <https://cml.dev/>` for training
+   models in the cloud or Kubernetes.
 
 
 Contributing
@@ -244,10 +252,6 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://dvc.org
    :alt: DVC logo
 
-.. |Release| image:: https://img.shields.io/badge/release-ok-brightgreen
-   :target: https://travis-ci.com/iterative/dvc/branches
-   :alt: Release
-
 .. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=master
    :target: https://github.com/iterative/dvc/actions
    :alt: GHA Tests

dvc/cli.py

@@ -30,6 +30,7 @@ from .command import (
     install,
     live,
     ls,
+    machine,
     metrics,
     move,
     params,
@@ -87,6 +88,7 @@ COMMANDS = [
     experiments,
     check_ignore,
     live,
+    machine,
 ]
 
 

dvc/command/experiments.py

@@ -750,6 +750,7 @@ class CmdExperimentsRemove(CmdBase):
             exp_names=self.args.experiment,
             queue=self.args.queue,
             clear_all=self.args.all,
+            remote=self.args.git_remote,
         )
 
         return 0
@@ -1231,7 +1232,7 @@ def add_parser(subparsers, parent_parser):
     )
     experiments_pull_parser.set_defaults(func=CmdExperimentsPull)
 
-    EXPERIMENTS_REMOVE_HELP = "Remove local experiments."
+    EXPERIMENTS_REMOVE_HELP = "Remove experiments."
     experiments_remove_parser = experiments_subparsers.add_parser(
         "remove",
         parents=[parent_parser],
@@ -1249,6 +1250,12 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         help="Remove all committed experiments.",
     )
+    remove_group.add_argument(
+        "-g",
+        "--git-remote",
+        metavar="<git_remote>",
+        help="Name or URL of the Git remote to remove the experiment from",
+    )
     experiments_remove_parser.add_argument(
         "experiment",
         nargs="*",

dvc/command/live.py

@@ -1,5 +1,5 @@
 import argparse
-import os
+from pathlib import Path
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, fix_subparsers
@@ -13,10 +13,11 @@ class CmdLive(CmdBase):
         metrics, plots = self.repo.live.show(target=target, revs=revs)
 
         if plots:
-            html_path = self.args.target + ".html"
-            self.repo.plots.write_html(html_path, plots, metrics)
+            html_path = Path.cwd() / (self.args.target + "_html")
+            from dvc.render.utils import render
 
-            ui.write("\nfile://", os.path.abspath(html_path), sep="")
+            index_path = render(self.repo, plots, metrics, html_path)
+            ui.write(index_path.as_uri())
             return 0
         return 1
 

dvc/command/machine.py

@@ -0,0 +1,296 @@
+import argparse
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.command.config import CmdConfig
+from dvc.config import ConfigError
+from dvc.ui import ui
+from dvc.utils import format_link
+
+
+class MachineDisabledError(ConfigError):
+    def __init__(self):
+        super().__init__("Machine feature is disabled")
+
+
+class CmdMachineConfig(CmdConfig):
+    def __init__(self, args):
+
+        super().__init__(args)
+        if not self.config["feature"].get("machine", False):
+            raise MachineDisabledError
+
+        if getattr(self.args, "name", None):
+            self.args.name = self.args.name.lower()
+
+    def _check_exists(self, conf):
+        if self.args.name not in conf["machine"]:
+            raise ConfigError(f"machine '{self.args.name}' doesn't exist.")
+
+
+class CmdMachineAdd(CmdMachineConfig):
+    def run(self):
+        from dvc.machine import validate_name
+
+        validate_name(self.args.name)
+
+        if self.args.default:
+            ui.write(f"Setting '{self.args.name}' as a default machine.")
+
+        with self.config.edit(self.args.level) as conf:
+            if self.args.name in conf["machine"] and not self.args.force:
+                raise ConfigError(
+                    "machine '{}' already exists. Use `-f|--force` to "
+                    "overwrite it.".format(self.args.name)
+                )
+
+            conf["machine"][self.args.name] = {"cloud": self.args.cloud}
+            if self.args.default:
+                conf["core"]["machine"] = self.args.name
+
+        return 0
+
+
+class CmdMachineRemove(CmdMachineConfig):
+    def run(self):
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            del conf["machine"][self.args.name]
+
+        up_to_level = self.args.level or "repo"
+        # Remove core.machine refs to this machine in any shadowing configs
+        for level in reversed(self.config.LEVELS):
+            with self.config.edit(level) as conf:
+                if conf["core"].get("machine") == self.args.name:
+                    del conf["core"]["machine"]
+
+            if level == up_to_level:
+                break
+
+        return 0
+
+
+class CmdMachineModify(CmdMachineConfig):
+    def run(self):
+        from dvc.config import merge
+
+        with self.config.edit(self.args.level) as conf:
+            merged = self.config.load_config_to_level(self.args.level)
+            merge(merged, conf)
+            self._check_exists(merged)
+
+            if self.args.name not in conf["machine"]:
+                conf["machine"][self.args.name] = {}
+            section = conf["machine"][self.args.name]
+            if self.args.unset:
+                section.pop(self.args.option, None)
+            else:
+                section[self.args.option] = self.args.value
+        return 0
+
+
+class CmdMachineDefault(CmdMachineConfig):
+    def run(self):
+        if self.args.name is None and not self.args.unset:
+            conf = self.config.read(self.args.level)
+            try:
+                print(conf["core"]["machine"])
+            except KeyError:
+                ui.write("No default machine set")
+                return 1
+        else:
+            with self.config.edit(self.args.level) as conf:
+                if self.args.unset:
+                    conf["core"].pop("machine", None)
+                else:
+                    merged_conf = self.config.load_config_to_level(
+                        self.args.level
+                    )
+                    if (
+                        self.args.name in conf["machine"]
+                        or self.args.name in merged_conf["machine"]
+                    ):
+                        conf["core"]["machine"] = self.args.name
+                    else:
+                        raise ConfigError(
+                            "default machine must be present in machine "
+                            "list."
+                        )
+        return 0
+
+
+class CmdMachineCreate(CmdBase):
+    def run(self):
+        if self.repo.machine is None:
+            raise MachineDisabledError
+
+        self.repo.machine.create(self.args.name)
+        return 0
+
+
+class CmdMachineDestroy(CmdBase):
+    def run(self):
+        if self.repo.machine is None:
+            raise MachineDisabledError
+
+        self.repo.machine.destroy(self.args.name)
+        return 0
+
+
+class CmdMachineSsh(CmdBase):
+    def run(self):
+        if self.repo.machine is None:
+            raise MachineDisabledError
+
+        self.repo.machine.run_shell(self.args.name)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    machine_HELP = "Set up and manage cloud machines."
+    machine_parser = subparsers.add_parser(
+        "machine",
+        parents=[parent_parser],
+        description=append_doc_link(machine_HELP, "machine"),
+        # NOTE: suppress help during development to hide command
+        # help=machine_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    machine_subparsers = machine_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc machine CMD --help` for " "command-specific help.",
+    )
+
+    fix_subparsers(machine_subparsers)
+
+    machine_ADD_HELP = "Add a new data machine."
+    machine_add_parser = machine_subparsers.add_parser(
+        "add",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_ADD_HELP, "machine/add"),
+        help=machine_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_add_parser.add_argument("name", help="Name of the machine")
+    machine_add_parser.add_argument(
+        "cloud",
+        help="Machine cloud. See full list of supported clouds at {}".format(
+            format_link(
+                "https://github.com/iterative/"
+                "terraform-provider-iterative#machine"
+            )
+        ),
+    )
+    machine_add_parser.add_argument(
+        "-d",
+        "--default",
+        action="store_true",
+        default=False,
+        help="Set as default machine.",
+    )
+    machine_add_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force overwriting existing configs",
+    )
+    machine_add_parser.set_defaults(func=CmdMachineAdd)
+
+    machine_DEFAULT_HELP = "Set/unset the default machine."
+    machine_default_parser = machine_subparsers.add_parser(
+        "default",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_DEFAULT_HELP, "machine/default"),
+        help=machine_DEFAULT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_default_parser.add_argument(
+        "name", nargs="?", help="Name of the machine"
+    )
+    machine_default_parser.add_argument(
+        "-u",
+        "--unset",
+        action="store_true",
+        default=False,
+        help="Unset default machine.",
+    )
+    machine_default_parser.set_defaults(func=CmdMachineDefault)
+
+    machine_MODIFY_HELP = "Modify the configuration of an machine."
+    machine_modify_parser = machine_subparsers.add_parser(
+        "modify",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_MODIFY_HELP, "machine/modify"),
+        help=machine_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_modify_parser.add_argument("name", help="Name of the machine")
+    machine_modify_parser.add_argument(
+        "option", help="Name of the option to modify."
+    )
+    machine_modify_parser.add_argument(
+        "value", nargs="?", help="(optional) Value of the option."
+    )
+    machine_modify_parser.add_argument(
+        "-u",
+        "--unset",
+        default=False,
+        action="store_true",
+        help="Unset option.",
+    )
+    machine_modify_parser.set_defaults(func=CmdMachineModify)
+
+    machine_REMOVE_HELP = "Remove an machine."
+    machine_remove_parser = machine_subparsers.add_parser(
+        "remove",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_REMOVE_HELP, "machine/remove"),
+        help=machine_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_remove_parser.add_argument(
+        "name", help="Name of the machine to remove."
+    )
+    machine_remove_parser.set_defaults(func=CmdMachineRemove)
+
+    machine_CREATE_HELP = "Create and start a machine instance."
+    machine_create_parser = machine_subparsers.add_parser(
+        "create",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_CREATE_HELP, "machine/create"),
+        help=machine_CREATE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_create_parser.add_argument(
+        "name", help="Name of the machine to create."
+    )
+    machine_create_parser.set_defaults(func=CmdMachineCreate)
+
+    machine_DESTROY_HELP = "Destroy an machine instance."
+    machine_destroy_parser = machine_subparsers.add_parser(
+        "destroy",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_DESTROY_HELP, "machine/destroy"),
+        help=machine_DESTROY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_destroy_parser.add_argument(
+        "name", help="Name of the machine instance to destroy."
+    )
+    machine_destroy_parser.set_defaults(func=CmdMachineDestroy)
+
+    machine_SSH_HELP = "Connect to a machine via SSH."
+    machine_ssh_parser = machine_subparsers.add_parser(
+        "ssh",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_SSH_HELP, "machine/ssh"),
+        help=machine_SSH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_ssh_parser.add_argument(
+        "name", help="Name of the machine instance to connect to."
+    )
+    machine_ssh_parser.set_defaults(func=CmdMachineSsh)

dvc/command/plots.py

@@ -4,6 +4,7 @@ import logging
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
+from dvc.render.utils import find_vega, render
 from dvc.ui import ui
 from dvc.utils import format_link
 
@@ -35,43 +36,50 @@ class CmdPlots(CmdBase):
                 return 1
 
         try:
-            plots = self._func(targets=self.args.targets, props=self._props())
+            plots_data = self._func(
+                targets=self.args.targets, props=self._props()
+            )
+
+            if not plots_data:
+                ui.error_write(
+                    "No plots were loaded, "
+                    "visualization file will not be created."
+                )
 
             if self.args.show_vega:
                 target = self.args.targets[0]
-                ui.write(plots[target])
+                plot_json = find_vega(self.repo, plots_data, target)
+                if plot_json:
+                    ui.write(plot_json)
                 return 0
 
-        except DvcException:
-            logger.exception("")
-            return 1
-
-        if plots:
-            rel: str = self.args.out or "plots.html"
+            rel: str = self.args.out or "dvc_plots"
             path: Path = (Path.cwd() / rel).resolve()
-            self.repo.plots.write_html(
-                path, plots=plots, html_template_path=self.args.html_template
+            index_path = render(
+                self.repo,
+                plots_data,
+                path=path,
+                html_template_path=self.args.html_template,
             )
 
-            assert (
-                path.is_absolute()
-            )  # as_uri throws ValueError if not absolute
-            url = path.as_uri()
+            assert index_path.is_absolute()
+            url = index_path.as_uri()
             ui.write(url)
+
             if self.args.open:
                 import webbrowser
 
-                opened = webbrowser.open(rel)
+                opened = webbrowser.open(index_path)
                 if not opened:
                     ui.error_write(
                         "Failed to open. Please try opening it manually."
                     )
                     return 1
-        else:
-            ui.error_write(
-                "No plots were loaded, visualization file will not be created."
-            )
-        return 0
+            return 0
+
+        except DvcException:
+            logger.exception("")
+            return 1
 
 
 class CmdPlotsShow(CmdPlots):

dvc/config.py

@@ -32,6 +32,18 @@ class RemoteNotFoundError(RemoteConfigError):
     pass
 
 
+class MachineConfigError(ConfigError):
+    pass
+
+
+class NoMachineError(MachineConfigError):
+    pass
+
+
+class MachineNotFoundError(MachineConfigError):
+    pass
+
+
 @memoize
 def get_compiled_schema():
     from voluptuous import Schema
@@ -167,7 +179,7 @@ class Config(dict):
                 conf_obj = ConfigObj(fobj)
         else:
             conf_obj = ConfigObj()
-        return _parse_remotes(_lower_keys(conf_obj.dict()))
+        return _parse_named(_lower_keys(conf_obj.dict()))
 
     def _save_config(self, level, conf_dict):
         from configobj import ConfigObj
@@ -179,7 +191,7 @@ class Config(dict):
 
         fs.makedirs(os.path.dirname(filename))
 
-        config = ConfigObj(_pack_remotes(conf_dict))
+        config = ConfigObj(_pack_named(conf_dict))
         with fs.open(filename, "wb") as fobj:
             config.write(fobj)
         config.filename = filename
@@ -300,27 +312,29 @@ class Config(dict):
             raise ConfigError(str(exc)) from None
 
 
-def _parse_remotes(conf):
-    result = {"remote": {}}
+def _parse_named(conf):
+    result = {"remote": {}, "machine": {}}
 
     for section, val in conf.items():
-        name = re_find(r'^\s*remote\s*"(.*)"\s*$', section)
-        if name:
-            result["remote"][name] = val
+        match = re_find(r'^\s*(remote|machine)\s*"(.*)"\s*$', section)
+        if match:
+            key, name = match
+            result[key][name] = val
         else:
             result[section] = val
 
     return result
 
 
-def _pack_remotes(conf):
+def _pack_named(conf):
     # Drop empty sections
     result = compact(conf)
 
     # Transform remote.name -> 'remote "name"'
-    for name, val in conf["remote"].items():
-        result[f'remote "{name}"'] = val
-    result.pop("remote", None)
+    for key in ("remote", "machine"):
+        for name, val in conf[key].items():
+            result[f'{key} "{name}"'] = val
+        result.pop(key, None)
 
     return result
 

dvc/config_schema.py

@@ -118,6 +118,7 @@ SCHEMA = {
         Optional("autostage", default=False): Bool,
         Optional("experiments"): Bool,  # obsoleted
         Optional("check_update", default=True): Bool,
+        "machine": Lower,
     },
     "cache": {
         "local": str,
@@ -154,6 +155,7 @@ SCHEMA = {
                     "grant_read_acp": str,
                     "grant_write_acp": str,
                     "grant_full_control": str,
+                    "cache_regions": bool,
                     **REMOTE_COMMON,
                 },
                 "gs": {
@@ -224,10 +226,16 @@ SCHEMA = {
         "row_limit": All(Coerce(int), Range(1)),  # obsoleted
         "row_cleanup_quota": All(Coerce(int), Range(0, 100)),  # obsoleted
     },
+    "machine": {
+        str: {
+            "cloud": All(Lower, Choices("aws", "azure")),
+        },
+    },
     # section for experimental features
     "feature": {
+        Optional("machine", default=False): Bool,
         # enabled by default. It's of no use, kept for backward compatibility.
-        Optional("parametrization", default=True): Bool
+        Optional("parametrization", default=True): Bool,
     },
     "plots": {"html_template": str},
 }

dvc/daemon.py

@@ -12,29 +12,34 @@ from dvc.utils import fix_env, is_binary
 
 logger = logging.getLogger(__name__)
 
-CREATE_NEW_PROCESS_GROUP = 0x00000200
-DETACHED_PROCESS = 0x00000008
+CREATE_NO_WINDOW = 0x08000000  # only available since Python 3.7 in subprocess.
 
 
 def _popen(cmd, **kwargs):
     prefix = [sys.executable]
     if not is_binary():
-        prefix += [sys.argv[0]]
-
+        main_entrypoint = os.path.join(
+            os.path.abspath(os.path.dirname(__file__)), "__main__.py"
+        )
+        prefix += [main_entrypoint]
     return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
 
 
 def _spawn_windows(cmd, env):
-    from subprocess import STARTF_USESHOWWINDOW, STARTUPINFO
+    from subprocess import (
+        CREATE_NEW_PROCESS_GROUP,
+        STARTF_USESHOWWINDOW,
+        STARTUPINFO,
+    )
 
-    creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
+    # https://stackoverflow.com/a/7006424
+    # https://bugs.python.org/issue41619
+    creationflags = CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW
 
     startupinfo = STARTUPINFO()
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
-    _popen(
-        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo
-    ).communicate()
+    _popen(cmd, env=env, creationflags=creationflags, startupinfo=startupinfo)
 
 
 def _spawn_posix(cmd, env):

dvc/fs/__init__.py

@@ -2,6 +2,7 @@ import posixpath
 from urllib.parse import urlparse
 
 from ..scheme import Schemes
+from . import utils  # noqa: F401
 from .azure import AzureFileSystem
 from .gdrive import GDriveFileSystem
 from .gs import GSFileSystem

dvc/fs/base.py

@@ -9,7 +9,6 @@ from dvc.path_info import URLInfo
 from dvc.progress import Tqdm
 from dvc.utils import tmp_fname
 from dvc.utils.fs import makedirs, move
-from dvc.utils.http import open_url
 
 logger = logging.getLogger(__name__)
 
@@ -125,13 +124,10 @@ class BaseFileSystem:
             f"dependencies: {missing}. {hint}"
         )
 
-    def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
-        if hasattr(self, "_generate_download_url"):
-            # pylint:disable=no-member
-            func = self._generate_download_url  # type: ignore[attr-defined]
-            get_url = partial(func, path_info)
-            return open_url(get_url, mode=mode, encoding=encoding, **kwargs)
+    def checksum(self, path_info) -> str:
+        raise NotImplementedError
 
+    def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
         raise RemoteActionNotImplemented("open", self.scheme)
 
     def exists(self, path_info) -> bool:

dvc/fs/dvc.py

@@ -261,3 +261,10 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         fs._download(  # pylint: disable=protected-access
             path, to_file, **kwargs
         )
+
+    def checksum(self, path_info):
+        info = self.info(path_info)
+        md5 = info.get("md5")
+        if md5:
+            return md5
+        raise NotImplementedError

dvc/fs/fsspec_wrapper.py

@@ -85,6 +85,9 @@ class FSSpecWrapper(BaseFileSystem):
     ):  # pylint: disable=arguments-differ
         return self.fs.open(self._with_bucket(path_info), mode=mode)
 
+    def checksum(self, path_info):
+        return self.fs.checksum(self._with_bucket(path_info))
+
     def copy(self, from_info, to_info):
         self.makedirs(to_info.parent)
         self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))

dvc/fs/git.py

@@ -1,6 +1,5 @@
 import errno
 import os
-import stat
 
 from dvc.utils import is_exec, relpath
 
@@ -96,30 +95,20 @@ class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         if not self.exists(path_info):
             return False
 
-        mode = self.stat(path_info).st_mode
+        mode = self.info(path_info)["mode"]
         return is_exec(mode)
 
     def info(self, path_info):
         key = self._get_key(path_info)
         try:
-            st = self.trie.stat(key)
+            return self.trie.info(key)
         except KeyError:
             raise FileNotFoundError(
                 errno.ENOENT, os.strerror(errno.ENOENT), path_info
             )
-        return {
-            "size": st.st_size,
-            "type": "directory" if stat.S_ISDIR(st.st_mode) else "file",
-        }
 
-    def stat(self, path):
-        key = self._get_key(path)
-        try:
-            return self.trie.stat(key)
-        except KeyError:
-            raise FileNotFoundError(
-                errno.ENOENT, os.strerror(errno.ENOENT), path
-            )
+    def checksum(self, path_info):
+        return self.info(path_info)["sha"]
 
     def walk_files(self, path_info, **kwargs):
         for root, _, files in self.walk(path_info, **kwargs):

dvc/fs/hdfs.py

@@ -65,6 +65,8 @@ class HDFSFileSystem(BaseFileSystem):
     PARAM_CHECKSUM = "checksum"
     TRAVERSE_PREFIX_LEN = 2
 
+    BLOCK_SIZE = 2 ** 26  # 128MB
+
     def __init__(self, **config):
         super().__init__(**config)
 
@@ -245,7 +247,7 @@ class HDFSFileSystem(BaseFileSystem):
     def _upload_fobj(self, fobj, to_info, **kwargs):
         with self.hdfs(to_info) as hdfs:
             with hdfs.open_output_stream(to_info.path) as fdest:
-                shutil.copyfileobj(fobj, fdest)
+                shutil.copyfileobj(fobj, fdest, self.BLOCK_SIZE)
 
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
@@ -262,7 +264,7 @@ class HDFSFileSystem(BaseFileSystem):
                     disable=no_progress_bar,
                 ) as wrapped:
                     with hdfs.open_output_stream(tmp_file) as sobj:
-                        sobj.write(wrapped.read())
+                        shutil.copyfileobj(wrapped, sobj, self.BLOCK_SIZE)
             hdfs.move(tmp_file, to_info.path)
 
     def _download(
@@ -280,4 +282,4 @@ class HDFSFileSystem(BaseFileSystem):
                     disable=no_progress_bar,
                 ) as wrapped:
                     with hdfs.open_input_stream(from_info.path) as sobj:
-                        wrapped.write(sobj.read())
+                        shutil.copyfileobj(sobj, wrapped, self.BLOCK_SIZE)

dvc/fs/http.py

@@ -65,9 +65,6 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 self.headers.update({self.custom_auth_header: self.password})
         return None
 
-    def _generate_download_url(self, path_info):
-        return path_info.url
-
     @wrap_prop(threading.Lock())
     @cached_property
     def _session(self):
@@ -190,6 +187,17 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 desc=name or to_info.url,
             )
 
+    def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
+        from dvc.utils.http import open_url
+
+        return open_url(
+            path_info.url,
+            mode=mode,
+            encoding=encoding,
+            auth=self._auth_method(path_info),
+            **kwargs,
+        )
+
     @staticmethod
     def _content_length(response) -> Optional[int]:
         res = response.headers.get("Content-Length")

dvc/fs/local.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import stat
 
 from dvc.path_info import PathInfo
 from dvc.scheme import Schemes
@@ -34,7 +33,12 @@ class LocalFileSystem(BaseFileSystem):
 
     def exists(self, path_info) -> bool:
         assert isinstance(path_info, str) or path_info.scheme == "local"
-        return self.fs.exists(path_info)
+        # TODO: replace this with os.path.exists once the problem is fixed on
+        # the fsspec https://github.com/intake/filesystem_spec/issues/742
+        return os.path.lexists(path_info)
+
+    def checksum(self, path_info) -> str:
+        return self.fs.checksum(path_info)
 
     def isfile(self, path_info) -> bool:
         return os.path.isfile(path_info)
@@ -83,12 +87,9 @@ class LocalFileSystem(BaseFileSystem):
         makedirs(path_info, exist_ok=kwargs.pop("exist_ok", True))
 
     def isexec(self, path_info):
-        mode = self.stat(path_info).st_mode
+        mode = self.info(path_info)["mode"]
         return is_exec(mode)
 
-    def stat(self, path):
-        return os.stat(path)
-
     def move(self, from_info, to_info):
         if (
             isinstance(from_info, PathInfo) and from_info.scheme != "local"
@@ -154,13 +155,8 @@ class LocalFileSystem(BaseFileSystem):
     def reflink(self, from_info, to_info):
         System.reflink(from_info, to_info)
 
-    @staticmethod
-    def info(path_info):
-        st = os.stat(path_info)
-        return {
-            "size": st.st_size,
-            "type": "directory" if stat.S_ISDIR(st.st_mode) else "file",
-        }
+    def info(self, path_info):
+        return self.fs.info(path_info)
 
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs

dvc/fs/repo.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import stat
 import threading
 from contextlib import suppress
 from itertools import takewhile
@@ -256,8 +255,8 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             return False
 
         try:
-            st = fs.stat(path)
-            return stat.S_ISDIR(st.st_mode)
+            info = fs.info(path)
+            return info["type"] == "directory"
         except (OSError, ValueError):
             # from CPython's os.path.isdir()
             pass
@@ -287,8 +286,8 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             return False
 
         try:
-            st = fs.stat(path)
-            return stat.S_ISREG(st.st_mode)
+            info = fs.info(path)
+            return info["type"] == "file"
         except (OSError, ValueError):
             # from CPython's os.path.isfile()
             pass
@@ -313,10 +312,6 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             return dvc_fs.isexec(path_info)
         return fs.isexec(path_info)
 
-    def stat(self, path):
-        fs, _ = self._get_fs_pair(path)
-        return fs.stat(path)
-
     def _dvc_walk(self, walk):
         try:
             root, dirs, files = next(walk)
@@ -478,11 +473,11 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             with suppress(FileNotFoundError):
                 dvc_meta = dvc_fs.metadata(path_info)
 
-        stat_result = None
+        info_result = None
         with suppress(FileNotFoundError):
-            stat_result = fs.stat(path_info)
+            info_result = fs.info(path_info)
 
-        if not stat_result and not dvc_meta:
+        if not info_result and not dvc_meta:
             raise FileNotFoundError
 
         from ._metadata import Metadata
@@ -492,13 +487,13 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             repo=self._get_repo(abspath) or self._main_repo,
         )
 
-        isdir = bool(stat_result) and stat.S_ISDIR(stat_result.st_mode)
+        isdir = bool(info_result) and info_result["type"] == "directory"
         meta.isdir = meta.isdir or isdir
 
         if not dvc_meta:
             from dvc.utils import is_exec
 
-            meta.is_exec = bool(stat_result) and is_exec(stat_result.st_mode)
+            meta.is_exec = bool(info_result) and is_exec(info_result["mode"])
         return meta
 
     def info(self, path_info):
@@ -508,3 +503,11 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             return fs.info(path_info)
         except FileNotFoundError:
             return dvc_fs.info(path_info)
+
+    def checksum(self, path_info):
+        fs, dvc_fs = self._get_fs_pair(path_info)
+
+        try:
+            return fs.checksum(path_info)
+        except FileNotFoundError:
+            return dvc_fs.checksum(path_info)

dvc/fs/s3.py

@@ -68,12 +68,12 @@ class BaseS3FileSystem(ObjectFSWrapper):
         self._transfer_config = None
         config_path = os.environ.get("AWS_CONFIG_FILE", _AWS_CONFIG_PATH)
         if not os.path.exists(config_path):
-            return None
+            return {}
 
         config = load_config(config_path)
         profile_config = config["profiles"].get(profile or "default")
         if not profile_config:
-            return None
+            return {}
 
         s3_config = profile_config.get("s3", {})
         return self._split_s3_config(s3_config)
@@ -123,6 +123,14 @@ class BaseS3FileSystem(ObjectFSWrapper):
         if shared_creds:
             os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
 
+        if (
+            client["region_name"] is None
+            and session_config["s3"].get("region_name") is None
+            and os.getenv("AWS_REGION") is None
+        ):
+            # Enable bucket region caching
+            login_info["cache_regions"] = config.get("cache_regions", True)
+
         config_path = config.get("configpath")
         if config_path:
             os.environ.setdefault("AWS_CONFIG_FILE", config_path)

dvc/fs/ssh.py

@@ -2,7 +2,7 @@ import getpass
 import os.path
 import threading
 
-from funcy import cached_property, first, memoize, silent, wrap_prop, wrap_with
+from funcy import cached_property, memoize, silent, wrap_prop, wrap_with
 
 from dvc import prompt
 from dvc.scheme import Schemes
@@ -68,14 +68,17 @@ class SSHFileSystem(CallbackMixin, FSSpecWrapper):
 
         login_info["password"] = config.get("password")
 
-        if user_ssh_config.get("IdentityFile"):
-            config.setdefault(
-                "keyfile", first(user_ssh_config.get("IdentityFile"))
-            )
+        raw_keys = []
+        if config.get("keyfile"):
+            raw_keys.append(config.get("keyfile"))
+        elif user_ssh_config.get("IdentityFile"):
+            raw_keys.extend(user_ssh_config.get("IdentityFile"))
+
+        if raw_keys:
+            login_info["client_keys"] = [
+                os.path.expanduser(key) for key in raw_keys
+            ]
 
-        keyfile = config.get("keyfile")
-        if keyfile:
-            login_info["client_keys"] = [os.path.expanduser(keyfile)]
         login_info["timeout"] = config.get("timeout", _SSH_TIMEOUT)
 
         # These two settings fine tune the asyncssh to use the

dvc/fs/utils.py

@@ -0,0 +1,55 @@
+import logging
+import os
+from typing import TYPE_CHECKING
+
+from .local import LocalFileSystem
+
+if TYPE_CHECKING:
+    from dvc.types import AnyPath, DvcPath
+
+    from .base import BaseFileSystem
+
+logger = logging.getLogger(__name__)
+
+
+def transfer(
+    from_fs: "BaseFileSystem",
+    from_info: "AnyPath",
+    to_fs: "BaseFileSystem",
+    to_info: "DvcPath",
+    move: bool = False,
+) -> None:
+    use_move = isinstance(from_fs, type(to_fs)) and move
+    try:
+        if use_move:
+            return to_fs.move(from_info, to_info)
+
+        if isinstance(from_fs, LocalFileSystem):
+            if not isinstance(from_info, from_fs.PATH_CLS):
+                from_info = from_fs.PATH_CLS(from_info)
+            return to_fs.upload(from_info, to_info)
+
+        if isinstance(to_fs, LocalFileSystem):
+            return from_fs.download_file(from_info, to_info)
+
+        with from_fs.open(from_info, mode="rb") as fobj:
+            return to_fs.upload_fobj(fobj, to_info)
+    except OSError as exc:
+        # If the target file already exists, we are going to simply
+        # ignore the exception (#4992).
+        #
+        # On Windows, it is not always guaranteed that you'll get
+        # FileExistsError (it might be PermissionError or a bare OSError)
+        # but all of those exceptions raised from the original
+        # FileExistsError so we have a separate check for that.
+        if isinstance(exc, FileExistsError) or (
+            os.name == "nt"
+            and exc.__context__
+            and isinstance(exc.__context__, FileExistsError)
+        ):
+            logger.debug("'%s' file already exists, skipping", to_info)
+            if use_move:
+                from_fs.remove(from_info)
+            return None
+
+        raise

dvc/machine/__init__.py

@@ -0,0 +1,184 @@
+import logging
+import os
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Iterable,
+    Mapping,
+    Optional,
+    Tuple,
+    Type,
+)
+
+from dvc.exceptions import DvcException
+from dvc.types import StrPath
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+    from .backend.base import BaseMachineBackend
+
+    BackendCls = Type[BaseMachineBackend]
+
+logger = logging.getLogger(__name__)
+
+
+RESERVED_NAMES = {"local", "localhost"}
+
+
+def validate_name(name: str):
+    from dvc.exceptions import InvalidArgumentError
+
+    name = name.lower()
+    if name in RESERVED_NAMES:
+        raise InvalidArgumentError(
+            f"Machine name '{name}' is reserved for internal DVC use."
+        )
+
+
+class MachineBackends(Mapping):
+    try:
+        from .backend.terraform import TerraformBackend
+    except ImportError:
+        TerraformBackend = None  # type: ignore[assignment, misc]
+
+    DEFAULT: Dict[str, Optional["BackendCls"]] = {
+        "terraform": TerraformBackend,
+    }
+
+    def __getitem__(self, key: str) -> "BaseMachineBackend":
+        """Lazily initialize backends and cache it afterwards"""
+        initialized = self.initialized.get(key)
+        if not initialized:
+            backend = self.backends[key]
+            initialized = backend(
+                os.path.join(self.tmp_dir, key), **self.kwargs
+            )
+            self.initialized[key] = initialized
+        return initialized
+
+    def __init__(
+        self,
+        selected: Optional[Iterable[str]],
+        tmp_dir: StrPath,
+        **kwargs,
+    ) -> None:
+        selected = selected or list(self.DEFAULT)
+        self.backends: Dict[str, "BackendCls"] = {}
+        for key in selected:
+            cls = self.DEFAULT.get(key)
+            if cls is None:
+                raise DvcException(
+                    f"'dvc machine' backend '{key}' is missing required "
+                    "dependencies. Install them with:\n"
+                    f"\tpip install dvc[{key}]"
+                )
+            self.backends[key] = cls
+
+        self.initialized: Dict[str, "BaseMachineBackend"] = {}
+
+        self.tmp_dir = tmp_dir
+        self.kwargs = kwargs
+
+    def __iter__(self):
+        return iter(self.backends)
+
+    def __len__(self) -> int:
+        return len(self.backends)
+
+    def close_initialized(self) -> None:
+        for backend in self.initialized.values():
+            backend.close()
+
+
+class MachineManager:
+    """Class that manages dvc cloud machines.
+
+    Args:
+        repo (dvc.repo.Repo): repo instance that belongs to the repo that
+            we are working on.
+
+    Raises:
+        config.ConfigError: thrown when config has invalid format.
+    """
+
+    CLOUD_BACKENDS = {
+        "aws": "terraform",
+        "azure": "terraform",
+    }
+
+    def __init__(
+        self, repo: "Repo", backends: Optional[Iterable[str]] = None, **kwargs
+    ):
+        self.repo = repo
+        tmp_dir = os.path.join(self.repo.tmp_dir, "machine")
+        self.backends = MachineBackends(backends, tmp_dir=tmp_dir, **kwargs)
+
+    def get_config_and_backend(
+        self,
+        name: Optional[str] = None,
+    ) -> Tuple[dict, "BaseMachineBackend"]:
+        from dvc.config import NoMachineError
+
+        if not name:
+            name = self.repo.config["core"].get("machine")
+
+        if name:
+            config = self._get_config(name=name)
+            backend = self._get_backend(config["cloud"])
+            return config, backend
+
+        if bool(self.repo.config["machine"]):
+            error_msg = (
+                "no machine specified. Setup default machine with\n"
+                "    dvc machine default <name>\n"
+            )
+        else:
+            error_msg = (
+                "no machine specified. Create a default machine with\n"
+                "    dvc machine add -d <name> <cloud>"
+            )
+
+        raise NoMachineError(error_msg)
+
+    def _get_config(self, **kwargs):
+        config = self.repo.config
+        name = kwargs.get("name")
+        if name:
+            try:
+                conf = config["machine"][name.lower()]
+                conf["name"] = name
+            except KeyError:
+                from dvc.config import MachineNotFoundError
+
+                raise MachineNotFoundError(f"machine '{name}' doesn't exist")
+        else:
+            conf = kwargs
+        return conf
+
+    def _get_backend(self, cloud: str) -> "BaseMachineBackend":
+        from dvc.config import NoMachineError
+
+        try:
+            backend = self.CLOUD_BACKENDS[cloud]
+            return self.backends[backend]
+        except KeyError:
+            raise NoMachineError(f"Machine platform '{cloud}' unsupported")
+
+    def create(self, name: Optional[str]):
+        """Create and start the specified machine instance."""
+        config, backend = self.get_config_and_backend(name)
+        return backend.create(**config)
+
+    def destroy(self, name: Optional[str]):
+        """Destroy the specified machine instance."""
+        config, backend = self.get_config_and_backend(name)
+        return backend.destroy(**config)
+
+    def get_sshfs(self, name: Optional[str]):
+        config, backend = self.get_config_and_backend(name)
+        return backend.get_sshfs(**config)
+
+    def run_shell(self, name: Optional[str]):
+        config, backend = self.get_config_and_backend(name)
+        return backend.run_shell(**config)

dvc/machine/backend/base.py

@@ -0,0 +1,27 @@
+from abc import abstractmethod
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Iterator, Optional
+
+from tpi import base
+
+if TYPE_CHECKING:
+    from dvc.fs.ssh import SSHFileSystem
+    from dvc.repo.experiments.executor.base import BaseExecutor
+
+
+class BaseMachineBackend(base.BaseMachineBackend):
+    @abstractmethod
+    def get_executor(
+        self, name: Optional[str] = None, **config
+    ) -> "BaseExecutor":
+        """Return an executor instance which can be used for DVC
+        experiment/pipeline execution on the specified machine.
+        """
+
+    @abstractmethod
+    @contextmanager
+    def get_sshfs(
+        self, name: Optional[str] = None, **config
+    ) -> Iterator["SSHFileSystem"]:
+        """Return an sshfs instance for the default directory on the
+        specified machine."""

dvc/machine/backend/terraform.py

@@ -0,0 +1,29 @@
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Iterator, Optional
+
+from tpi import TerraformProviderIterative, terraform
+
+from dvc.fs.ssh import SSHFileSystem
+
+if TYPE_CHECKING:
+    from dvc.repo.experiments.executor.base import BaseExecutor
+
+
+class TerraformBackend(terraform.TerraformBackend):
+    def get_executor(
+        self, name: Optional[str] = None, **config
+    ) -> "BaseExecutor":
+        raise NotImplementedError
+
+    @contextmanager
+    def get_sshfs(  # pylint: disable=unused-argument
+        self, name: Optional[str] = None, **config
+    ) -> Iterator["SSHFileSystem"]:
+        resource = self._default_resource(name)
+        with TerraformProviderIterative.pemfile(resource) as pem:
+            fs = SSHFileSystem(
+                host=resource["instance_ip"],
+                user="ubuntu",
+                keyfile=pem,
+            )
+            yield fs

dvc/objects/db/base.py

@@ -1,12 +1,10 @@
 import itertools
 import logging
-import os
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import suppress
 from copy import copy
 from typing import TYPE_CHECKING, Optional
 
-from dvc.fs.local import LocalFileSystem
 from dvc.objects.errors import ObjectDBPermissionError, ObjectFormatError
 from dvc.objects.file import HashFile
 from dvc.progress import Tqdm
@@ -85,38 +83,12 @@ class ObjectDB:
         _hash_info: "HashInfo",
         move: bool = False,
     ):
+        from dvc import fs
+
         self.makedirs(to_info.parent)
-        use_move = isinstance(from_fs, type(self.fs)) and move
-        try:
-            if use_move:
-                self.fs.move(from_info, to_info)
-            elif isinstance(from_fs, LocalFileSystem):
-                if not isinstance(from_info, from_fs.PATH_CLS):
-                    from_info = from_fs.PATH_CLS(from_info)
-                self.fs.upload(from_info, to_info)
-            elif isinstance(self.fs, LocalFileSystem):
-                from_fs.download_file(from_info, to_info)
-            else:
-                with from_fs.open(from_info, mode="rb") as fobj:
-                    self.fs.upload_fobj(fobj, to_info)
-        except OSError as exc:
-            # If the target file already exists, we are going to simply
-            # ignore the exception (#4992).
-            #
-            # On Windows, it is not always guaranteed that you'll get
-            # FileExistsError (it might be PermissionError or a bare OSError)
-            # but all of those exceptions raised from the original
-            # FileExistsError so we have a separate check for that.
-            if isinstance(exc, FileExistsError) or (
-                os.name == "nt"
-                and exc.__context__
-                and isinstance(exc.__context__, FileExistsError)
-            ):
-                logger.debug("'%s' file already exists, skipping", to_info)
-                if use_move:
-                    from_fs.remove(from_info)
-            else:
-                raise
+        return fs.utils.transfer(
+            from_fs, from_info, self.fs, to_info, move=move
+        )
 
     def add(
         self,
@@ -139,7 +111,7 @@ class ObjectDB:
             pass
 
         cache_info = self.hash_to_path_info(hash_info.value)
-        self._add_file(fs, path_info, cache_info, hash_info, move)
+        self._add_file(fs, path_info, cache_info, hash_info, move=move)
 
         try:
             if verify:

dvc/objects/reference.py

@@ -2,7 +2,7 @@ import errno
 import logging
 import os
 import pickle
-from typing import TYPE_CHECKING, Optional, Tuple
+from typing import TYPE_CHECKING, Optional
 
 from .errors import ObjectFormatError
 from .file import HashFile
@@ -20,8 +20,7 @@ logger = logging.getLogger(__name__)
 class ReferenceHashFile(HashFile):
     PARAM_PATH = "path"
     PARAM_HASH = "hash"
-    PARAM_MTIME = "mtime"
-    PARAM_SIZE = "size"
+    PARAM_CHECKSUM = "checksum"
     PARAM_FS_CONFIG = "fs_config"
 
     def __init__(
@@ -29,14 +28,11 @@ class ReferenceHashFile(HashFile):
         path_info: "AnyPath",
         fs: "BaseFileSystem",
         hash_info: "HashInfo",
-        mtime: Optional[str] = None,
-        size: Optional[int] = None,
+        checksum: Optional[str] = None,
         **kwargs,
     ):
         super().__init__(path_info, fs, hash_info, **kwargs)
-        cur_mtime, cur_size = self._get_mtime_and_size()
-        self.mtime = cur_mtime if mtime is None else mtime
-        self.hash_info.size = cur_size if size is None else size
+        self.checksum = checksum or fs.checksum(path_info)
 
     def __str__(self):
         return f"ref object {self.hash_info} -> {self.path_info}"
@@ -47,21 +43,14 @@ class ReferenceHashFile(HashFile):
             raise FileNotFoundError(
                 errno.ENOENT, os.strerror(errno.ENOENT), self.path_info
             )
-        if not (self.mtime, self.size) == self._get_mtime_and_size():
+        if self.checksum != self._get_checksum():
             raise ObjectFormatError(f"{self} is changed")
         if check_hash:
             self._check_hash(odb)
 
-    def _get_mtime_and_size(self) -> Tuple[Optional[str], int]:
-        from dvc.utils.fs import get_mtime_and_size
-
+    def _get_checksum(self) -> str:
         assert self.fs
-        if hasattr(self.fs, "stat"):
-            try:
-                return get_mtime_and_size(self.path_info, self.fs)
-            except FileNotFoundError:
-                pass
-        return None, self.fs.getsize(self.path_info)
+        return self.fs.checksum(self.path_info)
 
     def to_bytes(self):
         from dvc.fs.repo import RepoFileSystem
@@ -79,8 +68,7 @@ class ReferenceHashFile(HashFile):
         dict_ = {
             self.PARAM_PATH: path_info,
             self.PARAM_HASH: self.hash_info,
-            self.PARAM_MTIME: self.mtime,
-            self.PARAM_SIZE: self.size,
+            self.PARAM_CHECKSUM: self.checksum,
             self.PARAM_FS_CONFIG: self.config_tuple(self.fs),
         }
         try:
@@ -118,8 +106,7 @@ class ReferenceHashFile(HashFile):
             path_info,
             fs,
             hash_info,
-            mtime=dict_.get(cls.PARAM_MTIME),
-            size=dict_.get(cls.PARAM_SIZE),
+            checksum=dict_.get(cls.PARAM_CHECKSUM),
         )
 
     @staticmethod

dvc/output.py

@@ -81,6 +81,7 @@ def loadd_from(stage, d_list):
         desc = d.pop(Output.PARAM_DESC, False)
         isexec = d.pop(Output.PARAM_ISEXEC, False)
         live = d.pop(Output.PARAM_LIVE, False)
+        remote = d.pop(Output.PARAM_REMOTE, None)
         ret.append(
             _get(
                 stage,
@@ -94,6 +95,7 @@ def loadd_from(stage, d_list):
                 desc=desc,
                 isexec=isexec,
                 live=live,
+                remote=remote,
             )
         )
     return ret
@@ -109,6 +111,7 @@ def loads_from(
     checkpoint=False,
     isexec=False,
     live=False,
+    remote=None,
 ):
     return [
         _get(
@@ -122,6 +125,7 @@ def loads_from(
             checkpoint=checkpoint,
             isexec=isexec,
             live=live,
+            remote=remote,
         )
         for s in s_list
     ]
@@ -162,7 +166,6 @@ def load_from_pipeline(stage, data, typ="outs"):
     metric = typ == stage.PARAM_METRICS
     plot = typ == stage.PARAM_PLOTS
     live = typ == stage.PARAM_LIVE
-
     if live:
         # `live` is single object
         data = [data]
@@ -185,6 +188,7 @@ def load_from_pipeline(stage, data, typ="outs"):
                 Output.PARAM_CACHE,
                 Output.PARAM_PERSIST,
                 Output.PARAM_CHECKPOINT,
+                Output.PARAM_REMOTE,
             ],
         )
 
@@ -255,6 +259,7 @@ class Output:
     PARAM_LIVE = "live"
     PARAM_LIVE_SUMMARY = "summary"
     PARAM_LIVE_HTML = "html"
+    PARAM_REMOTE = "remote"
 
     METRIC_SCHEMA = Any(
         None,
@@ -283,6 +288,7 @@ class Output:
         live=False,
         desc=None,
         isexec=False,
+        remote=None,
     ):
         self.repo = stage.repo if stage else None
 
@@ -326,7 +332,7 @@ class Output:
         self.obj = None
         self.isexec = False if self.IS_DEPENDENCY else isexec
 
-        self.def_remote = None
+        self.remote = remote
 
     def _parse_path(self, fs, path_info):
         if fs.scheme != "local":
@@ -843,7 +849,8 @@ class Output:
         try:
             objects.check(self.odb, obj)
         except FileNotFoundError:
-            self.repo.cloud.pull([obj.hash_info], **kwargs)
+            remote = self.repo.cloud.get_remote_odb(self.remote)
+            self.repo.cloud.pull([obj.hash_info], odb=remote, **kwargs)
 
         if self.obj:
             return self.obj
@@ -855,9 +862,9 @@ class Output:
 
         return self.obj
 
-    def collect_used_dir_cache(
+    def _collect_used_dir_cache(
         self, remote=None, force=False, jobs=None, filter_info=None
-    ) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
+    ) -> Optional["Tree"]:
         """Fetch dir cache and return used object IDs for this out."""
 
         try:
@@ -878,13 +885,13 @@ class Output:
                     "unable to fully collect used cache"
                     " without cache for directory '{}'".format(self)
                 )
-            return {}
+            return None
 
         obj = self.get_obj()
         if filter_info and filter_info != self.path_info:
             prefix = filter_info.relative_to(self.path_info).parts
             obj = obj.filter(prefix)
-        return {None: set(self._named_obj_ids(obj))}
+        return obj
 
     def get_used_objs(
         self, **kwargs
@@ -917,22 +924,31 @@ class Output:
             return {}
 
         if self.is_dir_checksum:
-            return self.collect_used_dir_cache(**kwargs)
+            obj = self._collect_used_dir_cache(**kwargs)
+        else:
+            obj = self.get_obj(filter_info=kwargs.get("filter_info"))
+            if not obj:
+                obj = self.odb.get(self.hash_info)
 
-        obj = self.get_obj(filter_info=kwargs.get("filter_info"))
         if not obj:
-            obj = self.odb.get(self.hash_info)
+            return {}
 
-        return {None: set(self._named_obj_ids(obj))}
+        if self.remote:
+            remote = self.repo.cloud.get_remote_odb(name=self.remote)
+        else:
+            remote = None
+
+        return {remote: self._named_obj_ids(obj)}
 
     def _named_obj_ids(self, obj):
         name = str(self)
         obj.hash_info.obj_name = name
-        yield obj.hash_info
+        oids = {obj.hash_info}
         if isinstance(obj, Tree):
             for key, entry_obj in obj:
                 entry_obj.hash_info.obj_name = self.fs.sep.join([name, *key])
-                yield entry_obj.hash_info
+                oids.add(entry_obj.hash_info)
+        return oids
 
     def get_used_external(
         self, **kwargs
@@ -1033,4 +1049,5 @@ SCHEMA = {
     Output.PARAM_CACHE: bool,
     Output.PARAM_METRIC: Output.METRIC_SCHEMA,
     Output.PARAM_DESC: str,
+    Output.PARAM_REMOTE: str,
 }

dvc/render/__init__.py

@@ -0,0 +1,33 @@
+import logging
+from typing import TYPE_CHECKING, Dict
+
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
+logger = logging.getLogger(__name__)
+
+
+class Renderer:
+    def __init__(self, data: Dict):
+        self.data = data
+
+        from dvc.render.utils import get_files
+
+        files = get_files(self.data)
+
+        # we assume comparison of same file between revisions for now
+        assert len(files) == 1
+        self.filename = files.pop()
+
+    def _convert(self, path: "StrPath"):
+        raise NotImplementedError
+
+    @property
+    def DIV(self):
+        raise NotImplementedError
+
+    def generate_html(self, path: "StrPath"):
+        """this method might edit content of path"""
+        partial = self._convert(path)
+        div_id = f"plot_{self.filename.replace('.', '_').replace('/', '_')}"
+        return self.DIV.format(id=div_id, partial=partial)

dvc/render/html.py

@@ -1,11 +1,17 @@
-from typing import Dict, List, Optional
+import os
+from pathlib import Path
+from typing import TYPE_CHECKING, Dict, List, Optional
 
 from dvc.exceptions import DvcException
-from dvc.types import StrPath
+
+if TYPE_CHECKING:
+    from dvc.render import Renderer
+    from dvc.types import StrPath
 
 PAGE_HTML = """<!DOCTYPE html>
 <html>
 <head>
+    {refresh_tag}
     <title>DVC Plot</title>
     <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
     <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
@@ -16,12 +22,6 @@ PAGE_HTML = """<!DOCTYPE html>
 </body>
 </html>"""
 
-VEGA_DIV_HTML = """<div id = "{id}"></div>
-<script type = "text/javascript">
-    var spec = {vega_json};
-    vegaEmbed('#{id}', spec);
-</script>"""
-
 
 class MissingPlaceholderError(DvcException):
     def __init__(self, placeholder):
@@ -31,14 +31,23 @@ class MissingPlaceholderError(DvcException):
 class HTML:
     PLACEHOLDER = "plot_divs"
     PLACEHOLDER_FORMAT_STR = f"{{{PLACEHOLDER}}}"
+    REFRESH_PLACEHOLDER = "refresh_tag"
+    REFRESH_TAG = '<meta http-equiv="refresh" content="{}">'
 
-    def __init__(self, template: str = None):
+    def __init__(
+        self,
+        template: Optional[str] = None,
+        refresh_seconds: Optional[int] = None,
+    ):
         template = template or PAGE_HTML
         if self.PLACEHOLDER_FORMAT_STR not in template:
             raise MissingPlaceholderError(self.PLACEHOLDER_FORMAT_STR)
 
         self.template = template
         self.elements: List[str] = []
+        self.refresh_tag = ""
+        if refresh_seconds is not None:
+            self.refresh_tag = self.REFRESH_TAG.format(refresh_seconds)
 
     def with_metrics(self, metrics: Dict[str, Dict]) -> "HTML":
         import tabulate
@@ -56,41 +65,43 @@ class HTML:
         self.elements.append(tabulate.tabulate(rows, header, tablefmt="html"))
         return self
 
-    def with_plots(self, plots: Dict[str, Dict]) -> "HTML":
-        self.elements.extend(
-            [
-                VEGA_DIV_HTML.format(id=f"plot{i}", vega_json=plot)
-                for i, plot in enumerate(plots.values())
-            ]
-        )
-        return self
-
     def with_element(self, html: str) -> "HTML":
         self.elements.append(html)
         return self
 
     def embed(self) -> str:
-        kwargs = {self.PLACEHOLDER: "\n".join(self.elements)}
+        kwargs = {
+            self.PLACEHOLDER: "\n".join(self.elements),
+            self.REFRESH_PLACEHOLDER: self.refresh_tag,
+        }
         return self.template.format(**kwargs)
 
 
 def write(
-    path: StrPath,
-    plots: Dict[str, Dict],
+    path: "StrPath",
+    renderers: List["Renderer"],
     metrics: Optional[Dict[str, Dict]] = None,
-    template_path: Optional[StrPath] = None,
+    template_path: Optional["StrPath"] = None,
+    refresh_seconds: Optional[int] = None,
 ):
+
+    os.makedirs(path, exist_ok=True)
+
     page_html = None
     if template_path:
         with open(template_path) as fobj:
             page_html = fobj.read()
 
-    document = HTML(page_html)
+    document = HTML(page_html, refresh_seconds=refresh_seconds)
     if metrics:
         document.with_metrics(metrics)
         document.with_element("<br>")
 
-    document.with_plots(plots)
+    for renderer in renderers:
+        document.with_element(renderer.generate_html(path))
 
-    with open(path, "w") as fd:
+    index = Path(os.path.join(path, "index.html"))
+
+    with open(index, "w") as fd:
         fd.write(document.embed())
+    return index

dvc/render/image.py

@@ -0,0 +1,64 @@
+import os
+from typing import TYPE_CHECKING
+
+from dvc.render import Renderer
+from dvc.render.utils import get_files
+from dvc.utils import relpath
+
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
+
+class ImageRenderer(Renderer):
+    DIV = """
+        <div
+            id="{id}"
+            style="border: 1px solid;">
+            {partial}
+        </div>"""
+
+    def _write_image(
+        self,
+        page_dir_path: "StrPath",
+        revision: str,
+        filename: str,
+        image_data: bytes,
+    ):
+        static = os.path.join(page_dir_path, "static")
+        os.makedirs(static, exist_ok=True)
+
+        img_path = os.path.join(
+            static, f"{revision}_{filename.replace(os.sep, '_')}"
+        )
+        rel_img_path = relpath(img_path, page_dir_path)
+        with open(img_path, "wb") as fd:
+            fd.write(image_data)
+        return """
+        <div>
+            <p>{title}</p>
+            <img src="{src}">
+        </div>""".format(
+            title=revision, src=rel_img_path
+        )
+
+    def _convert(self, path: "StrPath"):
+        div_content = []
+        for rev, rev_data in self.data.items():
+            if "data" in rev_data:
+                for file, file_data in rev_data.get("data", {}).items():
+                    if "data" in file_data:
+                        div_content.append(
+                            self._write_image(
+                                path, rev, file, file_data["data"]
+                            )
+                        )
+        if div_content:
+            div_content.insert(0, f"<p>{self.filename}</p>")
+            return "\n".join(div_content)
+        return ""
+
+    @staticmethod
+    def matches(data):
+        files = get_files(data)
+        extensions = set(map(lambda f: os.path.splitext(f)[1], files))
+        return extensions.issubset({".jpg", ".jpeg", ".gif", ".png"})

dvc/render/utils.py

@@ -0,0 +1,91 @@
+import os.path
+from typing import Dict, List, Set
+
+
+def get_files(data: Dict) -> Set:
+    files = set()
+    for rev in data.keys():
+        for file in data[rev].get("data", {}).keys():
+            files.add(file)
+    return files
+
+
+def group_by_filename(plots_data: Dict) -> List[Dict]:
+    # TODO use dpath.util.search once
+    #  https://github.com/dpath-maintainers/dpath-python/issues/147 is released
+    #  now cannot search when errors are present in data
+    files = get_files(plots_data)
+    grouped = []
+
+    for file in files:
+        tmp: Dict = {}
+        for revision, revision_data in plots_data.items():
+            if file in revision_data.get("data", {}):
+                if "data" not in tmp:
+                    tmp[revision] = {"data": {}}
+                tmp[revision]["data"].update(
+                    {file: revision_data["data"][file]}
+                )
+        grouped.append(tmp)
+
+    return grouped
+
+
+def find_vega(repo, plots_data, target):
+    # TODO same as group_by_filename
+    grouped = group_by_filename(plots_data)
+    found = None
+    for plot_group in grouped:
+        files = get_files(plot_group)
+        assert len(files) == 1
+        file = files.pop()
+        if file == target:
+            found = plot_group
+            break
+
+    from dvc.render.vega import VegaRenderer
+
+    if found and VegaRenderer.matches(found):
+        return VegaRenderer(found, repo.plots.templates).get_vega()
+    return ""
+
+
+def match_renderers(plots_data, templates):
+    from dvc.render.image import ImageRenderer
+    from dvc.render.vega import VegaRenderer
+
+    renderers = []
+    for g in group_by_filename(plots_data):
+        if VegaRenderer.matches(g):
+            renderers.append(VegaRenderer(g, templates))
+        if ImageRenderer.matches(g):
+            renderers.append(ImageRenderer(g))
+    return renderers
+
+
+def render(
+    repo,
+    plots_data,
+    metrics=None,
+    path=None,
+    html_template_path=None,
+    refresh_seconds=None,
+):
+    # TODO we could probably remove repo usages (here and in VegaRenderer)
+    renderers = match_renderers(plots_data, repo.plots.templates)
+    if not html_template_path:
+        html_template_path = repo.config.get("plots", {}).get(
+            "html_template", None
+        )
+        if html_template_path and not os.path.isabs(html_template_path):
+            html_template_path = os.path.join(repo.dvc_dir, html_template_path)
+
+    from dvc.render.html import write
+
+    return write(
+        path,
+        renderers,
+        metrics=metrics,
+        template_path=html_template_path,
+        refresh_seconds=refresh_seconds,
+    )

dvc/render/vega.py

@@ -0,0 +1,175 @@
+import os
+from copy import copy, deepcopy
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
+
+from funcy import first
+
+from dvc.exceptions import DvcException
+from dvc.render import Renderer
+from dvc.render.utils import get_files
+
+if TYPE_CHECKING:
+    from dvc.repo.plots.template import PlotTemplates
+    from dvc.types import StrPath
+
+REVISION_FIELD = "rev"
+INDEX_FIELD = "step"
+
+
+class PlotMetricTypeError(DvcException):
+    def __init__(self, file):
+        super().__init__(
+            "'{}' - file type error\n"
+            "Only JSON, YAML, CSV and TSV formats are supported.".format(file)
+        )
+
+
+class PlotDataStructureError(DvcException):
+    def __init__(self):
+        super().__init__(
+            "Plot data extraction failed. Please see "
+            "https://man.dvc.org/plot for supported data formats."
+        )
+
+
+def _filter_fields(
+    datapoints: List[Dict], filename, revision, fields=None
+) -> List[Dict]:
+    if not fields:
+        return datapoints
+    assert isinstance(fields, set)
+
+    new_data = []
+    for data_point in datapoints:
+        new_dp = copy(data_point)
+
+        keys = set(data_point.keys())
+        if keys & fields != fields:
+            raise DvcException(
+                "Could not find fields: '{}' for '{}' at '{}'.".format(
+                    ", ".join(fields), filename, revision
+                )
+            )
+
+        to_del = keys - fields
+        for key in to_del:
+            del new_dp[key]
+        new_data.append(new_dp)
+    return new_data
+
+
+def _lists(dictionary):
+    for _, value in dictionary.items():
+        if isinstance(value, dict):
+            yield from _lists(value)
+        elif isinstance(value, list):
+            yield value
+
+
+def _find_data(data: Union[Dict, List], fields=None) -> List[Dict]:
+    if not isinstance(data, dict):
+        return data
+
+    if not fields:
+        # just look for first list of dicts
+        fields = set()
+
+    for lst in _lists(data):
+        if (
+            all(isinstance(dp, dict) for dp in lst)
+            and set(first(lst).keys()) & fields == fields
+        ):
+            return lst
+    raise PlotDataStructureError()
+
+
+def _append_index(datapoints: List[Dict], append_index=False) -> List[Dict]:
+    if not append_index or INDEX_FIELD in first(datapoints).keys():
+        return datapoints
+
+    for index, data_point in enumerate(datapoints):
+        data_point[INDEX_FIELD] = index
+    return datapoints
+
+
+def _append_revision(datapoints: List[Dict], revision) -> List[Dict]:
+    for data_point in datapoints:
+        data_point[REVISION_FIELD] = revision
+    return datapoints
+
+
+class VegaRenderer(Renderer):
+    DIV = """
+    <div id = "{id}">
+        <script type = "text/javascript">
+            var spec = {partial};
+            vegaEmbed('#{id}', spec);
+        </script>
+    </div>
+    """
+
+    def __init__(self, data: Dict, templates: "PlotTemplates"):
+        super().__init__(data)
+        self.templates = templates
+
+    def _squash_props(self) -> Dict:
+        resolved: Dict[str, str] = {}
+        for rev_data in self.data.values():
+            for file_data in rev_data.get("data", {}).values():
+                props = file_data.get("props", {})
+                resolved = {**resolved, **props}
+        return resolved
+
+    def _datapoints(self, props: Dict):
+        fields = props.get("fields", set())
+        if fields:
+            fields = {*fields, props.get("x"), props.get("y")} - {None}
+
+        datapoints = []
+        for revision, rev_data in self.data.items():
+            for filename, file_data in rev_data.get("data", {}).items():
+                if "data" in file_data:
+                    tmp = deepcopy(file_data.get("data"))
+                    tmp = _find_data(tmp, fields=fields - {INDEX_FIELD})
+                    tmp = _append_index(
+                        tmp, append_index=props.get("append_index", False)
+                    )
+                    tmp = _filter_fields(
+                        tmp,
+                        filename=filename,
+                        revision=revision,
+                        fields=fields,
+                    )
+                    tmp = _append_revision(tmp, revision=revision)
+                    datapoints.extend(tmp)
+        return datapoints
+
+    def get_vega(self) -> Optional[str]:
+        props = self._squash_props()
+
+        template = self.templates.load(props.get("template") or "default")
+
+        if not props.get("x") and template.has_anchor("x"):
+            props["append_index"] = True
+            props["x"] = INDEX_FIELD
+
+        datapoints = self._datapoints(props)
+
+        if datapoints:
+            if not props.get("y") and template.has_anchor("y"):
+                fields = list(first(datapoints))
+                skip = (REVISION_FIELD, props.get("x"))
+                props["y"] = first(
+                    f for f in reversed(fields) if f not in skip
+                )
+            return template.render(datapoints, props=props)
+        return None
+
+    def _convert(self, path: "StrPath"):
+        return self.get_vega()
+
+    @staticmethod
+    def matches(data):
+        files = get_files(data)
+        extensions = set(map(lambda f: os.path.splitext(f)[1], files))
+        return extensions.issubset({".yml", ".yaml", ".json", ".csv", ".tsv"})

dvc/repo/__init__.py

@@ -12,6 +12,7 @@ from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
 from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
 from dvc.ignore import DvcIgnoreFilter
 from dvc.path_info import PathInfo
+from dvc.utils import env2bool
 from dvc.utils.fs import path_isin
 
 if TYPE_CHECKING:
@@ -134,6 +135,7 @@ class Repo:
         from dvc.data_cloud import DataCloud
         from dvc.fs.local import LocalFileSystem
         from dvc.lock import LockNoop, make_lock
+        from dvc.machine import MachineManager
         from dvc.objects.db import ODBManager
         from dvc.repo.live import Live
         from dvc.repo.metrics import Metrics
@@ -196,6 +198,14 @@ class Repo:
         self.params = Params(self)
         self.live = Live(self)
 
+        if self.tmp_dir and (
+            self.config["feature"].get("machine", False)
+            or env2bool("DVC_TEST")
+        ):
+            self.machine = MachineManager(self)
+        else:
+            self.machine = None
+
         self.stage_collection_error_handler: Optional[
             Callable[[str, Exception], None]
         ] = None

dvc/repo/collect.py

@@ -49,7 +49,6 @@ def _collect_paths(
                 )
             else:
                 logger.warning("'%s' was not found at: '%s'.", path_info, rev)
-            continue
         target_infos.append(path_info)
     return target_infos
 

dvc/repo/experiments/pull.py

@@ -4,7 +4,7 @@ from dvc.exceptions import DvcException, InvalidArgumentError
 from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
 
-from .utils import exp_commits, remote_exp_refs_by_name
+from .utils import exp_commits, resolve_exp_ref
 
 logger = logging.getLogger(__name__)
 
@@ -14,7 +14,11 @@ logger = logging.getLogger(__name__)
 def pull(
     repo, git_remote, exp_name, *args, force=False, pull_cache=False, **kwargs
 ):
-    exp_ref = _get_exp_ref(repo, git_remote, exp_name)
+    exp_ref = resolve_exp_ref(repo.scm, exp_name, git_remote)
+    if not exp_ref:
+        raise InvalidArgumentError(
+            f"Experiment '{exp_name}' does not exist in '{git_remote}'"
+        )
 
     def on_diverged(refname: str, rev: str) -> bool:
         if repo.scm.get_ref(refname) == rev:
@@ -35,33 +39,6 @@ def pull(
         _pull_cache(repo, exp_ref, **kwargs)
 
 
-def _get_exp_ref(repo, git_remote, exp_name):
-    if exp_name.startswith("refs/"):
-        return exp_name
-
-    exp_refs = list(remote_exp_refs_by_name(repo.scm, git_remote, exp_name))
-    if not exp_refs:
-        raise InvalidArgumentError(
-            f"Experiment '{exp_name}' does not exist in '{git_remote}'"
-        )
-    if len(exp_refs) > 1:
-        cur_rev = repo.scm.get_rev()
-        for info in exp_refs:
-            if info.baseline_sha == cur_rev:
-                return info
-        msg = [
-            (
-                f"Ambiguous name '{exp_name}' refers to multiple "
-                "experiments in '{git_remote}'. Use full refname to pull one "
-                "of the following:"
-            ),
-            "",
-        ]
-        msg.extend([f"\t{info}" for info in exp_refs])
-        raise InvalidArgumentError("\n".join(msg))
-    return exp_refs[0]
-
-
 def _pull_cache(repo, exp_ref, dvc_remote=None, jobs=None, run_cache=False):
     revs = list(exp_commits(repo.scm, [exp_ref]))
     logger.debug("dvc fetch experiment '%s'", exp_ref)

dvc/repo/experiments/push.py

@@ -2,10 +2,9 @@ import logging
 
 from dvc.exceptions import DvcException, InvalidArgumentError
 from dvc.repo import locked
-from dvc.repo.experiments.base import ExpRefInfo
 from dvc.repo.scm_context import scm_context
 
-from .utils import exp_commits, exp_refs_by_name
+from .utils import exp_commits, resolve_exp_ref
 
 logger = logging.getLogger(__name__)
 
@@ -21,7 +20,11 @@ def push(
     push_cache=False,
     **kwargs,
 ):
-    exp_ref = _get_exp_ref(repo, exp_name)
+    exp_ref = resolve_exp_ref(repo.scm, exp_name)
+    if not exp_ref:
+        raise InvalidArgumentError(
+            f"'{exp_name}' is not a valid experiment name"
+        )
 
     def on_diverged(refname: str, rev: str) -> bool:
         if repo.scm.get_ref(refname) == rev:
@@ -42,33 +45,6 @@ def push(
         _push_cache(repo, exp_ref, **kwargs)
 
 
-def _get_exp_ref(repo, exp_name: str) -> ExpRefInfo:
-    if exp_name.startswith("refs/"):
-        return ExpRefInfo.from_ref(exp_name)
-
-    exp_refs = list(exp_refs_by_name(repo.scm, exp_name))
-    if not exp_refs:
-        raise InvalidArgumentError(
-            f"'{exp_name}' is not a valid experiment name"
-        )
-    if len(exp_refs) > 1:
-        cur_rev = repo.scm.get_rev()
-        for info in exp_refs:
-            if info.baseline_sha == cur_rev:
-                return info
-        msg = [
-            (
-                f"Ambiguous name '{exp_name}' refers to multiple "
-                "experiments. Use full refname to push one of the "
-                "following:"
-            ),
-            "",
-        ]
-        msg.extend([f"\t{info}" for info in exp_refs])
-        raise InvalidArgumentError("\n".join(msg))
-    return exp_refs[0]
-
-
 def _push_cache(repo, exp_ref, dvc_remote=None, jobs=None, run_cache=False):
     revs = list(exp_commits(repo.scm, [exp_ref]))
     logger.debug("dvc push experiment '%s'", exp_ref)

dvc/repo/experiments/remove.py

@@ -6,8 +6,7 @@ from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
 from dvc.scm.base import RevError
 
-from .base import EXPS_NAMESPACE, ExpRefInfo
-from .utils import exp_refs, exp_refs_by_name, remove_exp_refs
+from .utils import exp_refs, remove_exp_refs, resolve_exp_ref
 
 logger = logging.getLogger(__name__)
 
@@ -19,6 +18,7 @@ def remove(
     exp_names=None,
     queue=False,
     clear_all=False,
+    remote=None,
     **kwargs,
 ):
     if not any([exp_names, queue, clear_all]):
@@ -31,13 +31,7 @@ def remove(
         removed += _clear_all(repo)
 
     if exp_names:
-        remained = _remove_commited_exps(repo, exp_names)
-        remained = _remove_queued_exps(repo, remained)
-        if remained:
-            raise InvalidArgumentError(
-                "'{}' is not a valid experiment".format(";".join(remained))
-            )
-        removed += len(exp_names) - len(remained)
+        removed += _remove_exp_by_names(repo, remote, exp_names)
     return removed
 
 
@@ -67,46 +61,24 @@ def _get_exp_stash_index(repo, ref_or_rev: str) -> Optional[int]:
     return None
 
 
-def _get_exp_ref(repo, exp_name: str) -> Optional[ExpRefInfo]:
-    cur_rev = repo.scm.get_rev()
-    if exp_name.startswith(EXPS_NAMESPACE):
-        if repo.scm.get_ref(exp_name):
-            return ExpRefInfo.from_ref(exp_name)
-    else:
-        exp_ref_list = list(exp_refs_by_name(repo.scm, exp_name))
-        if exp_ref_list:
-            return _get_ref(exp_ref_list, exp_name, cur_rev)
-    return None
-
-
-def _get_ref(ref_infos, name, cur_rev) -> Optional[ExpRefInfo]:
-    if len(ref_infos) > 1:
-        for info in ref_infos:
-            if info.baseline_sha == cur_rev:
-                return info
-        msg = [
-            (
-                f"Ambiguous name '{name}' refers to multiple "
-                "experiments. Use full refname to remove one of "
-                "the following:"
-            )
-        ]
-        msg.extend([f"\t{info}" for info in ref_infos])
-        raise InvalidArgumentError("\n".join(msg))
-    return ref_infos[0]
-
-
-def _remove_commited_exps(repo, refs: List[str]) -> List[str]:
+def _remove_commited_exps(
+    repo, remote: Optional[str], exp_names: List[str]
+) -> List[str]:
     remain_list = []
     remove_list = []
-    for ref in refs:
-        ref_info = _get_exp_ref(repo, ref)
+    for exp_name in exp_names:
+        ref_info = resolve_exp_ref(repo.scm, exp_name, remote)
+
         if ref_info:
             remove_list.append(ref_info)
         else:
-            remain_list.append(ref)
+            remain_list.append(exp_name)
     if remove_list:
-        remove_exp_refs(repo.scm, remove_list)
+        if not remote:
+            remove_exp_refs(repo.scm, remove_list)
+        else:
+            for ref_info in remove_list:
+                repo.scm.push_refspec(remote, None, str(ref_info))
     return remain_list
 
 
@@ -119,3 +91,14 @@ def _remove_queued_exps(repo, refs_or_revs: List[str]) -> List[str]:
         else:
             repo.experiments.stash.drop(stash_index)
     return remain_list
+
+
+def _remove_exp_by_names(repo, remote, exp_names: List[str]) -> int:
+    remained = _remove_commited_exps(repo, remote, exp_names)
+    if not remote:
+        remained = _remove_queued_exps(repo, remained)
+    if remained:
+        raise InvalidArgumentError(
+            "'{}' is not a valid experiment".format(";".join(remained))
+        )
+    return len(exp_names) - len(remained)

dvc/repo/experiments/utils.py

@@ -1,5 +1,6 @@
 from typing import Generator, Iterable, Optional, Set
 
+from dvc.exceptions import InvalidArgumentError
 from dvc.scm.git import Git
 
 from .base import (
@@ -115,3 +116,44 @@ def fix_exp_head(scm: "Git", ref: Optional[str]) -> Optional[str]:
         if name == "HEAD" and scm.get_ref(EXEC_BASELINE):
             return "".join((EXEC_BASELINE, tail))
     return ref
+
+
+def resolve_exp_ref(
+    scm, exp_name: str, git_remote: Optional[str] = None
+) -> Optional[ExpRefInfo]:
+    if exp_name.startswith("refs/"):
+        return ExpRefInfo.from_ref(exp_name)
+
+    if git_remote:
+        exp_ref_list = list(remote_exp_refs_by_name(scm, git_remote, exp_name))
+    else:
+        exp_ref_list = list(exp_refs_by_name(scm, exp_name))
+
+    if not exp_ref_list:
+        return None
+    if len(exp_ref_list) > 1:
+        cur_rev = scm.get_rev()
+        for info in exp_ref_list:
+            if info.baseline_sha == cur_rev:
+                return info
+        if git_remote:
+            msg = [
+                (
+                    f"Ambiguous name '{exp_name}' refers to multiple "
+                    "experiments. Use full refname to push one of the "
+                    "following:"
+                ),
+                "",
+            ]
+        else:
+            msg = [
+                (
+                    f"Ambiguous name '{exp_name}' refers to multiple "
+                    f"experiments in '{git_remote}'. Use full refname to pull "
+                    "one of the following:"
+                ),
+                "",
+            ]
+        msg.extend([f"\t{info}" for info in exp_ref_list])
+        raise InvalidArgumentError("\n".join(msg))
+    return exp_ref_list[0]

dvc/repo/live.py

@@ -2,6 +2,8 @@ import logging
 import os
 from typing import TYPE_CHECKING, List, Optional
 
+from dvc.render.utils import render
+
 logger = logging.getLogger(__name__)
 
 if TYPE_CHECKING:
@@ -15,10 +17,12 @@ def create_summary(out):
 
     metrics, plots = out.repo.live.show(str(out.path_info))
 
-    html_path = out.path_info.with_suffix(".html")
+    html_path = out.path_info.fspath + "_dvc_plots"
 
-    out.repo.plots.write_html(html_path, plots, metrics)
-    logger.info(f"\nfile://{os.path.abspath(html_path)}")
+    index_path = render(
+        out.repo, plots, metrics=metrics, path=html_path, refresh_seconds=5
+    )
+    logger.info(f"\nfile://{os.path.abspath(index_path)}")
 
 
 def summary_path_info(out: "Output") -> Optional["PathInfo"]:

dvc/repo/plots/__init__.py

@@ -8,8 +8,7 @@ from typing import TYPE_CHECKING, Callable, Dict, List, Optional
 from funcy import cached_property, first, project
 
 from dvc.exceptions import DvcException
-from dvc.repo.plots.data import PlotMetricTypeError
-from dvc.types import StrPath
+from dvc.render.vega import PlotMetricTypeError
 from dvc.utils import (
     error_handler,
     errored_revisions,
@@ -113,58 +112,33 @@ class Plots:
             props = props or {}
 
             for path, repo_path in plot_files:
-                res[repo_path] = {"props": rev_props}
+                joined_props = {**rev_props, **props}
+                res[repo_path] = {"props": joined_props}
                 res[repo_path].update(
                     parse(
                         fs,
                         path,
-                        rev_props=rev_props,
-                        props=props,
+                        props=joined_props,
                         onerror=onerror,
                     )
                 )
         return res
 
-    @staticmethod
-    def render(data, revs=None, props=None, templates=None):
-        """Renders plots"""
-        props = props or {}
-
-        # Merge data by plot file and apply overriding props
-        plots = _prepare_plots(data, revs, props)
-
-        result = {}
-        for datafile, desc in plots.items():
-            rendered = _render(
-                datafile,
-                desc["data"],
-                desc["props"],
-                templates,
-            )
-            if rendered:
-                result[datafile] = rendered
-
-        return result
-
     def show(
         self,
         targets: List[str] = None,
         revs=None,
         props=None,
-        templates=None,
         recursive=False,
         onerror=None,
     ):
         if onerror is None:
             onerror = onerror_collect
-        data = self.collect(
+
+        return self.collect(
             targets, revs, recursive, onerror=onerror, props=props
         )
 
-        if templates is None:
-            templates = self.templates
-        return self.render(data, revs, props, templates)
-
     def diff(self, *args, **kwargs):
         from .diff import diff
 
@@ -217,26 +191,6 @@ class Plots:
 
         return PlotTemplates(self.repo.dvc_dir)
 
-    def write_html(
-        self,
-        path: StrPath,
-        plots: Dict[str, Dict],
-        metrics: Optional[Dict[str, Dict]] = None,
-        html_template_path: Optional[StrPath] = None,
-    ):
-        if not html_template_path:
-            html_template_path = self.repo.config.get("plots", {}).get(
-                "html_template", None
-            )
-            if html_template_path and not os.path.isabs(html_template_path):
-                html_template_path = os.path.join(
-                    self.repo.dvc_dir, html_template_path
-                )
-
-        from dvc.utils.html import write
-
-        write(path, plots, metrics, html_template_path)
-
 
 def _is_plot(out: "Output") -> bool:
     return bool(out.plot) or bool(out.live)
@@ -264,17 +218,19 @@ def _collect_plots(
 
 
 @error_handler
-def parse(fs, path, props=None, rev_props=None, **kwargs):
+def parse(fs, path, props=None, **kwargs):
     props = props or {}
-    rev_props = rev_props or {}
     _, extension = os.path.splitext(path)
     if extension in (".tsv", ".csv"):
-        header = {**rev_props, **props}.get("header", True)
+        header = props.get("header", True)
         if extension == ".csv":
             return _load_sv(path=path, fs=fs, delimiter=",", header=header)
         return _load_sv(path=path, fs=fs, delimiter="\t", header=header)
     if extension in LOADERS or extension in (".yml", ".yaml"):
         return LOADERS[extension](path=path, fs=fs)
+    if extension in (".jpeg", ".jpg", ".gif", ".png"):
+        with fs.open(path, "rb") as fd:
+            return fd.read()
     raise PlotMetricTypeError(path)
 
 
@@ -291,89 +247,6 @@ def _plot_props(out: "Output") -> Dict:
     return project(out.plot, PLOT_PROPS)
 
 
-def _prepare_plots(data, revs, props):
-    """Groups data by plot file.
-
-    Also resolves props conflicts between revs and applies global props.
-    """
-    # we go in order revs are supplied on props conflict first ones win.
-    revs = iter(data) if revs is None else revs
-
-    plots, props_revs = {}, {}
-    for rev in revs:
-        # Asked for revision without data
-        if rev not in data:
-            continue
-
-        for datafile, desc in data[rev].get("data", {}).items():
-            # We silently skip on an absent data file,
-            # see also try/except/pass in .collect()
-            if "data" not in desc:
-                continue
-
-            # props from command line overwrite plot props from out definition
-            full_props = {**desc["props"], **props}
-
-            if datafile in plots:
-                saved = plots[datafile]
-                if saved["props"] != full_props:
-                    logger.warning(
-                        f"Inconsistent plot props for '{datafile}' in "
-                        f"'{props_revs[datafile]}' and '{rev}'. "
-                        f"Going to use ones from '{props_revs[datafile]}'"
-                    )
-
-                saved["data"][rev] = desc["data"]
-            else:
-                plots[datafile] = {
-                    "props": full_props,
-                    "data": {rev: desc["data"]},
-                }
-                # Save rev we got props from
-                props_revs[datafile] = rev
-
-    return plots
-
-
-def _render(datafile, datas, props, templates):
-    from .data import PlotData, plot_data
-
-    # Copy it to not modify a passed value
-    props = props.copy()
-
-    # Add x and y to fields if set
-    fields = props.get("fields")
-    if fields is not None:
-        fields = {*fields, props.get("x"), props.get("y")} - {None}
-
-    template = templates.load(props.get("template") or "default")
-
-    # If x is not set add index field
-    if not props.get("x") and template.has_anchor("x"):
-        props["append_index"] = True
-        props["x"] = PlotData.INDEX_FIELD
-
-    # Parse all data, preprocess it and collect as a list of dicts
-    data = []
-    for rev, unstructured_data in datas.items():
-        rev_data = plot_data(datafile, rev, unstructured_data).to_datapoints(
-            fields=fields,
-            path=props.get("path"),
-            append_index=props.get("append_index", False),
-        )
-        data.extend(rev_data)
-
-    # If y is not set then use last field not used yet
-    if data:
-        if not props.get("y") and template.has_anchor("y"):
-            fields = list(first(data))
-            skip = (PlotData.REVISION_FIELD, props.get("x"))
-            props["y"] = first(f for f in reversed(fields) if f not in skip)
-
-        return template.render(data, props=props)
-    return None
-
-
 def _load_sv(path, fs, delimiter=",", header=True):
     with fs.open(path, "r") as fd:
         content = fd.read()

dvc/repo/plots/data.py

@@ -1,138 +0,0 @@
-import os
-from copy import copy
-
-from funcy import first
-
-from dvc.exceptions import DvcException
-from dvc.utils.serialize import ParseError
-
-
-class PlotMetricTypeError(DvcException):
-    def __init__(self, file):
-        super().__init__(
-            "'{}' - file type error\n"
-            "Only JSON, YAML, CSV and TSV formats are supported.".format(file)
-        )
-
-
-class PlotDataStructureError(DvcException):
-    def __init__(self):
-        super().__init__(
-            "Plot data extraction failed. Please see "
-            "https://man.dvc.org/plot for supported data formats."
-        )
-
-
-class PlotParsingError(ParseError):
-    def __init__(self, path, revision):
-        self.path = path
-        self.revision = revision
-
-        super().__init__(path, f"revision: '{revision}'")
-
-
-def plot_data(filename, revision, content):
-    _, extension = os.path.splitext(filename.lower())
-    if extension in (".json", ".yaml"):
-        return DictData(filename, revision, content)
-    if extension in (".csv", ".tsv"):
-        return ListData(filename, revision, content)
-    raise PlotMetricTypeError(filename)
-
-
-def _filter_fields(data_points, filename, revision, fields=None, **kwargs):
-    if not fields:
-        return data_points
-    assert isinstance(fields, set)
-
-    new_data = []
-    for data_point in data_points:
-        new_dp = copy(data_point)
-
-        keys = set(data_point.keys())
-        if keys & fields != fields:
-            raise DvcException(
-                "Could not find fields: '{}' for '{}' at '{}'.".format(
-                    ", ".join(fields), filename, revision
-                )
-            )
-
-        to_del = keys - fields
-        for key in to_del:
-            del new_dp[key]
-        new_data.append(new_dp)
-    return new_data
-
-
-def _lists(dictionary):
-    for _, value in dictionary.items():
-        if isinstance(value, dict):
-            yield from _lists(value)
-        elif isinstance(value, list):
-            yield value
-
-
-def _find_data(data, fields=None, **kwargs):
-    if not isinstance(data, dict):
-        return data
-
-    if not fields:
-        # just look for first list of dicts
-        fields = set()
-
-    for lst in _lists(data):
-        if (
-            all(isinstance(dp, dict) for dp in lst)
-            and set(first(lst).keys()) & fields == fields
-        ):
-            return lst
-    raise PlotDataStructureError()
-
-
-def _append_index(data_points, append_index=False, **kwargs):
-    if not append_index or PlotData.INDEX_FIELD in first(data_points).keys():
-        return data_points
-
-    for index, data_point in enumerate(data_points):
-        data_point[PlotData.INDEX_FIELD] = index
-    return data_points
-
-
-def _append_revision(data_points, revision, **kwargs):
-    for data_point in data_points:
-        data_point[PlotData.REVISION_FIELD] = revision
-    return data_points
-
-
-class PlotData:
-    REVISION_FIELD = "rev"
-    INDEX_FIELD = "step"
-
-    def __init__(self, filename, revision, content, **kwargs):
-        self.filename = filename
-        self.revision = revision
-        self.content = content
-
-    def _processors(self):
-        return [_filter_fields, _append_index, _append_revision]
-
-    def to_datapoints(self, **kwargs):
-        data = self.content
-
-        for data_proc in self._processors():
-            data = data_proc(
-                data, filename=self.filename, revision=self.revision, **kwargs
-            )
-        return data
-
-
-class DictData(PlotData):
-    # For files usually parsed as dicts: eg JSON, Yaml
-    def _processors(self):
-        parent_processors = super()._processors()
-        return [_find_data] + parent_processors
-
-
-class ListData(PlotData):
-    # For files parsed as list: CSV, TSV
-    pass

dvc/schema.py

@@ -49,6 +49,7 @@ OUT_PSTAGE_DETAILED_SCHEMA = {
         Output.PARAM_PERSIST: bool,
         Output.PARAM_CHECKPOINT: bool,
         Output.PARAM_DESC: str,
+        Output.PARAM_REMOTE: str,
     }
 }
 

dvc/scm/git/backend/dulwich.py

@@ -39,7 +39,7 @@ class DulwichObject(GitObject):
         self.repo = repo
         self._name = name
         self._mode = mode
-        self.sha = sha
+        self._sha = sha
 
     def open(self, mode: str = "r", encoding: str = None):
         if not encoding:
@@ -68,6 +68,14 @@ class DulwichObject(GitObject):
                 self.repo, entry.path.decode(), entry.mode, entry.sha
             )
 
+    @property
+    def size(self) -> int:
+        return len(self.repo[self.sha].as_raw_string())
+
+    @property
+    def sha(self) -> str:
+        return self._sha
+
 
 class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
     """Dulwich Git backend."""
@@ -399,9 +407,11 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
             ) from exc
 
         def update_refs(refs):
+            from dulwich.objects import ZERO_SHA
+
             new_refs = {}
             for ref, value in zip(dest_refs, values):
-                if ref in refs:
+                if ref in refs and value != ZERO_SHA:
                     local_sha = self.repo.refs[ref]
                     remote_sha = refs[ref]
                     try:

dvc/scm/git/backend/gitpython.py

@@ -92,6 +92,14 @@ class GitPythonObject(GitObject):
         for obj in self.obj:
             yield GitPythonObject(obj)
 
+    @property
+    def size(self) -> int:
+        return self.obj.size
+
+    @property
+    def sha(self) -> str:
+        return self.obj.hexsha
+
 
 class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
     """git-python Git backend."""

dvc/scm/git/backend/pygit2.py

@@ -58,6 +58,14 @@ class Pygit2Object(GitObject):
             return stat.S_IFDIR
         return self.obj.filemode
 
+    @property
+    def size(self) -> int:
+        return len(self.obj.read_raw())
+
+    @property
+    def sha(self) -> str:
+        return self.obj.hex
+
     def scandir(self) -> Iterable["Pygit2Object"]:
         for entry in self.obj:  # noqa: B301
             yield Pygit2Object(entry)

dvc/scm/git/objects.py

@@ -1,4 +1,3 @@
-import os
 import stat
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
@@ -44,6 +43,16 @@ class GitObject(ABC):
     def issubmodule(self) -> bool:
         return S_ISGITLINK(self.mode)
 
+    @property
+    @abstractmethod
+    def size(self) -> int:
+        pass
+
+    @property
+    @abstractmethod
+    def sha(self) -> str:
+        pass
+
 
 class GitTrie:
     def __init__(self, tree: GitObject, rev: str):
@@ -116,9 +125,14 @@ class GitTrie:
         if not topdown:
             yield top, dirs, nondirs
 
-    def stat(self, key: tuple) -> os.stat_result:
+    def info(self, key: tuple) -> dict:
         obj = self.trie[key]
-        return os.stat_result((obj.mode, 0, 0, 0, 0, 0, 0, 0, 0, 0))
+        return {
+            "size": obj.size,
+            "type": "directory" if stat.S_ISDIR(obj.mode) else "file",
+            "sha": obj.sha,
+            "mode": obj.mode,
+        }
 
 
 @dataclass

dvc/stage/serialize.py

@@ -28,6 +28,7 @@ PARAM_PLOT = Output.PARAM_PLOT
 PARAM_PERSIST = Output.PARAM_PERSIST
 PARAM_CHECKPOINT = Output.PARAM_CHECKPOINT
 PARAM_DESC = Output.PARAM_DESC
+PARAM_REMOTE = Output.PARAM_REMOTE
 
 DEFAULT_PARAMS_FILE = ParamsDependency.DEFAULT_PARAMS_FILE
 
@@ -52,6 +53,8 @@ def _get_flags(out):
         yield from out.plot.items()
     if out.live and isinstance(out.live, dict):
         yield from out.live.items()
+    if out.remote:
+        yield PARAM_REMOTE, out.remote
 
 
 def _serialize_out(out):

dvc/utils/fs.py

@@ -47,22 +47,22 @@ def get_mtime_and_size(path, fs, dvcignore=None):
             walk_iterator = fs.walk_files(path)
         for file_path in walk_iterator:
             try:
-                stats = fs.stat(file_path)
+                stats = fs.info(file_path)
             except OSError as exc:
                 # NOTE: broken symlink case.
                 if exc.errno != errno.ENOENT:
                     raise
                 continue
-            size += stats.st_size
-            files_mtimes[os.fspath(file_path)] = stats.st_mtime
+            size += stats["size"]
+            files_mtimes[os.fspath(file_path)] = stats["mtime"]
 
         # We track file changes and moves, which cannot be detected with simply
         # max(mtime(f) for f in non_ignored_files)
         mtime = dict_md5(files_mtimes)
     else:
-        base_stat = fs.stat(path)
-        size = base_stat.st_size
-        mtime = base_stat.st_mtime
+        base_stat = fs.info(path)
+        size = base_stat["size"]
+        mtime = base_stat["mtime"]
         mtime = int(nanotime.timestamp(mtime))
 
     return str(mtime), size

dvc/utils/http.py

@@ -24,13 +24,15 @@ def open_url(url, mode="r", encoding=None, **iter_opts):
 
 
 @contextmanager
-def iter_url(url, chunk_size=io.DEFAULT_BUFFER_SIZE):
+def iter_url(url, chunk_size=io.DEFAULT_BUFFER_SIZE, auth=None):
     """Iterate over chunks requested from url."""
     import requests
 
     def request(headers=None):
         the_url = url() if callable(url) else url
-        response = requests.get(the_url, stream=True, headers=headers)
+        response = requests.get(
+            the_url, stream=True, headers=headers, auth=auth
+        )
         if response.status_code == 404:
             raise FileNotFoundError(f"Can't open {the_url}")
         response.raise_for_status()

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.6.4"
+_BASE_VERSION = "2.7.0"
 
 
 def _generate_version(base_version):

setup.py

@@ -87,20 +87,22 @@ install_requires = [
     "dictdiffer>=0.8.1",
     "python-benedict>=0.21.1",
     "pyparsing==2.4.7",
-    "typing_extensions>=3.7.4",
-    "fsspec>=2021.7.0",
+    "typing_extensions>=3.7.4; python_version < '3.10'",
+    # until https://github.com/python/typing/issues/865 is fixed for python3.10
+    "typing_extensions==3.10.0.0; python_version >= '3.10'",
+    "fsspec>=2021.8.1",
     "diskcache>=5.2.1",
 ]
 
 
 # Extra dependencies for remote integrations
 
-gs = ["gcsfs==2021.7.0"]
-gdrive = ["pydrive2[fsspec]>=1.9.1"]
-s3 = ["s3fs==2021.8.0", "aiobotocore[boto3]>1.0.1"]
-azure = ["adlfs==2021.7.1", "azure-identity>=1.4.0", "knack"]
-oss = ["ossfs==2021.7.5"]
-ssh = ["sshfs>=2021.7.1"]
+gs = ["gcsfs==2021.8.1"]
+gdrive = ["pydrive2[fsspec]>=1.9.2"]
+s3 = ["s3fs==2021.8.1", "aiobotocore[boto3]>1.0.1"]
+azure = ["adlfs==2021.8.2", "azure-identity>=1.4.0", "knack"]
+oss = ["ossfs==2021.8.0"]
+ssh = ["sshfs>=2021.8.1"]
 
 hdfs = ["pyarrow>=2.0.0; python_version < '3.10'"]
 webhdfs = ["hdfs==2.5.8"]
@@ -109,12 +111,14 @@ webdav = ["webdav4>=0.9.0"]
 # for linux and mac, so it will fail to compile if user doesn't have all the
 # requirements, including kerberos itself. Once all the wheels are available,
 # we can start shipping it by default.
-ssh_gssapi = ["sshfs[gssapi]>=2021.7.1"]
+ssh_gssapi = ["sshfs[gssapi]>=2021.8.1"]
 all_remotes = gs + s3 + azure + ssh + oss + gdrive + hdfs + webhdfs + webdav
 
+terraform = ["tpi[ssh]>=0.0.0"]
+
 tests_requirements = (
     Path("test_requirements.txt").read_text().strip().splitlines()
-)
+) + terraform
 
 setup(
     name="dvc",
@@ -123,6 +127,8 @@ setup(
     long_description=open("README.rst", encoding="UTF-8").read(),
     author="Dmitry Petrov",
     author_email="[email protected]",
+    maintainer="Iterative",
+    maintainer_email="[email protected]",
     download_url="https://github.com/iterative/dvc",
     license="Apache License 2.0",
     install_requires=install_requires,
@@ -138,6 +144,7 @@ setup(
         "hdfs": hdfs,
         "webhdfs": webhdfs,
         "webdav": webdav,
+        "terraform": terraform,
         "tests": tests_requirements,
     },
     keywords="data-science data-version-control machine-learning git"

test_requirements.txt

@@ -1,7 +1,7 @@
 wheel==0.37.0
 
 # Test requirements
-pytest==6.2.4
+pytest==6.2.5
 pytest-cov==2.12.1
 pytest-xdist==2.3.0
 pytest-mock==3.6.1
@@ -21,8 +21,8 @@ crc32c==2.2.post0
 xmltodict==0.12.0
 google-compute-engine==2.8.13
 google-cloud-storage==1.42.0
-
 urllib3==1.26.6
+dvclive>=0.2.1
 
 # required by collective.checkdocs
 Pygments==2.10.0
@@ -44,6 +44,3 @@ types-requests
 types-paramiko
 types-tabulate
 types-toml
-
-# pypi doesn't allow for direct dependencies
-# gdrivefs @ git+https://github.com/intake/gdrivefs.git

tests/conftest.py

@@ -155,3 +155,12 @@ def pytest_configure(config):
             enabled_remotes.discard(remote_name)
         if enabled:
             enabled_remotes.add(remote_name)
+
+
+@pytest.fixture()
+def custom_template(tmp_dir, dvc):
+    import shutil
+
+    template = tmp_dir / "custom_template.json"
+    shutil.copy(tmp_dir / ".dvc" / "plots" / "default.json", template)
+    return template

tests/dir_helpers.py

@@ -67,6 +67,8 @@ __all__ = [
     "erepo_dir",
     "git_dir",
     "git_init",
+    "git_upstream",
+    "git_downstream",
 ]
 
 
@@ -407,3 +409,23 @@ def git_dir(make_tmp_dir):
     path = make_tmp_dir("git-erepo", scm=True)
     path.scm.commit("init repo")
     return path
+
+
+@pytest.fixture
+def git_upstream(tmp_dir, erepo_dir, git_dir, request):
+    remote = erepo_dir if "dvc" in request.fixturenames else git_dir
+    url = "file://{}".format(remote.resolve().as_posix())
+    tmp_dir.scm.gitpython.repo.create_remote("upstream", url)
+    remote.remote = "upstream"
+    remote.url = url
+    return remote
+
+
+@pytest.fixture
+def git_downstream(tmp_dir, erepo_dir, git_dir, request):
+    remote = erepo_dir if "dvc" in request.fixturenames else git_dir
+    url = "file://{}".format(tmp_dir.resolve().as_posix())
+    remote.scm.gitpython.repo.create_remote("upstream", url)
+    remote.remote = "upstream"
+    remote.url = url
+    return remote

tests/func/experiments/conftest.py

@@ -88,21 +88,3 @@ def checkpoint_stage(tmp_dir, scm, dvc, mocker):
     scm.commit("init")
     stage.iterations = DEFAULT_ITERATIONS
     return stage
-
-
-@pytest.fixture
-def git_upstream(tmp_dir, erepo_dir):
-    url = "file://{}".format(erepo_dir.resolve().as_posix())
-    tmp_dir.scm.gitpython.repo.create_remote("upstream", url)
-    erepo_dir.remote = "upstream"
-    erepo_dir.url = url
-    return erepo_dir
-
-
-@pytest.fixture
-def git_downstream(tmp_dir, erepo_dir):
-    url = "file://{}".format(tmp_dir.resolve().as_posix())
-    erepo_dir.scm.gitpython.repo.create_remote("upstream", url)
-    erepo_dir.remote = "upstream"
-    erepo_dir.url = url
-    return erepo_dir

tests/func/experiments/test_remove.py

@@ -90,3 +90,29 @@ def test_remove_all(tmp_dir, scm, dvc, exp_stage):
     assert len(dvc.experiments.stash) == 2
     assert scm.get_ref(str(ref_info2)) is None
     assert scm.get_ref(str(ref_info1)) is None
+
+
+@pytest.mark.parametrize("use_url", [True, False])
+def test_remove_remote(tmp_dir, scm, dvc, exp_stage, git_upstream, use_url):
+    remote = git_upstream.url if use_url else git_upstream.remote
+
+    ref_info_list = []
+    exp_list = []
+    for i in range(3):
+        results = dvc.experiments.run(
+            exp_stage.addressing, params=[f"foo={i}"]
+        )
+        exp = first(results)
+        exp_list.append(exp)
+        ref_info = first(exp_refs_by_rev(scm, exp))
+        ref_info_list.append(ref_info)
+        dvc.experiments.push(remote, ref_info.name)
+        assert git_upstream.scm.get_ref(str(ref_info)) == exp
+
+    dvc.experiments.remove(
+        remote=remote, exp_names=[str(ref_info_list[0]), ref_info_list[1].name]
+    )
+
+    assert git_upstream.scm.get_ref(str(ref_info_list[0])) is None
+    assert git_upstream.scm.get_ref(str(ref_info_list[1])) is None
+    assert git_upstream.scm.get_ref(str(ref_info_list[2])) == exp_list[2]

tests/func/plots/conftest.py

@@ -1,10 +0,0 @@
-import shutil
-
-import pytest
-
-
-@pytest.fixture()
-def custom_template(tmp_dir, dvc):
-    template = tmp_dir / "custom_template.json"
-    shutil.copy(tmp_dir / ".dvc" / "plots" / "default.json", template)
-    return template

tests/func/plots/test_diff.py

@@ -1,12 +1,8 @@
-import json
-
-from dvc.repo.plots.data import PlotData
 from tests.func.plots.utils import _write_json
 
 
 def test_diff_dirty(tmp_dir, scm, dvc, run_copy_metrics):
-    metric_1 = [{"y": 2}, {"y": 3}]
-    _write_json(tmp_dir, metric_1, "metric_t.json")
+    _write_json(tmp_dir, [{"y": 2}, {"y": 3}], "metric_t.json")
     run_copy_metrics(
         "metric_t.json",
         "metric.json",
@@ -14,8 +10,8 @@ def test_diff_dirty(tmp_dir, scm, dvc, run_copy_metrics):
         commit="init",
     )
 
-    metric_2 = [{"y": 3}, {"y": 5}]
-    _write_json(tmp_dir, metric_2, "metric_t.json")
+    metric_head = [{"y": 3}, {"y": 5}]
+    _write_json(tmp_dir, metric_head, "metric_t.json")
     run_copy_metrics(
         "metric_t.json",
         "metric.json",
@@ -23,34 +19,30 @@ def test_diff_dirty(tmp_dir, scm, dvc, run_copy_metrics):
         commit="second",
     )
 
-    metric_3 = [{"y": 5}, {"y": 6}]
-    _write_json(tmp_dir, metric_3, "metric_t.json")
+    metric_1 = [{"y": 5}, {"y": 6}]
+    _write_json(tmp_dir, metric_1, "metric_t.json")
     run_copy_metrics(
         "metric_t.json", "metric.json", plots_no_cache=["metric.json"]
     )
+    props = {"fields": {"y"}}
+    diff_result = dvc.plots.diff(props=props)
+    assert diff_result == {
+        "workspace": {
+            "data": {"metric.json": {"data": metric_1, "props": props}}
+        },
+        "HEAD": {
+            "data": {"metric.json": {"data": metric_head, "props": props}}
+        },
+    }
+    metric_2 = [{"y": 7}, {"y": 8}]
+    _write_json(tmp_dir, metric_2, "metric.json")
 
-    plot_string = dvc.plots.diff(props={"fields": {"y"}})["metric.json"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {"y": 3, PlotData.INDEX_FIELD: 0, "rev": "HEAD"},
-        {"y": 5, PlotData.INDEX_FIELD: 1, "rev": "HEAD"},
-        {"y": 5, PlotData.INDEX_FIELD: 0, "rev": "workspace"},
-        {"y": 6, PlotData.INDEX_FIELD: 1, "rev": "workspace"},
-    ]
-    assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
-    assert plot_content["encoding"]["y"]["field"] == "y"
-
-    _write_json(tmp_dir, [{"y": 7}, {"y": 8}], "metric.json")
-
-    plot_string = dvc.plots.diff(props={"fields": {"y"}})["metric.json"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {"y": 3, PlotData.INDEX_FIELD: 0, "rev": "HEAD"},
-        {"y": 5, PlotData.INDEX_FIELD: 1, "rev": "HEAD"},
-        {"y": 7, PlotData.INDEX_FIELD: 0, "rev": "workspace"},
-        {"y": 8, PlotData.INDEX_FIELD: 1, "rev": "workspace"},
-    ]
-    assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
-    assert plot_content["encoding"]["y"]["field"] == "y"
+    diff_result = dvc.plots.diff(props=props)
+    assert diff_result == {
+        "workspace": {
+            "data": {"metric.json": {"data": metric_2, "props": props}}
+        },
+        "HEAD": {
+            "data": {"metric.json": {"data": metric_head, "props": props}}
+        },
+    }

tests/func/plots/test_modify.py

@@ -1,4 +1,3 @@
-import json
 import os
 
 import pytest
@@ -94,8 +93,5 @@ def test_dir_plots(tmp_dir, dvc, run_copy_metrics):
     dvc.plots.modify("subdir", {"title": "TITLE"})
 
     result = dvc.plots.show()
-    p1_content = json.loads(result[p1])
-    p2_content = json.loads(result[p2])
-
-    assert p1_content["title"] == p2_content["title"] == "TITLE"
-    assert p1_content == p2_content
+    assert result["workspace"]["data"][p1]["props"]["title"] == "TITLE"
+    assert result["workspace"]["data"][p2]["props"]["title"] == "TITLE"

tests/func/plots/test_show.py

@@ -1,8 +1,4 @@
-import json
-import logging
 import os
-import shutil
-from collections import OrderedDict
 
 import pytest
 from funcy import get_in
@@ -11,13 +7,8 @@ from dvc.dvcfile import PIPELINE_FILE
 from dvc.exceptions import OverlappingOutputPathsError
 from dvc.main import main
 from dvc.path_info import PathInfo
+from dvc.render.vega import PlotMetricTypeError
 from dvc.repo import Repo
-from dvc.repo.plots.data import PlotData, PlotMetricTypeError
-from dvc.repo.plots.template import (
-    BadTemplateError,
-    NoFieldInDataError,
-    TemplateNotFoundError,
-)
 from dvc.utils import onerror_collect
 from dvc.utils.fs import remove
 from dvc.utils.serialize import (
@@ -26,346 +17,13 @@ from dvc.utils.serialize import (
     dump_yaml,
     modify_yaml,
 )
-from tests.func.plots.utils import _write_csv, _write_json
-
-
-def test_plot_csv_one_column(tmp_dir, scm, dvc, run_copy_metrics):
-    # no header
-    metric = [{"val": 2}, {"val": 3}]
-    _write_csv(metric, "metric_t.csv", header=False)
-    run_copy_metrics(
-        "metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
-    )
-
-    props = {
-        "header": False,
-        "x_label": "x_title",
-        "y_label": "y_title",
-        "title": "mytitle",
-    }
-    plot_string = dvc.plots.show(props=props)["metric.csv"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["title"] == "mytitle"
-    assert plot_content["data"]["values"] == [
-        {"0": "2", PlotData.INDEX_FIELD: 0, "rev": "workspace"},
-        {"0": "3", PlotData.INDEX_FIELD: 1, "rev": "workspace"},
-    ]
-    assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
-    assert plot_content["encoding"]["y"]["field"] == "0"
-    assert plot_content["encoding"]["x"]["title"] == "x_title"
-    assert plot_content["encoding"]["y"]["title"] == "y_title"
-
-
-def test_plot_csv_multiple_columns(tmp_dir, scm, dvc, run_copy_metrics):
-    metric = [
-        OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
-        OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
-    ]
-    _write_csv(metric, "metric_t.csv")
-    run_copy_metrics(
-        "metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
-    )
-
-    plot_string = dvc.plots.show()["metric.csv"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {
-            "val": "2",
-            PlotData.INDEX_FIELD: 0,
-            "rev": "workspace",
-            "first_val": "100",
-            "second_val": "100",
-        },
-        {
-            "val": "3",
-            PlotData.INDEX_FIELD: 1,
-            "rev": "workspace",
-            "first_val": "200",
-            "second_val": "300",
-        },
-    ]
-    assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
-    assert plot_content["encoding"]["y"]["field"] == "val"
-
-
-def test_plot_csv_choose_axes(tmp_dir, scm, dvc, run_copy_metrics):
-    metric = [
-        OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
-        OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
-    ]
-    _write_csv(metric, "metric_t.csv")
-    run_copy_metrics(
-        "metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
-    )
-
-    props = {"x": "first_val", "y": "second_val"}
-    plot_string = dvc.plots.show(props=props)["metric.csv"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {
-            "val": "2",
-            "rev": "workspace",
-            "first_val": "100",
-            "second_val": "100",
-        },
-        {
-            "val": "3",
-            "rev": "workspace",
-            "first_val": "200",
-            "second_val": "300",
-        },
-    ]
-    assert plot_content["encoding"]["x"]["field"] == "first_val"
-    assert plot_content["encoding"]["y"]["field"] == "second_val"
-
-
-def test_plot_json_single_val(tmp_dir, scm, dvc, run_copy_metrics):
-    metric = [{"val": 2}, {"val": 3}]
-    _write_json(tmp_dir, metric, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="first run",
-    )
-
-    plot_string = dvc.plots.show()["metric.json"]
-
-    plot_json = json.loads(plot_string)
-    assert plot_json["data"]["values"] == [
-        {"val": 2, PlotData.INDEX_FIELD: 0, "rev": "workspace"},
-        {"val": 3, PlotData.INDEX_FIELD: 1, "rev": "workspace"},
-    ]
-    assert plot_json["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
-    assert plot_json["encoding"]["y"]["field"] == "val"
-
-
-def test_plot_json_multiple_val(tmp_dir, scm, dvc, run_copy_metrics):
-    metric = [{"first_val": 100, "val": 2}, {"first_val": 200, "val": 3}]
-    _write_json(tmp_dir, metric, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="first run",
-    )
-
-    plot_string = dvc.plots.show()["metric.json"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {
-            "val": 2,
-            PlotData.INDEX_FIELD: 0,
-            "first_val": 100,
-            "rev": "workspace",
-        },
-        {
-            "val": 3,
-            PlotData.INDEX_FIELD: 1,
-            "first_val": 200,
-            "rev": "workspace",
-        },
-    ]
-    assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
-    assert plot_content["encoding"]["y"]["field"] == "val"
-
-
-def test_plot_confusion(tmp_dir, dvc, run_copy_metrics):
-    confusion_matrix = [
-        {"predicted": "B", "actual": "A"},
-        {"predicted": "A", "actual": "A"},
-    ]
-    _write_json(tmp_dir, confusion_matrix, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="first run",
-    )
-
-    props = {"template": "confusion", "x": "predicted", "y": "actual"}
-    show = dvc.plots.show(props=props)
-    plot_string = show["metric.json"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {"predicted": "B", "actual": "A", "rev": "workspace"},
-        {"predicted": "A", "actual": "A", "rev": "workspace"},
-    ]
-    assert plot_content["spec"]["transform"][0]["groupby"] == [
-        "actual",
-        "predicted",
-    ]
-    assert plot_content["spec"]["encoding"]["x"]["field"] == "predicted"
-    assert plot_content["spec"]["encoding"]["y"]["field"] == "actual"
-
-
-def test_plot_confusion_normalized(tmp_dir, dvc, run_copy_metrics):
-    confusion_matrix = [
-        {"predicted": "B", "actual": "A"},
-        {"predicted": "A", "actual": "A"},
-    ]
-    _write_json(tmp_dir, confusion_matrix, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="first run",
-    )
-
-    props = {
-        "template": "confusion_normalized",
-        "x": "predicted",
-        "y": "actual",
-    }
-    plot_string = dvc.plots.show(props=props)["metric.json"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {"predicted": "B", "actual": "A", "rev": "workspace"},
-        {"predicted": "A", "actual": "A", "rev": "workspace"},
-    ]
-    assert plot_content["spec"]["transform"][0]["groupby"] == [
-        "actual",
-        "predicted",
-    ]
-    assert plot_content["spec"]["transform"][1]["groupby"] == ["rev", "actual"]
-    assert plot_content["spec"]["encoding"]["x"]["field"] == "predicted"
-    assert plot_content["spec"]["encoding"]["y"]["field"] == "actual"
-
-
-def test_plot_multiple_revs_default(tmp_dir, scm, dvc, run_copy_metrics):
-    metric_1 = [{"y": 2}, {"y": 3}]
-    _write_json(tmp_dir, metric_1, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="init",
-        tag="v1",
-    )
-
-    metric_2 = [{"y": 3}, {"y": 5}]
-    _write_json(tmp_dir, metric_2, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="second",
-        tag="v2",
-    )
-
-    metric_3 = [{"y": 5}, {"y": 6}]
-    _write_json(tmp_dir, metric_3, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="third",
-    )
-    plot_string = dvc.plots.show(
-        revs=["HEAD", "v2", "v1"], props={"fields": {"y"}}
-    )["metric.json"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {"y": 5, PlotData.INDEX_FIELD: 0, "rev": "HEAD"},
-        {"y": 6, PlotData.INDEX_FIELD: 1, "rev": "HEAD"},
-        {"y": 3, PlotData.INDEX_FIELD: 0, "rev": "v2"},
-        {"y": 5, PlotData.INDEX_FIELD: 1, "rev": "v2"},
-        {"y": 2, PlotData.INDEX_FIELD: 0, "rev": "v1"},
-        {"y": 3, PlotData.INDEX_FIELD: 1, "rev": "v1"},
-    ]
-    assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
-    assert plot_content["encoding"]["y"]["field"] == "y"
-
-
-def test_plot_multiple_revs(tmp_dir, scm, dvc, run_copy_metrics):
-    templates_dir = dvc.plots.templates.templates_dir
-    shutil.copy(
-        os.path.join(templates_dir, "default.json"),
-        os.path.join(templates_dir, "template.json"),
-    )
-
-    metric_1 = [{"y": 2}, {"y": 3}]
-    _write_json(tmp_dir, metric_1, "metric_t.json")
-    stage = run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="init",
-        tag="v1",
-    )
-
-    metric_2 = [{"y": 3}, {"y": 5}]
-    _write_json(tmp_dir, metric_2, "metric_t.json")
-    assert dvc.reproduce(stage.addressing) == [stage]
-    scm.add(["metric.json", stage.path])
-    scm.commit("second")
-    scm.tag("v2")
-
-    metric_3 = [{"y": 5}, {"y": 6}]
-    _write_json(tmp_dir, metric_3, "metric_t.json")
-    assert dvc.reproduce(stage.addressing) == [stage]
-    scm.add(["metric.json", stage.path])
-    scm.commit("third")
-
-    props = {"template": "template.json"}
-    plot_string = dvc.plots.show(revs=["HEAD", "v2", "v1"], props=props)[
-        "metric.json"
-    ]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {"y": 5, PlotData.INDEX_FIELD: 0, "rev": "HEAD"},
-        {"y": 6, PlotData.INDEX_FIELD: 1, "rev": "HEAD"},
-        {"y": 3, PlotData.INDEX_FIELD: 0, "rev": "v2"},
-        {"y": 5, PlotData.INDEX_FIELD: 1, "rev": "v2"},
-        {"y": 2, PlotData.INDEX_FIELD: 0, "rev": "v1"},
-        {"y": 3, PlotData.INDEX_FIELD: 1, "rev": "v1"},
-    ]
-    assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
-    assert plot_content["encoding"]["y"]["field"] == "y"
-
-
-def test_plot_even_if_metric_missing(
-    tmp_dir, scm, dvc, caplog, run_copy_metrics
-):
-    tmp_dir.scm_gen("some_file", "content", commit="there is no metric")
-    scm.tag("v1")
-
-    metric = [{"y": 2}, {"y": 3}]
-    _write_json(tmp_dir, metric, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="there is metric",
-        tag="v2",
-    )
-
-    caplog.clear()
-    with caplog.at_level(logging.WARNING, "dvc"):
-        plots = dvc.plots.show(revs=["v1", "v2"], targets=["metric.json"])
-        assert "'metric.json' was not found at: 'v1'." in caplog.text
-
-    plot_content = json.loads(plots["metric.json"])
-    assert plot_content["data"]["values"] == [
-        {"y": 2, PlotData.INDEX_FIELD: 0, "rev": "v2"},
-        {"y": 3, PlotData.INDEX_FIELD: 1, "rev": "v2"},
-    ]
-    assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
-    assert plot_content["encoding"]["y"]["field"] == "y"
+from tests.func.plots.utils import _write_json
 
 
 def test_plot_cache_missing(tmp_dir, scm, dvc, caplog, run_copy_metrics):
-    metric = [{"y": 2}, {"y": 3}]
-    _write_json(tmp_dir, metric, "metric_t.json")
-    stage = run_copy_metrics(
+    metric1 = [{"y": 2}, {"y": 3}]
+    _write_json(tmp_dir, metric1, "metric_t.json")
+    run_copy_metrics(
         "metric_t.json",
         "metric.json",
         plots=["metric.json"],
@@ -374,8 +32,8 @@ def test_plot_cache_missing(tmp_dir, scm, dvc, caplog, run_copy_metrics):
     scm.tag("v1")
 
     # Make a different plot and then remove its datafile
-    metric = [{"y": 3}, {"y": 4}]
-    _write_json(tmp_dir, metric, "metric_t.json")
+    metric2 = [{"y": 3}, {"y": 4}]
+    _write_json(tmp_dir, metric2, "metric_t.json")
     stage = run_copy_metrics(
         "metric_t.json",
         "metric.json",
@@ -386,72 +44,12 @@ def test_plot_cache_missing(tmp_dir, scm, dvc, caplog, run_copy_metrics):
     remove(stage.outs[0].fspath)
     remove(stage.outs[0].cache_path)
 
-    plots = dvc.plots.show(revs=["v1", "v2"], targets=["metric.json"])
-    plot_content = json.loads(plots["metric.json"])
-    assert plot_content["data"]["values"] == [
-        {"y": 2, PlotData.INDEX_FIELD: 0, "rev": "v1"},
-        {"y": 3, PlotData.INDEX_FIELD: 1, "rev": "v1"},
-    ]
-
-
-def test_custom_template(tmp_dir, scm, dvc, custom_template, run_copy_metrics):
-    metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
-    _write_json(tmp_dir, metric, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="init",
-        tag="v1",
+    plots_data = dvc.plots.show(revs=["v1", "v2"], targets=["metric.json"])
+    assert plots_data["v1"]["data"]["metric.json"]["data"] == metric1
+    assert isinstance(
+        plots_data["v2"]["data"]["metric.json"]["error"], FileNotFoundError
     )
 
-    props = {"template": os.fspath(custom_template), "x": "a", "y": "b"}
-    plot_string = dvc.plots.show(props=props)["metric.json"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {"a": 1, "b": 2, "rev": "workspace"},
-        {"a": 2, "b": 3, "rev": "workspace"},
-    ]
-    assert plot_content["encoding"]["x"]["field"] == "a"
-    assert plot_content["encoding"]["y"]["field"] == "b"
-
-
-def _replace(path, src, dst):
-    path.write_text(path.read_text().replace(src, dst))
-
-
-def test_should_raise_on_no_template(tmp_dir, dvc, run_copy_metrics):
-    metric = [{"val": 2}, {"val": 3}]
-    _write_json(tmp_dir, metric, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="first run",
-    )
-
-    with pytest.raises(TemplateNotFoundError):
-        props = {"template": "non_existing_template.json"}
-        dvc.plots.show("metric.json", props=props)
-
-
-def test_bad_template(tmp_dir, dvc, run_copy_metrics):
-    metric = [{"val": 2}, {"val": 3}]
-    _write_json(tmp_dir, metric, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="first run",
-    )
-
-    tmp_dir.gen("template.json", json.dumps({"a": "b", "c": "d"}))
-
-    with pytest.raises(BadTemplateError):
-        props = {"template": "template.json"}
-        dvc.plots.show("metric.json", props=props)
-
 
 def test_plot_wrong_metric_type(tmp_dir, scm, dvc, run_copy_metrics):
     tmp_dir.gen("metric_t.txt", "some text")
@@ -470,112 +68,6 @@ def test_plot_wrong_metric_type(tmp_dir, scm, dvc, run_copy_metrics):
     )
 
 
-def test_plot_choose_columns(
-    tmp_dir, scm, dvc, custom_template, run_copy_metrics
-):
-    metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
-    _write_json(tmp_dir, metric, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="init",
-        tag="v1",
-    )
-
-    props = {
-        "template": os.fspath(custom_template),
-        "fields": {"b", "c"},
-        "x": "b",
-        "y": "c",
-    }
-    plot_string = dvc.plots.show(props=props)["metric.json"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {"b": 2, "c": 3, "rev": "workspace"},
-        {"b": 3, "c": 4, "rev": "workspace"},
-    ]
-    assert plot_content["encoding"]["x"]["field"] == "b"
-    assert plot_content["encoding"]["y"]["field"] == "c"
-
-
-def test_plot_default_choose_column(tmp_dir, scm, dvc, run_copy_metrics):
-    metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
-    _write_json(tmp_dir, metric, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="init",
-        tag="v1",
-    )
-
-    plot_string = dvc.plots.show(props={"fields": {"b"}})["metric.json"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {PlotData.INDEX_FIELD: 0, "b": 2, "rev": "workspace"},
-        {PlotData.INDEX_FIELD: 1, "b": 3, "rev": "workspace"},
-    ]
-    assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
-    assert plot_content["encoding"]["y"]["field"] == "b"
-
-
-def test_plot_yaml(tmp_dir, scm, dvc, run_copy_metrics):
-    metric = [{"val": 2}, {"val": 3}]
-    dump_yaml("metric_t.yaml", metric)
-    run_copy_metrics(
-        "metric_t.yaml", "metric.yaml", plots_no_cache=["metric.yaml"]
-    )
-
-    plot_string = dvc.plots.show()["metric.yaml"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {"val": 2, PlotData.INDEX_FIELD: 0, "rev": "workspace"},
-        {"val": 3, PlotData.INDEX_FIELD: 1, "rev": "workspace"},
-    ]
-
-
-def test_raise_on_wrong_field(tmp_dir, scm, dvc, run_copy_metrics):
-    metric = [{"val": 2}, {"val": 3}]
-    _write_json(tmp_dir, metric, "metric_t.json")
-    run_copy_metrics(
-        "metric_t.json",
-        "metric.json",
-        plots_no_cache=["metric.json"],
-        commit="first run",
-    )
-
-    with pytest.raises(NoFieldInDataError):
-        dvc.plots.show("metric.json", props={"x": "no_val"})
-
-    with pytest.raises(NoFieldInDataError):
-        dvc.plots.show("metric.json", props={"y": "no_val"})
-
-
-def test_multiple_plots(tmp_dir, scm, dvc, run_copy_metrics):
-    metric1 = [
-        OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
-        OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
-    ]
-    metric2 = [
-        OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
-        OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
-    ]
-    _write_csv(metric1, "metric_t1.csv")
-    _write_json(tmp_dir, metric2, "metric_t2.json")
-    run_copy_metrics(
-        "metric_t1.csv", "metric1.csv", plots_no_cache=["metric1.csv"]
-    )
-    run_copy_metrics(
-        "metric_t2.json", "metric2.json", plots_no_cache=["metric2.json"]
-    )
-
-    assert len(dvc.plots.show().keys()) == 2
-
-
 @pytest.mark.parametrize("use_dvc", [True, False])
 def test_show_non_plot(tmp_dir, scm, use_dvc):
     metric = [{"first_val": 100, "val": 2}, {"first_val": 200, "val": 3}]
@@ -586,28 +78,9 @@ def test_show_non_plot(tmp_dir, scm, use_dvc):
     else:
         dvc = Repo(uninitialized=True)
 
-    plot_string = dvc.plots.show(targets=["metric.json"])["metric.json"]
+    plots = dvc.plots.show(targets=["metric.json"])
 
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {
-            "val": 2,
-            PlotData.INDEX_FIELD: 0,
-            "first_val": 100,
-            "rev": "workspace",
-        },
-        {
-            "val": 3,
-            PlotData.INDEX_FIELD: 1,
-            "first_val": 200,
-            "rev": "workspace",
-        },
-    ]
-    assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
-    assert plot_content["encoding"]["y"]["field"] == "val"
-
-    if not use_dvc:
-        assert not (tmp_dir / ".dvc").exists()
+    assert plots["workspace"]["data"]["metric.json"]["data"] == metric
 
 
 def test_show_non_plot_and_plot_with_params(
@@ -618,28 +91,13 @@ def test_show_non_plot_and_plot_with_params(
     run_copy_metrics(
         "metric.json", "metric2.json", plots_no_cache=["metric2.json"]
     )
+    props = {"title": "TITLE"}
+    dvc.plots.modify("metric2.json", props=props)
 
-    dvc.plots.modify("metric2.json", props={"title": "TITLE"})
     result = dvc.plots.show(targets=["metric.json", "metric2.json"])
-
-    plot_content = json.loads(result["metric.json"])
-    plot2_content = json.loads(result["metric2.json"])
-
-    assert plot2_content["title"] == "TITLE"
-
-    assert plot_content != plot2_content
-    plot_content.pop("title")
-    plot2_content.pop("title")
-    assert plot_content == plot2_content
-
-
-def test_show_no_repo(tmp_dir):
-    metric = [{"first_val": 100, "val": 2}, {"first_val": 200, "val": 3}]
-    _write_json(tmp_dir, metric, "metric.json")
-
-    dvc = Repo(uninitialized=True)
-
-    dvc.plots.show(["metric.json"])
+    assert "metric.json" in result["workspace"]["data"]
+    assert "metric2.json" in result["workspace"]["data"]
+    assert result["workspace"]["data"]["metric2.json"]["props"] == props
 
 
 def test_show_from_subdir(tmp_dir, dvc, capsys):
@@ -654,26 +112,17 @@ def test_show_from_subdir(tmp_dir, dvc, capsys):
 
     out, _ = capsys.readouterr()
     assert subdir.as_uri() in out
-    assert (subdir / "plots.html").exists()
+    assert (subdir / "dvc_plots").is_dir()
+    assert (subdir / "dvc_plots" / "index.html").is_file()
 
 
-def test_show_malformed_plots(tmp_dir, scm, dvc, caplog):
-    tmp_dir.gen("plot.json", '[{"m":1}]')
-    scm.add(["plot.json"])
-    scm.commit("initial")
+def test_plots_show_non_existing(tmp_dir, dvc, caplog):
+    result = dvc.plots.show(targets=["plot.json"])
+    assert isinstance(
+        result["workspace"]["data"]["plot.json"]["error"], FileNotFoundError
+    )
 
-    tmp_dir.gen("plot.json", '[{"m":1]')
-
-    result = dvc.plots.show(targets=["plot.json"], revs=["workspace", "HEAD"])
-    plot_content = json.loads(result["plot.json"])
-
-    assert plot_content["data"]["values"] == [
-        {"m": 1, "rev": "HEAD", "step": 0}
-    ]
-
-
-def test_plots_show_non_existing(tmp_dir, dvc):
-    assert dvc.plots.show(targets=["plot.json"]) == {}
+    assert "'plot.json' was not found in current workspace." in caplog.text
 
 
 @pytest.mark.parametrize("clear_before_run", [True, False])
@@ -732,49 +181,13 @@ def test_dir_plots(tmp_dir, dvc, run_copy_metrics):
         plots=["subdir"],
         name="copy_double",
     )
+    props = {"title": "TITLE"}
     dvc.plots.modify("subdir", {"title": "TITLE"})
 
     result = dvc.plots.show()
-    p1_content = json.loads(result[p1])
-    p2_content = json.loads(result[p2])
-
-    assert p1_content["title"] == p2_content["title"] == "TITLE"
-
-
-def test_show_dir_plots(tmp_dir, dvc, run_copy_metrics):
-    subdir = tmp_dir / "subdir"
-    subdir.mkdir()
-    metric = [{"first_val": 100, "val": 2}, {"first_val": 200, "val": 3}]
-
-    fname = "file.json"
-    _write_json(tmp_dir, metric, fname)
-
-    p1 = os.path.join("subdir", "p1.json")
-    p2 = os.path.join("subdir", "p2.json")
-    tmp_dir.dvc.run(
-        cmd=(
-            f"mkdir subdir && python copy.py {fname} {p1} && "
-            f"python copy.py {fname} {p2}"
-        ),
-        deps=[fname],
-        single_stage=False,
-        plots=["subdir"],
-        name="copy_double",
-    )
-
-    result = dvc.plots.show(targets=["subdir"])
-    p1_content = json.loads(result[p1])
-    p2_content = json.loads(result[p2])
-
-    assert p1_content == p2_content
-
-    result = dvc.plots.show(targets=[p1])
-    assert set(result.keys()) == {p1}
-
-    remove(dvc.odb.local.cache_dir)
-    remove(subdir)
-
-    assert dvc.plots.show() == {}
+    assert set(result["workspace"]["data"]) == {p1, p2}
+    assert result["workspace"]["data"][p1]["props"] == props
+    assert result["workspace"]["data"][p2]["props"] == props
 
 
 def test_ignore_binary_file(tmp_dir, dvc, run_copy_metrics):
@@ -821,3 +234,30 @@ def test_log_errors(
         "DVC failed to load some plots for following revisions: 'workspace'."
         in error
     )
+
+
+def test_plots_binary(tmp_dir, scm, dvc, run_copy_metrics, custom_template):
+    with open("image.jpg", "wb") as fd:
+        fd.write(b"content")
+
+    dvc.add(["image.jpg"])
+    run_copy_metrics(
+        "image.jpg",
+        "plot.jpg",
+        commit="run training",
+        plots=["plot.jpg"],
+        name="s2",
+        single_stage=False,
+    )
+
+    scm.add(["dvc.yaml", "dvc.lock"])
+    scm.commit("initial")
+
+    scm.tag("v1")
+
+    with open("plot.jpg", "wb") as fd:
+        fd.write(b"content2")
+
+    result = dvc.plots.show(revs=["v1", "workspace"])
+    assert result["v1"]["data"]["plot.jpg"]["data"] == b"content"
+    assert result["workspace"]["data"]["plot.jpg"]["data"] == b"content2"

tests/func/test_data_cloud.py

@@ -609,3 +609,47 @@ def test_pull_no_00_prefix(tmp_dir, dvc, remote, monkeypatch):
     stats = dvc.pull()
     assert stats["fetched"] == 2
     assert set(stats["added"]) == {"foo", "bar"}
+
+
+def test_output_remote(tmp_dir, dvc, make_remote):
+    from dvc.utils.serialize import modify_yaml
+
+    make_remote("default", default=True)
+    make_remote("for_foo", default=False)
+    make_remote("for_data", default=False)
+
+    tmp_dir.dvc_gen("foo", "foo")
+    tmp_dir.dvc_gen("bar", "bar")
+    tmp_dir.dvc_gen("data", {"one": "one", "two": "two"})
+
+    with modify_yaml("foo.dvc") as d:
+        d["outs"][0]["remote"] = "for_foo"
+
+    with modify_yaml("data.dvc") as d:
+        d["outs"][0]["remote"] = "for_data"
+
+    dvc.push()
+
+    default = dvc.cloud.get_remote_odb("default")
+    for_foo = dvc.cloud.get_remote_odb("for_foo")
+    for_data = dvc.cloud.get_remote_odb("for_data")
+
+    assert set(default.all()) == {"37b51d194a7513e45b56f6524f2d51f2"}
+    assert set(for_foo.all()) == {"acbd18db4cc2f85cedef654fccc4a4d8"}
+    assert set(for_data.all()) == {
+        "f97c5d29941bfb1b2fdab0874906ab82",
+        "6b18131dc289fd37006705affe961ef8.dir",
+        "b8a9f715dbb64fd5c56e7783c6820a61",
+    }
+
+    clean(["foo", "bar", "data"], dvc)
+
+    dvc.pull()
+
+    assert set(dvc.odb.local.all()) == {
+        "37b51d194a7513e45b56f6524f2d51f2",
+        "acbd18db4cc2f85cedef654fccc4a4d8",
+        "f97c5d29941bfb1b2fdab0874906ab82",
+        "6b18131dc289fd37006705affe961ef8.dir",
+        "b8a9f715dbb64fd5c56e7783c6820a61",
+    }

tests/func/test_live.py

@@ -6,6 +6,7 @@ import pytest
 from funcy import first
 
 from dvc import stage as stage_module
+from dvc.render.utils import get_files
 
 LIVE_SCRIPT = dedent(
     """
@@ -122,9 +123,10 @@ def test_live_provides_metrics(tmp_dir, dvc, live_stage):
     }
 
     assert (tmp_dir / "logs").is_dir()
-    plots = dvc.plots.show()
-    assert os.path.join("logs", "accuracy.tsv") in plots
-    assert os.path.join("logs", "loss.tsv") in plots
+    plots_data = dvc.plots.show()
+    files = get_files(plots_data)
+    assert os.path.join("logs", "accuracy.tsv") in files
+    assert os.path.join("logs", "loss.tsv") in files
 
 
 def test_live_provides_no_metrics(tmp_dir, dvc, live_stage):
@@ -134,9 +136,10 @@ def test_live_provides_no_metrics(tmp_dir, dvc, live_stage):
     assert dvc.metrics.show() == {"": {}}
 
     assert (tmp_dir / "logs").is_dir()
-    plots = dvc.plots.show()
-    assert os.path.join("logs", "accuracy.tsv") in plots
-    assert os.path.join("logs", "loss.tsv") in plots
+    plots_data = dvc.plots.show()
+    files = get_files(plots_data)
+    assert os.path.join("logs", "accuracy.tsv") in files
+    assert os.path.join("logs", "loss.tsv") in files
 
 
 @pytest.mark.parametrize("typ", ("live", "live_no_cache"))
@@ -156,7 +159,10 @@ def test_experiments_track_summary(tmp_dir, scm, dvc, live_stage, typ):
 def test_live_html(tmp_dir, dvc, live_stage, html):
     live_stage(html=html, live="logs")
 
-    assert (tmp_dir / "logs.html").is_file() == html
+    assert (tmp_dir / "logs_dvc_plots" / "index.html").is_file() == html
+    if html:
+        html_text = (tmp_dir / "logs_dvc_plots" / "index.html").read_text()
+        assert 'http-equiv="refresh"' in html_text
 
 
 @pytest.fixture
@@ -243,7 +249,7 @@ def test_dvc_generates_html_during_run(tmp_dir, dvc, mocker, live_stage):
         dvclive.log("accuracy", 1/2)
         dvclive.next_step()
         time.sleep({})""".format(
-            str(monitor_await_time * 2)
+            str(monitor_await_time * 10)
         )
     )
     live_stage(summary=True, live="logs", code=script)

tests/func/test_remote.py

@@ -200,7 +200,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
     dvc.push()
     remove(dvc.odb.local.cache_dir)
 
-    baz.collect_used_dir_cache()
+    baz._collect_used_dir_cache()
     with patch.object(LocalFileSystem, "upload", side_effect=Exception):
         with pytest.raises(DownloadError) as download_error_info:
             dvc.pull()

tests/remotes/__init__.py

@@ -16,7 +16,7 @@ from .gs import (  # noqa: F401; noqa: F401
 )
 from .hdfs import HDFS, hadoop, hdfs, hdfs_server, real_hdfs  # noqa: F401
 from .http import HTTP, http, http_server  # noqa: F401
-from .local import Local, local_cloud, local_remote  # noqa: F401
+from .local import Local, local_cloud, local_remote, make_local  # noqa: F401
 from .oss import (  # noqa: F401
     OSS,
     TEST_OSS_REPO_BUCKET,
@@ -100,6 +100,24 @@ def docker_services(
     return Services(executor)
 
 
+@pytest.fixture
+def make_cloud(request):
+    def _make_cloud(typ):
+        return request.getfixturevalue(f"make_{typ}")()
+
+    return _make_cloud
+
+
+@pytest.fixture
+def make_remote(tmp_dir, dvc, make_cloud):
+    def _make_remote(name, typ="local", **kwargs):
+        cloud = make_cloud(typ)
+        tmp_dir.add_remote(name=name, config=cloud.config, **kwargs)
+        return cloud
+
+    return _make_remote
+
+
 @pytest.fixture
 def remote(tmp_dir, dvc, request):
     cloud = request.param

tests/remotes/local.py

@@ -17,11 +17,19 @@ class Local(Base):
 
 
 @pytest.fixture
-def local_cloud(make_tmp_dir):
-    ret = make_tmp_dir("local-cloud")
-    ret.url = str(ret)
-    ret.config = {"url": ret.url}
-    return ret
+def make_local(make_tmp_dir):
+    def _make_local():
+        ret = make_tmp_dir("local-cloud")
+        ret.url = str(ret)
+        ret.config = {"url": ret.url}
+        return ret
+
+    return _make_local
+
+
+@pytest.fixture
+def local_cloud(make_local):
+    return make_local()
 
 
 @pytest.fixture

tests/unit/command/test_experiments.py

@@ -253,15 +253,17 @@ def test_experiments_pull(dvc, scm, mocker):
 
 
 @pytest.mark.parametrize(
-    "queue,clear_all",
-    [(True, False), (False, True)],
+    "queue,clear_all,remote",
+    [(True, False, None), (False, True, None), (False, False, True)],
 )
-def test_experiments_remove(dvc, scm, mocker, queue, clear_all):
+def test_experiments_remove(dvc, scm, mocker, queue, clear_all, remote):
     if queue:
-        args = "--queue"
+        args = ["--queue"]
     if clear_all:
-        args = "--all"
-    cli_args = parse_args(["experiments", "remove", args])
+        args = ["--all"]
+    if remote:
+        args = ["--git-remote", "myremote", "exp-123", "exp-234"]
+    cli_args = parse_args(["experiments", "remove"] + args)
     assert cli_args.func == CmdExperimentsRemove
 
     cmd = cli_args.func(cli_args)
@@ -270,7 +272,8 @@ def test_experiments_remove(dvc, scm, mocker, queue, clear_all):
     assert cmd.run() == 0
     m.assert_called_once_with(
         cmd.repo,
-        exp_names=[],
+        exp_names=["exp-123", "exp-234"] if remote else [],
         queue=queue,
         clear_all=clear_all,
+        remote="myremote" if remote else None,
     )

tests/unit/command/test_machine.py

@@ -0,0 +1,80 @@
+import configobj
+
+from dvc.cli import parse_args
+from dvc.command.machine import (
+    CmdMachineAdd,
+    CmdMachineCreate,
+    CmdMachineDestroy,
+    CmdMachineRemove,
+    CmdMachineSsh,
+)
+
+
+def test_add(tmp_dir):
+    tmp_dir.gen({".dvc": {"config": "[feature]\n  machine = true"}})
+    cli_args = parse_args(["machine", "add", "foo", "aws"])
+    assert cli_args.func == CmdMachineAdd
+    cmd = cli_args.func(cli_args)
+    assert cmd.run() == 0
+    config = configobj.ConfigObj(str(tmp_dir / ".dvc" / "config"))
+    assert config['machine "foo"']["cloud"] == "aws"
+
+
+def test_remove(tmp_dir):
+    tmp_dir.gen(
+        {
+            ".dvc": {
+                "config": (
+                    "[feature]\n"
+                    "  machine = true\n"
+                    "['machine \"foo\"']\n"
+                    "  cloud = aws"
+                )
+            }
+        }
+    )
+    cli_args = parse_args(["machine", "remove", "foo"])
+    assert cli_args.func == CmdMachineRemove
+    cmd = cli_args.func(cli_args)
+    assert cmd.run() == 0
+    config = configobj.ConfigObj(str(tmp_dir / ".dvc" / "config"))
+    assert list(config.keys()) == ["feature"]
+
+
+def test_create(tmp_dir, dvc, mocker):
+    cli_args = parse_args(["machine", "create", "foo"])
+    assert cli_args.func == CmdMachineCreate
+
+    cmd = cli_args.func(cli_args)
+    m = mocker.patch.object(
+        cmd.repo.machine, "create", autospec=True, return_value=0
+    )
+
+    assert cmd.run() == 0
+    m.assert_called_once_with("foo")
+
+
+def test_destroy(tmp_dir, dvc, mocker):
+    cli_args = parse_args(["machine", "destroy", "foo"])
+    assert cli_args.func == CmdMachineDestroy
+
+    cmd = cli_args.func(cli_args)
+    m = mocker.patch.object(
+        cmd.repo.machine, "destroy", autospec=True, return_value=0
+    )
+
+    assert cmd.run() == 0
+    m.assert_called_once_with("foo")
+
+
+def test_ssh(tmp_dir, dvc, mocker):
+    cli_args = parse_args(["machine", "ssh", "foo"])
+    assert cli_args.func == CmdMachineSsh
+
+    cmd = cli_args.func(cli_args)
+    m = mocker.patch.object(
+        cmd.repo.machine, "run_shell", autospec=True, return_value=0
+    )
+
+    assert cmd.run() == 0
+    m.assert_called_once_with("foo")

tests/unit/command/test_plots.py

@@ -7,7 +7,19 @@ from dvc.cli import parse_args
 from dvc.command.plots import CmdPlotsDiff, CmdPlotsShow
 
 
-def test_plots_diff(dvc, mocker):
+@pytest.fixture
+def plots_data():
+    yield {
+        "revision": {
+            "data": {
+                "plot.csv": {"data": [{"val": 1}, {"val": 2}], "props": {}},
+                "other.jpg": {"data": b"content"},
+            }
+        }
+    }
+
+
+def test_plots_diff(dvc, mocker, plots_data):
     cli_args = parse_args(
         [
             "plots",
@@ -38,8 +50,9 @@ def test_plots_diff(dvc, mocker):
     assert cli_args.func == CmdPlotsDiff
 
     cmd = cli_args.func(cli_args)
-    m = mocker.patch(
-        "dvc.repo.plots.diff.diff", return_value={"datafile": "filledtemplate"}
+    m = mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data)
+    render_mock = mocker.patch(
+        "dvc.command.plots.render", return_value="html_path"
     )
 
     assert cmd.run() == 0
@@ -58,9 +71,10 @@ def test_plots_diff(dvc, mocker):
         },
         experiment=True,
     )
+    render_mock.assert_not_called()
 
 
-def test_plots_show_vega(dvc, mocker):
+def test_plots_show_vega(dvc, mocker, plots_data):
     cli_args = parse_args(
         [
             "plots",
@@ -80,7 +94,10 @@ def test_plots_show_vega(dvc, mocker):
 
     m = mocker.patch(
         "dvc.repo.plots.Plots.show",
-        return_value={"datafile": "filledtemplate"},
+        return_value=plots_data,
+    )
+    render_mock = mocker.patch(
+        "dvc.command.plots.render", return_value="html_path"
     )
 
     assert cmd.run() == 0
@@ -89,9 +106,10 @@ def test_plots_show_vega(dvc, mocker):
         targets=["datafile"],
         props={"template": "template", "header": False},
     )
+    render_mock.assert_not_called()
 
 
-def test_plots_diff_vega(dvc, mocker, capsys):
+def test_plots_diff_vega(dvc, mocker, capsys, plots_data):
     cli_args = parse_args(
         [
             "plots",
@@ -100,72 +118,87 @@ def test_plots_diff_vega(dvc, mocker, capsys):
             "HEAD~1",
             "--show-vega",
             "--targets",
-            "plots.csv",
+            "plot.csv",
         ]
     )
     cmd = cli_args.func(cli_args)
+    mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data)
     mocker.patch(
-        "dvc.repo.plots.diff.diff", return_value={"plots.csv": "plothtml"}
+        "dvc.command.plots.find_vega", return_value="vega_json_content"
+    )
+    render_mock = mocker.patch(
+        "dvc.command.plots.render", return_value="html_path"
     )
     assert cmd.run() == 0
 
     out, _ = capsys.readouterr()
-    assert "plothtml" in out
+
+    assert "vega_json_content" in out
+    render_mock.assert_not_called()
 
 
-def test_plots_diff_open(tmp_dir, dvc, mocker, capsys):
+def test_plots_diff_open(tmp_dir, dvc, mocker, capsys, plots_data):
     mocked_open = mocker.patch("webbrowser.open", return_value=True)
-    cli_args = parse_args(["plots", "diff", "--targets", "datafile", "--open"])
-    cmd = cli_args.func(cli_args)
-    mocker.patch(
-        "dvc.repo.plots.diff.diff", return_value={"datafile": "filledtemplate"}
+    cli_args = parse_args(
+        ["plots", "diff", "--targets", "plots.csv", "--open"]
     )
+    cmd = cli_args.func(cli_args)
+    mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data)
+
+    index_path = tmp_dir / "dvc_plots" / "index.html"
+    mocker.patch("dvc.command.plots.render", return_value=index_path)
 
     assert cmd.run() == 0
-    mocked_open.assert_called_once_with("plots.html")
-
-    expected_url = posixpath.join(tmp_dir.as_uri(), "plots.html")
+    mocked_open.assert_called_once_with(index_path)
 
     out, _ = capsys.readouterr()
-    assert expected_url in out
+    assert index_path.as_uri() in out
 
 
-def test_plots_diff_open_failed(tmp_dir, dvc, mocker, capsys):
+def test_plots_diff_open_failed(tmp_dir, dvc, mocker, capsys, plots_data):
     mocked_open = mocker.patch("webbrowser.open", return_value=False)
-    cli_args = parse_args(["plots", "diff", "--targets", "datafile", "--open"])
+    cli_args = parse_args(
+        ["plots", "diff", "--targets", "plots.csv", "--open"]
+    )
     cmd = cli_args.func(cli_args)
     mocker.patch(
-        "dvc.repo.plots.diff.diff", return_value={"datafile": "filledtemplate"}
+        "dvc.repo.plots.diff.diff", return_value={"datafile": plots_data}
     )
 
     assert cmd.run() == 1
-    mocked_open.assert_called_once_with("plots.html")
+    expected_url = tmp_dir / "dvc_plots" / "index.html"
+    mocked_open.assert_called_once_with(expected_url)
 
     error_message = "Failed to open. Please try opening it manually."
-    expected_url = posixpath.join(tmp_dir.as_uri(), "plots.html")
 
     out, err = capsys.readouterr()
-    assert expected_url in out
+    assert expected_url.as_uri() in out
     assert error_message in err
 
 
 @pytest.mark.parametrize(
     "output, expected_url_path",
     [
-        ("plots file with spaces.html", "plots%20file%20with%20spaces.html"),
-        (os.path.join("dir", "..", "plots.html"), "plots.html"),
+        (
+            "plots file with spaces",
+            posixpath.join("plots%20file%20with%20spaces", "index.html"),
+        ),
+        (
+            os.path.join("dir", "..", "plots"),
+            posixpath.join("plots", "index.html"),
+        ),
     ],
     ids=["quote", "resolve"],
 )
 def test_plots_path_is_quoted_and_resolved_properly(
-    tmp_dir, dvc, mocker, capsys, output, expected_url_path
+    tmp_dir, dvc, mocker, capsys, output, expected_url_path, plots_data
 ):
     cli_args = parse_args(
         ["plots", "diff", "--targets", "datafile", "--out", output]
     )
     cmd = cli_args.func(cli_args)
     mocker.patch(
-        "dvc.repo.plots.diff.diff", return_value={"datafile": "filledtemplate"}
+        "dvc.repo.plots.diff.diff", return_value={"datafile": plots_data}
     )
 
     assert cmd.run() == 0
@@ -173,3 +206,29 @@ def test_plots_path_is_quoted_and_resolved_properly(
 
     out, _ = capsys.readouterr()
     assert expected_url in out
+
+
+@pytest.mark.parametrize(
+    "output", ("some_out", os.path.join("to", "subdir"), None)
+)
+def test_should_call_render(tmp_dir, mocker, capsys, plots_data, output):
+    cli_args = parse_args(
+        ["plots", "diff", "--targets", "plots.csv", "--out", output]
+    )
+    cmd = cli_args.func(cli_args)
+    mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data)
+
+    output = output or "dvc_plots"
+    index_path = tmp_dir / output / "index.html"
+    render_mock = mocker.patch(
+        "dvc.command.plots.render", return_value=index_path
+    )
+
+    assert cmd.run() == 0
+
+    out, _ = capsys.readouterr()
+    assert index_path.as_uri() in out
+
+    render_mock.assert_called_once_with(
+        cmd.repo, plots_data, path=tmp_dir / output, html_template_path=None
+    )

tests/unit/fs/test_ssh.py

@@ -135,6 +135,24 @@ def test_ssh_keyfile(mock_file, config, expected_keyfile):
     assert fs.fs_args.get("client_keys") == expected_keyfile
 
 
+mock_ssh_multi_key_config = """
+IdentityFile file_1
+
+Host example.com
+    IdentityFile file_2
+"""
+
+
+@patch(
+    "builtins.open",
+    new_callable=mock_open,
+    read_data=mock_ssh_multi_key_config,
+)
+def test_ssh_multi_identity_files(mock_file):
+    fs = SSHFileSystem(host="example.com")
+    assert fs.fs_args.get("client_keys") == ["file_1", "file_2"]
+
+
 @pytest.mark.parametrize(
     "config,expected_gss_auth",
     [

tests/unit/machine/test_machine.py

@@ -0,0 +1,36 @@
+import pytest
+
+
+def test_validate_name():
+    from dvc.exceptions import InvalidArgumentError
+    from dvc.machine import RESERVED_NAMES, validate_name
+
+    for name in RESERVED_NAMES:
+        with pytest.raises(InvalidArgumentError):
+            validate_name(name)
+
+
+@pytest.mark.parametrize("cloud", ["aws", "azure"])
+def test_get_config_and_backend(tmp_dir, dvc, cloud):
+    from dvc.machine.backend.terraform import TerraformBackend
+
+    name = "foo"
+    with dvc.config.edit() as conf:
+        conf["machine"][name] = {"cloud": cloud}
+    config, backend = dvc.machine.get_config_and_backend(name)
+    assert config == {"cloud": cloud, "name": name}
+    assert isinstance(backend, TerraformBackend)
+
+
+def test_get_config_and_backend_nonexistent(tmp_dir, dvc):
+    from dvc.config import MachineNotFoundError
+
+    with pytest.raises(MachineNotFoundError):
+        dvc.machine.get_config_and_backend("foo")
+
+
+def test_get_config_and_backend_default(tmp_dir, dvc):
+    from dvc.config import NoMachineError
+
+    with pytest.raises(NoMachineError):
+        dvc.machine.get_config_and_backend()

tests/unit/output/test_load.py

@@ -79,6 +79,16 @@ def test_load_remote_files_from_pipeline(dvc):
     assert not out.hash_info
 
 
+def test_load_remote(dvc):
+    stage = Stage(dvc)
+    (foo, bar) = output.load_from_pipeline(
+        stage,
+        ["foo", {"bar": {"remote": "myremote"}}],
+    )
+    assert foo.remote is None
+    assert bar.remote == "myremote"
+
+
 @pytest.mark.parametrize("typ", [None, "", "illegal"])
 def test_load_from_pipeline_error_on_typ(dvc, typ):
     with pytest.raises(ValueError):

tests/unit/render/test_html.py

@@ -1,6 +1,6 @@
 import pytest
 
-from dvc.utils.html import HTML, PAGE_HTML, MissingPlaceholderError
+from dvc.render.html import HTML, PAGE_HTML, MissingPlaceholderError
 
 CUSTOM_PAGE_HTML = """<!DOCTYPE html>
 <html>
@@ -19,7 +19,11 @@ CUSTOM_PAGE_HTML = """<!DOCTYPE html>
 @pytest.mark.parametrize(
     "template,page_elements,expected_page",
     [
-        (None, ["content"], PAGE_HTML.format(plot_divs="content")),
+        (
+            None,
+            ["content"],
+            PAGE_HTML.format(plot_divs="content", refresh_tag=""),
+        ),
         (
             CUSTOM_PAGE_HTML,
             ["content"],

tests/unit/render/test_image.py

@@ -0,0 +1,46 @@
+import os
+
+import pytest
+
+from dvc.render.image import ImageRenderer
+
+
+@pytest.mark.parametrize(
+    "extension, matches",
+    (
+        (".csv", False),
+        (".json", False),
+        (".tsv", False),
+        (".yaml", False),
+        (".jpg", True),
+        (".gif", True),
+        (".jpeg", True),
+        (".png", True),
+    ),
+)
+def test_matches(extension, matches):
+    filename = "file" + extension
+    data = {
+        "HEAD": {"data": {filename: {}}},
+        "v1": {"data": {filename: {}}},
+    }
+    assert ImageRenderer.matches(data) == matches
+
+
+def test_render(tmp_dir):
+    data = {"workspace": {"data": {"file.jpg": {"data": b"content"}}}}
+
+    page_dir = os.path.join("some", "path")
+    html = ImageRenderer(data).generate_html(page_dir)
+
+    assert (tmp_dir / page_dir).is_dir()
+    image_file = tmp_dir / page_dir / "static" / "workspace_file.jpg"
+    assert image_file.is_file()
+
+    with open(image_file, "rb") as fobj:
+        assert fobj.read() == b"content"
+
+    assert "<p>file.jpg</p>" in html
+    assert (
+        f'<img src="{os.path.join("static", "workspace_file.jpg")}">' in html
+    )

tests/unit/render/test_render.py

@@ -0,0 +1,67 @@
+import os
+
+from dvc.render.utils import find_vega, render
+
+
+def assert_website_has_image(page_path, revision, filename, image_content):
+    index_path = page_path / "index.html"
+    assert index_path.is_file()
+    index_content = index_path.read_text()
+
+    resources_filename = f"{revision}_{filename.replace(os.sep, '_')}"
+    image_path = page_path / "static" / resources_filename
+    assert image_path.is_file()
+
+    img_html = f'<img src="{os.path.join("static", resources_filename)}">'
+    assert img_html in index_content
+    with open(image_path, "rb") as fobj:
+        assert fobj.read() == image_content
+
+
+def test_render(tmp_dir, dvc):
+    data = {
+        "HEAD": {
+            "data": {
+                "file.json": {
+                    "data": [{"y": 5}, {"y": 6}],
+                    "props": {"fields": {"y"}},
+                },
+                os.path.join("sub", "other_file.jpg"): {"data": b"content"},
+            }
+        },
+        "v2": {
+            "data": {
+                "file.json": {
+                    "data": [{"y": 3}, {"y": 5}],
+                    "props": {"fields": {"y"}},
+                },
+                "other_file.jpg": {"data": b"content2"},
+            }
+        },
+        "v1": {
+            "data": {
+                "some.csv": {
+                    "data": [{"y": 2}, {"y": 3}],
+                    "props": {"fields": {"y"}},
+                },
+                "another.gif": {"data": b"content3"},
+            }
+        },
+    }
+
+    render(dvc, data, path=os.path.join("results", "dir"))
+    page_path = tmp_dir / "results" / "dir"
+    index_path = page_path / "index.html"
+
+    assert index_path.is_file()
+    assert_website_has_image(
+        page_path, "HEAD", os.path.join("sub", "other_file.jpg"), b"content"
+    )
+    assert_website_has_image(page_path, "v2", "other_file.jpg", b"content2")
+    assert_website_has_image(page_path, "v1", "another.gif", b"content3")
+
+    index_content = index_path.read_text()
+    file_vega = find_vega(dvc, data, "file.json")
+    some_vega = find_vega(dvc, data, "some.csv")
+    assert file_vega in index_content.strip()
+    assert some_vega in index_content.strip()

tests/unit/render/test_vega.py

@@ -0,0 +1,436 @@
+import json
+import os
+from collections import OrderedDict
+
+import pytest
+
+from dvc.render.utils import find_vega, group_by_filename
+from dvc.render.vega import (
+    INDEX_FIELD,
+    REVISION_FIELD,
+    VegaRenderer,
+    _find_data,
+    _lists,
+)
+from dvc.repo.plots.template import (
+    BadTemplateError,
+    NoFieldInDataError,
+    TemplateNotFoundError,
+)
+
+
+@pytest.mark.parametrize(
+    "dictionary, expected_result",
+    [
+        ({}, []),
+        ({"x": ["a", "b", "c"]}, [["a", "b", "c"]]),
+        (
+            OrderedDict([("x", {"y": ["a", "b"]}), ("z", {"w": ["c", "d"]})]),
+            [["a", "b"], ["c", "d"]],
+        ),
+    ],
+)
+def test_finding_lists(dictionary, expected_result):
+    result = _lists(dictionary)
+
+    assert list(result) == expected_result
+
+
+def test_find_data_in_dict(tmp_dir):
+    m1 = [{"accuracy": 1, "loss": 2}, {"accuracy": 3, "loss": 4}]
+    m2 = [{"x": 1}, {"x": 2}]
+    dmetric = OrderedDict([("t1", m1), ("t2", m2)])
+
+    assert _find_data(dmetric) == m1
+    assert _find_data(dmetric, fields={"x"}) == m2
+
+
+def test_group_plots_data():
+    error = FileNotFoundError()
+    data = {
+        "v2": {
+            "data": {
+                "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}},
+                "other_file.jpg": {"data": "content"},
+            }
+        },
+        "v1": {
+            "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}}
+        },
+        "workspace": {
+            "data": {
+                "file.json": {"error": error, "props": {}},
+                "other_file.jpg": {"data": "content2"},
+            }
+        },
+    }
+
+    results = group_by_filename(data)
+    assert {
+        "v2": {
+            "data": {
+                "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}},
+            }
+        },
+        "v1": {
+            "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}}
+        },
+        "workspace": {
+            "data": {
+                "file.json": {"error": error, "props": {}},
+            }
+        },
+    } in results
+    assert {
+        "v2": {
+            "data": {
+                "other_file.jpg": {"data": "content"},
+            }
+        },
+        "workspace": {
+            "data": {
+                "other_file.jpg": {"data": "content2"},
+            }
+        },
+    } in results
+
+
+def test_one_column(tmp_dir, scm, dvc):
+    props = {
+        "x_label": "x_title",
+        "y_label": "y_title",
+        "title": "mytitle",
+    }
+    data = {
+        "workspace": {
+            "data": {
+                "file.json": {"data": [{"val": 2}, {"val": 3}], "props": props}
+            }
+        }
+    }
+
+    plot_string = VegaRenderer(data, dvc.plots.templates).get_vega()
+
+    plot_content = json.loads(plot_string)
+    assert plot_content["title"] == "mytitle"
+    assert plot_content["data"]["values"] == [
+        {"val": 2, INDEX_FIELD: 0, REVISION_FIELD: "workspace"},
+        {"val": 3, INDEX_FIELD: 1, REVISION_FIELD: "workspace"},
+    ]
+    assert plot_content["encoding"]["x"]["field"] == INDEX_FIELD
+    assert plot_content["encoding"]["y"]["field"] == "val"
+    assert plot_content["encoding"]["x"]["title"] == "x_title"
+    assert plot_content["encoding"]["y"]["title"] == "y_title"
+
+
+def test_multiple_columns(tmp_dir, scm, dvc):
+    metric = [
+        OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
+        OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
+    ]
+
+    data = {
+        "workspace": {"data": {"file.json": {"data": metric, "props": {}}}}
+    }
+
+    plot_string = VegaRenderer(data, dvc.plots.templates).get_vega()
+
+    plot_content = json.loads(plot_string)
+    assert plot_content["data"]["values"] == [
+        {
+            "val": 2,
+            INDEX_FIELD: 0,
+            REVISION_FIELD: "workspace",
+            "first_val": 100,
+            "second_val": 100,
+        },
+        {
+            "val": 3,
+            INDEX_FIELD: 1,
+            REVISION_FIELD: "workspace",
+            "first_val": 200,
+            "second_val": 300,
+        },
+    ]
+    assert plot_content["encoding"]["x"]["field"] == INDEX_FIELD
+    assert plot_content["encoding"]["y"]["field"] == "val"
+
+
+def test_choose_axes(tmp_dir, scm, dvc):
+    metric = [
+        OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
+        OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
+    ]
+
+    props = {"x": "first_val", "y": "second_val"}
+
+    data = {
+        "workspace": {"data": {"file.json": {"data": metric, "props": props}}}
+    }
+    plot_string = VegaRenderer(data, dvc.plots.templates).get_vega()
+
+    plot_content = json.loads(plot_string)
+    assert plot_content["data"]["values"] == [
+        {
+            "val": 2,
+            REVISION_FIELD: "workspace",
+            "first_val": 100,
+            "second_val": 100,
+        },
+        {
+            "val": 3,
+            REVISION_FIELD: "workspace",
+            "first_val": 200,
+            "second_val": 300,
+        },
+    ]
+    assert plot_content["encoding"]["x"]["field"] == "first_val"
+    assert plot_content["encoding"]["y"]["field"] == "second_val"
+
+
+def test_confusion(tmp_dir, dvc):
+    confusion_matrix = [
+        {"predicted": "B", "actual": "A"},
+        {"predicted": "A", "actual": "A"},
+    ]
+    props = {"template": "confusion", "x": "predicted", "y": "actual"}
+
+    data = {
+        "workspace": {
+            "data": {"file.json": {"data": confusion_matrix, "props": props}}
+        }
+    }
+
+    plot_string = VegaRenderer(data, dvc.plots.templates).get_vega()
+
+    plot_content = json.loads(plot_string)
+    assert plot_content["data"]["values"] == [
+        {"predicted": "B", "actual": "A", REVISION_FIELD: "workspace"},
+        {"predicted": "A", "actual": "A", REVISION_FIELD: "workspace"},
+    ]
+    assert plot_content["spec"]["transform"][0]["groupby"] == [
+        "actual",
+        "predicted",
+    ]
+    assert plot_content["spec"]["encoding"]["x"]["field"] == "predicted"
+    assert plot_content["spec"]["encoding"]["y"]["field"] == "actual"
+
+
+def test_multiple_revs_default(tmp_dir, scm, dvc):
+    metric_1 = [{"y": 2}, {"y": 3}]
+    metric_2 = [{"y": 3}, {"y": 5}]
+    metric_3 = [{"y": 5}, {"y": 6}]
+
+    data = {
+        "HEAD": {
+            "data": {
+                "file.json": {"data": metric_3, "props": {"fields": {"y"}}}
+            }
+        },
+        "v2": {
+            "data": {
+                "file.json": {"data": metric_2, "props": {"fields": {"y"}}}
+            }
+        },
+        "v1": {
+            "data": {
+                "file.json": {"data": metric_1, "props": {"fields": {"y"}}}
+            }
+        },
+    }
+
+    plot_string = VegaRenderer(data, dvc.plots.templates).get_vega()
+
+    plot_content = json.loads(plot_string)
+    assert plot_content["data"]["values"] == [
+        {"y": 5, INDEX_FIELD: 0, REVISION_FIELD: "HEAD"},
+        {"y": 6, INDEX_FIELD: 1, REVISION_FIELD: "HEAD"},
+        {"y": 3, INDEX_FIELD: 0, REVISION_FIELD: "v2"},
+        {"y": 5, INDEX_FIELD: 1, REVISION_FIELD: "v2"},
+        {"y": 2, INDEX_FIELD: 0, REVISION_FIELD: "v1"},
+        {"y": 3, INDEX_FIELD: 1, REVISION_FIELD: "v1"},
+    ]
+    assert plot_content["encoding"]["x"]["field"] == INDEX_FIELD
+    assert plot_content["encoding"]["y"]["field"] == "y"
+
+
+def test_metric_missing(tmp_dir, scm, dvc, caplog):
+
+    metric = [{"y": 2}, {"y": 3}]
+    data = {
+        "v2": {"data": {"file.json": {"data": metric, "props": {}}}},
+        "workspace": {
+            "data": {"file.json": {"error": FileNotFoundError(), "props": {}}}
+        },
+    }
+    plot_string = VegaRenderer(data, dvc.plots.templates).get_vega()
+
+    plot_content = json.loads(plot_string)
+    assert plot_content["data"]["values"] == [
+        {"y": 2, INDEX_FIELD: 0, REVISION_FIELD: "v2"},
+        {"y": 3, INDEX_FIELD: 1, REVISION_FIELD: "v2"},
+    ]
+    assert plot_content["encoding"]["x"]["field"] == INDEX_FIELD
+    assert plot_content["encoding"]["y"]["field"] == "y"
+
+
+def test_custom_template(tmp_dir, scm, dvc, custom_template):
+    metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
+    props = {"template": os.fspath(custom_template), "x": "a", "y": "b"}
+    data = {
+        "workspace": {"data": {"file.json": {"data": metric, "props": props}}}
+    }
+
+    plot_string = VegaRenderer(data, dvc.plots.templates).get_vega()
+
+    plot_content = json.loads(plot_string)
+    assert plot_content["data"]["values"] == [
+        {"a": 1, "b": 2, REVISION_FIELD: "workspace"},
+        {"a": 2, "b": 3, REVISION_FIELD: "workspace"},
+    ]
+    assert plot_content["encoding"]["x"]["field"] == "a"
+    assert plot_content["encoding"]["y"]["field"] == "b"
+
+
+def test_raise_on_no_template(tmp_dir, dvc):
+    metric = [{"val": 2}, {"val": 3}]
+    props = {"template": "non_existing_template.json"}
+    data = {
+        "workspace": {"data": {"file.json": {"data": metric, "props": props}}}
+    }
+
+    with pytest.raises(TemplateNotFoundError):
+        VegaRenderer(data, dvc.plots.templates).get_vega()
+
+
+def test_bad_template(tmp_dir, dvc):
+    metric = [{"val": 2}, {"val": 3}]
+    tmp_dir.gen("template.json", json.dumps({"a": "b", "c": "d"}))
+    props = {"template": "template.json"}
+    data = {
+        "workspace": {"data": {"file.json": {"data": metric, "props": props}}}
+    }
+
+    with pytest.raises(BadTemplateError):
+        VegaRenderer(data, dvc.plots.templates).get_vega()
+
+
+def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template):
+    metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
+    props = {
+        "template": os.fspath(custom_template),
+        "fields": {"b", "c"},
+        "x": "b",
+        "y": "c",
+    }
+    data = {
+        "workspace": {"data": {"file.json": {"data": metric, "props": props}}}
+    }
+
+    plot_string = VegaRenderer(data, dvc.plots.templates).get_vega()
+
+    plot_content = json.loads(plot_string)
+    assert plot_content["data"]["values"] == [
+        {"b": 2, "c": 3, REVISION_FIELD: "workspace"},
+        {"b": 3, "c": 4, REVISION_FIELD: "workspace"},
+    ]
+    assert plot_content["encoding"]["x"]["field"] == "b"
+    assert plot_content["encoding"]["y"]["field"] == "c"
+
+
+def test_plot_default_choose_column(tmp_dir, scm, dvc):
+    metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
+    data = {
+        "workspace": {
+            "data": {"file.json": {"data": metric, "props": {"fields": {"b"}}}}
+        }
+    }
+
+    plot_string = VegaRenderer(data, dvc.plots.templates).get_vega()
+    plot_content = json.loads(plot_string)
+
+    assert plot_content["data"]["values"] == [
+        {INDEX_FIELD: 0, "b": 2, REVISION_FIELD: "workspace"},
+        {INDEX_FIELD: 1, "b": 3, REVISION_FIELD: "workspace"},
+    ]
+    assert plot_content["encoding"]["x"]["field"] == INDEX_FIELD
+    assert plot_content["encoding"]["y"]["field"] == "b"
+
+
+def test_raise_on_wrong_field(tmp_dir, scm, dvc):
+    metric = [{"val": 2}, {"val": 3}]
+    data = {
+        "workspace": {
+            "data": {"file.json": {"data": metric, "props": {"x": "no_val"}}}
+        }
+    }
+
+    with pytest.raises(NoFieldInDataError):
+        VegaRenderer(data, dvc.plots.templates).get_vega()
+
+
+@pytest.mark.parametrize(
+    "extension, matches",
+    (
+        (".csv", True),
+        (".json", True),
+        (".tsv", True),
+        (".yaml", True),
+        (".jpg", False),
+        (".gif", False),
+        (".jpeg", False),
+        (".png", False),
+    ),
+)
+def test_matches(extension, matches):
+    filename = "file" + extension
+    data = {
+        "HEAD": {"data": {filename: {}}},
+        "v1": {"data": {filename: {}}},
+    }
+    assert VegaRenderer.matches(data) == matches
+
+
+def test_find_vega(tmp_dir, dvc):
+    data = {
+        "HEAD": {
+            "data": {
+                "file.json": {
+                    "data": [{"y": 5}, {"y": 6}],
+                    "props": {"fields": {"y"}},
+                },
+                "other_file.jpg": {"data": b"content"},
+            }
+        },
+        "v2": {
+            "data": {
+                "file.json": {
+                    "data": [{"y": 3}, {"y": 5}],
+                    "props": {"fields": {"y"}},
+                },
+                "other_file.jpg": {"data": b"content2"},
+            }
+        },
+        "v1": {
+            "data": {
+                "file.json": {
+                    "data": [{"y": 2}, {"y": 3}],
+                    "props": {"fields": {"y"}},
+                },
+                "another.gif": {"data": b"content2"},
+            }
+        },
+    }
+
+    plot_content = json.loads(find_vega(dvc, data, "file.json"))
+
+    assert plot_content["data"]["values"] == [
+        {"y": 5, INDEX_FIELD: 0, REVISION_FIELD: "HEAD"},
+        {"y": 6, INDEX_FIELD: 1, REVISION_FIELD: "HEAD"},
+        {"y": 3, INDEX_FIELD: 0, REVISION_FIELD: "v2"},
+        {"y": 5, INDEX_FIELD: 1, REVISION_FIELD: "v2"},
+        {"y": 2, INDEX_FIELD: 0, REVISION_FIELD: "v1"},
+        {"y": 3, INDEX_FIELD: 1, REVISION_FIELD: "v1"},
+    ]
+    assert plot_content["encoding"]["x"]["field"] == INDEX_FIELD
+    assert plot_content["encoding"]["y"]["field"] == "y"

tests/unit/repo/experiments/test_utils.py

@@ -0,0 +1,27 @@
+import pytest
+
+from dvc.repo.experiments.base import EXPS_NAMESPACE, ExpRefInfo
+from dvc.repo.experiments.utils import resolve_exp_ref
+
+
+def commit_exp_ref(tmp_dir, scm, file="foo", contents="foo", name="foo"):
+    tmp_dir.scm_gen(file, contents, commit="init")
+    rev = scm.get_rev()
+    ref = "/".join([EXPS_NAMESPACE, "ab", "c123", name])
+    scm.gitpython.set_ref(ref, rev)
+    return ref, rev
+
+
+@pytest.mark.parametrize("use_url", [True, False])
+@pytest.mark.parametrize("name_only", [True, False])
+def test_resolve_exp_ref(tmp_dir, scm, git_upstream, name_only, use_url):
+    ref, _ = commit_exp_ref(tmp_dir, scm)
+    ref_info = resolve_exp_ref(scm, "foo" if name_only else ref)
+    assert isinstance(ref_info, ExpRefInfo)
+    assert str(ref_info) == ref
+
+    scm.push_refspec(git_upstream.url, ref, ref)
+    remote = git_upstream.url if use_url else git_upstream.remote
+    remote_ref_info = resolve_exp_ref(scm, "foo" if name_only else ref, remote)
+    assert isinstance(remote_ref_info, ExpRefInfo)
+    assert str(remote_ref_info) == ref

tests/unit/repo/plots/test_data.py

@@ -1,44 +0,0 @@
-from collections import OrderedDict
-from typing import Dict, List
-
-import pytest
-
-from dvc.repo.plots.data import DictData, _lists
-
-
-@pytest.mark.parametrize(
-    "dictionary, expected_result",
-    [
-        ({}, []),
-        ({"x": ["a", "b", "c"]}, [["a", "b", "c"]]),
-        (
-            OrderedDict([("x", {"y": ["a", "b"]}), ("z", {"w": ["c", "d"]})]),
-            [["a", "b"], ["c", "d"]],
-        ),
-    ],
-)
-def test_finding_lists(dictionary, expected_result):
-    result = _lists(dictionary)
-
-    assert list(result) == expected_result
-
-
-def test_find_data_in_dict(tmp_dir):
-    m1 = [{"accuracy": 1, "loss": 2}, {"accuracy": 3, "loss": 4}]
-    m2 = [{"x": 1}, {"x": 2}]
-    dmetric = OrderedDict([("t1", m1), ("t2", m2)])
-
-    plot_data = DictData("-", "revision", dmetric)
-
-    def points_with(datapoints: List, additional_info: Dict):
-        for datapoint in datapoints:
-            datapoint.update(additional_info)
-
-        return datapoints
-
-    assert list(map(dict, plot_data.to_datapoints())) == points_with(
-        m1, {"rev": "revision"}
-    )
-    assert list(
-        map(dict, plot_data.to_datapoints(fields={"x"}))
-    ) == points_with(m2, {"rev": "revision"})

tests/unit/test_plots.py

@@ -1,6 +1,8 @@
 import json
 import os
 
+from dvc.render.utils import get_files
+
 
 def test_plots_order(tmp_dir, dvc):
     tmp_dir.gen(
@@ -26,9 +28,9 @@ def test_plots_order(tmp_dir, dvc):
             name="stage2",
         )
 
-    assert list(dvc.plots.show()) == [
+    assert get_files(dvc.plots.show()) == {
         "p.json",
         os.path.join("sub", "p4.json"),
         "p1.json",
         os.path.join("sub", "p3.json"),
-    ]
+    }