dvc: bump to 1.0.0a9

This one doesn't allow us to release new pre-releases, because pypi
forbids that:

https://travis-ci.com/github/iterative/dvc/jobs/343343967#L2447

Let's install it separately for now, until there is a new official
mock-ssh-server release.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,10 +1,19 @@
 ---
-name: Bug report
-about: Create a report to help us improve
-
+name: "\U0001F41B Bug Report"
+about: Create a bug report to help us improve DVC
+labels: bug, triage
 ---
 
+## Bug Report
 
-**Please provide information about your setup**
-DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
+### Please provide information about your setup
 
+**Output of `dvc version`:**
+
+```console
+$ dvc version
+```
+
+**Additional Information (if any):**
+
+If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: "🤗 Need help?"
+    url: https://dvc.org/chat
+    about: If you have a question, ask us on Discord. Please join with this invite 👉

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,6 @@
 ---
-name: Feature request
+name: "\U0001F680 Feature Request"
 about: Suggest an idea for this project
-
+labels: feature request, triage
 ---
 

.pre-commit-config.yaml

@@ -27,18 +27,37 @@ repos:
   repo: https://github.com/lovesegfault/beautysh
   rev: master
 - hooks:
-  - id: dvc-pre-commit
-    language_version: python3
+  - args:
+    - git-hook
+    - pre-commit
+    entry: dvc
+    id: dvc-pre-commit
+    language: system
+    name: DVC pre-commit
     stages:
     - commit
-  - id: dvc-pre-push
-    language_version: python3
+    verbose: true
+    require_serial: true
+  - args:
+    - git-hook
+    - pre-push
+    entry: dvc
+    id: dvc-pre-push
+    language: system
+    name: DVC pre-push
     stages:
     - push
+    require_serial: true
   - always_run: true
+    args:
+    - git-hook
+    - post-checkout
+    entry: dvc
     id: dvc-post-checkout
-    language_version: python3
+    language: system
+    minimum_pre_commit_version: 2.2.0
+    name: DVC post-checkout
     stages:
     - post-checkout
-  repo: https://github.com/iterative/dvc
-  rev: master
+    require_serial: true
+  repo: local

.pre-commit-hooks.yaml

@@ -6,6 +6,7 @@
   language: python
   language_version: python3
   name: DVC pre-commit
+  require_serial: true
   stages:
   - commit
   verbose: true
@@ -17,6 +18,7 @@
   language: python
   language_version: python3
   name: DVC pre-push
+  require_serial: true
   stages:
   - push
 - always_run: true
@@ -29,5 +31,6 @@
   language_version: python3
   minimum_pre_commit_version: 2.2.0
   name: DVC post-checkout
+  require_serial: true
   stages:
   - post-checkout

dvc/cli.py

@@ -10,9 +10,11 @@ from .command import (
     commit,
     config,
     daemon,
+    dag,
     data_sync,
     destroy,
     diff,
+    freeze,
     gc,
     get,
     get_url,
@@ -21,12 +23,10 @@ from .command import (
     imp_url,
     init,
     install,
-    lock,
     ls,
     metrics,
     move,
     params,
-    pipeline,
     plots,
     remote,
     remove,
@@ -66,8 +66,8 @@ COMMANDS = [
     install,
     root,
     ls,
-    lock,
-    pipeline,
+    freeze,
+    dag,
     daemon,
     commit,
     diff,

dvc/command/add.py

@@ -18,6 +18,7 @@ class CmdAdd(CmdBase):
                 recursive=self.args.recursive,
                 no_commit=self.args.no_commit,
                 fname=self.args.file,
+                external=self.args.external,
             )
 
         except DvcException:
@@ -49,6 +50,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Don't put files/directories into cache.",
     )
+    parser.add_argument(
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow targets that are outside of the DVC repository.",
+    )
     parser.add_argument(
         "-f",
         "--file",

dvc/command/dag.py

@@ -0,0 +1,115 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _show_ascii(G):
+    from dvc.repo.graph import get_pipelines
+    from dvc.dagascii import draw
+
+    pipelines = get_pipelines(G)
+
+    ret = []
+    for pipeline in pipelines:
+        ret.append(draw(pipeline.nodes, pipeline.edges))
+
+    return "\n".join(ret)
+
+
+def _show_dot(G):
+    import io
+    from networkx.drawing.nx_pydot import write_dot
+
+    dot_file = io.StringIO()
+    write_dot(G, dot_file)
+    return dot_file.getvalue()
+
+
+def _build(G, target=None, full=False):
+    import networkx as nx
+    from dvc.repo.graph import get_pipeline, get_pipelines
+
+    if target:
+        H = get_pipeline(get_pipelines(G), target)
+        if not full:
+            descendants = nx.descendants(G, target)
+            descendants.add(target)
+            H.remove_nodes_from(set(G.nodes()) - descendants)
+    else:
+        H = G
+
+    def _relabel(stage):
+        return stage.addressing
+
+    return nx.relabel_nodes(H, _relabel, copy=False)
+
+
+class CmdDAG(CmdBase):
+    def run(self):
+        try:
+            target = None
+            if self.args.target:
+                stages = self.repo.collect(self.args.target)
+                if len(stages) > 1:
+                    logger.error(
+                        f"'{self.args.target}' contains more than one stage "
+                        "{stages}, please specify one stage"
+                    )
+                    return 1
+                target = stages[0]
+
+            G = _build(self.repo.graph, target=target, full=self.args.full,)
+
+            if self.args.dot:
+                logger.info(_show_dot(G))
+            else:
+                from dvc.utils.pager import pager
+
+                pager(_show_ascii(G))
+
+            return 0
+        except DvcException:
+            msg = "failed to show"
+            if self.args.target:
+                msg += f"a pipeline for '{target}'"
+            else:
+                msg += "pipelines"
+            logger.exception(msg)
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    DAG_HELP = "Visualize DVC project DAG."
+    dag_parser = subparsers.add_parser(
+        "dag",
+        parents=[parent_parser],
+        description=append_doc_link(DAG_HELP, "dag"),
+        help=DAG_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    dag_parser.add_argument(
+        "--dot",
+        action="store_true",
+        default=False,
+        help="Print DAG with .dot format.",
+    )
+    dag_parser.add_argument(
+        "--full",
+        action="store_true",
+        default=False,
+        help=(
+            "Show full DAG that the target belongs too, instead of "
+            "showing DAG consisting only of ancestors."
+        ),
+    )
+    dag_parser.add_argument(
+        "target",
+        nargs="?",
+        help="Stage or output to show pipeline for (optional). "
+        "Finds all stages in the workspace by default.",
+    )
+    dag_parser.set_defaults(func=CmdDAG)

dvc/command/freeze.py

@@ -0,0 +1,57 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+class CmdFreezeBase(CmdBase):
+    def _run(self, func, name):
+        ret = 0
+        for target in self.args.targets:
+            try:
+                func(target)
+            except DvcException:
+                logger.exception(f"failed to {name} '{target}'")
+                ret = 1
+        return ret
+
+
+class CmdFreeze(CmdFreezeBase):
+    def run(self):
+        return self._run(self.repo.freeze, "freeze")
+
+
+class CmdUnfreeze(CmdFreezeBase):
+    def run(self):
+        return self._run(self.repo.unfreeze, "unfreeze")
+
+
+def add_parser(subparsers, parent_parser):
+    FREEZE_HELP = "Freeze DVC-files."
+    freeze_parser = subparsers.add_parser(
+        "freeze",
+        parents=[parent_parser],
+        description=append_doc_link(FREEZE_HELP, "freeze"),
+        help=FREEZE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    freeze_parser.add_argument(
+        "targets", nargs="+", help="DVC-files to freeze."
+    )
+    freeze_parser.set_defaults(func=CmdFreeze)
+
+    UNFREEZE_HELP = "Unfreeze DVC-files."
+    unfreeze_parser = subparsers.add_parser(
+        "unfreeze",
+        parents=[parent_parser],
+        description=append_doc_link(UNFREEZE_HELP, "unfreeze"),
+        help=UNFREEZE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    unfreeze_parser.add_argument(
+        "targets", nargs="+", help="DVC-files to unfreeze."
+    )
+    unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/gc.py

@@ -21,7 +21,7 @@ class CmdGC(CmdBase):
 
         msg = "This will remove all cache except items used in "
 
-        msg += "the working tree"
+        msg += "the workspace"
         if self.args.all_commits:
             msg += " and all git commits"
         elif self.args.all_branches and self.args.all_tags:
@@ -77,7 +77,7 @@ def add_parser(subparsers, parent_parser):
         "--workspace",
         action="store_true",
         default=False,
-        help="Keep data files used in the current workspace (working tree).",
+        help="Keep data files used in the current workspace.",
     )
     gc_parser.add_argument(
         "-a",

dvc/command/lock.py

@@ -1,59 +0,0 @@
-import argparse
-import logging
-
-from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
-
-logger = logging.getLogger(__name__)
-
-
-class CmdLockBase(CmdBase):
-    def _run(self, unlock):
-        for target in self.args.targets:
-            try:
-                self.repo.lock_stage(target, unlock=unlock)
-            except DvcException:
-                logger.exception(
-                    "failed to {}lock '{}'".format(
-                        "un" if unlock else "", target
-                    )
-                )
-
-                return 1
-        return 0
-
-
-class CmdLock(CmdLockBase):
-    def run(self):
-        return self._run(False)
-
-
-class CmdUnlock(CmdLockBase):
-    def run(self):
-        return self._run(True)
-
-
-def add_parser(subparsers, parent_parser):
-    LOCK_HELP = "Lock DVC-files."
-    lock_parser = subparsers.add_parser(
-        "lock",
-        parents=[parent_parser],
-        description=append_doc_link(LOCK_HELP, "lock"),
-        help=LOCK_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    lock_parser.add_argument("targets", nargs="+", help="DVC-files to lock.")
-    lock_parser.set_defaults(func=CmdLock)
-
-    UNLOCK_HELP = "Unlock DVC-files."
-    unlock_parser = subparsers.add_parser(
-        "unlock",
-        parents=[parent_parser],
-        description=append_doc_link(UNLOCK_HELP, "unlock"),
-        help=UNLOCK_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    unlock_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to unlock."
-    )
-    unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/metrics.py

@@ -63,30 +63,6 @@ class CmdMetricsShow(CmdBase):
         return 0
 
 
-class CmdMetricsAdd(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.add(self.args.path)
-        except DvcException:
-            msg = f"failed to add metric file '{self.args.path}'"
-            logger.exception(msg)
-            return 1
-
-        return 0
-
-
-class CmdMetricsRemove(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.remove(self.args.path)
-        except DvcException:
-            msg = f"failed to remove metric file '{self.args.path}'"
-            logger.exception(msg)
-            return 1
-
-        return 0
-
-
 def _show_diff(diff, markdown=False, no_path=False, old=False):
     from collections import OrderedDict
 
@@ -145,7 +121,7 @@ class CmdMetricsDiff(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    METRICS_HELP = "Commands to add, manage, collect, and display metrics."
+    METRICS_HELP = "Commands to display and compare metrics."
 
     metrics_parser = subparsers.add_parser(
         "metrics",
@@ -162,17 +138,6 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(metrics_subparsers)
 
-    METRICS_ADD_HELP = "Mark a DVC-tracked file as a metric."
-    metrics_add_parser = metrics_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
-        help=METRICS_ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_add_parser.add_argument("path", help="Path to a metric file.")
-    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
-
     METRICS_SHOW_HELP = "Print metrics, with optional formatting."
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
@@ -291,14 +256,3 @@ def add_parser(subparsers, parent_parser):
         help="Show old metric value.",
     )
     metrics_diff_parser.set_defaults(func=CmdMetricsDiff)
-
-    METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
-    metrics_remove_parser = metrics_subparsers.add_parser(
-        "remove",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
-        help=METRICS_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
-    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)

dvc/command/pipeline.py

@@ -1,267 +0,0 @@
-import argparse
-import logging
-
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
-from dvc.exceptions import DvcException
-
-logger = logging.getLogger(__name__)
-
-
-class CmdPipelineShow(CmdBase):
-    def _show(self, target, commands, outs, locked):
-        import networkx
-        from dvc import dvcfile
-        from dvc.utils import parse_target
-
-        path, name = parse_target(target)
-        stage = dvcfile.Dvcfile(self.repo, path).stages[name]
-        G = self.repo.graph
-        stages = networkx.dfs_postorder_nodes(G, stage)
-        if locked:
-            stages = [s for s in stages if s.locked]
-
-        for stage in stages:
-            if commands:
-                if stage.cmd is None:
-                    continue
-                logger.info(stage.cmd)
-            elif outs:
-                for out in stage.outs:
-                    logger.info(str(out))
-            else:
-                logger.info(stage.addressing)
-
-    @staticmethod
-    def _build_output_graph(G, target_stage):
-        import networkx
-        from itertools import product
-
-        nodes = {str(out) for out in target_stage.outs}
-        edges = []
-
-        for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
-            from_stage_deps = {dep.path_info.parts for dep in from_stage.deps}
-            to_outs = {
-                to_out
-                for to_out in to_stage.outs
-                if to_out.path_info.parts in from_stage_deps
-            }
-            from_outs = {
-                from_out
-                for from_out in from_stage.outs
-                if str(from_out) in nodes
-            }
-            nodes |= {str(to_out) for to_out in to_outs}
-            for from_out, to_out in product(from_outs, to_outs):
-                edges.append((str(from_out), str(to_out)))
-        return nodes, edges
-
-    def _build_graph(self, target, commands=False, outs=False):
-        import networkx
-        from dvc import dvcfile
-        from dvc.repo.graph import get_pipeline
-        from dvc.utils import parse_target
-
-        path, name = parse_target(target)
-        target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
-        G = get_pipeline(self.repo.pipelines, target_stage)
-
-        nodes = set()
-        for stage in networkx.dfs_preorder_nodes(G, target_stage):
-            if commands:
-                if stage.cmd is None:
-                    continue
-                nodes.add(stage.cmd)
-            elif not outs:
-                nodes.add(stage.addressing)
-
-        edges = []
-        for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
-            if commands:
-                if to_stage.cmd is None:
-                    continue
-                edges.append((from_stage.cmd, to_stage.cmd))
-            elif not outs:
-                edges.append((from_stage.addressing, to_stage.addressing))
-
-        if outs:
-            nodes, edges = self._build_output_graph(G, target_stage)
-
-        return list(nodes), edges, networkx.is_tree(G)
-
-    def _show_ascii(self, target, commands, outs):
-        from dvc.dagascii import draw
-
-        nodes, edges, _ = self._build_graph(target, commands, outs)
-
-        if not nodes:
-            return
-
-        draw(nodes, edges)
-
-    def _show_dependencies_tree(self, target, commands, outs):
-        from treelib import Tree
-
-        nodes, edges, is_tree = self._build_graph(target, commands, outs)
-        if not nodes:
-            return
-        if not is_tree:
-            raise DvcException(
-                "DAG is not a tree, can not print it in tree-structure way, "
-                "please use --ascii instead"
-            )
-
-        tree = Tree()
-        tree.create_node(target, target)  # Root node
-        observe_list = [target]
-        while len(observe_list) > 0:
-            current_root = observe_list[0]
-            for edge in edges:
-                if edge[0] == current_root:
-                    tree.create_node(edge[1], edge[1], parent=current_root)
-                    observe_list.append(edge[1])
-            observe_list.pop(0)
-        tree.show()
-
-    def _write_dot(self, target, commands, outs):
-        import io
-        import networkx
-        from networkx.drawing.nx_pydot import write_dot
-
-        _, edges, _ = self._build_graph(target, commands, outs)
-        edges = [edge[::-1] for edge in edges]
-
-        simple_g = networkx.DiGraph()
-        simple_g.add_edges_from(edges)
-
-        dot_file = io.StringIO()
-        write_dot(simple_g, dot_file)
-        logger.info(dot_file.getvalue())
-
-    def run(self):
-        from dvc.dvcfile import DVC_FILE
-
-        if not self.args.targets:
-            self.args.targets = [DVC_FILE]
-
-        for target in self.args.targets:
-            try:
-                if self.args.ascii:
-                    self._show_ascii(
-                        target, self.args.commands, self.args.outs
-                    )
-                elif self.args.dot:
-                    self._write_dot(target, self.args.commands, self.args.outs)
-                elif self.args.tree:
-                    self._show_dependencies_tree(
-                        target, self.args.commands, self.args.outs
-                    )
-                else:
-                    self._show(
-                        target,
-                        self.args.commands,
-                        self.args.outs,
-                        self.args.locked,
-                    )
-            except DvcException:
-                msg = f"failed to show pipeline for '{target}'"
-                logger.exception(msg)
-                return 1
-        return 0
-
-
-class CmdPipelineList(CmdBase):
-    def run(self):
-        pipelines = self.repo.pipelines
-        for pipeline in pipelines:
-            for stage in pipeline:
-                logger.info(stage.addressing)
-            if len(pipeline) != 0:
-                logger.info("=" * 80)
-        logger.info("{} pipelines total".format(len(pipelines)))
-
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    PIPELINE_HELP = "Manage pipelines."
-    pipeline_parser = subparsers.add_parser(
-        "pipeline",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_HELP, "pipeline"),
-        help=PIPELINE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-
-    pipeline_subparsers = pipeline_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc pipeline CMD --help` for command-specific help.",
-    )
-
-    fix_subparsers(pipeline_subparsers)
-
-    PIPELINE_LIST_HELP = "List connected groups of stages (pipelines)."
-    pipeline_list_parser = pipeline_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
-        help=PIPELINE_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pipeline_list_parser.set_defaults(func=CmdPipelineList)
-
-    PIPELINE_SHOW_HELP = "Show stages in a pipeline."
-    pipeline_show_parser = pipeline_subparsers.add_parser(
-        "show",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline/show"),
-        help=PIPELINE_SHOW_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
-    pipeline_show_group.add_argument(
-        "-c",
-        "--commands",
-        action="store_true",
-        default=False,
-        help="Print commands instead of paths to DVC-files.",
-    )
-    pipeline_show_group.add_argument(
-        "-o",
-        "--outs",
-        action="store_true",
-        default=False,
-        help="Print output files instead of paths to DVC-files.",
-    )
-    pipeline_show_parser.add_argument(
-        "-l",
-        "--locked",
-        action="store_true",
-        default=False,
-        help="Print locked DVC stages",
-    )
-    pipeline_show_parser.add_argument(
-        "--ascii",
-        action="store_true",
-        default=False,
-        help="Output DAG as ASCII.",
-    )
-    pipeline_show_parser.add_argument(
-        "--dot",
-        action="store_true",
-        default=False,
-        help="Print DAG with .dot format.",
-    )
-    pipeline_show_parser.add_argument(
-        "--tree",
-        action="store_true",
-        default=False,
-        help="Output DAG as Dependencies Tree.",
-    )
-    pipeline_show_parser.add_argument(
-        "targets",
-        nargs="*",
-        help="DVC-files to show pipeline for. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
-    )
-    pipeline_show_parser.set_defaults(func=CmdPipelineShow)

dvc/command/plots.py

@@ -4,6 +4,7 @@ import os
 
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
+from dvc.schema import PLOT_PROPS
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
@@ -32,23 +33,28 @@ class CmdPlots(CmdBase):
     def _func(self, *args, **kwargs):
         raise NotImplementedError
 
+    def _props(self):
+        # Pass only props specified by user, to not shadow ones from plot def
+        props = {p: getattr(self.args, p) for p in PLOT_PROPS}
+        return {k: v for k, v in props.items() if v is not None}
+
     def run(self):
+        if self.args.show_vega:
+            if not self.args.targets:
+                logger.error("please specify a target for `--show-vega`")
+                return 1
+            if len(self.args.targets) > 1:
+                logger.error(
+                    "you can only specify one target for `--show-vega`"
+                )
+                return 1
+
         try:
-            plots = self._func(
-                targets=self.args.targets,
-                template=self.args.template,
-                x_field=self.args.x,
-                y_field=self.args.y,
-                csv_header=not self.args.no_csv_header,
-                title=self.args.title,
-                x_title=self.args.xlab,
-                y_title=self.args.ylab,
-            )
+            plots = self._func(targets=self.args.targets, props=self._props())
 
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(plots))
+            if self.args.show_vega:
+                target = self.args.targets[0]
+                logger.info(plots[target])
                 return 0
 
             divs = [
@@ -82,6 +88,14 @@ class CmdPlotsDiff(CmdPlots):
         return self.repo.plots.diff(*args, revs=self.args.revisions, **kwargs)
 
 
+class CmdPlotsModify(CmdPlots):
+    def run(self):
+        self.repo.plots.modify(
+            self.args.target, props=self._props(), unset=self.args.unset,
+        )
+        return 0
+
+
 def add_parser(subparsers, parent_parser):
     PLOTS_HELP = (
         "Generating plots for metrics stored in structured files "
@@ -110,51 +124,13 @@ def add_parser(subparsers, parent_parser):
         help=SHOW_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    plots_show_parser.add_argument(
-        "-t",
-        "--template",
-        nargs="?",
-        default=None,
-        help=(
-            "Special JSON or HTML schema file to inject with the data. "
-            "See {}".format(
-                format_link("https://man.dvc.org/plots#plot-templates")
-            )
-        ),
-    )
-    plots_show_parser.add_argument(
-        "-o", "--out", default=None, help="Destination path to save plots to.",
-    )
-    plots_show_parser.add_argument(
-        "-x", default=None, help="Field name for x axis."
-    )
-    plots_show_parser.add_argument(
-        "-y", default=None, help="Field name for y axis."
-    )
-    plots_show_parser.add_argument(
-        "--no-csv-header",
-        action="store_true",
-        default=False,
-        help="Required when CSV or TSV datafile does not have a header.",
-    )
-    plots_show_parser.add_argument(
-        "--show-json",
-        action="store_true",
-        default=False,
-        help="Show output in JSON format.",
-    )
-    plots_show_parser.add_argument("--title", default=None, help="Plot title.")
-    plots_show_parser.add_argument(
-        "--xlab", default=None, help="X axis title."
-    )
-    plots_show_parser.add_argument(
-        "--ylab", default=None, help="Y axis title."
-    )
     plots_show_parser.add_argument(
         "targets",
         nargs="*",
-        help="Metrics files to visualize. Shows all plots by default.",
+        help="Plots files to visualize. Shows all plots by default.",
     )
+    _add_props_arguments(plots_show_parser)
+    _add_output_arguments(plots_show_parser)
     plots_show_parser.set_defaults(func=CmdPlotsShow)
 
     PLOTS_DIFF_HELP = (
@@ -169,6 +145,37 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help="Plots file to visualize. Shows all plots by default.",
+    )
+    plots_diff_parser.add_argument(
+        "revisions", nargs="*", default=None, help="Git commits to plot from",
+    )
+    _add_props_arguments(plots_diff_parser)
+    _add_output_arguments(plots_diff_parser)
+    plots_diff_parser.set_defaults(func=CmdPlotsDiff)
+
+    PLOTS_MODIFY_HELP = "Modify plot props associated with a target file."
+    plots_modify_parser = plots_subparsers.add_parser(
+        "modify",
+        parents=[parent_parser],
+        description=append_doc_link(PLOTS_MODIFY_HELP, "plots/modify"),
+        help=PLOTS_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_modify_parser.add_argument(
+        "target", help="Plot file to set props to.",
+    )
+    _add_props_arguments(plots_modify_parser)
+    plots_modify_parser.add_argument(
+        "--unset", nargs="*", help="Props to unset.",
+    )
+    plots_modify_parser.set_defaults(func=CmdPlotsModify)
+
+
+def _add_props_arguments(parser):
+    parser.add_argument(
         "-t",
         "--template",
         nargs="?",
@@ -180,40 +187,27 @@ def add_parser(subparsers, parent_parser):
             )
         ),
     )
-    plots_diff_parser.add_argument(
-        "--targets",
-        nargs="*",
-        help="Metrics file to visualize. Shows all plots by default.",
-    )
-    plots_diff_parser.add_argument(
-        "-o", "--out", default=None, help="Destination path to save plots to.",
-    )
-    plots_diff_parser.add_argument(
-        "-x", default=None, help="Field name for x axis."
-    )
-    plots_diff_parser.add_argument(
-        "-y", default=None, help="Field name for y axis."
-    )
-    plots_diff_parser.add_argument(
+    parser.add_argument("-x", default=None, help="Field name for x axis.")
+    parser.add_argument("-y", default=None, help="Field name for y axis.")
+    parser.add_argument(
         "--no-csv-header",
-        action="store_true",
-        default=False,
+        action="store_false",
+        dest="csv_header",
+        default=None,  # Use default None to distinguish when it's not used
         help="Provided CSV ot TSV datafile does not have a header.",
     )
-    plots_diff_parser.add_argument(
-        "--show-json",
+    parser.add_argument("--title", default=None, help="Plot title.")
+    parser.add_argument("--x-label", default=None, help="X axis label.")
+    parser.add_argument("--y-label", default=None, help="Y axis lebel.")
+
+
+def _add_output_arguments(parser):
+    parser.add_argument(
+        "-o", "--out", default=None, help="Destination path to save plots to.",
+    )
+    parser.add_argument(
+        "--show-vega",
         action="store_true",
         default=False,
-        help="Show output in JSON format.",
+        help="Show output in Vega format.",
     )
-    plots_diff_parser.add_argument("--title", default=None, help="Plot title.")
-    plots_diff_parser.add_argument(
-        "--xlab", default=None, help="X axis title."
-    )
-    plots_diff_parser.add_argument(
-        "--ylab", default=None, help="Y axis title."
-    )
-    plots_diff_parser.add_argument(
-        "revisions", nargs="*", default=None, help="Git commits to plot from",
-    )
-    plots_diff_parser.set_defaults(func=CmdPlotsDiff)

dvc/command/remote.py

@@ -62,7 +62,11 @@ class CmdRemoteModify(CmdRemote):
     def run(self):
         with self.config.edit(self.args.level) as conf:
             self._check_exists(conf)
-            conf["remote"][self.args.name][self.args.option] = self.args.value
+            section = conf["remote"][self.args.name]
+            if self.args.unset:
+                section.pop(self.args.option, None)
+            else:
+                section[self.args.option] = self.args.value
         return 0
 
 

dvc/command/remove.py

@@ -1,7 +1,6 @@
 import argparse
 import logging
 
-import dvc.prompt as prompt
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -9,30 +8,10 @@ logger = logging.getLogger(__name__)
 
 
 class CmdRemove(CmdBase):
-    def _is_dvc_only(self, target):
-        if not self.args.purge:
-            return True
-
-        if self.args.force:
-            return False
-
-        msg = "Are you sure you want to remove '{}' with its outputs?".format(
-            target
-        )
-
-        if prompt.confirm(msg):
-            return False
-
-        raise DvcException(
-            "Cannot purge without a confirmation from the user."
-            " Use `-f` to force."
-        )
-
     def run(self):
         for target in self.args.targets:
             try:
-                dvc_only = self._is_dvc_only(target)
-                self.repo.remove(target, dvc_only=dvc_only)
+                self.repo.remove(target, outs=self.args.outs)
             except DvcException:
                 logger.exception(f"failed to remove '{target}'")
                 return 1
@@ -40,7 +19,7 @@ class CmdRemove(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REMOVE_HELP = "Remove DVC-tracked files or directories."
+    REMOVE_HELP = "Remove stage entry and unprotect outputs"
     remove_parser = subparsers.add_parser(
         "remove",
         parents=[parent_parser],
@@ -48,27 +27,11 @@ def add_parser(subparsers, parent_parser):
         help=REMOVE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remove_parser_group = remove_parser.add_mutually_exclusive_group()
-    remove_parser_group.add_argument(
-        "-o",
+    remove_parser.add_argument(
         "--outs",
         action="store_true",
-        default=True,
-        help="Only remove DVC-file outputs. (Default)",
-    )
-    remove_parser_group.add_argument(
-        "-p",
-        "--purge",
-        action="store_true",
         default=False,
-        help="Remove DVC-file and all its outputs.",
-    )
-    remove_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Force purge.",
+        help="Remove outputs as well.",
     )
     remove_parser.add_argument(
         "targets", nargs="+", help="DVC-files to remove."

dvc/command/run.py

@@ -45,7 +45,7 @@ class CmdRun(CmdBase):
                 fname=self.args.file,
                 wdir=self.args.wdir,
                 no_exec=self.args.no_exec,
-                overwrite=self.args.overwrite_dvcfile,
+                overwrite=self.args.overwrite,
                 run_cache=not self.args.no_run_cache,
                 no_commit=self.args.no_commit,
                 outs_persist=self.args.outs_persist,
@@ -53,6 +53,7 @@ class CmdRun(CmdBase):
                 always_changed=self.args.always_changed,
                 name=self.args.name,
                 single_stage=self.args.single_stage,
+                external=self.args.external,
             )
         except DvcException:
             logger.exception("")
@@ -177,10 +178,10 @@ def add_parser(subparsers, parent_parser):
         help="Only create stage file without actually running it.",
     )
     run_parser.add_argument(
-        "--overwrite-dvcfile",
+        "--overwrite",
         action="store_true",
         default=False,
-        help="Overwrite existing DVC-file without asking for confirmation.",
+        help="Overwrite existing stage",
     )
     run_parser.add_argument(
         "--no-run-cache",
@@ -225,6 +226,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=argparse.SUPPRESS,
     )
+    run_parser.add_argument(
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow outputs that are outside of the DVC repository.",
+    )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."
     )

dvc/dagascii.py

@@ -96,10 +96,9 @@ class AsciiCanvas:
 
     def draw(self):
         """Draws ASCII canvas on the screen."""
-        pager = find_pager()
         lines = map("".join, self.canvas)
         joined_lines = os.linesep.join(lines)
-        pager(joined_lines)
+        return joined_lines
 
     def point(self, x, y, char):
         """Create a point on ASCII canvas.
@@ -316,4 +315,4 @@ def draw(vertexes, edges):
             int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
         )
 
-    canvas.draw()
+    return canvas.draw()

dvc/data_cloud.py

@@ -93,7 +93,7 @@ class DataCloud:
     def _save_pulled_checksums(self, cache):
         for checksum in cache.scheme_keys("local"):
             cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
-            if self.repo.cache.local.exists(cache_file):
+            if self.repo.cache.local.tree.exists(cache_file):
                 # We can safely save here, as existing corrupted files will
                 # be removed upon status, while files corrupted during
                 # download will not be moved from tmp_file

dvc/dvcfile.py

@@ -17,11 +17,7 @@ from dvc.stage.exceptions import (
 from dvc.stage.loader import SingleStageLoader, StageLoader
 from dvc.utils import relpath
 from dvc.utils.collections import apply_diff
-from dvc.utils.stage import (
-    dump_stage_file,
-    parse_stage,
-    parse_stage_for_update,
-)
+from dvc.utils.yaml import dump_yaml, parse_yaml, parse_yaml_for_update
 
 logger = logging.getLogger(__name__)
 
@@ -101,7 +97,7 @@ class FileMixin:
 
         with self.repo.tree.open(self.path) as fd:
             stage_text = fd.read()
-        d = parse_stage(stage_text, self.path)
+        d = parse_yaml(stage_text, self.path)
         self.validate(d, self.relpath)
         return d, stage_text
 
@@ -149,8 +145,8 @@ class SingleStageFile(FileMixin):
         logger.debug(
             "Saving information to '{file}'.".format(file=relpath(self.path))
         )
-        dump_stage_file(self.path, serialize.to_single_stage_file(stage))
-        self.repo.scm.track_file(relpath(self.path))
+        dump_yaml(self.path, serialize.to_single_stage_file(stage))
+        self.repo.scm.track_file(self.relpath)
 
     def remove_with_prompt(self, force=False):
         if not self.exists():
@@ -165,6 +161,9 @@ class SingleStageFile(FileMixin):
 
         self.remove()
 
+    def remove_stage(self, stage):
+        self.remove()
+
 
 class PipelineFile(FileMixin):
     """Abstraction for pipelines file, .yaml + .lock combined."""
@@ -182,12 +181,12 @@ class PipelineFile(FileMixin):
         assert isinstance(stage, PipelineStage)
         check_dvc_filename(self.path)
 
-        if not no_lock:
-            self._dump_lockfile(stage)
-
         if update_pipeline and not stage.is_data_source:
             self._dump_pipeline_file(stage)
 
+        if not no_lock:
+            self._dump_lockfile(stage)
+
     def _dump_lockfile(self, stage):
         self._lockfile.dump(stage)
 
@@ -195,8 +194,9 @@ class PipelineFile(FileMixin):
         data = {}
         if self.exists():
             with open(self.path) as fd:
-                data = parse_stage_for_update(fd.read(), self.path)
+                data = parse_yaml_for_update(fd.read(), self.path)
         else:
+            logger.info("Creating '%s'", self.relpath)
             open(self.path, "w+").close()
 
         data["stages"] = data.get("stages", {})
@@ -207,8 +207,11 @@ class PipelineFile(FileMixin):
         else:
             data["stages"].update(stage_data)
 
-        dump_stage_file(self.path, data)
-        self.repo.scm.track_file(relpath(self.path))
+        logger.info(
+            "Adding stage '%s' to '%s'", stage.name, self.relpath,
+        )
+        dump_yaml(self.path, data)
+        self.repo.scm.track_file(self.relpath)
 
     @property
     def stage(self):
@@ -230,6 +233,22 @@ class PipelineFile(FileMixin):
         super().remove()
         self._lockfile.remove()
 
+    def remove_stage(self, stage):
+        self._lockfile.remove_stage(stage)
+        if not self.exists():
+            return
+
+        with open(self.path, "r") as f:
+            d = parse_yaml_for_update(f.read(), self.path)
+
+        self.validate(d, self.path)
+        if stage.name not in d.get("stages", {}):
+            return
+
+        logger.debug("Removing '%s' from '%s'", stage.name, self.path)
+        del d["stages"][stage.name]
+        dump_yaml(self.path, d)
+
 
 class Lockfile(FileMixin):
     from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
@@ -238,7 +257,7 @@ class Lockfile(FileMixin):
         if not self.exists():
             return {}
         with self.repo.tree.open(self.path) as fd:
-            data = parse_stage(fd.read(), self.path)
+            data = parse_yaml(fd.read(), self.path)
         try:
             self.validate(data, fname=self.relpath)
         except StageFileFormatError:
@@ -250,15 +269,38 @@ class Lockfile(FileMixin):
     def dump(self, stage, **kwargs):
         stage_data = serialize.to_lockfile(stage)
         if not self.exists():
+            modified = True
+            logger.info("Generating lock file '%s'", self.relpath)
             data = stage_data
             open(self.path, "w+").close()
         else:
             with self.repo.tree.open(self.path, "r") as fd:
-                data = parse_stage_for_update(fd.read(), self.path)
+                data = parse_yaml_for_update(fd.read(), self.path)
+            modified = data.get(stage.name, {}) != stage_data.get(
+                stage.name, {}
+            )
+            if modified:
+                logger.info("Updating lock file '%s'", self.relpath)
             data.update(stage_data)
+        dump_yaml(self.path, data)
+        if modified:
+            self.repo.scm.track_file(self.relpath)
 
-        dump_stage_file(self.path, data)
-        self.repo.scm.track_file(relpath(self.path))
+    def remove_stage(self, stage):
+        if not self.exists():
+            return
+
+        with open(self.path) as f:
+            d = parse_yaml_for_update(f.read(), self.path)
+        self.validate(d, self.path)
+
+        if stage.name not in d:
+            return
+
+        logger.debug("Removing '%s' from '%s'", stage.name, self.path)
+        del d[stage.name]
+
+        dump_yaml(self.path, d)
 
 
 class Dvcfile:

dvc/exceptions.py

@@ -173,17 +173,24 @@ class BadMetricError(DvcException):
 class NoMetricsError(DvcException):
     def __init__(self):
         super().__init__(
-            "no metric files in this repository. "
-            "Use `dvc metrics add` to add a metric file to track."
+            "no metric files in this repository. Use `-m/-M` options for "
+            "`dvc run` to mark stage outputs as  metrics."
         )
 
 
-class StageFileCorruptedError(DvcException):
+class NoPlotsError(DvcException):
+    def __init__(self):
+        super().__init__(
+            "no plots in this repository. Use `--plots/--plots-no-cache` "
+            "options for `dvc run` to mark stage outputs as plots."
+        )
+
+
+class YAMLFileCorruptedError(DvcException):
     def __init__(self, path):
         path = relpath(path)
         super().__init__(
-            "unable to read DVC-file: {} "
-            "YAML file structure is corrupted".format(path)
+            f"unable to read: '{path}', YAML file structure is corrupted"
         )
 
 
@@ -322,3 +329,15 @@ class RemoteCacheRequiredError(DvcException):
 
 class IsADirectoryError(DvcException):
     """Raised when a file operation is requested on a directory."""
+
+
+class NoOutputOrStageError(DvcException):
+    """
+    Raised when the target is neither an output nor a stage name in dvc.yaml
+    """
+
+    def __init__(self, target, file):
+        super().__init__(
+            f"'{target}' "
+            f"does not exist as an output or a stage name in '{file}'"
+        )

dvc/ignore.py

@@ -5,7 +5,9 @@ from funcy import cached_property
 from pathspec import PathSpec
 from pathspec.patterns import GitWildMatchPattern
 
+from dvc.path_info import PathInfo
 from dvc.scm.tree import BaseTree
+from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -82,7 +84,7 @@ class DvcIgnoreRepo(DvcIgnore):
         def is_dvc_repo(directory):
             from dvc.repo import Repo
 
-            return os.path.isdir(os.path.join(directory, Repo.DVC_DIR))
+            return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
 
         dirs = [d for d in dirs if not is_dvc_repo(d)]
 
@@ -90,13 +92,14 @@ class DvcIgnoreRepo(DvcIgnore):
 
 
 class DvcIgnoreFilter:
-    def __init__(self, tree):
+    def __init__(self, tree, root_dir):
         self.tree = tree
+        self.root_dir = root_dir
         self.ignores = {
             DvcIgnoreDirs([".git", ".hg", ".dvc"]),
             DvcIgnoreRepo(),
         }
-        for root, dirs, files in self.tree.walk(self.tree.tree_root):
+        for root, dirs, files in self.tree.walk(self.root_dir):
             self._update(root)
             dirs[:], files[:] = self(root, dirs, files)
 
@@ -113,31 +116,98 @@ class DvcIgnoreFilter:
 
 
 class CleanTree(BaseTree):
-    def __init__(self, tree):
+    def __init__(self, tree, tree_root=None):
         self.tree = tree
+        if tree_root:
+            self._tree_root = tree_root
+        else:
+            self._tree_root = self.tree.tree_root
 
     @cached_property
     def dvcignore(self):
-        return DvcIgnoreFilter(self.tree)
+        return DvcIgnoreFilter(self.tree, self.tree_root)
 
     @property
     def tree_root(self):
-        return self.tree.tree_root
+        return self._tree_root
 
     def open(self, path, mode="r", encoding="utf-8"):
-        return self.tree.open(path, mode, encoding)
+        if self.isfile(path):
+            return self.tree.open(path, mode, encoding)
+        raise FileNotFoundError
 
     def exists(self, path):
-        return self.tree.exists(path)
+        if self.tree.exists(path) and self._parents_exist(path):
+            if self.tree.isdir(path):
+                return self._valid_dirname(path)
+            return self._valid_filename(path)
+        return False
 
     def isdir(self, path):
-        return self.tree.isdir(path)
+        return (
+            self.tree.isdir(path)
+            and self._parents_exist(path)
+            and self._valid_dirname(path)
+        )
+
+    def _valid_dirname(self, path):
+        path = os.path.abspath(path)
+        if path == self.tree_root:
+            return True
+        dirname, basename = os.path.split(path)
+        dirs, _ = self.dvcignore(dirname, [basename], [])
+        if dirs:
+            return True
+        return False
 
     def isfile(self, path):
-        return self.tree.isfile(path)
+        return (
+            self.tree.isfile(path)
+            and self._parents_exist(path)
+            and self._valid_filename(path)
+        )
+
+    def _valid_filename(self, path):
+        dirname, basename = os.path.split(os.path.normpath(path))
+        _, files = self.dvcignore(os.path.abspath(dirname), [], [basename])
+        if files:
+            return True
+        return False
 
     def isexec(self, path):
-        return self.tree.isexec(path)
+        return self.exists(path) and self.tree.isexec(path)
+
+    def _parents_exist(self, path):
+        from dvc.repo import Repo
+
+        path = PathInfo(path)
+
+        # if parent is tree_root or inside a .dvc dir we can skip this check
+        if path.parent == self.tree_root or Repo.DVC_DIR in path.parts:
+            return True
+
+        # if path is outside of tree, assume this is a local remote/local cache
+        # link/move operation where we do not need to filter ignores
+        path = relpath(path, self.tree_root)
+        if path.startswith("..") or (
+            os.name == "nt"
+            and not os.path.commonprefix(
+                [os.path.abspath(path), self.tree_root]
+            )
+        ):
+            return True
+
+        # check if parent directories are in our ignores, starting from
+        # tree_root
+        for parent_dir in reversed(PathInfo(path).parents):
+            dirname, basename = os.path.split(parent_dir)
+            if basename == ".":
+                # parent_dir == tree_root
+                continue
+            dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
+            if not dirs:
+                return False
+        return True
 
     def walk(self, top, topdown=True):
         for root, dirs, files in self.tree.walk(top, topdown):
@@ -148,4 +218,6 @@ class CleanTree(BaseTree):
             yield root, dirs, files
 
     def stat(self, path):
-        return self.tree.stat(path)
+        if self.exists(path):
+            return self.tree.stat(path)
+        raise FileNotFoundError

dvc/output/base.py

@@ -56,6 +56,12 @@ class BaseOutput:
     PARAM_METRIC_XPATH = "xpath"
     PARAM_PLOT = "plot"
     PARAM_PLOT_TEMPLATE = "template"
+    PARAM_PLOT_X = "x"
+    PARAM_PLOT_Y = "y"
+    PARAM_PLOT_X_LABEL = "x_label"
+    PARAM_PLOT_Y_LABEL = "y_label"
+    PARAM_PLOT_TITLE = "title"
+    PARAM_PLOT_CSV_HEADER = "csv_header"
     PARAM_PERSIST = "persist"
 
     METRIC_SCHEMA = Any(
@@ -175,7 +181,7 @@ class BaseOutput:
 
     @property
     def exists(self):
-        return self.remote.exists(self.path_info)
+        return self.remote.tree.exists(self.path_info)
 
     def save_info(self):
         return self.remote.save_info(self.path_info)
@@ -211,13 +217,13 @@ class BaseOutput:
 
     @property
     def is_empty(self):
-        return self.remote.is_empty(self.path_info)
+        return self.remote.tree.is_empty(self.path_info)
 
     def isdir(self):
-        return self.remote.isdir(self.path_info)
+        return self.remote.tree.isdir(self.path_info)
 
     def isfile(self):
-        return self.remote.isfile(self.path_info)
+        return self.remote.tree.isfile(self.path_info)
 
     def ignore(self):
         if not self.use_scm_ignore:
@@ -245,17 +251,15 @@ class BaseOutput:
             if self.metric or self.plot:
                 self.verify_metric()
             if not self.IS_DEPENDENCY:
-                logger.info(
-                    "Output '{}' doesn't use cache. Skipping saving.".format(
-                        self
-                    )
+                logger.debug(
+                    "Output '%s' doesn't use cache. Skipping saving.", self
                 )
             return
 
         assert not self.IS_DEPENDENCY
 
         if not self.changed():
-            logger.info(f"Output '{self}' didn't change. Skipping saving.")
+            logger.debug("Output '%s' didn't change. Skipping saving.", self)
             return
 
         self.info = self.save_info()
@@ -322,7 +326,7 @@ class BaseOutput:
         )
 
     def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+        self.remote.tree.remove(self.path_info)
         if self.scheme != "local":
             return
 
@@ -333,7 +337,7 @@ class BaseOutput:
         if self.scheme == "local" and self.use_scm_ignore:
             self.repo.scm.ignore_remove(self.fspath)
 
-        self.remote.move(self.path_info, out.path_info)
+        self.remote.tree.move(self.path_info, out.path_info)
         self.def_path = out.def_path
         self.path_info = out.path_info
         self.save()

dvc/remote/azure.py

@@ -8,12 +8,45 @@ from funcy import cached_property, wrap_prop
 
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
 
 
+class AzureRemoteTree(BaseRemoteTree):
+    @property
+    def blob_service(self):
+        return self.remote.blob_service
+
+    def _generate_download_url(self, path_info, expires=3600):
+        from azure.storage.blob import BlobPermissions
+
+        expires_at = datetime.utcnow() + timedelta(seconds=expires)
+
+        sas_token = self.blob_service.generate_blob_shared_access_signature(
+            path_info.bucket,
+            path_info.path,
+            permission=BlobPermissions.READ,
+            expiry=expires_at,
+        )
+        download_url = self.blob_service.make_blob_url(
+            path_info.bucket, path_info.path, sas_token=sas_token
+        )
+        return download_url
+
+    def exists(self, path_info):
+        paths = self.remote.list_paths(path_info.bucket, path_info.path)
+        return any(path_info.path == path for path in paths)
+
+    def remove(self, path_info):
+        if path_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        logger.debug(f"Removing {path_info}")
+        self.blob_service.delete_blob(path_info.bucket, path_info.path)
+
+
 class AzureRemote(BaseRemote):
     scheme = Schemes.AZURE
     path_cls = CloudURLInfo
@@ -21,6 +54,7 @@ class AzureRemote(BaseRemote):
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
     LIST_OBJECT_PAGE_SIZE = 5000
+    TREE_CLS = AzureRemoteTree
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
@@ -65,14 +99,7 @@ class AzureRemote(BaseRemote):
     def get_file_checksum(self, path_info):
         return self.get_etag(path_info)
 
-    def remove(self, path_info):
-        if path_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        logger.debug(f"Removing {path_info}")
-        self.blob_service.delete_blob(path_info.bucket, path_info.path)
-
-    def _list_paths(self, bucket, prefix, progress_callback=None):
+    def list_paths(self, bucket, prefix, progress_callback=None):
         blob_service = self.blob_service
         next_marker = None
         while True:
@@ -97,7 +124,7 @@ class AzureRemote(BaseRemote):
             )
         else:
             prefix = self.path_info.path
-        return self._list_paths(
+        return self.list_paths(
             self.path_info.bucket, prefix, progress_callback
         )
 
@@ -122,23 +149,3 @@ class AzureRemote(BaseRemote):
                 to_file,
                 progress_callback=pbar.update_to,
             )
-
-    def exists(self, path_info):
-        paths = self._list_paths(path_info.bucket, path_info.path)
-        return any(path_info.path == path for path in paths)
-
-    def _generate_download_url(self, path_info, expires=3600):
-        from azure.storage.blob import BlobPermissions
-
-        expires_at = datetime.utcnow() + timedelta(seconds=expires)
-
-        sas_token = self.blob_service.generate_blob_shared_access_signature(
-            path_info.bucket,
-            path_info.path,
-            permission=BlobPermissions.READ,
-            expiry=expires_at,
-        )
-        download_url = self.blob_service.make_blob_url(
-            path_info.bucket, path_info.path, sas_token=sas_token
-        )
-        return download_url

dvc/remote/base.py

@@ -83,12 +83,99 @@ def index_locked(f):
     return wrapper
 
 
+class BaseRemoteTree:
+    SHARED_MODE_MAP = {None: (None, None), "group": (None, None)}
+
+    def __init__(self, remote, config):
+        self.remote = remote
+        shared = config.get("shared")
+        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
+
+    @property
+    def file_mode(self):
+        return self._file_mode
+
+    @property
+    def dir_mode(self):
+        return self._dir_mode
+
+    @property
+    def scheme(self):
+        return self.remote.scheme
+
+    @property
+    def path_cls(self):
+        return self.remote.path_cls
+
+    def open(self, path_info, mode="r", encoding=None):
+        if hasattr(self, "_generate_download_url"):
+            get_url = partial(self._generate_download_url, path_info)
+            return open_url(get_url, mode=mode, encoding=encoding)
+
+        raise RemoteActionNotImplemented("open", self.scheme)
+
+    def exists(self, path_info):
+        raise NotImplementedError
+
+    def isdir(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return False
+
+    def isfile(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return True
+
+    def iscopy(self, path_info):
+        """Check if this file is an independent copy."""
+        return False  # We can't be sure by default
+
+    def walk_files(self, path_info):
+        """Return a generator with `PathInfo`s to all the files"""
+        raise NotImplementedError
+
+    def is_empty(self, path_info):
+        return False
+
+    def remove(self, path_info):
+        raise RemoteActionNotImplemented("remove", self.scheme)
+
+    def makedirs(self, path_info):
+        """Optional: Implement only if the remote needs to create
+        directories before copying/linking/moving data
+        """
+
+    def move(self, from_info, to_info, mode=None):
+        assert mode is None
+        self.copy(from_info, to_info)
+        self.remove(from_info)
+
+    def copy(self, from_info, to_info):
+        raise RemoteActionNotImplemented("copy", self.scheme)
+
+    def copy_fobj(self, fobj, to_info):
+        raise RemoteActionNotImplemented("copy_fobj", self.scheme)
+
+    def symlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("symlink", self.scheme)
+
+    def hardlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("hardlink", self.scheme)
+
+    def reflink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("reflink", self.scheme)
+
+
 class BaseRemote:
     scheme = "base"
     path_cls = URLInfo
     REQUIRES = {}
     JOBS = 4 * cpu_count()
     INDEX_CLS = RemoteIndex
+    TREE_CLS = BaseRemoteTree
 
     PARAM_RELPATH = "relpath"
     CHECKSUM_DIR_SUFFIX = ".dir"
@@ -102,7 +189,6 @@ class BaseRemote:
     CAN_TRAVERSE = True
 
     CACHE_MODE = None
-    SHARED_MODE_MAP = {None: (None, None), "group": (None, None)}
 
     state = StateNoop()
 
@@ -111,9 +197,6 @@ class BaseRemote:
 
         self._check_requires(config)
 
-        shared = config.get("shared")
-        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
-
         self.checksum_jobs = (
             config.get("checksum_jobs")
             or (self.repo and self.repo.config["core"].get("checksum_jobs"))
@@ -134,6 +217,8 @@ class BaseRemote:
         else:
             self.index = RemoteIndexNoop()
 
+        self.tree = self.TREE_CLS(self, config)
+
     @classmethod
     def get_missing_deps(cls):
         import importlib
@@ -218,7 +303,7 @@ class BaseRemote:
         if tree:
             walk_files = tree.walk_files
         else:
-            walk_files = self.walk_files
+            walk_files = self.tree.walk_files
 
         for fname in walk_files(path_info, **kwargs):
             if DvcIgnore.DVCIGNORE_FILE == fname.name:
@@ -273,8 +358,8 @@ class BaseRemote:
         checksum, tmp_info = self._get_dir_info_checksum(dir_info)
         new_info = self.cache.checksum_to_path_info(checksum)
         if self.cache.changed_cache_file(checksum):
-            self.cache.makedirs(new_info.parent)
-            self.cache.move(tmp_info, new_info, mode=self.CACHE_MODE)
+            self.cache.tree.makedirs(new_info.parent)
+            self.cache.tree.move(tmp_info, new_info, mode=self.CACHE_MODE)
 
         if path_info:
             self.state.save(path_info, checksum)
@@ -344,7 +429,7 @@ class BaseRemote:
     def get_checksum(self, path_info):
         assert isinstance(path_info, str) or path_info.scheme == self.scheme
 
-        if not self.exists(path_info):
+        if not self.tree.exists(path_info):
             return None
 
         checksum = self.state.get(path_info)
@@ -355,14 +440,16 @@ class BaseRemote:
         if (
             checksum
             and self.is_dir_checksum(checksum)
-            and not self.exists(self.cache.checksum_to_path_info(checksum))
+            and not self.tree.exists(
+                self.cache.checksum_to_path_info(checksum)
+            )
         ):
             checksum = None
 
         if checksum:
             return checksum
 
-        if self.isdir(path_info):
+        if self.tree.isdir(path_info):
             checksum = self.get_dir_checksum(path_info)
         else:
             checksum = self.get_file_checksum(path_info)
@@ -396,7 +483,7 @@ class BaseRemote:
             "checking if '%s'('%s') has changed.", path_info, checksum_info
         )
 
-        if not self.exists(path_info):
+        if not self.tree.exists(path_info):
             logger.debug("'%s' doesn't exist.", path_info)
             return True
 
@@ -428,9 +515,9 @@ class BaseRemote:
         self._link(from_info, to_info, self.cache_types)
 
     def _link(self, from_info, to_info, link_types):
-        assert self.isfile(from_info)
+        assert self.tree.isfile(from_info)
 
-        self.makedirs(to_info.parent)
+        self.tree.makedirs(to_info.parent)
 
         self._try_links(from_info, to_info, link_types)
 
@@ -438,9 +525,9 @@ class BaseRemote:
         if self.cache_type_confirmed:
             return
 
-        is_link = getattr(self, f"is_{link_type}", None)
+        is_link = getattr(self.tree, f"is_{link_type}", None)
         if is_link and not is_link(path_info):
-            self.remove(path_info)
+            self.tree.remove(path_info)
             raise DvcException(f"failed to verify {link_type}")
 
         self.cache_type_confirmed = True
@@ -448,7 +535,7 @@ class BaseRemote:
     @slow_link_guard
     def _try_links(self, from_info, to_info, link_types):
         while link_types:
-            link_method = getattr(self, link_types[0])
+            link_method = getattr(self.tree, link_types[0])
             try:
                 self._do_link(from_info, to_info, link_method)
                 self._verify_link(to_info, link_types[0])
@@ -463,7 +550,7 @@ class BaseRemote:
         raise DvcException("no possible cache types left to try out.")
 
     def _do_link(self, from_info, to_info, link_method):
-        if self.exists(to_info):
+        if self.tree.exists(to_info):
             raise DvcException(f"Link '{to_info}' already exists!")
 
         link_method(from_info, to_info)
@@ -486,19 +573,21 @@ class BaseRemote:
                     if not (
                         tree.isdvc(path_info, strict=False) and tree.fetch
                     ):
-                        self.copy_fobj(fobj, cache_info)
+                        self.tree.copy_fobj(fobj, cache_info)
                 callback = kwargs.get("download_callback")
                 if callback:
                     callback(1)
         else:
             if self.changed_cache(checksum):
-                self.move(path_info, cache_info, mode=self.CACHE_MODE)
+                self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
                 self.link(cache_info, path_info)
-            elif self.iscopy(path_info) and self._cache_is_copy(path_info):
+            elif self.tree.iscopy(path_info) and self._cache_is_copy(
+                path_info
+            ):
                 # Default relink procedure involves unneeded copy
                 self.unprotect(path_info)
             else:
-                self.remove(path_info)
+                self.tree.remove(path_info)
                 self.link(cache_info, path_info)
 
             if save_link:
@@ -522,14 +611,14 @@ class BaseRemote:
 
         workspace_file = path_info.with_name("." + uuid())
         test_cache_file = self.path_info / ".cache_type_test_file"
-        if not self.exists(test_cache_file):
-            with self.open(test_cache_file, "wb") as fobj:
+        if not self.tree.exists(test_cache_file):
+            with self.tree.open(test_cache_file, "wb") as fobj:
                 fobj.write(bytes(1))
         try:
             self.link(test_cache_file, workspace_file)
         finally:
-            self.remove(workspace_file)
-            self.remove(test_cache_file)
+            self.tree.remove(workspace_file)
+            self.tree.remove(test_cache_file)
 
         self.cache_type_confirmed = True
         return self.cache_types[0] == "copy"
@@ -561,29 +650,6 @@ class BaseRemote:
             self.state.save(path_info, checksum)
         return {self.PARAM_CHECKSUM: checksum}
 
-    def is_empty(self, path_info):
-        return False
-
-    def isfile(self, path_info):
-        """Optional: Overwrite only if the remote has a way to distinguish
-        between a directory and a file.
-        """
-        return True
-
-    def isdir(self, path_info):
-        """Optional: Overwrite only if the remote has a way to distinguish
-        between a directory and a file.
-        """
-        return False
-
-    def iscopy(self, path_info):
-        """Check if this file is an independent copy."""
-        return False  # We can't be sure by default
-
-    def walk_files(self, path_info):
-        """Return a generator with `PathInfo`s to all the files"""
-        raise NotImplementedError
-
     @staticmethod
     def protect(path_info):
         pass
@@ -617,7 +683,7 @@ class BaseRemote:
             isdir = tree.isdir
             save_link = False
         else:
-            isdir = self.isdir
+            isdir = self.tree.isdir
 
         if isdir(path_info):
             return self._save_dir(
@@ -625,6 +691,9 @@ class BaseRemote:
             )
         return self._save_file(path_info, checksum, save_link, tree, **kwargs)
 
+    def open(self, *args, **kwargs):
+        return self.tree.open(*args, **kwargs)
+
     def _handle_transfer_exception(
         self, from_info, to_info, exception, operation
     ):
@@ -680,13 +749,13 @@ class BaseRemote:
             raise NotImplementedError
 
         if to_info.scheme == self.scheme != "local":
-            self.copy(from_info, to_info)
+            self.tree.copy(from_info, to_info)
             return 0
 
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        if self.isdir(from_info):
+        if self.tree.isdir(from_info):
             return self._download_dir(
                 from_info, to_info, name, no_progress_bar, file_mode, dir_mode
             )
@@ -697,7 +766,7 @@ class BaseRemote:
     def _download_dir(
         self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
     ):
-        from_infos = list(self.walk_files(from_info))
+        from_infos = list(self.tree.walk_files(from_info))
         to_infos = (
             to_info / info.relative_to(from_info) for info in from_infos
         )
@@ -744,39 +813,6 @@ class BaseRemote:
 
         return 0
 
-    def open(self, path_info, mode="r", encoding=None):
-        if hasattr(self, "_generate_download_url"):
-            get_url = partial(self._generate_download_url, path_info)
-            return open_url(get_url, mode=mode, encoding=encoding)
-
-        raise RemoteActionNotImplemented("open", self.scheme)
-
-    def remove(self, path_info):
-        raise RemoteActionNotImplemented("remove", self.scheme)
-
-    def move(self, from_info, to_info, mode=None):
-        assert mode is None
-        self.copy(from_info, to_info)
-        self.remove(from_info)
-
-    def copy(self, from_info, to_info):
-        raise RemoteActionNotImplemented("copy", self.scheme)
-
-    def copy_fobj(self, fobj, to_info):
-        raise RemoteActionNotImplemented("copy_fobj", self.scheme)
-
-    def symlink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("symlink", self.scheme)
-
-    def hardlink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("hardlink", self.scheme)
-
-    def reflink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("reflink", self.scheme)
-
-    def exists(self, path_info):
-        raise NotImplementedError
-
     def path_to_checksum(self, path):
         parts = self.path_cls(path).parts[-2:]
 
@@ -849,7 +885,7 @@ class BaseRemote:
             if self.is_dir_checksum(checksum):
                 # backward compatibility
                 self._remove_unpacked_dir(checksum)
-            self.remove(path_info)
+            self.tree.remove(path_info)
             removed = True
         if removed:
             self.index.clear()
@@ -896,9 +932,9 @@ class BaseRemote:
             self.protect(cache_info)
             return False
 
-        if self.exists(cache_info):
+        if self.tree.exists(cache_info):
             logger.warning("corrupted cache file '%s'.", cache_info)
-            self.remove(cache_info)
+            self.tree.remove(cache_info)
 
         return True
 
@@ -1142,7 +1178,7 @@ class BaseRemote:
         ) as pbar:
 
             def exists_with_progress(path_info):
-                ret = self.exists(path_info)
+                ret = self.tree.exists(path_info)
                 pbar.update_msg(str(path_info))
                 return ret
 
@@ -1161,7 +1197,7 @@ class BaseRemote:
         return not self.changed_cache(current)
 
     def safe_remove(self, path_info, force=False):
-        if not self.exists(path_info):
+        if not self.tree.exists(path_info):
             return
 
         if not force and not self.already_cached(path_info):
@@ -1173,7 +1209,7 @@ class BaseRemote:
             if not prompt.confirm(msg):
                 raise ConfirmRemoveError(str(path_info))
 
-        self.remove(path_info)
+        self.tree.remove(path_info)
 
     def _checkout_file(
         self, path_info, checksum, force, progress_callback=None, relink=False
@@ -1181,7 +1217,7 @@ class BaseRemote:
         """The file is changed we need to checkout a new copy"""
         added, modified = True, False
         cache_info = self.checksum_to_path_info(checksum)
-        if self.exists(path_info):
+        if self.tree.exists(path_info):
             logger.debug("data '%s' will be replaced.", path_info)
             self.safe_remove(path_info, force=force)
             added, modified = False, True
@@ -1194,11 +1230,6 @@ class BaseRemote:
 
         return added, modified and not relink
 
-    def makedirs(self, path_info):
-        """Optional: Implement only if the remote needs to create
-        directories before copying/linking/moving data
-        """
-
     def _checkout_dir(
         self,
         path_info,
@@ -1211,9 +1242,9 @@ class BaseRemote:
         added, modified = False, False
         # Create dir separately so that dir is created
         # even if there are no files in it
-        if not self.exists(path_info):
+        if not self.tree.exists(path_info):
             added = True
-            self.makedirs(path_info)
+            self.tree.makedirs(path_info)
 
         dir_info = self.get_dir_cache(checksum)
 
@@ -1249,7 +1280,7 @@ class BaseRemote:
         return added, not added and modified and not relink
 
     def _remove_redundant_files(self, path_info, dir_info, force):
-        existing_files = set(self.walk_files(path_info))
+        existing_files = set(self.tree.walk_files(path_info))
 
         needed_files = {
             path_info / entry[self.PARAM_RELPATH] for entry in dir_info

dvc/remote/gdrive.py

@@ -12,7 +12,7 @@ from funcy.py3 import cat
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.scheme import Schemes
 from dvc.utils import format_link, tmp_fname
 
@@ -87,6 +87,20 @@ class GDriveURLInfo(CloudURLInfo):
         self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
 
 
+class GDriveRemoteTree(BaseRemoteTree):
+    def exists(self, path_info):
+        try:
+            self.remote.get_item_id(path_info)
+        except GDrivePathNotFound:
+            return False
+        else:
+            return True
+
+    def remove(self, path_info):
+        item_id = self.remote.get_item_id(path_info)
+        self.remote.gdrive_delete_file(item_id)
+
+
 class GDriveRemote(BaseRemote):
     scheme = Schemes.GDRIVE
     path_cls = GDriveURLInfo
@@ -95,6 +109,7 @@ class GDriveRemote(BaseRemote):
     # Always prefer traverse for GDrive since API usage quotas are a concern.
     TRAVERSE_WEIGHT_MULTIPLIER = 1
     TRAVERSE_PREFIX_LEN = 2
+    TREE_CLS = GDriveRemoteTree
 
     GDRIVE_CREDENTIALS_DATA = "GDRIVE_CREDENTIALS_DATA"
     DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"
@@ -281,7 +296,7 @@ class GDriveRemote(BaseRemote):
         cache = {
             "dirs": defaultdict(list),
             "ids": {},
-            "root_id": self._get_item_id(
+            "root_id": self.get_item_id(
                 self.path_info,
                 use_cache=False,
                 hint="Confirm the directory exists and you can access it.",
@@ -340,7 +355,7 @@ class GDriveRemote(BaseRemote):
                     "use the `gdrive_user_credentials_file` config\n"
                     "   option if you use multiple GDrive remotes with "
                     "different email accounts.\n\nDetails".format(
-                        item_id, self.path_info, self._credentials_location
+                        item_id, self.path_info, self.credentials_location
                     )
                 ) from exc
             raise
@@ -394,7 +409,7 @@ class GDriveRemote(BaseRemote):
             gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
 
     @_gdrive_retry
-    def _gdrive_delete_file(self, item_id):
+    def gdrive_delete_file(self, item_id):
         from pydrive2.files import ApiRequestError
 
         param = {"id": item_id}
@@ -498,7 +513,7 @@ class GDriveRemote(BaseRemote):
             [self._create_dir(min(parent_ids), title, path)] if create else []
         )
 
-    def _get_item_id(self, path_info, create=False, use_cache=True, hint=None):
+    def get_item_id(self, path_info, create=False, use_cache=True, hint=None):
         assert path_info.bucket == self._bucket
 
         item_ids = self._path_to_item_ids(path_info.path, create, use_cache)
@@ -508,25 +523,17 @@ class GDriveRemote(BaseRemote):
         assert not create
         raise GDrivePathNotFound(path_info, hint)
 
-    def exists(self, path_info):
-        try:
-            self._get_item_id(path_info)
-        except GDrivePathNotFound:
-            return False
-        else:
-            return True
-
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
         dirname = to_info.parent
         assert dirname
-        parent_id = self._get_item_id(dirname, True)
+        parent_id = self.get_item_id(dirname, True)
 
         self._gdrive_upload_file(
             parent_id, to_info.name, no_progress_bar, from_file, name
         )
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        item_id = self._get_item_id(from_info)
+        item_id = self.get_item_id(from_info)
         self._gdrive_download_file(item_id, to_file, name, no_progress_bar)
 
     def list_cache_paths(self, prefix=None, progress_callback=None):
@@ -552,12 +559,5 @@ class GDriveRemote(BaseRemote):
                 self._ids_cache["ids"][parent_id], item["title"]
             )
 
-    def remove(self, path_info):
-        item_id = self._get_item_id(path_info)
-        self._gdrive_delete_file(item_id)
-
     def get_file_checksum(self, path_info):
         raise NotImplementedError
-
-    def walk_files(self, path_info):
-        raise NotImplementedError

dvc/remote/gs.py

@@ -10,7 +10,7 @@ from funcy import cached_property, wrap_prop
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
@@ -65,11 +65,82 @@ def _upload_to_bucket(
             blob.upload_from_file(wrapped)
 
 
+class GSRemoteTree(BaseRemoteTree):
+    @property
+    def gs(self):
+        return self.remote.gs
+
+    def _generate_download_url(self, path_info, expires=3600):
+        expiration = timedelta(seconds=int(expires))
+
+        bucket = self.gs.bucket(path_info.bucket)
+        blob = bucket.get_blob(path_info.path)
+        if blob is None:
+            raise FileNotFoundError
+        return blob.generate_signed_url(expiration=expiration)
+
+    def exists(self, path_info):
+        """Check if the blob exists. If it does not exist,
+        it could be a part of a directory path.
+
+        eg: if `data/file.txt` exists, check for `data` should return True
+        """
+        return self.isfile(path_info) or self.isdir(path_info)
+
+    def isdir(self, path_info):
+        dir_path = path_info / ""
+        return bool(list(self.remote.list_paths(dir_path, max_items=1)))
+
+    def isfile(self, path_info):
+        if path_info.path.endswith("/"):
+            return False
+
+        blob = self.gs.bucket(path_info.bucket).blob(path_info.path)
+        return blob.exists()
+
+    def walk_files(self, path_info):
+        for fname in self.remote.list_paths(path_info / ""):
+            # skip nested empty directories
+            if fname.endswith("/"):
+                continue
+            yield path_info.replace(fname)
+
+    def remove(self, path_info):
+        if path_info.scheme != "gs":
+            raise NotImplementedError
+
+        logger.debug(f"Removing gs://{path_info}")
+        blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
+        if not blob:
+            return
+
+        blob.delete()
+
+    def makedirs(self, path_info):
+        if not path_info.path:
+            return
+
+        self.gs.bucket(path_info.bucket).blob(
+            (path_info / "").path
+        ).upload_from_string("")
+
+    def copy(self, from_info, to_info):
+        from_bucket = self.gs.bucket(from_info.bucket)
+        blob = from_bucket.get_blob(from_info.path)
+        if not blob:
+            msg = f"'{from_info.path}' doesn't exist in the cloud"
+            raise DvcException(msg)
+
+        to_bucket = self.gs.bucket(to_info.bucket)
+        from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path)
+
+
 class GSRemote(BaseRemote):
     scheme = Schemes.GS
     path_cls = CloudURLInfo
     REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
     PARAM_CHECKSUM = "md5"
+    TREE_CLS = GSRemoteTree
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
@@ -105,28 +176,7 @@ class GSRemote(BaseRemote):
         md5 = base64.b64decode(b64_md5)
         return codecs.getencoder("hex")(md5)[0].decode("utf-8")
 
-    def copy(self, from_info, to_info):
-        from_bucket = self.gs.bucket(from_info.bucket)
-        blob = from_bucket.get_blob(from_info.path)
-        if not blob:
-            msg = f"'{from_info.path}' doesn't exist in the cloud"
-            raise DvcException(msg)
-
-        to_bucket = self.gs.bucket(to_info.bucket)
-        from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path)
-
-    def remove(self, path_info):
-        if path_info.scheme != "gs":
-            raise NotImplementedError
-
-        logger.debug(f"Removing gs://{path_info}")
-        blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
-        if not blob:
-            return
-
-        blob.delete()
-
-    def _list_paths(
+    def list_paths(
         self, path_info, max_items=None, prefix=None, progress_callback=None
     ):
         if prefix:
@@ -141,44 +191,10 @@ class GSRemote(BaseRemote):
             yield blob.name
 
     def list_cache_paths(self, prefix=None, progress_callback=None):
-        return self._list_paths(
+        return self.list_paths(
             self.path_info, prefix=prefix, progress_callback=progress_callback
         )
 
-    def walk_files(self, path_info):
-        for fname in self._list_paths(path_info / ""):
-            # skip nested empty directories
-            if fname.endswith("/"):
-                continue
-            yield path_info.replace(fname)
-
-    def makedirs(self, path_info):
-        if not path_info.path:
-            return
-
-        self.gs.bucket(path_info.bucket).blob(
-            (path_info / "").path
-        ).upload_from_string("")
-
-    def isdir(self, path_info):
-        dir_path = path_info / ""
-        return bool(list(self._list_paths(dir_path, max_items=1)))
-
-    def isfile(self, path_info):
-        if path_info.path.endswith("/"):
-            return False
-
-        blob = self.gs.bucket(path_info.bucket).blob(path_info.path)
-        return blob.exists()
-
-    def exists(self, path_info):
-        """Check if the blob exists. If it does not exist,
-        it could be a part of a directory path.
-
-        eg: if `data/file.txt` exists, check for `data` should return True
-        """
-        return self.isfile(path_info) or self.isdir(path_info)
-
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
         bucket = self.gs.bucket(to_info.bucket)
         _upload_to_bucket(
@@ -201,12 +217,3 @@ class GSRemote(BaseRemote):
                 disable=no_progress_bar,
             ) as wrapped:
                 blob.download_to_file(wrapped)
-
-    def _generate_download_url(self, path_info, expires=3600):
-        expiration = timedelta(seconds=int(expires))
-
-        bucket = self.gs.bucket(path_info.bucket)
-        blob = bucket.get_blob(path_info.path)
-        if blob is None:
-            raise FileNotFoundError
-        return blob.generate_signed_url(expiration=expiration)

dvc/remote/hdfs.py

@@ -11,18 +11,75 @@ from urllib.parse import urlparse
 from dvc.scheme import Schemes
 from dvc.utils import fix_env, tmp_fname
 
-from .base import BaseRemote, RemoteCmdError
+from .base import BaseRemote, BaseRemoteTree, RemoteCmdError
 from .pool import get_connection
 
 logger = logging.getLogger(__name__)
 
 
+class HDFSRemoteTree(BaseRemoteTree):
+    @property
+    def hdfs(self):
+        return self.remote.hdfs
+
+    @contextmanager
+    def open(self, path_info, mode="r", encoding=None):
+        assert mode in {"r", "rt", "rb"}
+
+        try:
+            with self.hdfs(path_info) as hdfs, closing(
+                hdfs.open(path_info.path, mode="rb")
+            ) as fd:
+                if mode == "rb":
+                    yield fd
+                else:
+                    yield io.TextIOWrapper(fd, encoding=encoding)
+        except OSError as e:
+            # Empty .errno and not specific enough error class in pyarrow,
+            # see https://issues.apache.org/jira/browse/ARROW-6248
+            if "file does not exist" in str(e):
+                raise FileNotFoundError(*e.args)
+            raise
+
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info.scheme == "hdfs"
+        with self.hdfs(path_info) as hdfs:
+            return hdfs.exists(path_info.path)
+
+    def remove(self, path_info):
+        if path_info.scheme != "hdfs":
+            raise NotImplementedError
+
+        if self.exists(path_info):
+            logger.debug(f"Removing {path_info.path}")
+            with self.hdfs(path_info) as hdfs:
+                hdfs.rm(path_info.path)
+
+    def copy(self, from_info, to_info, **_kwargs):
+        dname = posixpath.dirname(to_info.path)
+        with self.hdfs(to_info) as hdfs:
+            hdfs.mkdir(dname)
+            # NOTE: this is how `hadoop fs -cp` works too: it copies through
+            # your local machine.
+            with hdfs.open(from_info.path, "rb") as from_fobj:
+                tmp_info = to_info.parent / tmp_fname(to_info.name)
+                try:
+                    with hdfs.open(tmp_info.path, "wb") as tmp_fobj:
+                        tmp_fobj.upload(from_fobj)
+                    hdfs.rename(tmp_info.path, to_info.path)
+                except Exception:
+                    self.remove(tmp_info)
+                    raise
+
+
 class HDFSRemote(BaseRemote):
     scheme = Schemes.HDFS
     REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
     PARAM_CHECKSUM = "checksum"
     REQUIRES = {"pyarrow": "pyarrow"}
     TRAVERSE_PREFIX_LEN = 2
+    TREE_CLS = HDFSRemoteTree
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
@@ -91,37 +148,6 @@ class HDFSRemote(BaseRemote):
         )
         return self._group(regex, stdout, "checksum")
 
-    def copy(self, from_info, to_info, **_kwargs):
-        dname = posixpath.dirname(to_info.path)
-        with self.hdfs(to_info) as hdfs:
-            hdfs.mkdir(dname)
-            # NOTE: this is how `hadoop fs -cp` works too: it copies through
-            # your local machine.
-            with hdfs.open(from_info.path, "rb") as from_fobj:
-                tmp_info = to_info.parent / tmp_fname(to_info.name)
-                try:
-                    with hdfs.open(tmp_info.path, "wb") as tmp_fobj:
-                        tmp_fobj.upload(from_fobj)
-                    hdfs.rename(tmp_info.path, to_info.path)
-                except Exception:
-                    self.remove(tmp_info)
-                    raise
-
-    def remove(self, path_info):
-        if path_info.scheme != "hdfs":
-            raise NotImplementedError
-
-        if self.exists(path_info):
-            logger.debug(f"Removing {path_info.path}")
-            with self.hdfs(path_info) as hdfs:
-                hdfs.rm(path_info.path)
-
-    def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info.scheme == "hdfs"
-        with self.hdfs(path_info) as hdfs:
-            return hdfs.exists(path_info.path)
-
     def _upload(self, from_file, to_info, **_kwargs):
         with self.hdfs(to_info) as hdfs:
             hdfs.mkdir(posixpath.dirname(to_info.path))
@@ -135,27 +161,8 @@ class HDFSRemote(BaseRemote):
             with open(to_file, "wb+") as fobj:
                 hdfs.download(from_info.path, fobj)
 
-    @contextmanager
-    def open(self, path_info, mode="r", encoding=None):
-        assert mode in {"r", "rt", "rb"}
-
-        try:
-            with self.hdfs(path_info) as hdfs, closing(
-                hdfs.open(path_info.path, mode="rb")
-            ) as fd:
-                if mode == "rb":
-                    yield fd
-                else:
-                    yield io.TextIOWrapper(fd, encoding=encoding)
-        except OSError as e:
-            # Empty .errno and not specific enough error class in pyarrow,
-            # see https://issues.apache.org/jira/browse/ARROW-6248
-            if "file does not exist" in str(e):
-                raise FileNotFoundError(*e.args)
-            raise
-
     def list_cache_paths(self, prefix=None, progress_callback=None):
-        if not self.exists(self.path_info):
+        if not self.tree.exists(self.path_info):
             return
 
         if prefix:

dvc/remote/http.py

@@ -8,7 +8,7 @@ import dvc.prompt as prompt
 from dvc.exceptions import DvcException, HTTPError
 from dvc.path_info import HTTPURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
@@ -23,6 +23,11 @@ def ask_password(host, user):
     )
 
 
+class HTTPRemoteTree(BaseRemoteTree):
+    def exists(self, path_info):
+        return bool(self.remote.request("HEAD", path_info.url))
+
+
 class HTTPRemote(BaseRemote):
     scheme = Schemes.HTTP
     path_cls = HTTPURLInfo
@@ -32,6 +37,7 @@ class HTTPRemote(BaseRemote):
     CHUNK_SIZE = 2 ** 16
     PARAM_CHECKSUM = "etag"
     CAN_TRAVERSE = False
+    TREE_CLS = HTTPRemoteTree
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
@@ -52,7 +58,7 @@ class HTTPRemote(BaseRemote):
         self.headers = {}
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        response = self._request("GET", from_info.url, stream=True)
+        response = self.request("GET", from_info.url, stream=True)
         if response.status_code != 200:
             raise HTTPError(response.status_code, response.reason)
         with open(to_file, "wb") as fd:
@@ -88,20 +94,17 @@ class HTTPRemote(BaseRemote):
                             break
                         yield chunk
 
-        response = self._request("POST", to_info.url, data=chunks())
+        response = self.request("POST", to_info.url, data=chunks())
         if response.status_code not in (200, 201):
             raise HTTPError(response.status_code, response.reason)
 
-    def exists(self, path_info):
-        return bool(self._request("HEAD", path_info.url))
-
     def _content_length(self, response):
         res = response.headers.get("Content-Length")
         return int(res) if res else None
 
     def get_file_checksum(self, path_info):
         url = path_info.url
-        headers = self._request("HEAD", url).headers
+        headers = self.request("HEAD", url).headers
         etag = headers.get("ETag") or headers.get("Content-MD5")
 
         if not etag:
@@ -155,7 +158,7 @@ class HTTPRemote(BaseRemote):
 
         return session
 
-    def _request(self, method, url, **kwargs):
+    def request(self, method, url, **kwargs):
         import requests
 
         kwargs.setdefault("allow_redirects", True)

dvc/remote/local.py

@@ -17,6 +17,7 @@ from dvc.remote.base import (
     STATUS_MISSING,
     STATUS_NEW,
     BaseRemote,
+    BaseRemoteTree,
     index_locked,
 )
 from dvc.remote.index import RemoteIndexNoop
@@ -36,37 +37,12 @@ from dvc.utils.fs import (
 logger = logging.getLogger(__name__)
 
 
-class LocalRemote(BaseRemote):
-    scheme = Schemes.LOCAL
-    path_cls = PathInfo
-    PARAM_CHECKSUM = "md5"
-    PARAM_PATH = "path"
-    TRAVERSE_PREFIX_LEN = 2
-    INDEX_CLS = RemoteIndexNoop
-
-    UNPACKED_DIR_SUFFIX = ".unpacked"
-
-    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
-
-    CACHE_MODE = 0o444
+class LocalRemoteTree(BaseRemoteTree):
     SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-        self.cache_dir = config.get("url")
-        self._dir_info = {}
-
     @property
-    def state(self):
-        return self.repo.state
-
-    @property
-    def cache_dir(self):
-        return self.path_info.fspath if self.path_info else None
-
-    @cache_dir.setter
-    def cache_dir(self, value):
-        self.path_info = PathInfo(value) if value else None
+    def repo(self):
+        return self.remote.repo
 
     @cached_property
     def _work_tree(self):
@@ -83,42 +59,9 @@ class LocalRemote(BaseRemote):
             return self._work_tree
         return None
 
-    @classmethod
-    def supported(cls, config):
-        return True
-
-    @cached_property
-    def cache_path(self):
-        return os.path.abspath(self.cache_dir)
-
-    def checksum_to_path(self, checksum):
-        # NOTE: `self.cache_path` is already normalized so we can simply use
-        # `os.sep` instead of `os.path.join`. This results in this helper
-        # being ~5.5 times faster.
-        return (
-            f"{self.cache_path}{os.sep}{checksum[0:2]}{os.sep}{checksum[2:]}"
-        )
-
-    def list_cache_paths(self, prefix=None, progress_callback=None):
-        assert self.path_info is not None
-        if prefix:
-            path_info = self.path_info / prefix[:2]
-            if not self.exists(path_info):
-                return
-        else:
-            path_info = self.path_info
-        if progress_callback:
-            for path in walk_files(path_info):
-                progress_callback()
-                yield path
-        else:
-            yield from walk_files(path_info)
-
-    def get(self, md5):
-        if not md5:
-            return None
-
-        return self.checksum_to_path_info(md5).url
+    @staticmethod
+    def open(path_info, mode="r", encoding=None):
+        return open(path_info, mode=mode, encoding=encoding)
 
     def exists(self, path_info):
         assert isinstance(path_info, str) or path_info.scheme == "local"
@@ -128,36 +71,6 @@ class LocalRemote(BaseRemote):
             return True
         return self.repo.tree.exists(path_info)
 
-    def makedirs(self, path_info):
-        makedirs(path_info, exist_ok=True, mode=self._dir_mode)
-
-    def already_cached(self, path_info):
-        assert path_info.scheme in ["", "local"]
-
-        current_md5 = self.get_checksum(path_info)
-
-        if not current_md5:
-            return False
-
-        return not self.changed_cache(current_md5)
-
-    def _verify_link(self, path_info, link_type):
-        if link_type == "hardlink" and self.getsize(path_info) == 0:
-            return
-
-        super()._verify_link(path_info, link_type)
-
-    def is_empty(self, path_info):
-        path = path_info.fspath
-
-        if self.isfile(path_info) and os.path.getsize(path) == 0:
-            return True
-
-        if self.isdir(path_info) and len(os.listdir(path)) == 0:
-            return True
-
-        return False
-
     def isfile(self, path_info):
         if not self.repo:
             return os.path.isfile(path_info)
@@ -177,10 +90,6 @@ class LocalRemote(BaseRemote):
             System.is_symlink(path_info) or System.is_hardlink(path_info)
         )
 
-    @staticmethod
-    def getsize(path_info):
-        return os.path.getsize(path_info)
-
     def walk_files(self, path_info):
         if self.work_tree:
             tree = self.work_tree
@@ -189,8 +98,16 @@ class LocalRemote(BaseRemote):
         for fname in tree.walk_files(path_info):
             yield PathInfo(fname)
 
-    def get_file_checksum(self, path_info):
-        return file_md5(path_info)[0]
+    def is_empty(self, path_info):
+        path = path_info.fspath
+
+        if self.isfile(path_info) and os.path.getsize(path) == 0:
+            return True
+
+        if self.isdir(path_info) and len(os.listdir(path)) == 0:
+            return True
+
+        return False
 
     def remove(self, path_info):
         if isinstance(path_info, PathInfo):
@@ -203,6 +120,9 @@ class LocalRemote(BaseRemote):
         if self.exists(path):
             remove(path)
 
+    def makedirs(self, path_info):
+        makedirs(path_info, exist_ok=True, mode=self.dir_mode)
+
     def move(self, from_info, to_info, mode=None):
         if from_info.scheme != "local" or to_info.scheme != "local":
             raise NotImplementedError
@@ -211,9 +131,9 @@ class LocalRemote(BaseRemote):
 
         if mode is None:
             if self.isfile(from_info):
-                mode = self._file_mode
+                mode = self.file_mode
             else:
-                mode = self._dir_mode
+                mode = self.dir_mode
 
         move(from_info, to_info, mode=mode)
 
@@ -221,7 +141,7 @@ class LocalRemote(BaseRemote):
         tmp_info = to_info.parent / tmp_fname(to_info.name)
         try:
             System.copy(from_info, tmp_info)
-            os.chmod(tmp_info, self._file_mode)
+            os.chmod(tmp_info, self.file_mode)
             os.rename(tmp_info, to_info)
         except Exception:
             self.remove(tmp_info)
@@ -232,7 +152,7 @@ class LocalRemote(BaseRemote):
         tmp_info = to_info.parent / tmp_fname(to_info.name)
         try:
             copy_fobj_to_file(fobj, tmp_info)
-            os.chmod(tmp_info, self._file_mode)
+            os.chmod(tmp_info, self.file_mode)
             os.rename(tmp_info, to_info)
         except Exception:
             self.remove(tmp_info)
@@ -281,9 +201,102 @@ class LocalRemote(BaseRemote):
         System.reflink(from_info, tmp_info)
         # NOTE: reflink has its own separate inode, so you can set permissions
         # that are different from the source.
-        os.chmod(tmp_info, self._file_mode)
+        os.chmod(tmp_info, self.file_mode)
         os.rename(tmp_info, to_info)
 
+    @staticmethod
+    def getsize(path_info):
+        return os.path.getsize(path_info)
+
+
+class LocalRemote(BaseRemote):
+    scheme = Schemes.LOCAL
+    path_cls = PathInfo
+    PARAM_CHECKSUM = "md5"
+    PARAM_PATH = "path"
+    TRAVERSE_PREFIX_LEN = 2
+    INDEX_CLS = RemoteIndexNoop
+    TREE_CLS = LocalRemoteTree
+
+    UNPACKED_DIR_SUFFIX = ".unpacked"
+
+    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
+
+    CACHE_MODE = 0o444
+
+    def __init__(self, repo, config):
+        super().__init__(repo, config)
+        self.cache_dir = config.get("url")
+        self._dir_info = {}
+
+    @property
+    def state(self):
+        return self.repo.state
+
+    @property
+    def cache_dir(self):
+        return self.path_info.fspath if self.path_info else None
+
+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.path_info = PathInfo(value) if value else None
+
+    @classmethod
+    def supported(cls, config):
+        return True
+
+    @cached_property
+    def cache_path(self):
+        return os.path.abspath(self.cache_dir)
+
+    def checksum_to_path(self, checksum):
+        # NOTE: `self.cache_path` is already normalized so we can simply use
+        # `os.sep` instead of `os.path.join`. This results in this helper
+        # being ~5.5 times faster.
+        return (
+            f"{self.cache_path}{os.sep}{checksum[0:2]}{os.sep}{checksum[2:]}"
+        )
+
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        assert self.path_info is not None
+        if prefix:
+            path_info = self.path_info / prefix[:2]
+            if not self.tree.exists(path_info):
+                return
+        else:
+            path_info = self.path_info
+        if progress_callback:
+            for path in walk_files(path_info):
+                progress_callback()
+                yield path
+        else:
+            yield from walk_files(path_info)
+
+    def get(self, md5):
+        if not md5:
+            return None
+
+        return self.checksum_to_path_info(md5).url
+
+    def already_cached(self, path_info):
+        assert path_info.scheme in ["", "local"]
+
+        current_md5 = self.get_checksum(path_info)
+
+        if not current_md5:
+            return False
+
+        return not self.changed_cache(current_md5)
+
+    def _verify_link(self, path_info, link_type):
+        if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
+            return
+
+        super()._verify_link(path_info, link_type)
+
+    def get_file_checksum(self, path_info):
+        return file_md5(path_info)[0]
+
     def cache_exists(self, checksums, jobs=None, name=None):
         return [
             checksum
@@ -316,10 +329,6 @@ class LocalRemote(BaseRemote):
             from_info, to_file, no_progress_bar=no_progress_bar, name=name
         )
 
-    @staticmethod
-    def open(path_info, mode="r", encoding=None):
-        return open(path_info, mode=mode, encoding=encoding)
-
     @index_locked
     def status(
         self,
@@ -503,8 +512,8 @@ class LocalRemote(BaseRemote):
         if download:
             func = partial(
                 remote.download,
-                dir_mode=self._dir_mode,
-                file_mode=self._file_mode,
+                dir_mode=self.tree.dir_mode,
+                file_mode=self.tree.file_mode,
             )
             status = STATUS_DELETED
             desc = "Downloading"
@@ -662,7 +671,7 @@ class LocalRemote(BaseRemote):
                 "a symlink or a hardlink.".format(path)
             )
 
-        os.chmod(path, self._file_mode)
+        os.chmod(path, self.tree.file_mode)
 
     def _unprotect_dir(self, path):
         assert is_working_tree(self.repo.tree)
@@ -703,7 +712,7 @@ class LocalRemote(BaseRemote):
     def _remove_unpacked_dir(self, checksum):
         info = self.checksum_to_path_info(checksum)
         path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
-        self.remove(path_info)
+        self.tree.remove(path_info)
 
     def is_protected(self, path_info):
         try:

dvc/remote/oss.py

@@ -7,12 +7,34 @@ from funcy import cached_property, wrap_prop
 
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
 
 
+class OSSRemoteTree(BaseRemoteTree):
+    @property
+    def oss_service(self):
+        return self.remote.oss_service
+
+    def _generate_download_url(self, path_info, expires=3600):
+        assert path_info.bucket == self.remote.path_info.bucket
+
+        return self.oss_service.sign_url("GET", path_info.path, expires)
+
+    def exists(self, path_info):
+        paths = self.remote.list_paths(path_info.path)
+        return any(path_info.path == path for path in paths)
+
+    def remove(self, path_info):
+        if path_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        logger.debug(f"Removing oss://{path_info}")
+        self.oss_service.delete_object(path_info.path)
+
+
 class OSSRemote(BaseRemote):
     """
     oss2 document:
@@ -38,6 +60,7 @@ class OSSRemote(BaseRemote):
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
     LIST_OBJECT_PAGE_SIZE = 100
+    TREE_CLS = OSSRemoteTree
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
@@ -83,14 +106,7 @@ class OSSRemote(BaseRemote):
             )
         return bucket
 
-    def remove(self, path_info):
-        if path_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        logger.debug(f"Removing oss://{path_info}")
-        self.oss_service.delete_object(path_info.path)
-
-    def _list_paths(self, prefix, progress_callback=None):
+    def list_paths(self, prefix, progress_callback=None):
         import oss2
 
         for blob in oss2.ObjectIterator(self.oss_service, prefix=prefix):
@@ -105,7 +121,7 @@ class OSSRemote(BaseRemote):
             )
         else:
             prefix = self.path_info.path
-        return self._list_paths(prefix, progress_callback)
+        return self.list_paths(prefix, progress_callback)
 
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
@@ -122,12 +138,3 @@ class OSSRemote(BaseRemote):
             self.oss_service.get_object_to_file(
                 from_info.path, to_file, progress_callback=pbar.update_to
             )
-
-    def _generate_download_url(self, path_info, expires=3600):
-        assert path_info.bucket == self.path_info.bucket
-
-        return self.oss_service.sign_url("GET", path_info.path, expires)
-
-    def exists(self, path_info):
-        paths = self._list_paths(path_info.path)
-        return any(path_info.path == path for path in paths)

dvc/remote/s3.py

@@ -9,17 +9,197 @@ from dvc.config import ConfigError
 from dvc.exceptions import DvcException, ETagMismatchError
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
 
 
+class S3RemoteTree(BaseRemoteTree):
+    @property
+    def s3(self):
+        return self.remote.s3
+
+    def _generate_download_url(self, path_info, expires=3600):
+        params = {"Bucket": path_info.bucket, "Key": path_info.path}
+        return self.s3.generate_presigned_url(
+            ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
+        )
+
+    def exists(self, path_info):
+        """Check if the blob exists. If it does not exist,
+        it could be a part of a directory path.
+
+        eg: if `data/file.txt` exists, check for `data` should return True
+        """
+        return self.isfile(path_info) or self.isdir(path_info)
+
+    def isdir(self, path_info):
+        # S3 doesn't have a concept for directories.
+        #
+        # Using `head_object` with a path pointing to a directory
+        # will throw a 404 error.
+        #
+        # A reliable way to know if a given path is a directory is by
+        # checking if there are more files sharing the same prefix
+        # with a `list_objects` call.
+        #
+        # We need to make sure that the path ends with a forward slash,
+        # since we can end with false-positives like the following example:
+        #
+        #       bucket
+        #       └── data
+        #          ├── alice
+        #          └── alpha
+        #
+        # Using `data/al` as prefix will return `[data/alice, data/alpha]`,
+        # While `data/al/` will return nothing.
+        #
+        dir_path = path_info / ""
+        return bool(list(self.remote.list_paths(dir_path, max_items=1)))
+
+    def isfile(self, path_info):
+        from botocore.exceptions import ClientError
+
+        if path_info.path.endswith("/"):
+            return False
+
+        try:
+            self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
+        except ClientError as exc:
+            if exc.response["Error"]["Code"] != "404":
+                raise
+            return False
+
+        return True
+
+    def walk_files(self, path_info, max_items=None):
+        for fname in self.remote.list_paths(path_info / "", max_items):
+            if fname.endswith("/"):
+                continue
+
+            yield path_info.replace(path=fname)
+
+    def remove(self, path_info):
+        if path_info.scheme != "s3":
+            raise NotImplementedError
+
+        logger.debug(f"Removing {path_info}")
+        self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
+
+    def makedirs(self, path_info):
+        # We need to support creating empty directories, which means
+        # creating an object with an empty body and a trailing slash `/`.
+        #
+        # We are not creating directory objects for every parent prefix,
+        # as it is not required.
+        if not path_info.path:
+            return
+
+        dir_path = path_info / ""
+        self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
+
+    def copy(self, from_info, to_info):
+        self._copy(self.s3, from_info, to_info, self.remote.extra_args)
+
+    @classmethod
+    def _copy_multipart(
+        cls, s3, from_info, to_info, size, n_parts, extra_args
+    ):
+        mpu = s3.create_multipart_upload(
+            Bucket=to_info.bucket, Key=to_info.path, **extra_args
+        )
+        mpu_id = mpu["UploadId"]
+
+        parts = []
+        byte_position = 0
+        for i in range(1, n_parts + 1):
+            obj = S3Remote.get_head_object(
+                s3, from_info.bucket, from_info.path, PartNumber=i
+            )
+            part_size = obj["ContentLength"]
+            lastbyte = byte_position + part_size - 1
+            if lastbyte > size:
+                lastbyte = size - 1
+
+            srange = f"bytes={byte_position}-{lastbyte}"
+
+            part = s3.upload_part_copy(
+                Bucket=to_info.bucket,
+                Key=to_info.path,
+                PartNumber=i,
+                UploadId=mpu_id,
+                CopySourceRange=srange,
+                CopySource={"Bucket": from_info.bucket, "Key": from_info.path},
+            )
+            parts.append(
+                {"PartNumber": i, "ETag": part["CopyPartResult"]["ETag"]}
+            )
+            byte_position += part_size
+
+        assert n_parts == len(parts)
+
+        s3.complete_multipart_upload(
+            Bucket=to_info.bucket,
+            Key=to_info.path,
+            UploadId=mpu_id,
+            MultipartUpload={"Parts": parts},
+        )
+
+    @classmethod
+    def _copy(cls, s3, from_info, to_info, extra_args):
+        # NOTE: object's etag depends on the way it was uploaded to s3 or the
+        # way it was copied within the s3. More specifically, it depends on
+        # the chunk size that was used to transfer it, which would affect
+        # whether an object would be uploaded as a single part or as a
+        # multipart.
+        #
+        # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
+        # then it was transferred as a single part, which means that the chunk
+        # size used to transfer it was greater or equal to the ContentLength
+        # of that object. So to preserve that tag over the next transfer, we
+        # could use any value >= ContentLength.
+        #
+        # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
+        # then it was transferred as a multipart, which means that the chunk
+        # size used to transfer it was less than ContentLength of that object.
+        # Unfortunately, in general, it doesn't mean that the chunk size was
+        # the same throughout the transfer, so it means that in order to
+        # preserve etag, we need to transfer each part separately, so the
+        # object is transfered in the same chunks as it was originally.
+        from boto3.s3.transfer import TransferConfig
+
+        obj = S3Remote.get_head_object(s3, from_info.bucket, from_info.path)
+        etag = obj["ETag"].strip('"')
+        size = obj["ContentLength"]
+
+        _, _, parts_suffix = etag.partition("-")
+        if parts_suffix:
+            n_parts = int(parts_suffix)
+            cls._copy_multipart(
+                s3, from_info, to_info, size, n_parts, extra_args=extra_args
+            )
+        else:
+            source = {"Bucket": from_info.bucket, "Key": from_info.path}
+            s3.copy(
+                source,
+                to_info.bucket,
+                to_info.path,
+                ExtraArgs=extra_args,
+                Config=TransferConfig(multipart_threshold=size + 1),
+            )
+
+        cached_etag = S3Remote.get_etag(s3, to_info.bucket, to_info.path)
+        if etag != cached_etag:
+            raise ETagMismatchError(etag, cached_etag)
+
+
 class S3Remote(BaseRemote):
     scheme = Schemes.S3
     path_cls = CloudURLInfo
     REQUIRES = {"boto3": "boto3"}
     PARAM_CHECKSUM = "etag"
+    TREE_CLS = S3RemoteTree
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
@@ -89,107 +269,6 @@ class S3Remote(BaseRemote):
             raise DvcException(f"s3://{bucket}/{path} does not exist") from exc
         return obj
 
-    @classmethod
-    def _copy_multipart(
-        cls, s3, from_info, to_info, size, n_parts, extra_args
-    ):
-        mpu = s3.create_multipart_upload(
-            Bucket=to_info.bucket, Key=to_info.path, **extra_args
-        )
-        mpu_id = mpu["UploadId"]
-
-        parts = []
-        byte_position = 0
-        for i in range(1, n_parts + 1):
-            obj = cls.get_head_object(
-                s3, from_info.bucket, from_info.path, PartNumber=i
-            )
-            part_size = obj["ContentLength"]
-            lastbyte = byte_position + part_size - 1
-            if lastbyte > size:
-                lastbyte = size - 1
-
-            srange = f"bytes={byte_position}-{lastbyte}"
-
-            part = s3.upload_part_copy(
-                Bucket=to_info.bucket,
-                Key=to_info.path,
-                PartNumber=i,
-                UploadId=mpu_id,
-                CopySourceRange=srange,
-                CopySource={"Bucket": from_info.bucket, "Key": from_info.path},
-            )
-            parts.append(
-                {"PartNumber": i, "ETag": part["CopyPartResult"]["ETag"]}
-            )
-            byte_position += part_size
-
-        assert n_parts == len(parts)
-
-        s3.complete_multipart_upload(
-            Bucket=to_info.bucket,
-            Key=to_info.path,
-            UploadId=mpu_id,
-            MultipartUpload={"Parts": parts},
-        )
-
-    @classmethod
-    def _copy(cls, s3, from_info, to_info, extra_args):
-        # NOTE: object's etag depends on the way it was uploaded to s3 or the
-        # way it was copied within the s3. More specifically, it depends on
-        # the chunk size that was used to transfer it, which would affect
-        # whether an object would be uploaded as a single part or as a
-        # multipart.
-        #
-        # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
-        # then it was transferred as a single part, which means that the chunk
-        # size used to transfer it was greater or equal to the ContentLength
-        # of that object. So to preserve that tag over the next transfer, we
-        # could use any value >= ContentLength.
-        #
-        # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
-        # then it was transferred as a multipart, which means that the chunk
-        # size used to transfer it was less than ContentLength of that object.
-        # Unfortunately, in general, it doesn't mean that the chunk size was
-        # the same throughout the transfer, so it means that in order to
-        # preserve etag, we need to transfer each part separately, so the
-        # object is transfered in the same chunks as it was originally.
-        from boto3.s3.transfer import TransferConfig
-
-        obj = cls.get_head_object(s3, from_info.bucket, from_info.path)
-        etag = obj["ETag"].strip('"')
-        size = obj["ContentLength"]
-
-        _, _, parts_suffix = etag.partition("-")
-        if parts_suffix:
-            n_parts = int(parts_suffix)
-            cls._copy_multipart(
-                s3, from_info, to_info, size, n_parts, extra_args=extra_args
-            )
-        else:
-            source = {"Bucket": from_info.bucket, "Key": from_info.path}
-            s3.copy(
-                source,
-                to_info.bucket,
-                to_info.path,
-                ExtraArgs=extra_args,
-                Config=TransferConfig(multipart_threshold=size + 1),
-            )
-
-        cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
-        if etag != cached_etag:
-            raise ETagMismatchError(etag, cached_etag)
-
-    def copy(self, from_info, to_info):
-        self._copy(self.s3, from_info, to_info, self.extra_args)
-
-    def remove(self, path_info):
-        if path_info.scheme != "s3":
-            raise NotImplementedError
-
-        logger.debug(f"Removing {path_info}")
-        self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
-
     def _list_objects(
         self, path_info, max_items=None, prefix=None, progress_callback=None
     ):
@@ -211,7 +290,7 @@ class S3Remote(BaseRemote):
             else:
                 yield from contents
 
-    def _list_paths(
+    def list_paths(
         self, path_info, max_items=None, prefix=None, progress_callback=None
     ):
         return (
@@ -222,69 +301,10 @@ class S3Remote(BaseRemote):
         )
 
     def list_cache_paths(self, prefix=None, progress_callback=None):
-        return self._list_paths(
+        return self.list_paths(
             self.path_info, prefix=prefix, progress_callback=progress_callback
         )
 
-    def isfile(self, path_info):
-        from botocore.exceptions import ClientError
-
-        if path_info.path.endswith("/"):
-            return False
-
-        try:
-            self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
-        except ClientError as exc:
-            if exc.response["Error"]["Code"] != "404":
-                raise
-            return False
-
-        return True
-
-    def exists(self, path_info):
-        """Check if the blob exists. If it does not exist,
-        it could be a part of a directory path.
-
-        eg: if `data/file.txt` exists, check for `data` should return True
-        """
-        return self.isfile(path_info) or self.isdir(path_info)
-
-    def makedirs(self, path_info):
-        # We need to support creating empty directories, which means
-        # creating an object with an empty body and a trailing slash `/`.
-        #
-        # We are not creating directory objects for every parent prefix,
-        # as it is not required.
-        if not path_info.path:
-            return
-
-        dir_path = path_info / ""
-        self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
-
-    def isdir(self, path_info):
-        # S3 doesn't have a concept for directories.
-        #
-        # Using `head_object` with a path pointing to a directory
-        # will throw a 404 error.
-        #
-        # A reliable way to know if a given path is a directory is by
-        # checking if there are more files sharing the same prefix
-        # with a `list_objects` call.
-        #
-        # We need to make sure that the path ends with a forward slash,
-        # since we can end with false-positives like the following example:
-        #
-        #       bucket
-        #       └── data
-        #          ├── alice
-        #          └── alpha
-        #
-        # Using `data/al` as prefix will return `[data/alice, data/alpha]`,
-        # While `data/al/` will return nothing.
-        #
-        dir_path = path_info / ""
-        return bool(list(self._list_paths(dir_path, max_items=1)))
-
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
         total = os.path.getsize(from_file)
         with Tqdm(
@@ -312,19 +332,6 @@ class S3Remote(BaseRemote):
                 from_info.bucket, from_info.path, to_file, Callback=pbar.update
             )
 
-    def _generate_download_url(self, path_info, expires=3600):
-        params = {"Bucket": path_info.bucket, "Key": path_info.path}
-        return self.s3.generate_presigned_url(
-            ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
-        )
-
-    def walk_files(self, path_info, max_items=None):
-        for fname in self._list_paths(path_info / "", max_items):
-            if fname.endswith("/"):
-                continue
-
-            yield path_info.replace(path=fname)
-
     def _append_aws_grants_to_extra_args(self, config):
         # Keys for extra_args can be one of the following list:
         # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS

dvc/remote/ssh/__init__.py

@@ -14,7 +14,7 @@ from funcy import first, memoize, silent, wrap_with
 
 import dvc.prompt as prompt
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.remote.pool import get_connection
 from dvc.scheme import Schemes
 from dvc.utils import to_chunks
@@ -33,6 +33,105 @@ def ask_password(host, user, port):
     )
 
 
+class SSHRemoteTree(BaseRemoteTree):
+    @property
+    def ssh(self):
+        return self.remote.ssh
+
+    @contextmanager
+    def open(self, path_info, mode="r", encoding=None):
+        assert mode in {"r", "rt", "rb", "wb"}
+
+        with self.ssh(path_info) as ssh, closing(
+            ssh.sftp.open(path_info.path, mode)
+        ) as fd:
+            if "b" in mode:
+                yield fd
+            else:
+                yield io.TextIOWrapper(fd, encoding=encoding)
+
+    def exists(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.exists(path_info.path)
+
+    def isdir(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.isdir(path_info.path)
+
+    def isfile(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.isfile(path_info.path)
+
+    def walk_files(self, path_info):
+        with self.ssh(path_info) as ssh:
+            for fname in ssh.walk_files(path_info.path):
+                yield path_info.replace(path=fname)
+
+    def remove(self, path_info):
+        if path_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(path_info) as ssh:
+            ssh.remove(path_info.path)
+
+    def makedirs(self, path_info):
+        with self.ssh(path_info) as ssh:
+            ssh.makedirs(path_info.path)
+
+    def move(self, from_info, to_info, mode=None):
+        assert mode is None
+        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.move(from_info.path, to_info.path)
+
+    def copy(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.atomic_copy(from_info.path, to_info.path)
+
+    def symlink(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.symlink(from_info.path, to_info.path)
+
+    def hardlink(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        # See dvc/remote/local/__init__.py - hardlink()
+        if self.getsize(from_info) == 0:
+
+            with self.ssh(to_info) as ssh:
+                ssh.sftp.open(to_info.path, "w").close()
+
+            logger.debug(
+                "Created empty file: {src} -> {dest}".format(
+                    src=str(from_info), dest=str(to_info)
+                )
+            )
+            return
+
+        with self.ssh(from_info) as ssh:
+            ssh.hardlink(from_info.path, to_info.path)
+
+    def reflink(self, from_info, to_info):
+        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.reflink(from_info.path, to_info.path)
+
+    def getsize(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.getsize(path_info.path)
+
+
 class SSHRemote(BaseRemote):
     scheme = Schemes.SSH
     REQUIRES = {"paramiko": "paramiko"}
@@ -46,6 +145,7 @@ class SSHRemote(BaseRemote):
     # We use conservative setting of 4 instead to not exhaust max sessions.
     CHECKSUM_JOBS = 4
     TRAVERSE_PREFIX_LEN = 2
+    TREE_CLS = SSHRemoteTree
 
     DEFAULT_CACHE_TYPES = ["copy"]
 
@@ -148,10 +248,6 @@ class SSHRemote(BaseRemote):
             sock=self.sock,
         )
 
-    def exists(self, path_info):
-        with self.ssh(path_info) as ssh:
-            return ssh.exists(path_info.path)
-
     def get_file_checksum(self, path_info):
         if path_info.scheme != self.scheme:
             raise NotImplementedError
@@ -159,74 +255,6 @@ class SSHRemote(BaseRemote):
         with self.ssh(path_info) as ssh:
             return ssh.md5(path_info.path)
 
-    def isdir(self, path_info):
-        with self.ssh(path_info) as ssh:
-            return ssh.isdir(path_info.path)
-
-    def isfile(self, path_info):
-        with self.ssh(path_info) as ssh:
-            return ssh.isfile(path_info.path)
-
-    def getsize(self, path_info):
-        with self.ssh(path_info) as ssh:
-            return ssh.getsize(path_info.path)
-
-    def copy(self, from_info, to_info):
-        if not from_info.scheme == to_info.scheme == self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(from_info) as ssh:
-            ssh.atomic_copy(from_info.path, to_info.path)
-
-    def symlink(self, from_info, to_info):
-        if not from_info.scheme == to_info.scheme == self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(from_info) as ssh:
-            ssh.symlink(from_info.path, to_info.path)
-
-    def hardlink(self, from_info, to_info):
-        if not from_info.scheme == to_info.scheme == self.scheme:
-            raise NotImplementedError
-
-        # See dvc/remote/local/__init__.py - hardlink()
-        if self.getsize(from_info) == 0:
-
-            with self.ssh(to_info) as ssh:
-                ssh.sftp.open(to_info.path, "w").close()
-
-            logger.debug(
-                "Created empty file: {src} -> {dest}".format(
-                    src=str(from_info), dest=str(to_info)
-                )
-            )
-            return
-
-        with self.ssh(from_info) as ssh:
-            ssh.hardlink(from_info.path, to_info.path)
-
-    def reflink(self, from_info, to_info):
-        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(from_info) as ssh:
-            ssh.reflink(from_info.path, to_info.path)
-
-    def remove(self, path_info):
-        if path_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(path_info) as ssh:
-            ssh.remove(path_info.path)
-
-    def move(self, from_info, to_info, mode=None):
-        assert mode is None
-        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(from_info) as ssh:
-            ssh.move(from_info.path, to_info.path)
-
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
         assert from_info.isin(self.path_info)
         with self.ssh(self.path_info) as ssh:
@@ -247,18 +275,6 @@ class SSHRemote(BaseRemote):
                 no_progress_bar=no_progress_bar,
             )
 
-    @contextmanager
-    def open(self, path_info, mode="r", encoding=None):
-        assert mode in {"r", "rt", "rb", "wb"}
-
-        with self.ssh(path_info) as ssh, closing(
-            ssh.sftp.open(path_info.path, mode)
-        ) as fd:
-            if "b" in mode:
-                yield fd
-            else:
-                yield io.TextIOWrapper(fd, encoding=encoding)
-
     def list_cache_paths(self, prefix=None, progress_callback=None):
         if prefix:
             root = posixpath.join(self.path_info.path, prefix[:2])
@@ -275,15 +291,6 @@ class SSHRemote(BaseRemote):
             else:
                 yield from ssh.walk_files(root)
 
-    def walk_files(self, path_info):
-        with self.ssh(path_info) as ssh:
-            for fname in ssh.walk_files(path_info.path):
-                yield path_info.replace(path=fname)
-
-    def makedirs(self, path_info):
-        with self.ssh(path_info) as ssh:
-            ssh.makedirs(path_info.path)
-
     def batch_exists(self, path_infos, callback):
         def _exists(chunk_and_channel):
             chunk, channel = chunk_and_channel

dvc/remote/ssh/connection.py

@@ -237,11 +237,15 @@ class SSHConnection:
             or channel.recv_ready()
             or channel.recv_stderr_ready()
         ):
-            import select
+            import selectors
+
+            selector = selectors.DefaultSelector()
+            selector.register(stdout.channel, selectors.EVENT_READ)
 
             got_chunk = False
-            readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
-            for c in readq:
+            events = selector.select(self.timeout)
+            for key, _ in events:
+                c = key.fileobj
                 if c.recv_ready():
                     stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
                     got_chunk = True

dvc/repo/__init__.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from contextlib import contextmanager
 from functools import wraps
@@ -5,17 +6,26 @@ from functools import wraps
 from funcy import cached_property, cat, first
 
 from dvc.config import Config
+from dvc.dvcfile import PIPELINE_FILE, Dvcfile, is_valid_filename
 from dvc.exceptions import FileMissingError
 from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
-from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
+from dvc.exceptions import (
+    NoOutputOrStageError,
+    NotDvcRepoError,
+    OutputNotFoundError,
+)
 from dvc.ignore import CleanTree
 from dvc.path_info import PathInfo
 from dvc.repo.tree import RepoTree
+from dvc.scm.tree import is_working_tree
 from dvc.utils.fs import path_isin
 
+from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
 from ..utils import parse_target
 from .graph import check_acyclic, get_pipeline, get_pipelines
 
+logger = logging.getLogger(__name__)
+
 
 def locked(f):
     @wraps(f)
@@ -38,7 +48,7 @@ class Repo:
     from dvc.repo.add import add
     from dvc.repo.remove import remove
     from dvc.repo.ls import ls
-    from dvc.repo.lock import lock as lock_stage
+    from dvc.repo.freeze import freeze, unfreeze
     from dvc.repo.move import move
     from dvc.repo.run import run
     from dvc.repo.imp import imp
@@ -124,7 +134,13 @@ class Repo:
 
     @tree.setter
     def tree(self, tree):
-        self._tree = tree if isinstance(tree, CleanTree) else CleanTree(tree)
+        if is_working_tree(tree) or tree.tree_root == self.root_dir:
+            root = None
+        else:
+            root = self.root_dir
+        self._tree = (
+            tree if isinstance(tree, CleanTree) else CleanTree(tree, root)
+        )
         # Our graph cache is no longer valid, as it was based on the previous
         # tree.
         self._reset()
@@ -181,6 +197,25 @@ class Repo:
 
         self.scm.ignore_list(flist)
 
+    def get_stage(self, path=None, name=None):
+        if not path:
+            path = PIPELINE_FILE
+            logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
+
+        dvcfile = Dvcfile(self, path)
+        return dvcfile.stages[name]
+
+    def get_stages(self, path=None, name=None):
+        if not path:
+            path = PIPELINE_FILE
+            logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
+
+        if name:
+            return [self.get_stage(path, name)]
+
+        dvcfile = Dvcfile(self, path)
+        return list(dvcfile.stages.values())
+
     def check_modified_graph(self, new_stages):
         """Generate graph including the new stage to check for errors"""
         # Building graph might be costly for the ones with many DVC-files,
@@ -204,10 +239,9 @@ class Repo:
         stages = nx.dfs_postorder_nodes(graph)
         return [stage for stage in stages if path_isin(stage.path, path)]
 
-    def collect(self, target, with_deps=False, recursive=False, graph=None):
-        import networkx as nx
-        from ..dvcfile import Dvcfile
-
+    def collect(
+        self, target=None, with_deps=False, recursive=False, graph=None
+    ):
         if not target:
             return list(graph) if graph else self.stages
 
@@ -217,36 +251,81 @@ class Repo:
             )
 
         path, name = parse_target(target)
-        dvcfile = Dvcfile(self, path)
-        stages = list(dvcfile.stages.filter(name).values())
+        stages = self.get_stages(path, name)
         if not with_deps:
             return stages
 
         res = set()
         for stage in stages:
-            pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
-            res.update(nx.dfs_postorder_nodes(pipeline, stage))
+            res.update(self._collect_pipeline(stage, graph=graph))
         return res
 
-    def collect_granular(self, target, *args, **kwargs):
-        from ..dvcfile import Dvcfile, is_valid_filename
+    def _collect_pipeline(self, stage, graph=None):
+        import networkx as nx
 
+        pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
+        return nx.dfs_postorder_nodes(pipeline, stage)
+
+    def _collect_from_default_dvcfile(self, target):
+        dvcfile = Dvcfile(self, PIPELINE_FILE)
+        if dvcfile.exists():
+            return dvcfile.stages.get(target)
+
+    def collect_granular(
+        self, target=None, with_deps=False, recursive=False, graph=None
+    ):
+        """
+        Priority is in the order of following in case of ambiguity:
+            - .dvc file or .yaml file
+            - dir if recursive and directory exists
+            - stage_name
+            - output file
+        """
         if not target:
             return [(stage, None) for stage in self.stages]
 
         file, name = parse_target(target)
-        if is_valid_filename(file) and not kwargs.get("with_deps"):
-            # Optimization: do not collect the graph for a specific .dvc target
-            stages = Dvcfile(self, file).stages.filter(name)
-            return [(stage, None) for stage in stages.values()]
+        stages = []
 
-        try:
-            (out,) = self.find_outs_by_path(file, strict=False)
-            filter_info = PathInfo(os.path.abspath(file))
-            return [(out.stage, filter_info)]
-        except OutputNotFoundError:
-            stages = self.collect(target, *args, **kwargs)
-            return [(stage, None) for stage in stages]
+        # Optimization: do not collect the graph for a specific target
+        if not file:
+            # parsing is ambiguous when it does not have a colon
+            # or if it's not a dvcfile, as it can be a stage name
+            # in `dvc.yaml` or, an output in a stage.
+            logger.debug(
+                "Checking if stage '%s' is in '%s'", target, PIPELINE_FILE
+            )
+            if not (recursive and os.path.isdir(target)):
+                stage = self._collect_from_default_dvcfile(target)
+                if stage:
+                    stages = (
+                        self._collect_pipeline(stage) if with_deps else [stage]
+                    )
+        elif not with_deps and is_valid_filename(file):
+            stages = self.get_stages(file, name)
+
+        if not stages:
+            if not (recursive and os.path.isdir(target)):
+                try:
+                    (out,) = self.find_outs_by_path(target, strict=False)
+                    filter_info = PathInfo(os.path.abspath(target))
+                    return [(out.stage, filter_info)]
+                except OutputNotFoundError:
+                    pass
+
+            try:
+                stages = self.collect(target, with_deps, recursive, graph)
+            except StageFileDoesNotExistError as exc:
+                # collect() might try to use `target` as a stage name
+                # and throw error that dvc.yaml does not exist, whereas it
+                # should say that both stage name and file does not exist.
+                if file and is_valid_filename(file):
+                    raise
+                raise NoOutputOrStageError(target, exc.file) from exc
+            except StageNotFound as exc:
+                raise NoOutputOrStageError(target, exc.file) from exc
+
+        return [(stage, None) for stage in stages]
 
     def used_cache(
         self,
@@ -362,7 +441,7 @@ class Repo:
         stages = stages or self.stages
         outs = Trie()  # Use trie to efficiently find overlapping outs and deps
 
-        for stage in filter(bool, stages):
+        for stage in filter(bool, stages):  # bug? not using it later
             for out in stage.outs:
                 out_key = out.path_info.parts
 
@@ -443,22 +522,18 @@ class Repo:
         return PlotTemplates(self.dvc_dir)
 
     def _collect_stages(self):
-        from dvc.dvcfile import Dvcfile, is_valid_filename
-
         stages = []
         outs = set()
 
         for root, dirs, files in self.tree.walk(self.root_dir):
             for file_name in filter(is_valid_filename, files):
-                path = os.path.join(root, file_name)
-                stage_loader = Dvcfile(self, path).stages
-                stages.extend(stage_loader.values())
+                new_stages = self.get_stages(os.path.join(root, file_name))
+                stages.extend(new_stages)
                 outs.update(
                     out.fspath
-                    for stage in stages
-                    for out in (
-                        out for out in stage.outs if out.scheme == "local"
-                    )
+                    for stage in new_stages
+                    for out in stage.outs
+                    if out.scheme == "local"
                 )
             dirs[:] = [d for d in dirs if os.path.join(root, d) not in outs]
         return stages

dvc/repo/add.py

@@ -21,7 +21,9 @@ logger = logging.getLogger(__name__)
 
 @locked
 @scm_context
-def add(repo, targets, recursive=False, no_commit=False, fname=None):
+def add(
+    repo, targets, recursive=False, no_commit=False, fname=None, external=False
+):
     if recursive and fname:
         raise RecursiveAddingWhileUsingFilename()
 
@@ -52,7 +54,9 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
                     )
                 )
 
-            stages = _create_stages(repo, sub_targets, fname, pbar=pbar)
+            stages = _create_stages(
+                repo, sub_targets, fname, pbar=pbar, external=external
+            )
 
             try:
                 repo.check_modified_graph(stages)
@@ -120,7 +124,7 @@ def _find_all_targets(repo, target, recursive):
     return [target]
 
 
-def _create_stages(repo, targets, fname, pbar=None):
+def _create_stages(repo, targets, fname, pbar=None, external=False):
     from dvc.stage import Stage, create_stage
 
     stages = []
@@ -132,7 +136,14 @@ def _create_stages(repo, targets, fname, pbar=None):
         unit="file",
     ):
         path, wdir, out = resolve_paths(repo, out)
-        stage = create_stage(Stage, repo, fname or path, wdir=wdir, outs=[out])
+        stage = create_stage(
+            Stage,
+            repo,
+            fname or path,
+            wdir=wdir,
+            outs=[out],
+            external=external,
+        )
         if stage:
             Dvcfile(repo, stage.path).remove_with_prompt(force=True)
 

dvc/repo/brancher.py

@@ -18,7 +18,7 @@ def brancher(  # noqa: E302
         str: the display name for the currently selected tree, it could be:
             - a git revision identifier
             - empty string it there is no branches to iterate over
-            - "Working Tree" if there are uncommitted changes in the SCM repo
+            - "workspace" if there are uncommitted changes in the SCM repo
     """
     if not any([revs, all_branches, all_tags, all_commits]):
         yield ""
@@ -30,10 +30,10 @@ def brancher(  # noqa: E302
     scm = self.scm
 
     self.tree = WorkingTree(self.root_dir)
-    yield "working tree"
+    yield "workspace"
 
-    if revs and "working tree" in revs:
-        revs.remove("working tree")
+    if revs and "workspace" in revs:
+        revs.remove("workspace")
 
     if all_commits:
         revs = scm.list_all_commits()
@@ -48,6 +48,9 @@ def brancher(  # noqa: E302
         if revs:
             for sha, names in group_by(scm.resolve_rev, revs).items():
                 self.tree = scm.get_tree(sha)
-                yield ", ".join(names)
+                # ignore revs that don't contain repo root
+                # (i.e. revs from before a subdir=True repo was init'ed)
+                if self.tree.exists(self.root_dir):
+                    yield ", ".join(names)
     finally:
         self.tree = saved_tree

dvc/repo/checkout.py

@@ -1,7 +1,11 @@
 import logging
 import os
 
-from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit
+from dvc.exceptions import (
+    CheckoutError,
+    CheckoutErrorSuggestGit,
+    NoOutputOrStageError,
+)
 from dvc.progress import Tqdm
 from dvc.utils import relpath
 
@@ -18,11 +22,11 @@ def _get_unused_links(repo):
     return repo.state.get_unused_links(used)
 
 
-def _fspath_dir(path, root):
+def _fspath_dir(path):
     if not os.path.exists(str(path)):
         return str(path)
 
-    path = relpath(path, root)
+    path = relpath(path)
     return os.path.join(path, "") if os.path.isdir(path) else path
 
 
@@ -56,7 +60,7 @@ def _checkout(
         targets = [None]
         unused = _get_unused_links(self)
 
-    stats["deleted"] = [_fspath_dir(u, self.root_dir) for u in unused]
+    stats["deleted"] = [_fspath_dir(u) for u in unused]
     self.state.remove_links(unused)
 
     if isinstance(targets, str):
@@ -70,7 +74,11 @@ def _checkout(
                     target, with_deps=with_deps, recursive=recursive
                 )
             )
-        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+        except (
+            StageFileDoesNotExistError,
+            StageFileBadNameError,
+            NoOutputOrStageError,
+        ) as exc:
             if not target:
                 raise
             raise CheckoutErrorSuggestGit(target) from exc
@@ -87,9 +95,7 @@ def _checkout(
                 filter_info=filter_info,
             )
             for key, items in result.items():
-                stats[key].extend(
-                    _fspath_dir(path, self.root_dir) for path in items
-                )
+                stats[key].extend(_fspath_dir(path) for path in items)
 
     if stats.get("failed"):
         raise CheckoutError(stats["failed"], stats)

dvc/repo/commit.py

@@ -46,3 +46,4 @@ def commit(self, target, with_deps=False, recursive=False, force=False):
         stage.commit()
 
         Dvcfile(self, stage.path).dump(stage)
+    return stages

dvc/repo/diff.py

@@ -7,7 +7,7 @@ from dvc.scm.tree import is_working_tree
 @locked
 def diff(self, a_rev="HEAD", b_rev=None):
     """
-    By default, it compares the working tree with the last commit's tree.
+    By default, it compares the workspace with the last commit's tree.
 
     This implementation differs from `git diff` since DVC doesn't have
     the concept of `index`, but it keeps the same interface, thus,

dvc/repo/freeze.py

@@ -0,0 +1,21 @@
+from . import locked
+
+
+@locked
+def _set(repo, target, frozen):
+    from dvc.utils import parse_target
+
+    path, name = parse_target(target)
+    stage = repo.get_stage(path, name)
+    stage.frozen = frozen
+    stage.dvcfile.dump(stage, update_pipeline=True)
+
+    return stage
+
+
+def freeze(repo, target):
+    return _set(repo, target, True)
+
+
+def unfreeze(repo, target):
+    return _set(repo, target, False)

dvc/repo/imp.py

@@ -3,4 +3,4 @@ def imp(self, url, path, out=None, rev=None):
     if rev is not None:
         erepo["rev"] = rev
 
-    return self.imp_url(path, out=out, erepo=erepo, locked=True)
+    return self.imp_url(path, out=out, erepo=erepo, frozen=True)

dvc/repo/imp_url.py

@@ -5,12 +5,12 @@ from dvc.utils import relpath, resolve_output, resolve_paths
 from dvc.utils.fs import path_isin
 
 from ..exceptions import OutputDuplicationError
-from . import locked as locked_repo
+from . import locked
 
 
-@locked_repo
+@locked
 @scm_context
-def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
+def imp_url(self, url, out=None, fname=None, erepo=None, frozen=True):
     from dvc.dvcfile import Dvcfile
     from dvc.stage import Stage, create_stage
 
@@ -48,7 +48,7 @@ def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
 
     stage.run()
 
-    stage.locked = locked
+    stage.frozen = frozen
 
     dvcfile.dump(stage)
 

dvc/repo/lock.py

@@ -1,15 +0,0 @@
-from . import locked
-
-
-@locked
-def lock(self, target, unlock=False):
-    from .. import dvcfile
-    from dvc.utils import parse_target
-
-    path, name = parse_target(target)
-    dvcfile = dvcfile.Dvcfile(self, path)
-    stage = dvcfile.stages[name]
-    stage.locked = False if unlock else True
-    dvcfile.dump(stage, update_pipeline=True)
-
-    return stage

dvc/repo/metrics/__init__.py

@@ -2,21 +2,11 @@ class Metrics:
     def __init__(self, repo):
         self.repo = repo
 
-    def add(self, *args, **kwargs):
-        from dvc.repo.metrics.add import add
-
-        return add(self.repo, *args, **kwargs)
-
     def show(self, *args, **kwargs):
         from dvc.repo.metrics.show import show
 
         return show(self.repo, *args, **kwargs)
 
-    def remove(self, *args, **kwargs):
-        from dvc.repo.metrics.remove import remove
-
-        return remove(self.repo, *args, **kwargs)
-
     def diff(self, *args, **kwargs):
         from .diff import diff
 

dvc/repo/metrics/add.py

@@ -1,5 +0,0 @@
-from dvc.repo.metrics.modify import modify
-
-
-def add(repo, path):
-    modify(repo, path)

dvc/repo/metrics/modify.py

@@ -1,22 +0,0 @@
-from dvc.dvcfile import Dvcfile
-from dvc.exceptions import DvcException
-from dvc.repo import locked
-
-
-@locked
-def modify(repo, path, delete=False):
-    outs = repo.find_outs_by_path(path)
-    assert len(outs) == 1
-    out = outs[0]
-
-    if out.scheme != "local":
-        msg = "output '{}' scheme '{}' is not supported for metrics"
-        raise DvcException(msg.format(out.path, out.path_info.scheme))
-
-    if delete:
-        out.metric = None
-
-    out.verify_metric()
-
-    dvcfile = Dvcfile(repo, out.stage.path)
-    dvcfile.dump(out.stage)

dvc/repo/metrics/remove.py

@@ -1,5 +0,0 @@
-from dvc.repo.metrics.modify import modify
-
-
-def remove(repo, path):
-    modify(repo, path, delete=True)

dvc/repo/metrics/show.py

@@ -104,13 +104,13 @@ def show(
     if not res:
         raise NoMetricsError()
 
-    # Hide working tree metrics if they are the same as in the active branch
+    # Hide workspace metrics if they are the same as in the active branch
     try:
         active_branch = repo.scm.active_branch()
     except TypeError:
         pass  # Detached head
     else:
-        if res.get("working tree") == res.get(active_branch):
-            res.pop("working tree", None)
+        if res.get("workspace") == res.get(active_branch):
+            res.pop("workspace", None)
 
     return res

dvc/repo/params/show.py

@@ -58,13 +58,13 @@ def show(repo, revs=None):
     if not res:
         raise NoParamsError("no parameter configs files in this repository")
 
-    # Hide working tree params if they are the same as in the active branch
+    # Hide workspace params if they are the same as in the active branch
     try:
         active_branch = repo.scm.active_branch()
     except TypeError:
         pass  # Detached head
     else:
-        if res.get("working tree") == res.get(active_branch):
-            res.pop("working tree", None)
+        if res.get("workspace") == res.get(active_branch):
+            res.pop("workspace", None)
 
     return res

dvc/repo/plots/__init__.py

@@ -11,3 +11,25 @@ class Plots:
         from .diff import diff
 
         return diff(self.repo, *args, **kwargs)
+
+    def modify(self, path, props=None, unset=None):
+        from dvc.dvcfile import Dvcfile
+
+        (out,) = self.repo.find_outs_by_path(path)
+
+        # This out will become a plot unless it is one already
+        if not isinstance(out.plot, dict):
+            out.plot = {}
+
+        for field in unset or ():
+            out.plot.pop(field, None)
+        out.plot.update(props or {})
+
+        # Empty dict will move it to non-plots
+        if not out.plot:
+            out.plot = True
+
+        out.verify_metric()
+
+        dvcfile = Dvcfile(self.repo, out.stage.path)
+        dvcfile.dump(out.stage, update_pipeline=True)

dvc/repo/plots/data.py

@@ -258,7 +258,7 @@ def _load_from_revisions(repo, datafile, revisions):
     exceptions = []
 
     for rev in repo.brancher(revs=revisions):
-        if rev == "working tree" and rev not in revisions:
+        if rev == "workspace" and rev not in revisions:
             continue
 
         try:

dvc/repo/plots/diff.py

@@ -3,7 +3,7 @@ def _revisions(revs, is_dirty):
     if len(revisions) <= 1:
         if len(revisions) == 0 and is_dirty:
             revisions.append("HEAD")
-        revisions.append("working tree")
+        revisions.append("workspace")
     return revisions
 
 

dvc/repo/plots/show.py

@@ -1,13 +1,14 @@
+import copy
 import logging
 import os
-from collections import defaultdict
 
-from funcy import first, last
+from funcy import first, last, project
 
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, NoPlotsError
 from dvc.repo import locked
+from dvc.schema import PLOT_PROPS
 
-from .data import PlotData
+from .data import NoMetricInHistoryError, PlotData
 from .template import NoDataForTemplateError, Template
 
 logger = logging.getLogger(__name__)
@@ -43,29 +44,24 @@ def _evaluate_templatepath(repo, template=None):
 
 @locked
 def fill_template(
-    repo,
-    datafile,
-    template_path,
-    revisions,
-    fields=None,
-    path=None,
-    csv_header=True,
-    x_field=None,
-    y_field=None,
-    **kwargs,
+    repo, datafile, template_path, revisions, props,
 ):
-    if x_field and fields:
-        fields.add(x_field)
+    # Copy things to not modify passed values
+    props = props.copy()
+    fields = copy.copy(props.get("fields"))
 
-    if y_field and fields:
-        fields.add(y_field)
+    if props.get("x") and fields:
+        fields.add(props.get("x"))
+
+    if props.get("y") and fields:
+        fields.add(props.get("y"))
 
     template_datafiles, x_anchor, y_anchor = _parse_template(
         template_path, datafile
     )
-    append_index = x_anchor and not x_field
+    append_index = x_anchor and not props.get("x")
     if append_index:
-        x_field = PlotData.INDEX_FIELD
+        props["x"] = PlotData.INDEX_FIELD
 
     template_data = {}
     for template_datafile in template_datafiles:
@@ -76,13 +72,13 @@ def fill_template(
         for pd in plot_datas:
             rev_data_points = pd.to_datapoints(
                 fields=fields,
-                path=path,
-                csv_header=csv_header,
+                path=props.get("path"),
+                csv_header=props.get("csv_header", True),
                 append_index=append_index,
             )
 
-            if y_anchor and not y_field:
-                y_field = _infer_y_field(rev_data_points, x_field)
+            if y_anchor and not props.get("y"):
+                props["y"] = _infer_y_field(rev_data_points, props.get("x"))
             tmp_data.extend(rev_data_points)
 
         template_data[template_datafile] = tmp_data
@@ -91,12 +87,7 @@ def fill_template(
         raise NoDataForTemplateError(template_path)
 
     content = Template.fill(
-        template_path,
-        template_data,
-        priority_datafile=datafile,
-        x_field=x_field,
-        y_field=y_field,
-        **kwargs,
+        template_path, template_data, priority_datafile=datafile, props=props,
     )
 
     path = datafile or ",".join(template_datafiles)
@@ -113,17 +104,20 @@ def _infer_y_field(rev_data_points, x_field):
     return y_field
 
 
-def _show(repo, datafile=None, template=None, revs=None, **kwargs):
+def _show(repo, datafile=None, revs=None, props=None):
     if revs is None:
-        revs = ["working tree"]
+        revs = ["workspace"]
 
-    if not datafile and not template:
+    if props is None:
+        props = {}
+
+    if not datafile and not props.get("template"):
         raise NoDataOrTemplateProvided()
 
-    template_path = _evaluate_templatepath(repo, template)
+    template_path = _evaluate_templatepath(repo, props.get("template"))
 
     plot_datafile, plot_content = fill_template(
-        repo, datafile, template_path, revs, **kwargs
+        repo, datafile, template_path, revs, props
     )
 
     return plot_datafile, plot_content
@@ -148,67 +142,76 @@ def _parse_template(template_path, priority_datafile):
     )
 
 
-def _collect_plots(repo):
-    plots = defaultdict(set)
+def _collect_plots(repo, targets=None):
+    from dvc.exceptions import OutputNotFoundError
+    from contextlib import suppress
 
-    for stage in repo.stages:
-        for out in stage.outs:
-            if not out.plot:
-                continue
-
-            if isinstance(out.plot, dict):
-                template = out.plot[out.PARAM_PLOT_TEMPLATE]
-            else:
-                template = None
-
-            plots[str(out)].add(template)
-
-    return plots
-
-
-def show(repo, targets=None, template=None, revs=None, **kwargs) -> dict:
-    if isinstance(targets, str):
-        targets = [targets]
+    def _targets_to_outs(targets):
+        for t in targets:
+            with suppress(OutputNotFoundError):
+                (out,) = repo.find_outs_by_path(t)
+                yield out
 
     if targets:
-        for target in targets:
-            return {
-                target: _show(
-                    repo,
-                    datafile=target,
-                    template=template,
-                    revs=revs,
-                    **kwargs,
-                )[1]
-            }
-
-    if not revs:
-        plots = _collect_plots(repo)
+        outs = _targets_to_outs(targets)
     else:
-        plots = defaultdict(set)
-        for rev in repo.brancher(revs=revs):
-            for plot, templates in _collect_plots(repo).items():
-                plots[plot].update(templates)
+        outs = (out for stage in repo.stages for out in stage.outs if out.plot)
+
+    return {str(out): _plot_props(out) for out in outs}
+
+
+def _plot_props(out):
+    if not out.plot:
+        raise DvcException(
+            f"'{out}' is not a plot. Use `dvc plots modify` to change that."
+        )
+    if isinstance(out.plot, list):
+        raise DvcException("Multiple plots per data file not supported yet.")
+    if isinstance(out.plot, bool):
+        return {}
+
+    return project(out.plot, PLOT_PROPS)
+
+
+def show(repo, targets=None, revs=None, props=None) -> dict:
+    if isinstance(targets, str):
+        targets = [targets]
+    if props is None:
+        props = {}
+
+    # Collect plot data files with associated props
+    plots = {}
+    for rev in repo.brancher(revs=revs):
+        if revs is not None and rev not in revs:
+            continue
+
+        for datafile, file_props in _collect_plots(repo, targets).items():
+            # props from command line overwrite plot props from out definition
+            full_props = {**file_props, **props}
+
+            if datafile in plots:
+                saved_rev, saved_props = plots[datafile]
+                if saved_props != props:
+                    logger.warning(
+                        f"Inconsistent plot props for '{datafile}' in "
+                        f"'{saved_rev}' and '{rev}'. "
+                        f"Going to use ones from '{saved_rev}'"
+                    )
+            else:
+                plots[datafile] = rev, full_props
 
     if not plots:
-        datafile, plot = _show(
-            repo, datafile=None, template=template, revs=revs, **kwargs
-        )
+        if targets:
+            raise NoMetricInHistoryError(", ".join(targets))
+
+        try:
+            datafile, plot = _show(repo, datafile=None, revs=revs, props=props)
+        except NoDataOrTemplateProvided:
+            raise NoPlotsError()
+
         return {datafile: plot}
 
-    ret = {}
-    for plot, templates in plots.items():
-        tmplt = template
-        if len(templates) == 1:
-            tmplt = list(templates)[0]
-        elif not template:
-            raise TooManyTemplatesError(
-                f"'{plot}' uses multiple templates '{templates}'. "
-                "Use `-t|--template` to specify the template to use. "
-            )
-
-        ret[plot] = _show(
-            repo, datafile=plot, template=tmplt, revs=revs, **kwargs
-        )[1]
-
-    return ret
+    return {
+        datafile: _show(repo, datafile=datafile, revs=revs, props=props)[1]
+        for datafile, (_, props) in plots.items()
+    }

dvc/repo/plots/template.py

@@ -38,8 +38,8 @@ class Template:
     X_ANCHOR = "<DVC_METRIC_X>"
     Y_ANCHOR = "<DVC_METRIC_Y>"
     TITLE_ANCHOR = "<DVC_METRIC_TITLE>"
-    X_TITLE_ANCHOR = "<DVC_METRIC_X_TITLE>"
-    Y_TITLE_ANCHOR = "<DVC_METRIC_Y_TITLE>"
+    X_LABEL_ANCHOR = "<DVC_METRIC_X_LABEL>"
+    Y_LABEL_ANCHOR = "<DVC_METRIC_Y_LABEL>"
 
     def __init__(self, templates_dir):
         self.plot_templates_dir = templates_dir
@@ -86,29 +86,24 @@ class Template:
 
     @staticmethod
     def fill(
-        template_path,
-        data,
-        priority_datafile=None,
-        x_field=None,
-        y_field=None,
-        title=None,
-        x_title=None,
-        y_title=None,
+        template_path, data, priority_datafile=None, props=None,
     ):
+        props = props or {}
+
         with open(template_path) as fobj:
             result_content = fobj.read()
 
-        if x_field:
-            Template._check_field_exists(data, x_field)
-        if y_field:
-            Template._check_field_exists(data, y_field)
+        if props.get("x"):
+            Template._check_field_exists(data, props.get("x"))
+        if props.get("y"):
+            Template._check_field_exists(data, props.get("y"))
 
         result_content = Template._replace_data_anchors(
             result_content, data, priority_datafile
         )
 
         result_content = Template._replace_metadata_anchors(
-            result_content, title, x_field, x_title, y_field, y_title
+            result_content, props
         )
 
         return result_content
@@ -122,34 +117,23 @@ class Template:
                 raise NoFieldInDataError(field)
 
     @staticmethod
-    def _replace_metadata_anchors(
-        result_content, title, x_field, x_title, y_field, y_title
-    ):
-        if Template.TITLE_ANCHOR in result_content:
-            if title:
-                result_content = result_content.replace(
-                    Template.TITLE_ANCHOR, title
-                )
-            else:
-                result_content = result_content.replace(
-                    Template.TITLE_ANCHOR, ""
-                )
-        if Template.X_ANCHOR in result_content and x_field:
-            result_content = result_content.replace(Template.X_ANCHOR, x_field)
-        if Template.Y_ANCHOR in result_content and y_field:
-            result_content = result_content.replace(Template.Y_ANCHOR, y_field)
-        if Template.X_TITLE_ANCHOR in result_content:
-            if not x_title and x_field:
-                x_title = x_field
-            result_content = result_content.replace(
-                Template.X_TITLE_ANCHOR, x_title
-            )
-        if Template.Y_TITLE_ANCHOR in result_content:
-            if not y_title and y_field:
-                y_title = y_field
-            result_content = result_content.replace(
-                Template.Y_TITLE_ANCHOR, y_title
-            )
+    def _replace_metadata_anchors(result_content, props):
+        props.setdefault("title", "")
+        props.setdefault("x_label", props.get("x"))
+        props.setdefault("y_label", props.get("y"))
+
+        replace_pairs = [
+            (Template.TITLE_ANCHOR, "title"),
+            (Template.X_ANCHOR, "x"),
+            (Template.Y_ANCHOR, "y"),
+            (Template.X_LABEL_ANCHOR, "x_label"),
+            (Template.Y_LABEL_ANCHOR, "y_label"),
+        ]
+        for anchor, key in replace_pairs:
+            value = props.get(key)
+            if anchor in result_content and value is not None:
+                result_content = result_content.replace(anchor, value)
+
         return result_content
 
     @staticmethod
@@ -186,12 +170,12 @@ class DefaultLinearTemplate(Template):
             "x": {
                 "field": Template.X_ANCHOR,
                 "type": "quantitative",
-                "title": Template.X_TITLE_ANCHOR,
+                "title": Template.X_LABEL_ANCHOR,
             },
             "y": {
                 "field": Template.Y_ANCHOR,
                 "type": "quantitative",
-                "title": Template.Y_TITLE_ANCHOR,
+                "title": Template.Y_LABEL_ANCHOR,
                 "scale": {"zero": False},
             },
             "color": {"field": "rev", "type": "nominal"},
@@ -211,13 +195,13 @@ class DefaultConfusionTemplate(Template):
                 "field": Template.X_ANCHOR,
                 "type": "nominal",
                 "sort": "ascending",
-                "title": Template.X_TITLE_ANCHOR,
+                "title": Template.X_LABEL_ANCHOR,
             },
             "y": {
                 "field": Template.Y_ANCHOR,
                 "type": "nominal",
                 "sort": "ascending",
-                "title": Template.Y_TITLE_ANCHOR,
+                "title": Template.Y_LABEL_ANCHOR,
             },
             "color": {"aggregate": "count", "type": "quantitative"},
             "facet": {"field": "rev", "type": "nominal"},
@@ -236,12 +220,12 @@ class DefaultScatterTemplate(Template):
             "x": {
                 "field": Template.X_ANCHOR,
                 "type": "quantitative",
-                "title": Template.X_TITLE_ANCHOR,
+                "title": Template.X_LABEL_ANCHOR,
             },
             "y": {
                 "field": Template.Y_ANCHOR,
                 "type": "quantitative",
-                "title": Template.Y_TITLE_ANCHOR,
+                "title": Template.Y_LABEL_ANCHOR,
                 "scale": {"zero": False},
             },
             "color": {"field": "rev", "type": "nominal"},

dvc/repo/remove.py

@@ -7,16 +7,11 @@ logger = logging.getLogger(__name__)
 
 
 @locked
-def remove(self, target, dvc_only=False):
-    from ..dvcfile import Dvcfile
-
+def remove(self, target, outs=False):
     path, name = parse_target(target)
-    dvcfile = Dvcfile(self, path)
-    stages = list(dvcfile.stages.filter(name).values())
-    for stage in stages:
-        stage.remove_outs(force=True)
+    stages = self.get_stages(path, name)
 
-    if not dvc_only:
-        dvcfile.remove()
+    for stage in stages:
+        stage.remove(remove_outs=outs, force=outs)
 
     return stages

dvc/repo/reproduce.py

@@ -10,9 +10,9 @@ logger = logging.getLogger(__name__)
 
 
 def _reproduce_stage(stage, **kwargs):
-    if stage.locked and not stage.is_import:
+    if stage.frozen and not stage.is_import:
         logger.warning(
-            "{} is locked. Its dependencies are"
+            "{} is frozen. Its dependencies are"
             " not going to be reproduced.".format(stage)
         )
 
@@ -34,7 +34,7 @@ def _get_active_graph(G):
 
     active = G.copy()
     for stage in G:
-        if not stage.locked:
+        if not stage.frozen:
             continue
         active.remove_edges_from(G.out_edges(stage))
         for edge in G.out_edges(stage):
@@ -61,7 +61,6 @@ def reproduce(
     all_pipelines=False,
     **kwargs
 ):
-    from ..dvcfile import Dvcfile
     from dvc.utils import parse_target
 
     if not target and not all_pipelines:
@@ -81,8 +80,7 @@ def reproduce(
         if all_pipelines:
             pipelines = active_pipelines
         else:
-            dvcfile = Dvcfile(self, path)
-            stage = dvcfile.stages[name]
+            stage = self.get_stage(path, name)
             pipelines = [get_pipeline(active_pipelines, stage)]
 
         targets = []
@@ -164,7 +162,12 @@ def _reproduce_stages(
 
     force_downstream = kwargs.pop("force_downstream", False)
     result = []
+    # `ret` is used to add a cosmetic newline.
+    ret = []
     for stage in pipeline:
+        if ret:
+            logger.info("")
+
         try:
             ret = _reproduce_stage(stage, **kwargs)
 

dvc/repo/run.py

@@ -3,7 +3,11 @@ import os
 from funcy import concat, first
 
 from dvc.exceptions import InvalidArgumentError
-from dvc.stage.exceptions import DuplicateStageName, InvalidStageName
+from dvc.stage.exceptions import (
+    DuplicateStageName,
+    InvalidStageName,
+    StageFileAlreadyExistsError,
+)
 
 from ..exceptions import OutputDuplicationError
 from . import locked
@@ -45,6 +49,9 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
     from dvc.stage import PipelineStage, Stage, create_stage
     from dvc.dvcfile import Dvcfile, PIPELINE_FILE
 
+    if not kwargs.get("cmd"):
+        raise InvalidArgumentError("command is not specified")
+
     stage_cls = PipelineStage
     path = PIPELINE_FILE
     stage_name = kwargs.get("name")
@@ -71,10 +78,12 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
 
     dvcfile = Dvcfile(self, stage.path)
     if dvcfile.exists():
-        if stage_name and stage_name in dvcfile.stages:
+        if kwargs.get("overwrite", True):
+            dvcfile.remove_stage(stage)
+        elif stage_cls != PipelineStage:
+            raise StageFileAlreadyExistsError(dvcfile.relpath)
+        elif stage_name and stage_name in dvcfile.stages:
             raise DuplicateStageName(stage_name, dvcfile)
-        if stage_cls != PipelineStage:
-            dvcfile.remove_with_prompt(force=kwargs.get("overwrite", True))
 
     try:
         self.check_modified_graph([stage])
@@ -82,8 +91,7 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
         raise OutputDuplicationError(exc.output, set(exc.stages) - {stage})
 
     if no_exec:
-        for out in stage.outs:
-            out.ignore()
+        stage.ignore_outs()
     else:
         stage.run(
             no_commit=kwargs.get("no_commit", False),

dvc/repo/status.py

@@ -14,9 +14,9 @@ def _joint_status(stages):
     status = {}
 
     for stage in stages:
-        if stage.locked and not stage.is_repo_import:
+        if stage.frozen and not stage.is_repo_import:
             logger.warning(
-                "{} is locked. Its dependencies are"
+                "{} is frozen. Its dependencies are"
                 " not going to be shown in the status output.".format(stage)
             )
 

dvc/schema.py

@@ -11,7 +11,8 @@ SINGLE_STAGE_SCHEMA = {
     StageParams.PARAM_WDIR: Any(str, None),
     StageParams.PARAM_DEPS: Any([dependency.SCHEMA], None),
     StageParams.PARAM_OUTS: Any([output.SCHEMA], None),
-    StageParams.PARAM_LOCKED: bool,
+    StageParams.PARAM_LOCKED: bool,  # backard compatibility
+    StageParams.PARAM_FROZEN: bool,
     StageParams.PARAM_META: object,
     StageParams.PARAM_ALWAYS_CHANGED: bool,
 }
@@ -29,8 +30,20 @@ OUT_PSTAGE_DETAILED_SCHEMA = {
     str: {BaseOutput.PARAM_CACHE: bool, BaseOutput.PARAM_PERSIST: bool}
 }
 
-PLOT_PSTAGE_SCHEMA = OUT_PSTAGE_DETAILED_SCHEMA.copy()
-PLOT_PSTAGE_SCHEMA[str][BaseOutput.PARAM_PLOT_TEMPLATE] = str
+PLOT_PROPS = {
+    BaseOutput.PARAM_PLOT_TEMPLATE: str,
+    BaseOutput.PARAM_PLOT_X: str,
+    BaseOutput.PARAM_PLOT_Y: str,
+    BaseOutput.PARAM_PLOT_X_LABEL: str,
+    BaseOutput.PARAM_PLOT_Y_LABEL: str,
+    BaseOutput.PARAM_PLOT_TITLE: str,
+    BaseOutput.PARAM_PLOT_CSV_HEADER: bool,
+}
+PLOT_PROPS_SCHEMA = {
+    **OUT_PSTAGE_DETAILED_SCHEMA[str],
+    **PLOT_PROPS,
+}
+PLOT_PSTAGE_SCHEMA = {str: Any(PLOT_PROPS_SCHEMA, [PLOT_PROPS_SCHEMA])}
 
 PARAM_PSTAGE_NON_DEFAULT_SCHEMA = {str: [str]}
 
@@ -42,7 +55,7 @@ SINGLE_PIPELINE_STAGE_SCHEMA = {
         Optional(StageParams.PARAM_PARAMS): [
             Any(str, PARAM_PSTAGE_NON_DEFAULT_SCHEMA)
         ],
-        Optional(StageParams.PARAM_LOCKED): bool,
+        Optional(StageParams.PARAM_FROZEN): bool,
         Optional(StageParams.PARAM_META): object,
         Optional(StageParams.PARAM_ALWAYS_CHANGED): bool,
         Optional(StageParams.PARAM_OUTS): [

dvc/serialize.py

@@ -10,7 +10,7 @@ from dvc.output import BaseOutput
 from dvc.stage.params import StageParams
 from dvc.stage.utils import resolve_wdir
 from dvc.utils.collections import apply_diff
-from dvc.utils.stage import parse_stage_for_update
+from dvc.utils.yaml import parse_yaml_for_update
 
 if TYPE_CHECKING:
     from dvc.stage import PipelineStage, Stage
@@ -115,7 +115,7 @@ def to_pipeline_file(stage: "PipelineStage"):
             stage.PARAM_PLOTS,
             _get_outs([out for out in stage.outs if out.plot]),
         ),
-        (stage.PARAM_LOCKED, stage.locked),
+        (stage.PARAM_FROZEN, stage.frozen),
         (stage.PARAM_ALWAYS_CHANGED, stage.always_changed),
     ]
     return {
@@ -165,7 +165,7 @@ def to_single_stage_file(stage: "Stage"):
     # - apply changes to a returned structure
     # - serialize it
     if stage._stage_text is not None:
-        saved_state = parse_stage_for_update(stage._stage_text, stage.path)
+        saved_state = parse_yaml_for_update(stage._stage_text, stage.path)
         # Stage doesn't work with meta in any way, so .dumpd() doesn't
         # have it. We simply copy it over.
         if "meta" in saved_state:

dvc/stage/__init__.py

@@ -19,6 +19,7 @@ from .utils import (
     check_circular_dependency,
     check_duplicated_arguments,
     check_missing_outputs,
+    check_no_externals,
     check_stage_path,
     compute_md5,
     fill_stage_dependencies,
@@ -42,6 +43,7 @@ def loads_from(cls, repo, path, wdir, data):
             [
                 Stage.PARAM_CMD,
                 Stage.PARAM_LOCKED,
+                Stage.PARAM_FROZEN,
                 Stage.PARAM_ALWAYS_CHANGED,
                 Stage.PARAM_MD5,
                 "name",
@@ -51,7 +53,7 @@ def loads_from(cls, repo, path, wdir, data):
     return cls(**kw)
 
 
-def create_stage(cls, repo, path, **kwargs):
+def create_stage(cls, repo, path, external=False, **kwargs):
     from dvc.dvcfile import check_dvc_filename
 
     wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
@@ -62,6 +64,8 @@ def create_stage(cls, repo, path, **kwargs):
 
     stage = loads_from(cls, repo, path, wdir, kwargs)
     fill_stage_outputs(stage, **kwargs)
+    if not external:
+        check_no_externals(stage)
     fill_stage_dependencies(
         stage, **project(kwargs, ["deps", "erepo", "params"])
     )
@@ -77,7 +81,7 @@ def create_stage(cls, repo, path, **kwargs):
             logger.warning("Build cache is ignored when persisting outputs.")
 
         if not ignore_run_cache and stage.can_be_skipped:
-            logger.info("Stage is cached, skipping.")
+            logger.info("Stage is cached, skipping")
             return None
 
     return stage
@@ -93,7 +97,8 @@ class Stage(params.StageParams):
         deps=None,
         outs=None,
         md5=None,
-        locked=False,
+        locked=False,  # backward compatibility
+        frozen=False,
         always_changed=False,
         stage_text=None,
         dvcfile=None,
@@ -110,7 +115,7 @@ class Stage(params.StageParams):
         self.outs = outs
         self.deps = deps
         self.md5 = md5
-        self.locked = locked
+        self.frozen = locked or frozen
         self.always_changed = always_changed
         self._stage_text = stage_text
         self._dvcfile = dvcfile
@@ -144,14 +149,10 @@ class Stage(params.StageParams):
         self._dvcfile = dvcfile
 
     def __repr__(self):
-        return "Stage: '{path}'".format(
-            path=self.path_in_repo if self.path else "No path"
-        )
+        return f"Stage: '{self.addressing}'"
 
     def __str__(self):
-        return "stage: '{path}'".format(
-            path=self.relpath if self.path else "No path"
-        )
+        return f"stage: '{self.addressing}'"
 
     @property
     def addressing(self):
@@ -159,7 +160,7 @@ class Stage(params.StageParams):
         Useful for alternative presentations where we don't need
         `Stage:` prefix.
         """
-        return self.relpath
+        return self.relpath if self.path else "No path"
 
     def __hash__(self):
         return hash(self.path_in_repo)
@@ -205,14 +206,15 @@ class Stage(params.StageParams):
         return isinstance(self.deps[0], dependency.RepoDependency)
 
     def changed_deps(self):
-        if self.locked:
+        if self.frozen:
             return False
 
         if self.is_callback:
-            logger.warning(
-                '{stage} is a "callback" stage '
+            logger.debug(
+                '%s is a "callback" stage '
                 "(has a command and no dependencies) and thus always "
-                "considered as changed.".format(stage=self)
+                "considered as changed.",
+                self,
             )
             return True
 
@@ -248,9 +250,9 @@ class Stage(params.StageParams):
 
         return False
 
-    def changed_stage(self, warn=False):
+    def changed_stage(self):
         changed = self.md5 != self.compute_md5()
-        if changed and warn:
+        if changed:
             logger.debug(self._changed_stage_entry())
         return changed
 
@@ -259,14 +261,12 @@ class Stage(params.StageParams):
         is_changed = (
             # Short-circuit order: stage md5 is fast,
             # deps are expected to change
-            self.changed_stage(warn=True)
+            self.changed_stage()
             or self.changed_deps()
             or self.changed_outs()
         )
         if is_changed:
-            logger.info("%s changed.", self)
-        else:
-            logger.info("%s didn't change.", self)
+            logger.debug("%s changed.", self)
         return is_changed
 
     @rwlocked(write=["outs"])
@@ -285,17 +285,18 @@ class Stage(params.StageParams):
             out.unprotect()
 
     @rwlocked(write=["outs"])
-    def remove(self, force=False, remove_outs=True):
+    def remove(self, force=False, remove_outs=True, purge=True):
         if remove_outs:
             self.remove_outs(ignore_remove=True, force=force)
         else:
             self.unprotect_outs()
-        self.dvcfile.remove()
+        if purge:
+            self.dvcfile.remove_stage(self)
 
     @rwlocked(read=["deps"], write=["outs"])
     def reproduce(self, interactive=False, **kwargs):
-
         if not (kwargs.get("force", False) or self.changed()):
+            logger.info("Stage '%s' didn't change, skipping", self.addressing)
             return None
 
         msg = (
@@ -379,6 +380,10 @@ class Stage(params.StageParams):
         for out in self.outs:
             out.save()
 
+    def ignore_outs(self):
+        for out in self.outs:
+            out.ignore()
+
     @staticmethod
     def _changed_entries(entries):
         return [str(entry) for entry in entries if entry.changed_checksum()]
@@ -402,16 +407,16 @@ class Stage(params.StageParams):
 
     @rwlocked(read=["deps"], write=["outs"])
     def run(self, dry=False, no_commit=False, force=False, run_cache=True):
-        if (self.cmd or self.is_import) and not self.locked and not dry:
+        if (self.cmd or self.is_import) and not self.frozen and not dry:
             self.remove_outs(ignore_remove=False, force=False)
 
-        if not self.locked and self.is_import:
+        if not self.frozen and self.is_import:
             sync_import(self, dry, force)
-        elif not self.locked and self.cmd:
+        elif not self.frozen and self.cmd:
             run_stage(self, dry, force, run_cache)
         else:
             args = (
-                ("outputs", "locked ") if self.locked else ("data sources", "")
+                ("outputs", "frozen ") if self.frozen else ("data sources", "")
             )
             logger.info("Verifying %s in %s%s", *args, self)
             if not dry:
@@ -453,7 +458,7 @@ class Stage(params.StageParams):
         ret = []
         show_import = self.is_repo_import and check_updates
 
-        if not self.locked or show_import:
+        if not self.frozen or show_import:
             self._status_deps(ret)
         self._status_outs(ret)
         self._status_always_changed(ret)
@@ -527,19 +532,13 @@ class PipelineStage(Stage):
     def __hash__(self):
         return hash((self.path_in_repo, self.name))
 
-    def __repr__(self):
-        return "Stage: '{path}:{name}'".format(
-            path=self.relpath if self.path else "No path", name=self.name
-        )
-
-    def __str__(self):
-        return "stage: '{path}:{name}'".format(
-            path=self.relpath if self.path else "No path", name=self.name
-        )
-
     @property
     def addressing(self):
-        return super().addressing + ":" + self.name
+        from dvc.dvcfile import PIPELINE_FILE
+
+        if self.path and self.relpath == PIPELINE_FILE:
+            return self.name
+        return f"{super().addressing}:{self.name}"
 
     def reload(self):
         return self.dvcfile.stages[self.name]
@@ -552,8 +551,8 @@ class PipelineStage(Stage):
         if self.cmd_changed:
             ret.append("changed command")
 
-    def changed_stage(self, warn=False):
-        if self.cmd_changed and warn:
+    def changed_stage(self):
+        if self.cmd_changed:
             logger.debug(self._changed_stage_entry())
         return self.cmd_changed
 

dvc/stage/cache.py

@@ -11,7 +11,7 @@ from dvc.serialize import to_single_stage_lockfile
 from dvc.stage.loader import StageLoader
 from dvc.utils import dict_sha256, relpath
 from dvc.utils.fs import makedirs
-from dvc.utils.stage import dump_stage_file
+from dvc.utils.yaml import dump_yaml
 
 logger = logging.getLogger(__name__)
 
@@ -77,7 +77,7 @@ class StageCache:
 
         return None
 
-    def _create_stage(self, cache):
+    def _create_stage(self, cache, wdir=None):
         from dvc.stage import create_stage, PipelineStage
 
         params = []
@@ -98,9 +98,11 @@ class StageCache:
             repo=self.repo,
             path="dvc.yaml",
             cmd=cache["cmd"],
+            wdir=wdir,
             params=params,
             deps=[dep["path"] for dep in cache.get("deps", [])],
             outs=[out["path"] for out in cache["outs"]],
+            external=True,
         )
         StageLoader.fill_from_lock(stage, cache)
         return stage
@@ -118,7 +120,7 @@ class StageCache:
         # NOTE: using temporary stage to avoid accidentally modifying original
         # stage and to workaround `commit/checkout` not working for uncached
         # outputs.
-        cached_stage = self._create_stage(cache)
+        cached_stage = self._create_stage(cache, wdir=stage.wdir)
 
         outs_no_cache = [
             out.def_path for out in stage.outs if not out.use_cache
@@ -153,7 +155,7 @@ class StageCache:
         path = self._get_cache_path(cache_key, cache_value)
         dpath = os.path.dirname(path)
         makedirs(dpath, exist_ok=True)
-        dump_stage_file(path, cache)
+        dump_yaml(path, cache)
 
     def is_cached(self, stage):
         return bool(self._load(stage))

dvc/stage/exceptions.py

@@ -15,15 +15,8 @@ class StageFileFormatError(DvcException):
 
 class StageFileDoesNotExistError(DvcException):
     def __init__(self, fname):
-        from dvc.dvcfile import DVC_FILE_SUFFIX, is_dvc_file
-
-        msg = f"'{fname}' does not exist."
-
-        sname = fname + DVC_FILE_SUFFIX
-        if is_dvc_file(sname):
-            msg += f" Do you mean '{sname}'?"
-
-        super().__init__(msg)
+        self.file = fname
+        super().__init__(f"'{self.file}' does not exist.")
 
 
 class StageFileAlreadyExistsError(DvcException):
@@ -65,6 +58,10 @@ class StageCommitError(DvcException):
     pass
 
 
+class StageExternalOutputsError(DvcException):
+    pass
+
+
 class StageUpdateError(DvcException):
     def __init__(self, path):
         super().__init__(
@@ -87,8 +84,10 @@ class MissingDataSource(DvcException):
 
 class StageNotFound(KeyError, DvcException):
     def __init__(self, file, name):
+        self.file = file.relpath
+        self.name = name
         super().__init__(
-            f"Stage '{name}' not found inside '{file.relpath}' file"
+            f"Stage '{self.name}' not found inside '{self.file}' file"
         )
 
 

dvc/stage/imports.py

@@ -5,12 +5,12 @@ logger = logging.getLogger(__name__)
 
 def update_import(stage, rev=None):
     stage.deps[0].update(rev=rev)
-    locked = stage.locked
-    stage.locked = False
+    frozen = stage.frozen
+    stage.frozen = False
     try:
         stage.reproduce()
     finally:
-        stage.locked = locked
+        stage.frozen = frozen
 
 
 def sync_import(stage, dry=False, force=False):
@@ -23,11 +23,7 @@ def sync_import(stage, dry=False, force=False):
     if dry:
         return
 
-    if (
-        not force
-        and not stage.changed_stage(warn=True)
-        and stage.already_cached()
-    ):
+    if not force and not stage.changed_stage() and stage.already_cached():
         stage.outs[0].checkout()
     else:
         stage.deps[0].download(stage.outs[0])

dvc/stage/loader.py

@@ -30,16 +30,6 @@ class StageLoader(Mapping):
         self.stages_data = stages_data or {}
         self.lockfile_data = lockfile_data or {}
 
-    def filter(self, item=None):
-        if not item:
-            return self
-
-        if item not in self:
-            raise StageNotFound(self.dvcfile, item)
-
-        data = {item: self.stages_data[item]} if item in self else {}
-        return self.__class__(self.dvcfile, data, self.lockfile_data)
-
     @staticmethod
     def fill_from_lock(stage, lock_data):
         from .params import StageParams
@@ -224,9 +214,6 @@ class SingleStageLoader(Mapping):
         self.stage_data = stage_data or {}
         self.stage_text = stage_text
 
-    def filter(self, item=None):
-        return self
-
     def __getitem__(self, item):
         if item:
             logger.warning(

dvc/stage/params.py

@@ -4,7 +4,8 @@ class StageParams:
     PARAM_WDIR = "wdir"
     PARAM_DEPS = "deps"
     PARAM_OUTS = "outs"
-    PARAM_LOCKED = "locked"
+    PARAM_LOCKED = "locked"  # backward compatibility
+    PARAM_FROZEN = "frozen"
     PARAM_META = "meta"
     PARAM_ALWAYS_CHANGED = "always_changed"
     PARAM_PARAMS = "params"

dvc/stage/run.py

@@ -79,11 +79,15 @@ def cmd_run(stage, *args, **kwargs):
 
 
 def _is_cached(stage):
-    return (
+    cached = (
         not stage.is_callback
         and not stage.always_changed
+        and not stage.changed_stage()
         and stage.already_cached()
     )
+    if cached:
+        logger.info("Stage '%s' is cached", stage.addressing)
+    return cached
 
 
 def restored_from_cache(stage):
@@ -93,7 +97,10 @@ def restored_from_cache(stage):
         return False
     # restore stage from build cache
     stage_cache.restore(stage)
-    return stage.outs_cached()
+    restored = stage.outs_cached()
+    if restored:
+        logger.info("Restored stage '%s' from run-cache", stage.addressing)
+    return restored
 
 
 def run_stage(stage, dry=False, force=False, run_cache=False):
@@ -102,10 +109,16 @@ def run_stage(stage, dry=False, force=False, run_cache=False):
             run_cache and restored_from_cache(stage)
         )
         if stage_cached:
-            logger.info("Stage is cached, skipping.")
+            logger.info("Skipping run, checking out outputs")
             stage.checkout()
             return
 
-    logger.info("Running command:\n\t%s", stage.cmd)
+    callback_str = "callback " if stage.is_callback else ""
+    logger.info(
+        "Running %s" "stage '%s' with command:",
+        callback_str,
+        stage.addressing,
+    )
+    logger.info("\t%s", stage.cmd)
     if not dry:
         cmd_run(stage)

dvc/stage/utils.py

@@ -6,9 +6,10 @@ from dvc import dependency, output
 from dvc.utils.fs import path_isin
 
 from ..remote import LocalRemote, S3Remote
-from ..utils import dict_md5, relpath
+from ..utils import dict_md5, format_link, relpath
 from .exceptions import (
     MissingDataSource,
+    StageExternalOutputsError,
     StagePathNotDirectoryError,
     StagePathNotFoundError,
     StagePathOutsideError,
@@ -68,6 +69,31 @@ def fill_stage_dependencies(stage, deps=None, erepo=None, params=None):
     stage.deps += dependency.loads_params(stage, params or [])
 
 
+def check_no_externals(stage):
+    from urllib.parse import urlparse
+
+    # NOTE: preventing users from accidentally using external outputs. See
+    # https://github.com/iterative/dvc/issues/1545 for more details.
+
+    def _is_external(out):
+        # NOTE: in case of `remote://` notation, the user clearly knows that
+        # this is an advanced feature and so we shouldn't error-out.
+        if out.is_in_repo or urlparse(out.def_path).scheme == "remote":
+            return False
+        return True
+
+    outs = [str(out) for out in stage.outs if _is_external(out)]
+    if not outs:
+        return
+
+    str_outs = ", ".join(outs)
+    link = format_link("https://dvc.org/doc/user-guide/managing-external-data")
+    raise StageExternalOutputsError(
+        f"Output(s) outside of DVC project: {str_outs}. "
+        f"See {link} for more info."
+    )
+
+
 def check_circular_dependency(stage):
     from dvc.exceptions import CircularDependencyError
 
@@ -136,7 +162,8 @@ def compute_md5(stage):
     return dict_md5(
         d,
         exclude=[
-            stage.PARAM_LOCKED,
+            stage.PARAM_LOCKED,  # backward compatibility
+            stage.PARAM_FROZEN,
             BaseOutput.PARAM_METRIC,
             BaseOutput.PARAM_PERSIST,
         ],
@@ -155,7 +182,7 @@ def get_dump(stage):
             stage.PARAM_MD5: stage.md5,
             stage.PARAM_CMD: stage.cmd,
             stage.PARAM_WDIR: resolve_wdir(stage.wdir, stage.path),
-            stage.PARAM_LOCKED: stage.locked,
+            stage.PARAM_FROZEN: stage.frozen,
             stage.PARAM_DEPS: [d.dumpd() for d in stage.deps],
             stage.PARAM_OUTS: [o.dumpd() for o in stage.outs],
             stage.PARAM_ALWAYS_CHANGED: stage.always_changed,

dvc/state.py

@@ -7,6 +7,7 @@ import sqlite3
 from urllib.parse import urlencode, urlunparse
 
 from dvc.exceptions import DvcException
+from dvc.scm.tree import WorkingTree
 from dvc.utils import current_timestamp, relpath, to_chunks
 from dvc.utils.fs import get_inode, get_mtime_and_size, remove
 
@@ -91,6 +92,7 @@ class State:  # pylint: disable=too-many-instance-attributes
     def __init__(self, repo):
         self.repo = repo
         self.root_dir = repo.root_dir
+        self.tree = WorkingTree(self.root_dir)
 
         state_config = repo.config.get("state", {})
         self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
@@ -364,9 +366,7 @@ class State:  # pylint: disable=too-many-instance-attributes
         assert checksum is not None
         assert os.path.exists(path_info)
 
-        actual_mtime, actual_size = get_mtime_and_size(
-            path_info, self.repo.tree
-        )
+        actual_mtime, actual_size = get_mtime_and_size(path_info, self.tree)
         actual_inode = get_inode(path_info)
 
         existing_record = self.get_state_record_for_inode(actual_inode)
@@ -397,7 +397,7 @@ class State:  # pylint: disable=too-many-instance-attributes
         if not os.path.exists(path):
             return None
 
-        actual_mtime, actual_size = get_mtime_and_size(path, self.repo.tree)
+        actual_mtime, actual_size = get_mtime_and_size(path, self.tree)
         actual_inode = get_inode(path)
 
         existing_record = self.get_state_record_for_inode(actual_inode)
@@ -423,7 +423,7 @@ class State:  # pylint: disable=too-many-instance-attributes
         if not os.path.exists(path_info):
             return
 
-        mtime, _ = get_mtime_and_size(path_info, self.repo.tree)
+        mtime, _ = get_mtime_and_size(path_info, self.tree)
         inode = get_inode(path_info)
         relative_path = relpath(path_info, self.root_dir)
 
@@ -450,7 +450,7 @@ class State:  # pylint: disable=too-many-instance-attributes
                 continue
 
             actual_inode = get_inode(path)
-            actual_mtime, _ = get_mtime_and_size(path, self.repo.tree)
+            actual_mtime, _ = get_mtime_and_size(path, self.tree)
 
             if (inode, mtime) == (actual_inode, actual_mtime):
                 logger.debug("Removing '%s' as unused link.", path)

dvc/utils/__init__.py

@@ -395,7 +395,7 @@ def format_link(link):
 
 
 def parse_target(target, default=None):
-    from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
+    from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK, is_valid_filename
     from dvc.exceptions import DvcException
 
     if not target:
@@ -409,13 +409,19 @@ def parse_target(target, default=None):
         match.group("path"),
         match.group("name"),
     )
+    if path:
+        if os.path.basename(path) == PIPELINE_LOCK:
+            raise DvcException(
+                "Did you mean: `{}`?".format(
+                    target.replace(".lock", ".yaml", 1)
+                )
+            )
+        if not name:
+            ret = (target, None)
+            return ret if is_valid_filename(target) else ret[::-1]
+
     if not path:
         path = default or PIPELINE_FILE
         logger.debug("Assuming file to be '%s'", path)
 
-    if os.path.basename(path) == PIPELINE_LOCK:
-        raise DvcException(
-            "Did you mean: `{}`?".format(target.replace(".lock", ".yaml", 1))
-        )
-
     return path, name

dvc/utils/pager.py

@@ -0,0 +1,48 @@
+"""Draws DAG in ASCII."""
+
+import logging
+import os
+import pydoc
+import sys
+
+from dvc.env import DVC_PAGER
+from dvc.utils import format_link
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_PAGER = "less"
+DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
+    DEFAULT_PAGER
+)
+
+
+def make_pager(cmd):
+    def _pager(text):
+        return pydoc.tempfilepager(pydoc.plain(text), cmd)
+
+    return _pager
+
+
+def find_pager():
+    if not sys.stdout.isatty():
+        return pydoc.plainpager
+
+    env_pager = os.getenv(DVC_PAGER)
+    if env_pager:
+        return make_pager(env_pager)
+
+    if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
+        return make_pager(DEFAULT_PAGER_FORMATTED)
+
+    logger.warning(
+        "Unable to find `less` in the PATH. Check out "
+        "{} for more info.".format(
+            format_link("https://man.dvc.org/pipeline/show")
+        )
+    )
+    return pydoc.plainpager
+
+
+def pager(text):
+    find_pager()(text)

dvc/utils/yaml.py

@@ -4,7 +4,7 @@ import yaml
 from ruamel.yaml import YAML
 from ruamel.yaml.error import YAMLError
 
-from dvc.exceptions import StageFileCorruptedError
+from dvc.exceptions import YAMLFileCorruptedError
 
 try:
     from yaml import CSafeLoader as SafeLoader
@@ -12,35 +12,35 @@ except ImportError:
     from yaml import SafeLoader
 
 
-def load_stage_file(path):
+def load_yaml(path):
     with open(path, encoding="utf-8") as fd:
-        return parse_stage(fd.read(), path)
+        return parse_yaml(fd.read(), path)
 
 
-def parse_stage(text, path):
+def parse_yaml(text, path):
     try:
         return yaml.load(text, Loader=SafeLoader) or {}
     except yaml.error.YAMLError as exc:
-        raise StageFileCorruptedError(path) from exc
+        raise YAMLFileCorruptedError(path) from exc
 
 
-def parse_stage_for_update(text, path):
+def parse_yaml_for_update(text, path):
     """Parses text into Python structure.
 
-    Unlike `parse_stage()` this returns ordered dicts, values have special
+    Unlike `parse_yaml()` this returns ordered dicts, values have special
     attributes to store comments and line breaks. This allows us to preserve
     all of those upon dump.
 
-    This one is, however, several times slower than simple `parse_stage()`.
+    This one is, however, several times slower than simple `parse_yaml()`.
     """
     try:
         yaml = YAML()
         return yaml.load(text) or {}
     except YAMLError as exc:
-        raise StageFileCorruptedError(path) from exc
+        raise YAMLFileCorruptedError(path) from exc
 
 
-def dump_stage_file(path, data):
+def dump_yaml(path, data):
     with open(path, "w", encoding="utf-8") as fd:
         yaml = YAML()
         yaml.default_flow_style = False

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "1.0.0a5"
+_BASE_VERSION = "1.0.0a9"
 
 
 def _generate_version(base_version):

scripts/ci/check_patch.sh

@@ -3,7 +3,7 @@
 set -x
 set -e
 
-pip install Pygments collective.checkdocs pre-commit
+pip install Pygments collective.checkdocs pre-commit .
 
 # stop the build if there are any readme formatting errors
 python setup.py checkdocs

scripts/ci/install.sh

@@ -10,6 +10,8 @@ scriptdir="$(dirname $0)"
 # work or not.
 $scriptdir/retry.sh pip install .[all,tests]
 
+$scriptdir/retry.sh pip install git+https://github.com/iterative/mock-ssh-server.git
+
 git config --global user.email "[email protected]"
 git config --global user.name "DVC Tester"
 

scripts/completion/dvc.bash

@@ -4,14 +4,15 @@
 #   - https://opensource.com/article/18/3/creating-bash-completion-script
 #   - https://stackoverflow.com/questions/12933362
 
-_dvc_commands='add cache checkout commit config destroy diff fetch get-url get gc \
-  import-url import init install lock list metrics move params pipeline plots pull push \
-  remote remove repro root run status unlock unprotect update version'
+_dvc_commands='add cache checkout commit config dag destroy diff fetch freeze \
+  get-url get gc import-url import init install list metrics move params \
+  plots pull push remote remove repro root run status unfreeze unprotect \
+  update version'
 
 _dvc_options='-h --help -V --version'
 _dvc_global_options='-h --help -q --quiet -v --verbose'
 
-_dvc_add='-R --recursive -f --file --no-commit'
+_dvc_add='-R --recursive -f --file --no-commit --external'
 _dvc_add_COMPGEN=_dvc_compgen_files
 _dvc_cache='dir'
 _dvc_cache_dir=' --global --system --local -u --unset'
@@ -20,6 +21,8 @@ _dvc_checkout_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_commit='-f --force -d --with-deps -R --recursive'
 _dvc_commit_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_config='-u --unset --local --system --global'
+_dvc_dag='--dot --full'
+_dvc_dag_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_destroy='-f --force'
 _dvc_diff='-t --show-json --show-hash --show-md'
 _dvc_fetch='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
@@ -33,29 +36,20 @@ _dvc_init='--no-scm -f --force'
 _dvc_install=''
 _dvc_list='-R --recursive --dvc-only --rev'
 _dvc_list_COMPGEN=_dvc_compgen_files
-_dvc_lock=''
-_dvc_lock_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_metrics='add diff modify remove show'
-_dvc_metrics_add='-t --type -x --xpath'
-_dvc_metrics_add_COMPGEN=_dvc_compgen_files
+_dvc_freeze=''
+_dvc_freeze_COMPGEN=_dvc_compgen_DVCFiles
+_dvc_metrics='diff show'
 _dvc_metrics_diff='--targets -t --type -x --xpath -R --show-json --show-md --no-path --old'
-_dvc_metrics_modify='-t --type -x --xpath'
-_dvc_metrics_modify_COMPGEN=_dvc_compgen_files
-_dvc_metrics_remove=''
-_dvc_metrics_remove_COMPGEN=_dvc_compgen_files
 _dvc_metrics_show='-t --type -x --xpath -a --all-branches -T --all-tags -R --recursive'
 _dvc_metrics_show_COMPGEN=_dvc_compgen_files
 _dvc_move=''
 _dvc_move_COMPGEN=_dvc_compgen_files
 _dvc_params='diff'
 _dvc_params_diff='--all --show-json --show-md --no-path'
-_dvc_pipeline='list show'
-_dvc_pipeline_list=''
-_dvc_pipeline_show='-c --commands -o --outs --ascii --dot --tree -l --locked'
-_dvc_pipeline_show_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_plots='show diff'
-_dvc_plots_show='-t --template -o --out -x -y --show-json --no-csv-header --title --xlab --ylab'
-_dvc_plots_diff='-t --template --targets -o --out -x -y --show-json --no-csv-header --title --xlab --ylab'
+_dvc_plots_show='-t --template -o --out -x -y --show-vega --no-csv-header --title --x-label --y-label'
+_dvc_plots_diff='-t --template --targets -o --out -x -y --show-vega --no-csv-header --title --x-label --y-label'
+_dvc_plots_modify='-t --template -x -y --no-csv-header --title --x-label --y-label'
 _dvc_pull='-j --jobs -r --remote -a --all-branches -T --all-tags -f --force -d --with-deps -R --recursive'
 _dvc_pull_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_push='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
@@ -72,11 +66,11 @@ _dvc_remove_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --no-run-cache --force-downstream --no-commit -R --recursive --downstream'
 _dvc_repro_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_root=''
-_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --no-run-cache --no-commit -w --wdir'
+_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --no-run-cache --no-commit -w --wdir --external'
 _dvc_run_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud'
 _dvc_status_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_unlock_COMPGEN=_dvc_compgen_DVCFiles
+_dvc_unfreeze_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_unprotect_COMPGEN=_dvc_compgen_files
 _dvc_update='--rev'
 _dvc_update_COMPGEN=_dvc_compgen_DVCFiles

scripts/completion/dvc.zsh

@@ -17,6 +17,7 @@ _dvc_commands() {
     "checkout:Checkout data files from cache."
     "commit:Save changed data to cache and update DVC-files."
     "config:Get or set config settings."
+    "dag:Visualize DVC project DAG."
     "destroy:Remove DVC-files, local DVC config and data cache."
     "diff:Show added, modified, or deleted data between commits in the DVC repository, or between a commit and the workspace."
     "fetch:Get files or directories tracked by DVC from remote storage into the cache."
@@ -29,10 +30,9 @@ _dvc_commands() {
     "install:Install DVC git hooks into the repository."
     "list:List repository contents, including files and directories tracked by DVC and by Git."
     "lock:Lock DVC-file."
-    "metrics:Commands to add, manage, collect and display metrics."
+    "metrics:Commands to display and compare metrics."
     "move:Rename or move a DVC controlled data file or a directory."
     "params:Commands to display params."
-    "pipeline:Manage pipelines."
     "pull:Pull data files from a DVC remote storage."
     "push:Push data files to a DVC remote storage."
     "plots:Generate plot for metrics structured as JSON, CSV or TSV."
@@ -66,6 +66,7 @@ _dvc_add=(
   {-R,--recursive}"[Recursively add each file under the directory.]"
   "--no-commit[Don't put files/directories into cache.]"
   {-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
+  "--external[Allow targets that are outside of the DVC project.]"
   "1:File:_files"
 )
 
@@ -96,6 +97,12 @@ _dvc_config=(
   {-u,--unset}"[Unset option.]"
 )
 
+_dvc_dag=(
+  "--dot[Print DAG in DOT format.]"
+  "--full[Show full DAG that the target belongs too, instead of showing DAG consisting only of ancestors.]"
+  "1:Stage:"
+)
+
 _dvc_destroy=(
   {-f,--force}"[Force destruction.]"
 )
@@ -132,7 +139,7 @@ _dvc_get=(
 )
 
 _dvc_gc=(
-  {-w,--workspace}"[Keep data files used in the current workspace (working tree).]"
+  {-w,--workspace}"[Keep data files used in the current workspace.]"
   {-a,--all-branches}"[Keep data files for the tips of all Git branches.]"
   "--all-commits[Keep data files for all Git commits.]"
   {-T,--all-tags}"[Keep data files for all Git tags.]"
@@ -163,7 +170,7 @@ _dvc_init=(
 
 _dvc_install=()
 
-_dvc_lock=(
+_dvc_freeze=(
   "*:Stages:_files -g '(*.dvc|Dvcfile)'"
 )
 
@@ -176,7 +183,7 @@ _dvc_list=(
 )
 
 _dvc_metrics=(
-  "1:Sub command:(add show diff modify remove)"
+  "1:Sub command:(show diff)"
 )
 
 _dvc_move=(
@@ -188,10 +195,6 @@ _dvc_params=(
   "1:Sub command:(diff)"
 )
 
-_dvc_pipeline=(
-  "1:Sub command:(show list)"
-)
-
 _dvc_pull=(
   {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
   {-r,--remote}"[Remote repository to pull from.]:Remote repository:"
@@ -214,7 +217,7 @@ _dvc_push=(
 )
 
 _dvc_plots=(
-  "1:Sub command:(show diff)"
+  "1:Sub command:(show diff modify)"
 )
 
 _dvc_remote=(
@@ -265,6 +268,7 @@ _dvc_run=(
   "--no-commit[Don't put files/directories into cache.]"
   "--outs-persist[Declare output file or directory that will not be removed upon repro.]:Output persistent:_files"
   "--outs-persist-no-cache[Declare output file or directory that will not be removed upon repro (do not put into DVC cache).]:Output persistent regular:_files"
+  "--external[Allow outputs that are outside of the DVC project.]"
 )
 
 _dvc_status=(
@@ -278,7 +282,7 @@ _dvc_status=(
   "*:Stages:_files -g '(*.dvc|Dvcfile)'"
 )
 
-_dvc_unlock=(
+_dvc_unfreeze=(
   "*:Stages:_files -g '(*.dvc|Dvcfile)'"
 )
 
@@ -305,9 +309,11 @@ case $words[1] in
   checkout) _arguments $_dvc_global_options $_dvc_checkout ;;
   commit) _arguments $_dvc_global_options $_dvc_commit ;;
   config) _arguments $_dvc_global_options $_dvc_config ;;
+  dag) _arguments $_dvc_global_options $_dvc_dag ;;
   destroy) _arguments $_dvc_global_options $_dvc_destroy ;;
   diff) _arguments $_dvc_global_options $_dvc_diff ;;
   fetch) _arguments $_dvc_global_options $_dvc_fetch ;;
+  freeze) _arguments $_dvc_global_options $_dvc_freeze ;;
   get-url) _arguments $_dvc_global_options $_dvc_geturl ;;
   get) _arguments $_dvc_global_options $_dvc_get ;;
   gc) _arguments $_dvc_global_options $_dvc_gc ;;
@@ -315,12 +321,10 @@ case $words[1] in
   import) _arguments $_dvc_global_options $_dvc_import ;;
   init) _arguments $_dvc_global_options $_dvc_init ;;
   install) _arguments $_dvc_global_options $_dvc_install ;;
-  lock) _arguments $_dvc_global_options $_dvc_lock ;;
   list) _arguments $_dvc_global_options $_dvc_list ;;
   metrics) _arguments $_dvc_global_options $_dvc_metrics ;;
   move) _arguments $_dvc_global_options $_dvc_move ;;
   params) _arguments $_dvc_global_options $_dvc_params ;;
-  pipeline) _arguments $_dvc_global_options $_dvc_pipeline ;;
   pull) _arguments $_dvc_global_options $_dvc_pull ;;
   push) _arguments $_dvc_global_options $_dvc_push ;;
   plots) _arguments $_dvc_global_options $_dvc_plots ;;
@@ -330,7 +334,7 @@ case $words[1] in
   root) _arguments $_dvc_global_options $_dvc_root ;;
   run) _arguments $_dvc_global_options $_dvc_run ;;
   status) _arguments $_dvc_global_options $_dvc_status ;;
-  unlock) _arguments $_dvc_global_options $_dvc_unlock ;;
+  unfreeze) _arguments $_dvc_global_options $_dvc_unfreeze ;;
   unprotect) _arguments $_dvc_global_options $_dvc_unprotect ;;
   update) _arguments $_dvc_global_options $_dvc_update ;;
 esac

setup.cfg

@@ -12,6 +12,6 @@ select=B,C,E,F,W,T4,B9
 [isort]
 include_trailing_comma=true
 known_first_party=dvc,tests
-known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,mockssh,moto,nanotime,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,tqdm,voluptuous,yaml,zc
+known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,mockssh,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,tqdm,voluptuous,yaml,zc
 line_length=79
 multi_line_output=3

setup.py

@@ -121,7 +121,7 @@ tests_requirements = [
     "flake8-docstrings",
     "pydocstyle<4.0",
     "jaraco.windows==3.9.2",
-    "mock-ssh-server>=0.6.0",
+    "mock-ssh-server",
     "moto==1.3.14.dev464",
     "rangehttpserver==1.2.0",
     "beautifulsoup4==4.4.0",

tests/conftest.py

@@ -34,7 +34,7 @@ user = "user"
 key_path = os.path.join(here, f"{user}.key")
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def ssh_server():
     users = {user: key_path}
     with mockssh.Server(users) as s:

tests/func/metrics/conftest.py

@@ -0,0 +1 @@
+from tests.func.plots import run_copy_metrics  # noqa: F401

tests/func/metrics/test_diff.py

@@ -3,11 +3,10 @@ import json
 import yaml
 
 
-def test_metrics_diff_simple(tmp_dir, scm, dvc):
+def test_metrics_diff_simple(tmp_dir, scm, dvc, run_copy_metrics):
     def _gen(val):
-        tmp_dir.gen({"m.yaml": str(val)})
-        dvc.run(cmd="", metrics=["m.yaml"], single_stage=True)
-        dvc.scm.add(["m.yaml.dvc"])
+        tmp_dir.gen({"m_temp.yaml": str(val)})
+        run_copy_metrics("m_temp.yaml", "m.yaml", metrics=["m.yaml"])
         dvc.scm.commit(str(val))
 
     _gen(1)
@@ -19,13 +18,13 @@ def test_metrics_diff_simple(tmp_dir, scm, dvc):
     assert dvc.metrics.diff(a_rev="HEAD~2") == expected
 
 
-def test_metrics_diff_yaml(tmp_dir, scm, dvc):
+def test_metrics_diff_yaml(tmp_dir, scm, dvc, run_copy_metrics):
     def _gen(val):
         metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
-        tmp_dir.gen({"m.yaml": yaml.dump(metrics)})
-        dvc.run(cmd="", metrics=["m.yaml"], single_stage=True)
-        dvc.scm.add(["m.yaml.dvc"])
-        dvc.scm.commit(str(val))
+        tmp_dir.gen({"m_temp.yaml": yaml.dump(metrics)})
+        run_copy_metrics(
+            "m_temp.yaml", "m.yaml", metrics=["m.yaml"], commit=str(val)
+        )
 
     _gen(1)
     _gen(2)
@@ -36,13 +35,13 @@ def test_metrics_diff_yaml(tmp_dir, scm, dvc):
     assert dvc.metrics.diff(a_rev="HEAD~2") == expected
 
 
-def test_metrics_diff_json(tmp_dir, scm, dvc):
+def test_metrics_diff_json(tmp_dir, scm, dvc, run_copy_metrics):
     def _gen(val):
         metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
-        tmp_dir.gen({"m.json": json.dumps(metrics)})
-        dvc.run(cmd="", metrics=["m.json"], single_stage=True)
-        dvc.scm.add(["m.json.dvc"])
-        dvc.scm.commit(str(val))
+        tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
+        run_copy_metrics(
+            "m_temp.json", "m.json", metrics=["m.json"], commit=str(val)
+        )
 
     _gen(1)
     _gen(2)
@@ -52,13 +51,13 @@ def test_metrics_diff_json(tmp_dir, scm, dvc):
     assert dvc.metrics.diff(a_rev="HEAD~2") == expected
 
 
-def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc):
+def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc, run_copy_metrics):
     def _gen(val):
         metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
-        tmp_dir.gen({"m.json": json.dumps(metrics)})
-        dvc.run(cmd="", metrics=["m.json"], single_stage=True)
-        dvc.scm.add(["m.json.dvc"])
-        dvc.scm.commit(str(val))
+        tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
+        run_copy_metrics(
+            "m_temp.json", "m.json", metrics=["m.json"], commit=str(val)
+        )
 
     _gen(1)
     _gen(2)
@@ -67,12 +66,15 @@ def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc):
     assert dvc.metrics.diff(a_rev="HEAD~2") == {}
 
 
-def test_metrics_diff_broken_json(tmp_dir, scm, dvc):
+def test_metrics_diff_broken_json(tmp_dir, scm, dvc, run_copy_metrics):
     metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
-    tmp_dir.gen({"m.json": json.dumps(metrics)})
-    dvc.run(cmd="", metrics_no_cache=["m.json"], single_stage=True)
-    dvc.scm.add(["m.json.dvc", "m.json"])
-    dvc.scm.commit("add metrics")
+    tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
+    run_copy_metrics(
+        "m_temp.json",
+        "m.json",
+        metrics_no_cache=["m.json"],
+        commit="add metrics",
+    )
 
     (tmp_dir / "m.json").write_text(json.dumps(metrics) + "ma\nlformed\n")
 
@@ -89,10 +91,10 @@ def test_metrics_diff_no_metrics(tmp_dir, scm, dvc):
     assert dvc.metrics.diff(a_rev="HEAD~1") == {}
 
 
-def test_metrics_diff_new_metric(tmp_dir, scm, dvc):
+def test_metrics_diff_new_metric(tmp_dir, scm, dvc, run_copy_metrics):
     metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
-    tmp_dir.gen({"m.json": json.dumps(metrics)})
-    dvc.run(cmd="", metrics_no_cache=["m.json"], single_stage=True)
+    tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
+    run_copy_metrics("m_temp.json", "m.json", metrics_no_cache=["m.json"])
 
     assert dvc.metrics.diff() == {
         "m.json": {
@@ -102,12 +104,15 @@ def test_metrics_diff_new_metric(tmp_dir, scm, dvc):
     }
 
 
-def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc):
+def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc, run_copy_metrics):
     metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
-    tmp_dir.gen({"m.json": json.dumps(metrics)})
-    dvc.run(cmd="", metrics_no_cache=["m.json"], single_stage=True)
-    dvc.scm.add(["m.json.dvc", "m.json"])
-    dvc.scm.commit("add metrics")
+    tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
+    run_copy_metrics(
+        "m_temp.json",
+        "m.json",
+        metrics_no_cache=["m.json"],
+        commit="add metrics",
+    )
 
     (tmp_dir / "m.json").unlink()
 
@@ -119,11 +124,14 @@ def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc):
     }
 
 
-def test_metrics_diff_with_unchanged(tmp_dir, scm, dvc):
-    tmp_dir.gen("metrics.yaml", "foo: 1\nxyz: 10")
-    dvc.run(metrics_no_cache=["metrics.yaml"], single_stage=True)
-    scm.add(["metrics.yaml", "metrics.yaml.dvc"])
-    scm.commit("1")
+def test_metrics_diff_with_unchanged(tmp_dir, scm, dvc, run_copy_metrics):
+    tmp_dir.gen("metrics_temp.yaml", "foo: 1\nxyz: 10")
+    run_copy_metrics(
+        "metrics_temp.yaml",
+        "metrics.yaml",
+        metrics_no_cache=["metrics.yaml"],
+        commit="1",
+    )
 
     tmp_dir.scm_gen("metrics.yaml", "foo: 2\nxyz: 10", commit="2")
     tmp_dir.scm_gen("metrics.yaml", "foo: 3\nxyz: 10", commit="3")

tests/func/metrics/test_show.py

@@ -1,5 +1,8 @@
+import os
+
 import pytest
 
+from dvc.repo import Repo
 from dvc.repo.metrics.show import NoMetricsError
 
 
@@ -8,36 +11,44 @@ def test_show_empty(dvc):
         dvc.metrics.show()
 
 
-def test_show_simple(tmp_dir, dvc):
-    tmp_dir.gen("metrics.yaml", "1.1")
-    dvc.run(metrics=["metrics.yaml"], single_stage=True)
+def test_show_simple(tmp_dir, dvc, run_copy_metrics):
+    tmp_dir.gen("metrics_t.yaml", "1.1")
+    run_copy_metrics(
+        "metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"],
+    )
     assert dvc.metrics.show() == {"": {"metrics.yaml": 1.1}}
 
 
-def test_show(tmp_dir, dvc):
-    tmp_dir.gen("metrics.yaml", "foo: 1.1")
-    dvc.run(metrics=["metrics.yaml"], single_stage=True)
+def test_show(tmp_dir, dvc, run_copy_metrics):
+    tmp_dir.gen("metrics_t.yaml", "foo: 1.1")
+    run_copy_metrics(
+        "metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"],
+    )
     assert dvc.metrics.show() == {"": {"metrics.yaml": {"foo": 1.1}}}
 
 
-def test_show_multiple(tmp_dir, dvc):
-    tmp_dir.gen("foo", "foo: 1\n")
-    tmp_dir.gen("baz", "baz: 2\n")
-    dvc.run(fname="foo.dvc", metrics=["foo"], single_stage=True)
-    dvc.run(fname="baz.dvc", metrics=["baz"], single_stage=True)
+def test_show_multiple(tmp_dir, dvc, run_copy_metrics):
+    tmp_dir.gen("foo_temp", "foo: 1\n")
+    tmp_dir.gen("baz_temp", "baz: 2\n")
+    run_copy_metrics("foo_temp", "foo", fname="foo.dvc", metrics=["foo"])
+    run_copy_metrics("baz_temp", "baz", fname="baz.dvc", metrics=["baz"])
     assert dvc.metrics.show() == {"": {"foo": {"foo": 1}, "baz": {"baz": 2}}}
 
 
-def test_show_invalid_metric(tmp_dir, dvc):
-    tmp_dir.gen("metrics.yaml", "foo:\n- bar\n- baz\nxyz: string")
-    dvc.run(metrics=["metrics.yaml"], single_stage=True)
+def test_show_invalid_metric(tmp_dir, dvc, run_copy_metrics):
+    tmp_dir.gen("metrics_temp.yaml", "foo:\n- bar\n- baz\nxyz: string")
+    run_copy_metrics(
+        "metrics_temp.yaml", "metrics.yaml", metrics=["metrics.yaml"]
+    )
     with pytest.raises(NoMetricsError):
         dvc.metrics.show()
 
 
-def test_show_branch(tmp_dir, scm, dvc):
-    tmp_dir.gen("metrics.yaml", "foo: 1")
-    dvc.run(metrics_no_cache=["metrics.yaml"], single_stage=True)
+def test_show_branch(tmp_dir, scm, dvc, run_copy_metrics):
+    tmp_dir.gen("metrics_temp.yaml", "foo: 1")
+    run_copy_metrics(
+        "metrics_temp.yaml", "metrics.yaml", metrics_no_cache=["metrics.yaml"]
+    )
     scm.add(["metrics.yaml", "metrics.yaml.dvc"])
     scm.commit("init")
 
@@ -45,6 +56,40 @@ def test_show_branch(tmp_dir, scm, dvc):
         tmp_dir.scm_gen("metrics.yaml", "foo: 2", commit="branch")
 
     assert dvc.metrics.show(revs=["branch"]) == {
-        "working tree": {"metrics.yaml": {"foo": 1}},
+        "workspace": {"metrics.yaml": {"foo": 1}},
         "branch": {"metrics.yaml": {"foo": 2}},
     }
+
+
+def test_show_subrepo_with_preexisting_tags(tmp_dir, scm):
+    tmp_dir.gen("foo", "foo")
+    scm.add("foo")
+    scm.commit("init")
+    scm.tag("no-metrics")
+
+    tmp_dir.gen({"subdir": {}})
+    subrepo_dir = tmp_dir / "subdir"
+    with subrepo_dir.chdir():
+        dvc = Repo.init(subdir=True)
+        scm.commit("init dvc")
+
+        dvc.run(
+            cmd="echo foo: 1 > metrics.yaml",
+            metrics=["metrics.yaml"],
+            single_stage=True,
+        )
+
+    scm.add(
+        [
+            str(subrepo_dir / "metrics.yaml"),
+            str(subrepo_dir / "metrics.yaml.dvc"),
+        ]
+    )
+    scm.commit("init metrics")
+    scm.tag("v1")
+
+    expected_path = os.path.join("subdir", "metrics.yaml")
+    assert dvc.metrics.show(all_tags=True) == {
+        "workspace": {expected_path: {"foo": 1}},
+        "v1": {expected_path: {"foo": 1}},
+    }

tests/func/params/test_diff.py

@@ -4,7 +4,7 @@ def test_diff_no_params(tmp_dir, scm, dvc):
 
 def test_diff_no_changes(tmp_dir, scm, dvc):
     tmp_dir.gen("params.yaml", "foo: bar")
-    dvc.run(params=["foo"], single_stage=True)
+    dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
     scm.add(["params.yaml", "Dvcfile"])
     scm.commit("bar")
     assert dvc.params.diff() == {}
@@ -12,7 +12,7 @@ def test_diff_no_changes(tmp_dir, scm, dvc):
 
 def test_diff(tmp_dir, scm, dvc):
     tmp_dir.gen("params.yaml", "foo: bar")
-    dvc.run(params=["foo"], single_stage=True)
+    dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
     scm.add(["params.yaml", "Dvcfile"])
     scm.commit("bar")
 
@@ -26,7 +26,7 @@ def test_diff(tmp_dir, scm, dvc):
 
 def test_diff_new(tmp_dir, scm, dvc):
     tmp_dir.gen("params.yaml", "foo: bar")
-    dvc.run(params=["foo"], single_stage=True)
+    dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
 
     assert dvc.params.diff() == {
         "params.yaml": {"foo": {"old": None, "new": "bar"}}
@@ -35,7 +35,7 @@ def test_diff_new(tmp_dir, scm, dvc):
 
 def test_diff_deleted(tmp_dir, scm, dvc):
     tmp_dir.gen("params.yaml", "foo: bar")
-    dvc.run(params=["foo"], single_stage=True)
+    dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
     scm.add(["params.yaml", "Dvcfile"])
     scm.commit("bar")
 
@@ -48,7 +48,7 @@ def test_diff_deleted(tmp_dir, scm, dvc):
 
 def test_diff_deleted_config(tmp_dir, scm, dvc):
     tmp_dir.gen("params.yaml", "foo: bar")
-    dvc.run(params=["foo"], single_stage=True)
+    dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
     scm.add(["params.yaml", "Dvcfile"])
     scm.commit("bar")
 
@@ -61,7 +61,7 @@ def test_diff_deleted_config(tmp_dir, scm, dvc):
 
 def test_diff_list(tmp_dir, scm, dvc):
     tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz")
-    dvc.run(params=["foo"], single_stage=True)
+    dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
     scm.add(["params.yaml", "Dvcfile"])
     scm.commit("foo")
 
@@ -76,7 +76,7 @@ def test_diff_list(tmp_dir, scm, dvc):
 
 def test_diff_dict(tmp_dir, scm, dvc):
     tmp_dir.gen("params.yaml", "foo:\n  bar: baz")
-    dvc.run(params=["foo"], single_stage=True)
+    dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
     scm.add(["params.yaml", "Dvcfile"])
     scm.commit("foo")
 
@@ -89,7 +89,7 @@ def test_diff_dict(tmp_dir, scm, dvc):
 
 def test_diff_with_unchanged(tmp_dir, scm, dvc):
     tmp_dir.gen("params.yaml", "foo: bar\nxyz: val")
-    dvc.run(params=["foo,xyz"], single_stage=True)
+    dvc.run(cmd="echo params.yaml", params=["foo,xyz"], single_stage=True)
     scm.add(["params.yaml", "Dvcfile"])
     scm.commit("bar")
 

tests/func/params/test_show.py

@@ -10,14 +10,24 @@ def test_show_empty(dvc):
 
 def test_show(tmp_dir, dvc):
     tmp_dir.gen("params.yaml", "foo: bar")
-    dvc.run(params=["foo"], single_stage=True)
+    dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
     assert dvc.params.show() == {"": {"params.yaml": {"foo": "bar"}}}
 
 
 def test_show_multiple(tmp_dir, dvc):
     tmp_dir.gen("params.yaml", "foo: bar\nbaz: qux\n")
-    dvc.run(fname="foo.dvc", params=["foo"], single_stage=True)
-    dvc.run(fname="baz.dvc", params=["baz"], single_stage=True)
+    dvc.run(
+        cmd="echo params.yaml",
+        fname="foo.dvc",
+        params=["foo"],
+        single_stage=True,
+    )
+    dvc.run(
+        cmd="echo params.yaml",
+        fname="baz.dvc",
+        params=["baz"],
+        single_stage=True,
+    )
     assert dvc.params.show() == {
         "": {"params.yaml": {"foo": "bar", "baz": "qux"}}
     }
@@ -25,13 +35,13 @@ def test_show_multiple(tmp_dir, dvc):
 
 def test_show_list(tmp_dir, dvc):
     tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz\n")
-    dvc.run(params=["foo"], single_stage=True)
+    dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
     assert dvc.params.show() == {"": {"params.yaml": {"foo": ["bar", "baz"]}}}
 
 
 def test_show_branch(tmp_dir, scm, dvc):
     tmp_dir.gen("params.yaml", "foo: bar")
-    dvc.run(params=["foo"], single_stage=True)
+    dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
     scm.add(["params.yaml", "Dvcfile"])
     scm.commit("init")
 
@@ -39,7 +49,7 @@ def test_show_branch(tmp_dir, scm, dvc):
         tmp_dir.scm_gen("params.yaml", "foo: baz", commit="branch")
 
     assert dvc.params.show(revs=["branch"]) == {
-        "working tree": {"params.yaml": {"foo": "bar"}},
+        "workspace": {"params.yaml": {"foo": "bar"}},
         "branch": {"params.yaml": {"foo": "baz"}},
     }
 

tests/func/plots/__init__.py

@@ -0,0 +1,30 @@
+import os
+
+import pytest
+
+
+@pytest.fixture
+def run_copy_metrics(tmp_dir, run_copy):
+    def run(file1, file2, commit=None, tag=None, **kwargs):
+        stage = tmp_dir.dvc.run(
+            cmd=f"python copy.py {file1} {file2}",
+            deps=[file1],
+            single_stage=True,
+            **kwargs,
+        )
+
+        if hasattr(tmp_dir.dvc, "scm"):
+            files = [stage.path]
+            files += [
+                os.fspath(out.path_info)
+                for out in stage.outs
+                if not out.use_cache
+            ]
+            tmp_dir.dvc.scm.add(files)
+            if commit:
+                tmp_dir.dvc.scm.commit(commit)
+            if tag:
+                tmp_dir.dvc.scm.tag(tag)
+        return stage
+
+    return run

tests/func/plots/conftest.py

@@ -0,0 +1 @@
+from . import run_copy_metrics  # noqa: F401

tests/func/plots/test_diff.py

@@ -1,27 +1,39 @@
 import json
 
-from .test_plots import PlotData, _run_with, _write_json
+from .test_plots import PlotData, _write_json
 
 
-def test_diff_dirty(tmp_dir, scm, dvc):
+def test_diff_dirty(tmp_dir, scm, dvc, run_copy_metrics):
     metric_1 = [{"y": 2}, {"y": 3}]
-    _write_json(tmp_dir, metric_1, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init")
+    _write_json(tmp_dir, metric_1, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="init",
+    )
 
     metric_2 = [{"y": 3}, {"y": 5}]
-    _write_json(tmp_dir, metric_2, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"], commit="second")
+    _write_json(tmp_dir, metric_2, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="second",
+    )
 
     metric_3 = [{"y": 5}, {"y": 6}]
-    _write_json(tmp_dir, metric_3, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"])
+    _write_json(tmp_dir, metric_3, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json", "metric.json", plots_no_cache=["metric.json"]
+    )
 
-    plot_string = dvc.plots.diff(fields={"y"})["metric.json"]
+    plot_string = dvc.plots.diff(props={"fields": {"y"}})["metric.json"]
 
     plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
-        {"y": 5, PlotData.INDEX_FIELD: 0, "rev": "working tree"},
-        {"y": 6, PlotData.INDEX_FIELD: 1, "rev": "working tree"},
+        {"y": 5, PlotData.INDEX_FIELD: 0, "rev": "workspace"},
+        {"y": 6, PlotData.INDEX_FIELD: 1, "rev": "workspace"},
         {"y": 3, PlotData.INDEX_FIELD: 0, "rev": "HEAD"},
         {"y": 5, PlotData.INDEX_FIELD: 1, "rev": "HEAD"},
     ]

tests/func/plots/test_plots.py

@@ -14,7 +14,6 @@ from dvc.repo.plots.data import (
     PlotData,
     PlotMetricTypeError,
 )
-from dvc.repo.plots.show import NoDataOrTemplateProvided
 from dvc.repo.plots.template import (
     NoDataForTemplateError,
     NoFieldInDataError,
@@ -22,24 +21,6 @@ from dvc.repo.plots.template import (
 )
 
 
-def _remove_whitespace(value):
-    return value.replace(" ", "").replace("\n", "")
-
-
-def _run_with(tmp_dir, commit=None, tag=None, **kwargs):
-    stage = tmp_dir.dvc.run(single_stage=True, **kwargs)
-    if hasattr(tmp_dir.dvc, "scm"):
-        files = [stage.path]
-        files += [
-            os.fspath(out.path_info) for out in stage.outs if not out.use_cache
-        ]
-        tmp_dir.dvc.scm.add(files)
-        if commit:
-            tmp_dir.dvc.scm.commit(commit)
-        if tag:
-            tmp_dir.dvc.scm.tag(tag)
-
-
 def _write_csv(metric, filename, header=True):
     with open(filename, "w", newline="") as csvobj:
         if header:
@@ -59,24 +40,27 @@ def _write_json(tmp_dir, metric, filename):
     tmp_dir.gen(filename, json.dumps(metric, sort_keys=True))
 
 
-def test_plot_csv_one_column(tmp_dir, scm, dvc):
+def test_plot_csv_one_column(tmp_dir, scm, dvc, run_copy_metrics):
     # no header
     metric = [{"val": 2}, {"val": 3}]
-    _write_csv(metric, "metric.csv", header=False)
-    _run_with(tmp_dir, plots_no_cache=["metric.csv"])
+    _write_csv(metric, "metric_t.csv", header=False)
+    run_copy_metrics(
+        "metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
+    )
 
-    plot_string = dvc.plots.show(
-        csv_header=False,
-        x_title="x_title",
-        y_title="y_title",
-        title="mytitle",
-    )["metric.csv"]
+    props = {
+        "csv_header": False,
+        "x_label": "x_title",
+        "y_label": "y_title",
+        "title": "mytitle",
+    }
+    plot_string = dvc.plots.show(props=props)["metric.csv"]
 
     plot_content = json.loads(plot_string)
     assert plot_content["title"] == "mytitle"
     assert plot_content["data"]["values"] == [
-        {"0": "2", PlotData.INDEX_FIELD: 0, "rev": "working tree"},
-        {"0": "3", PlotData.INDEX_FIELD: 1, "rev": "working tree"},
+        {"0": "2", PlotData.INDEX_FIELD: 0, "rev": "workspace"},
+        {"0": "3", PlotData.INDEX_FIELD: 1, "rev": "workspace"},
     ]
     assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
     assert plot_content["encoding"]["y"]["field"] == "0"
@@ -84,13 +68,15 @@ def test_plot_csv_one_column(tmp_dir, scm, dvc):
     assert plot_content["encoding"]["y"]["title"] == "y_title"
 
 
-def test_plot_csv_multiple_columns(tmp_dir, scm, dvc):
+def test_plot_csv_multiple_columns(tmp_dir, scm, dvc, run_copy_metrics):
     metric = [
         OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
         OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
     ]
-    _write_csv(metric, "metric.csv")
-    _run_with(tmp_dir, plots_no_cache=["metric.csv"])
+    _write_csv(metric, "metric_t.csv")
+    run_copy_metrics(
+        "metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
+    )
 
     plot_string = dvc.plots.show()["metric.csv"]
 
@@ -99,14 +85,14 @@ def test_plot_csv_multiple_columns(tmp_dir, scm, dvc):
         {
             "val": "2",
             PlotData.INDEX_FIELD: 0,
-            "rev": "working tree",
+            "rev": "workspace",
             "first_val": "100",
             "second_val": "100",
         },
         {
             "val": "3",
             PlotData.INDEX_FIELD: 1,
-            "rev": "working tree",
+            "rev": "workspace",
             "first_val": "200",
             "second_val": "300",
         },
@@ -115,29 +101,30 @@ def test_plot_csv_multiple_columns(tmp_dir, scm, dvc):
     assert plot_content["encoding"]["y"]["field"] == "val"
 
 
-def test_plot_csv_choose_axes(tmp_dir, scm, dvc):
+def test_plot_csv_choose_axes(tmp_dir, scm, dvc, run_copy_metrics):
     metric = [
         OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
         OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
     ]
-    _write_csv(metric, "metric.csv")
-    _run_with(tmp_dir, plots_no_cache=["metric.csv"])
+    _write_csv(metric, "metric_t.csv")
+    run_copy_metrics(
+        "metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
+    )
 
-    plot_string = dvc.plots.show(x_field="first_val", y_field="second_val")[
-        "metric.csv"
-    ]
+    props = {"x": "first_val", "y": "second_val"}
+    plot_string = dvc.plots.show(props=props)["metric.csv"]
 
     plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
         {
             "val": "2",
-            "rev": "working tree",
+            "rev": "workspace",
             "first_val": "100",
             "second_val": "100",
         },
         {
             "val": "3",
-            "rev": "working tree",
+            "rev": "workspace",
             "first_val": "200",
             "second_val": "300",
         },
@@ -146,29 +133,39 @@ def test_plot_csv_choose_axes(tmp_dir, scm, dvc):
     assert plot_content["encoding"]["y"]["field"] == "second_val"
 
 
-def test_plot_json_single_val(tmp_dir, scm, dvc):
+def test_plot_json_single_val(tmp_dir, scm, dvc, run_copy_metrics):
     metric = [{"val": 2}, {"val": 3}]
-    _write_json(tmp_dir, metric, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
+    _write_json(tmp_dir, metric, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="first run",
+    )
 
     plot_string = dvc.plots.show()["metric.json"]
 
     plot_json = json.loads(plot_string)
     assert plot_json["data"]["values"] == [
-        {"val": 2, PlotData.INDEX_FIELD: 0, "rev": "working tree"},
-        {"val": 3, PlotData.INDEX_FIELD: 1, "rev": "working tree"},
+        {"val": 2, PlotData.INDEX_FIELD: 0, "rev": "workspace"},
+        {"val": 3, PlotData.INDEX_FIELD: 1, "rev": "workspace"},
     ]
     assert plot_json["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
     assert plot_json["encoding"]["y"]["field"] == "val"
 
 
-def test_plot_json_multiple_val(tmp_dir, scm, dvc):
+def test_plot_json_multiple_val(tmp_dir, scm, dvc, run_copy_metrics):
     metric = [
         {"first_val": 100, "val": 2},
         {"first_val": 200, "val": 3},
     ]
-    _write_json(tmp_dir, metric, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
+    _write_json(tmp_dir, metric, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="first run",
+    )
 
     plot_string = dvc.plots.show()["metric.json"]
 
@@ -178,56 +175,118 @@ def test_plot_json_multiple_val(tmp_dir, scm, dvc):
             "val": 2,
             PlotData.INDEX_FIELD: 0,
             "first_val": 100,
-            "rev": "working tree",
+            "rev": "workspace",
         },
         {
             "val": 3,
             PlotData.INDEX_FIELD: 1,
             "first_val": 200,
-            "rev": "working tree",
+            "rev": "workspace",
         },
     ]
     assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
     assert plot_content["encoding"]["y"]["field"] == "val"
 
 
-def test_plot_confusion(tmp_dir, dvc):
+def test_plot_confusion(tmp_dir, dvc, run_copy_metrics):
     confusion_matrix = [
         {"predicted": "B", "actual": "A"},
         {"predicted": "A", "actual": "A"},
     ]
-    _write_json(tmp_dir, confusion_matrix, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
+    _write_json(tmp_dir, confusion_matrix, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="first run",
+    )
 
-    plot_string = dvc.plots.show(
-        template="confusion", x_field="predicted", y_field="actual",
-    )["metric.json"]
+    props = {"template": "confusion", "x": "predicted", "y": "actual"}
+    plot_string = dvc.plots.show(props=props)["metric.json"]
 
     plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
-        {"predicted": "B", "actual": "A", "rev": "working tree"},
-        {"predicted": "A", "actual": "A", "rev": "working tree"},
+        {"predicted": "B", "actual": "A", "rev": "workspace"},
+        {"predicted": "A", "actual": "A", "rev": "workspace"},
     ]
     assert plot_content["encoding"]["x"]["field"] == "predicted"
     assert plot_content["encoding"]["y"]["field"] == "actual"
 
 
-def test_plot_multiple_revs_default(tmp_dir, scm, dvc):
+def test_plot_multiple_revs_default(tmp_dir, scm, dvc, run_copy_metrics):
     metric_1 = [{"y": 2}, {"y": 3}]
-    _write_json(tmp_dir, metric_1, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
+    _write_json(tmp_dir, metric_1, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="init",
+        tag="v1",
+    )
 
     metric_2 = [{"y": 3}, {"y": 5}]
-    _write_json(tmp_dir, metric_2, "metric.json")
-    _run_with(
-        tmp_dir, plots_no_cache=["metric.json"], commit="second", tag="v2"
+    _write_json(tmp_dir, metric_2, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="second",
+        tag="v2",
     )
 
     metric_3 = [{"y": 5}, {"y": 6}]
-    _write_json(tmp_dir, metric_3, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"], commit="third")
+    _write_json(tmp_dir, metric_3, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="third",
+    )
+    plot_string = dvc.plots.show(
+        revs=["HEAD", "v2", "v1"], props={"fields": {"y"}}
+    )["metric.json"]
 
-    plot_string = dvc.plots.show(fields={"y"}, revs=["HEAD", "v2", "v1"],)[
+    plot_content = json.loads(plot_string)
+    assert plot_content["data"]["values"] == [
+        {"y": 5, PlotData.INDEX_FIELD: 0, "rev": "HEAD"},
+        {"y": 6, PlotData.INDEX_FIELD: 1, "rev": "HEAD"},
+        {"y": 3, PlotData.INDEX_FIELD: 0, "rev": "v2"},
+        {"y": 5, PlotData.INDEX_FIELD: 1, "rev": "v2"},
+        {"y": 2, PlotData.INDEX_FIELD: 0, "rev": "v1"},
+        {"y": 3, PlotData.INDEX_FIELD: 1, "rev": "v1"},
+    ]
+    assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
+    assert plot_content["encoding"]["y"]["field"] == "y"
+
+
+def test_plot_multiple_revs(tmp_dir, scm, dvc, run_copy_metrics):
+    shutil.copy(tmp_dir / ".dvc" / "plots" / "default.json", "template.json")
+
+    metric_1 = [{"y": 2}, {"y": 3}]
+    _write_json(tmp_dir, metric_1, "metric_t.json")
+    stage = run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="init",
+        tag="v1",
+    )
+
+    metric_2 = [{"y": 3}, {"y": 5}]
+    _write_json(tmp_dir, metric_2, "metric_t.json")
+    assert dvc.reproduce(stage.addressing) == [stage]
+    scm.add(["metric.json", stage.path])
+    scm.commit("second")
+    scm.tag("v2")
+
+    metric_3 = [{"y": 5}, {"y": 6}]
+    _write_json(tmp_dir, metric_3, "metric_t.json")
+    assert dvc.reproduce(stage.addressing) == [stage]
+    scm.add(["metric.json", stage.path])
+    scm.commit("third")
+
+    props = {"template": "template.json"}
+    plot_string = dvc.plots.show(revs=["HEAD", "v2", "v1"], props=props)[
         "metric.json"
     ]
 
@@ -244,48 +303,17 @@ def test_plot_multiple_revs_default(tmp_dir, scm, dvc):
     assert plot_content["encoding"]["y"]["field"] == "y"
 
 
-def test_plot_multiple_revs(tmp_dir, scm, dvc):
-    shutil.copy(tmp_dir / ".dvc" / "plots" / "default.json", "template.json")
-
-    metric_1 = [{"y": 2}, {"y": 3}]
-    _write_json(tmp_dir, metric_1, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
-
-    metric_2 = [{"y": 3}, {"y": 5}]
-    _write_json(tmp_dir, metric_2, "metric.json")
-    _run_with(
-        tmp_dir, plots_no_cache=["metric.json"], commit="second", tag="v2"
-    )
-
-    metric_3 = [{"y": 5}, {"y": 6}]
-    _write_json(tmp_dir, metric_3, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"], commit="third")
-
-    plot_string = dvc.plots.show(
-        template="template.json", revs=["HEAD", "v2", "v1"],
-    )["metric.json"]
-
-    plot_content = json.loads(plot_string)
-    assert plot_content["data"]["values"] == [
-        {"y": 5, PlotData.INDEX_FIELD: 0, "rev": "HEAD"},
-        {"y": 6, PlotData.INDEX_FIELD: 1, "rev": "HEAD"},
-        {"y": 3, PlotData.INDEX_FIELD: 0, "rev": "v2"},
-        {"y": 5, PlotData.INDEX_FIELD: 1, "rev": "v2"},
-        {"y": 2, PlotData.INDEX_FIELD: 0, "rev": "v1"},
-        {"y": 3, PlotData.INDEX_FIELD: 1, "rev": "v1"},
-    ]
-    assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
-    assert plot_content["encoding"]["y"]["field"] == "y"
-
-
-def test_plot_even_if_metric_missing(tmp_dir, scm, dvc, caplog):
+def test_plot_even_if_metric_missing(
+    tmp_dir, scm, dvc, caplog, run_copy_metrics
+):
     tmp_dir.scm_gen("some_file", "content", commit="there is no metric")
     scm.tag("v1")
 
     metric = [{"y": 2}, {"y": 3}]
-    _write_json(tmp_dir, metric, "metric.json")
-    _run_with(
-        tmp_dir,
+    _write_json(tmp_dir, metric, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
         plots_no_cache=["metric.json"],
         commit="there is metric",
         tag="v2",
@@ -335,19 +363,24 @@ def custom_template(tmp_dir, dvc):
     return custom_template
 
 
-def test_custom_template(tmp_dir, scm, dvc, custom_template):
+def test_custom_template(tmp_dir, scm, dvc, custom_template, run_copy_metrics):
     metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
-    _write_json(tmp_dir, metric, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
+    _write_json(tmp_dir, metric, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="init",
+        tag="v1",
+    )
 
-    plot_string = dvc.plots.show(
-        template=os.fspath(custom_template), x_field="a", y_field="b"
-    )["metric.json"]
+    props = {"template": os.fspath(custom_template), "x": "a", "y": "b"}
+    plot_string = dvc.plots.show(props=props)["metric.json"]
 
     plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
-        {"a": 1, "b": 2, "rev": "working tree"},
-        {"a": 2, "b": 3, "rev": "working tree"},
+        {"a": 1, "b": 2, "rev": "workspace"},
+        {"a": 2, "b": 3, "rev": "workspace"},
     ]
     assert plot_content["encoding"]["x"]["field"] == "a"
     assert plot_content["encoding"]["y"]["field"] == "b"
@@ -358,30 +391,37 @@ def _replace(path, src, dst):
 
 
 def test_custom_template_with_specified_data(
-    tmp_dir, scm, dvc, custom_template
+    tmp_dir, scm, dvc, custom_template, run_copy_metrics
 ):
     _replace(
         custom_template, "DVC_METRIC_DATA", "DVC_METRIC_DATA,metric.json",
     )
 
     metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
-    _write_json(tmp_dir, metric, "metric.json")
-    _run_with(tmp_dir, outs_no_cache=["metric.json"], commit="init", tag="v1")
+    _write_json(tmp_dir, metric, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        outs_no_cache=["metric.json"],
+        commit="init",
+        tag="v1",
+    )
 
-    plot_string = dvc.plots.show(
-        template=os.fspath(custom_template), x_field="a", y_field="b",
-    )["metric.json"]
+    props = {"template": os.fspath(custom_template), "x": "a", "y": "b"}
+    plot_string = dvc.plots.show(props=props)["metric.json"]
 
     plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
-        {"a": 1, "b": 2, "rev": "working tree"},
-        {"a": 2, "b": 3, "rev": "working tree"},
+        {"a": 1, "b": 2, "rev": "workspace"},
+        {"a": 2, "b": 3, "rev": "workspace"},
     ]
     assert plot_content["encoding"]["x"]["field"] == "a"
     assert plot_content["encoding"]["y"]["field"] == "b"
 
 
-def test_plot_override_specified_data_source(tmp_dir, scm, dvc):
+def test_plot_override_specified_data_source(
+    tmp_dir, scm, dvc, run_copy_metrics
+):
     shutil.copy(
         tmp_dir / ".dvc" / "plots" / "default.json",
         tmp_dir / "newtemplate.json",
@@ -393,103 +433,151 @@ def test_plot_override_specified_data_source(tmp_dir, scm, dvc):
     )
 
     metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
-    _write_json(tmp_dir, metric, "metric2.json")
-    _run_with(tmp_dir, outs_no_cache=["metric2.json"], commit="init", tag="v1")
+    _write_json(tmp_dir, metric, "metric1.json")
+    run_copy_metrics(
+        "metric1.json",
+        "metric2.json",
+        plots_no_cache=["metric2.json"],
+        commit="init",
+        tag="v1",
+    )
 
-    plot_string = dvc.plots.show(
-        targets=["metric2.json"], template="newtemplate.json", x_field="a"
-    )["metric2.json"]
+    props = {"template": "newtemplate.json", "x": "a"}
+    plot_string = dvc.plots.show(targets=["metric2.json"], props=props)[
+        "metric2.json"
+    ]
 
     plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
-        {"a": 1, "b": 2, "rev": "working tree"},
-        {"a": 2, "b": 3, "rev": "working tree"},
+        {"a": 1, "b": 2, "rev": "workspace"},
+        {"a": 2, "b": 3, "rev": "workspace"},
     ]
     assert plot_content["encoding"]["x"]["field"] == "a"
     assert plot_content["encoding"]["y"]["field"] == "b"
 
 
-def test_should_raise_on_no_template_and_datafile(tmp_dir, dvc):
-    with pytest.raises(NoDataOrTemplateProvided):
+def test_no_plots(tmp_dir, dvc):
+    from dvc.exceptions import NoPlotsError
+
+    with pytest.raises(NoPlotsError):
         dvc.plots.show()
 
 
-def test_should_raise_on_no_template(tmp_dir, dvc):
+def test_should_raise_on_no_template(tmp_dir, dvc, run_copy_metrics):
+    metric = [{"val": 2}, {"val": 3}]
+    _write_json(tmp_dir, metric, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="first run",
+    )
+
     with pytest.raises(TemplateNotFoundError):
-        dvc.plots.show("metric.json", template="non_existing_template.json")
+        props = {"template": "non_existing_template.json"}
+        dvc.plots.show("metric.json", props=props)
 
 
 def test_plot_no_data(tmp_dir, dvc):
     with pytest.raises(NoDataForTemplateError):
-        dvc.plots.show(template="default")
+        dvc.plots.show(props={"template": "default"})
 
 
-def test_plot_wrong_metric_type(tmp_dir, scm, dvc):
-    tmp_dir.scm_gen("metric.txt", "content", commit="initial")
+def test_plot_wrong_metric_type(tmp_dir, scm, dvc, run_copy_metrics):
+    tmp_dir.gen("metric_t.txt", "some text")
+    run_copy_metrics(
+        "metric_t.txt",
+        "metric.txt",
+        plots_no_cache=["metric.txt"],
+        commit="add text metric",
+    )
+
     with pytest.raises(PlotMetricTypeError):
         dvc.plots.show(targets=["metric.txt"])
 
 
-def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template):
+def test_plot_choose_columns(
+    tmp_dir, scm, dvc, custom_template, run_copy_metrics
+):
     metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
-    _write_json(tmp_dir, metric, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
+    _write_json(tmp_dir, metric, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="init",
+        tag="v1",
+    )
 
-    plot_string = dvc.plots.show(
-        template=os.fspath(custom_template),
-        fields={"b", "c"},
-        x_field="b",
-        y_field="c",
-    )["metric.json"]
+    props = {
+        "template": os.fspath(custom_template),
+        "fields": {"b", "c"},
+        "x": "b",
+        "y": "c",
+    }
+    plot_string = dvc.plots.show(props=props)["metric.json"]
 
     plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
-        {"b": 2, "c": 3, "rev": "working tree"},
-        {"b": 3, "c": 4, "rev": "working tree"},
+        {"b": 2, "c": 3, "rev": "workspace"},
+        {"b": 3, "c": 4, "rev": "workspace"},
     ]
     assert plot_content["encoding"]["x"]["field"] == "b"
     assert plot_content["encoding"]["y"]["field"] == "c"
 
 
-def test_plot_default_choose_column(tmp_dir, scm, dvc):
+def test_plot_default_choose_column(tmp_dir, scm, dvc, run_copy_metrics):
     metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
-    _write_json(tmp_dir, metric, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
+    _write_json(tmp_dir, metric, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="init",
+        tag="v1",
+    )
 
-    plot_string = dvc.plots.show(fields={"b"})["metric.json"]
+    plot_string = dvc.plots.show(props={"fields": {"b"}})["metric.json"]
 
     plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
-        {PlotData.INDEX_FIELD: 0, "b": 2, "rev": "working tree"},
-        {PlotData.INDEX_FIELD: 1, "b": 3, "rev": "working tree"},
+        {PlotData.INDEX_FIELD: 0, "b": 2, "rev": "workspace"},
+        {PlotData.INDEX_FIELD: 1, "b": 3, "rev": "workspace"},
     ]
     assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
     assert plot_content["encoding"]["y"]["field"] == "b"
 
 
-def test_plot_yaml(tmp_dir, scm, dvc):
+def test_plot_yaml(tmp_dir, scm, dvc, run_copy_metrics):
     metric = [{"val": 2}, {"val": 3}]
-    with open("metric.yaml", "w") as fobj:
+    with open("metric_t.yaml", "w") as fobj:
         yaml.dump(metric, fobj)
 
-    _run_with(tmp_dir, plots_no_cache=["metric.yaml"])
+    run_copy_metrics(
+        "metric_t.yaml", "metric.yaml", plots_no_cache=["metric.yaml"]
+    )
 
     plot_string = dvc.plots.show()["metric.yaml"]
 
     plot_content = json.loads(plot_string)
     assert plot_content["data"]["values"] == [
-        {"val": 2, PlotData.INDEX_FIELD: 0, "rev": "working tree"},
-        {"val": 3, PlotData.INDEX_FIELD: 1, "rev": "working tree"},
+        {"val": 2, PlotData.INDEX_FIELD: 0, "rev": "workspace"},
+        {"val": 3, PlotData.INDEX_FIELD: 1, "rev": "workspace"},
     ]
 
 
-def test_raise_on_wrong_field(tmp_dir, scm, dvc):
+def test_raise_on_wrong_field(tmp_dir, scm, dvc, run_copy_metrics):
     metric = [{"val": 2}, {"val": 3}]
-    _write_json(tmp_dir, metric, "metric.json")
-    _run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
+    _write_json(tmp_dir, metric, "metric_t.json")
+    run_copy_metrics(
+        "metric_t.json",
+        "metric.json",
+        plots_no_cache=["metric.json"],
+        commit="first run",
+    )
 
     with pytest.raises(NoFieldInDataError):
-        dvc.plots.show("metric.json", x_field="no_val")
+        dvc.plots.show("metric.json", props={"x": "no_val"})
 
     with pytest.raises(NoFieldInDataError):
-        dvc.plots.show("metric.json", y_field="no_val")
+        dvc.plots.show("metric.json", props={"y": "no_val"})

tests/func/test_add.py

@@ -17,17 +17,17 @@ from dvc.exceptions import (
     OutputDuplicationError,
     OverlappingOutputPathsError,
     RecursiveAddingWhileUsingFilename,
-    StageFileCorruptedError,
+    YAMLFileCorruptedError,
 )
 from dvc.main import main
 from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError
-from dvc.remote import LocalRemote
+from dvc.remote.local import LocalRemote, LocalRemoteTree
 from dvc.repo import Repo as DvcRepo
 from dvc.stage import Stage
 from dvc.system import System
 from dvc.utils import LARGE_DIR_SIZE, file_md5, relpath
 from dvc.utils.fs import path_isin
-from dvc.utils.stage import load_stage_file
+from dvc.utils.yaml import load_yaml
 from tests.basic_env import TestDvc
 from tests.utils import get_gitignore_content
 
@@ -46,7 +46,7 @@ def test_add(tmp_dir, dvc):
     assert stage.outs[0].info["md5"] == md5
     assert stage.md5 is None
 
-    assert load_stage_file("foo.dvc") == {
+    assert load_yaml("foo.dvc") == {
         "outs": [{"md5": "acbd18db4cc2f85cedef654fccc4a4d8", "path": "foo"}],
     }
 
@@ -187,11 +187,16 @@ def test_add_file_in_dir(tmp_dir, dvc):
 
 class TestAddExternalLocalFile(TestDvc):
     def test(self):
+        from dvc.stage.exceptions import StageExternalOutputsError
+
         dname = TestDvc.mkdtemp()
         fname = os.path.join(dname, "foo")
         shutil.copyfile(self.FOO, fname)
 
-        stages = self.dvc.add(fname)
+        with self.assertRaises(StageExternalOutputsError):
+            self.dvc.add(fname)
+
+        stages = self.dvc.add(fname, external=True)
         self.assertEqual(len(stages), 1)
         stage = stages[0]
         self.assertNotEqual(stage, None)
@@ -220,14 +225,14 @@ class TestAddLocalRemoteFile(TestDvc):
         ret = main(["add", foo])
         self.assertEqual(ret, 0)
 
-        d = load_stage_file("foo.dvc")
+        d = load_yaml("foo.dvc")
         self.assertEqual(d["outs"][0]["path"], foo)
 
         bar = os.path.join(cwd, self.BAR)
         ret = main(["add", bar])
         self.assertEqual(ret, 0)
 
-        d = load_stage_file("bar.dvc")
+        d = load_yaml("bar.dvc")
         self.assertEqual(d["outs"][0]["path"], self.BAR)
 
 
@@ -371,7 +376,7 @@ class SymlinkAddTestBase(TestDvc):
         stage_file = self.data_file_name + DVC_FILE_SUFFIX
         self.assertTrue(os.path.exists(stage_file))
 
-        d = load_stage_file(stage_file)
+        d = load_yaml(stage_file)
         relative_data_path = posixpath.join(
             self.link_name, self.data_file_name
         )
@@ -436,8 +441,7 @@ class TestShouldThrowProperExceptionOnCorruptedStageFile(TestDvc):
         assert 1 == ret
 
         expected_error = (
-            "unable to read DVC-file: {} "
-            "YAML file structure is corrupted".format(foo_stage)
+            f"unable to read: '{foo_stage}', YAML file structure is corrupted"
         )
 
         assert expected_error in self._caplog.text
@@ -478,7 +482,7 @@ def test_failed_add_cleanup(tmp_dir, scm, dvc):
     dvc.add("foo")
     tmp_dir.gen("foo.dvc", "- broken\nyaml")
 
-    with pytest.raises(StageFileCorruptedError):
+    with pytest.raises(YAMLFileCorruptedError):
         dvc.add("bar")
 
     assert not os.path.exists("bar.dvc")
@@ -593,7 +597,7 @@ def test_should_not_checkout_when_adding_cached_copy(tmp_dir, dvc, mocker):
 
     shutil.copy("bar", "foo")
 
-    copy_spy = mocker.spy(dvc.cache.local, "copy")
+    copy_spy = mocker.spy(dvc.cache.local.tree, "copy")
 
     dvc.add("foo")
 
@@ -619,7 +623,7 @@ def test_should_relink_on_repeated_add(
     tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
 
     os.remove("foo")
-    getattr(dvc.cache.local, link)(PathInfo("bar"), PathInfo("foo"))
+    getattr(dvc.cache.local.tree, link)(PathInfo("bar"), PathInfo("foo"))
 
     dvc.cache.local.cache_types = [new_link]
 
@@ -699,7 +703,7 @@ def test_add_empty_files(tmp_dir, dvc, link):
 def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker):
     dvc.cache.local.cache_types = ["hardlink"]
     tmp_dir.gen({"foo": "", "bar": "", "lorem": "lorem", "ipsum": "ipsum"})
-    m = mocker.spy(LocalRemote, "is_hardlink")
+    m = mocker.spy(LocalRemoteTree, "is_hardlink")
     stages = dvc.add(["foo", "bar", "lorem", "ipsum"])
 
     assert m.call_count == 1

tests/func/test_checkout.py

@@ -7,26 +7,24 @@ import shutil
 import pytest
 from mock import patch
 
-from dvc.dvcfile import DVC_FILE_SUFFIX, Dvcfile
+from dvc.dvcfile import DVC_FILE_SUFFIX, PIPELINE_FILE, Dvcfile
 from dvc.exceptions import (
     CheckoutError,
     CheckoutErrorSuggestGit,
     ConfirmRemoveError,
     DvcException,
+    NoOutputOrStageError,
 )
 from dvc.main import main
 from dvc.remote import S3Remote
 from dvc.remote.local import LocalRemote
 from dvc.repo import Repo as DvcRepo
 from dvc.stage import Stage
-from dvc.stage.exceptions import (
-    StageFileBadNameError,
-    StageFileDoesNotExistError,
-)
+from dvc.stage.exceptions import StageFileDoesNotExistError
 from dvc.system import System
 from dvc.utils import relpath
 from dvc.utils.fs import walk_files
-from dvc.utils.stage import dump_stage_file, load_stage_file
+from dvc.utils.yaml import dump_yaml, load_yaml
 from tests.basic_env import TestDvc, TestDvcGit
 from tests.func.test_repro import TestRepro
 from tests.remotes import S3
@@ -223,7 +221,7 @@ class TestCheckoutSelectiveRemove(CheckoutBase):
         self.assertEqual(0, ret)
 
         stage_path = self.DATA_DIR + DVC_FILE_SUFFIX
-        stage = load_stage_file(stage_path)
+        stage = load_yaml(stage_path)
         staged_files = self.outs_info(stage)
 
         # move instead of remove, to lock inode assigned to stage_files[0].path
@@ -306,10 +304,10 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
 
 class TestCheckoutMissingMd5InStageFile(TestRepro):
     def test(self):
-        d = load_stage_file(self.file1_stage)
+        d = load_yaml(self.file1_stage)
         del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM]
         del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM]
-        dump_stage_file(self.file1_stage, d)
+        dump_yaml(self.file1_stage, d)
 
         with pytest.raises(CheckoutError):
             self.dvc.checkout(force=True)
@@ -403,17 +401,26 @@ class TestCheckoutSuggestGit(TestRepro):
     def test(self):
 
         try:
-            self.dvc.checkout(targets=["gitbranch"])
+            self.dvc.checkout(targets="gitbranch")
         except DvcException as exc:
             self.assertIsInstance(exc, CheckoutErrorSuggestGit)
-            self.assertIsInstance(exc.__cause__, StageFileDoesNotExistError)
+            self.assertIsInstance(exc.__cause__, NoOutputOrStageError)
+            self.assertIsInstance(
+                exc.__cause__.__cause__, StageFileDoesNotExistError
+            )
+
+        try:
+            self.dvc.checkout(targets=self.FOO)
+        except DvcException as exc:
+            self.assertIsInstance(exc, CheckoutErrorSuggestGit)
+            self.assertIsInstance(exc.__cause__, NoOutputOrStageError)
             self.assertIsNone(exc.__cause__.__cause__)
 
         try:
-            self.dvc.checkout(targets=[self.FOO])
+            self.dvc.checkout(targets="looks-like-dvcfile.dvc")
         except DvcException as exc:
             self.assertIsInstance(exc, CheckoutErrorSuggestGit)
-            self.assertIsInstance(exc.__cause__, StageFileBadNameError)
+            self.assertIsInstance(exc.__cause__, StageFileDoesNotExistError)
             self.assertIsNone(exc.__cause__.__cause__)
 
 
@@ -756,12 +763,12 @@ def test_checkout_for_external_outputs(tmp_dir, dvc):
         Bucket=remote.path_info.bucket, Key=file_path.path, Body="foo"
     )
 
-    dvc.add(str(remote.path_info / "foo"))
+    dvc.add(str(remote.path_info / "foo"), external=True)
 
-    remote.remove(file_path)
+    remote.tree.remove(file_path)
     stats = dvc.checkout(force=True)
     assert stats == {**empty_checkout, "added": [str(file_path)]}
-    assert remote.exists(file_path)
+    assert remote.tree.exists(file_path)
 
     remote.s3.put_object(
         Bucket=remote.path_info.bucket, Key=file_path.path, Body="foo\nfoo"
@@ -770,9 +777,51 @@ def test_checkout_for_external_outputs(tmp_dir, dvc):
     assert stats == {**empty_checkout, "modified": [str(file_path)]}
 
 
-def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
-    from dvc.dvcfile import PIPELINE_FILE
+def test_checkouts_with_different_addressing(tmp_dir, dvc, run_copy):
+    tmp_dir.gen({"foo": "foo", "lorem": "lorem"})
+    run_copy("foo", "bar", name="copy-foo-bar")
+    run_copy("lorem", "ipsum", name="copy-lorem-ipsum")
 
+    (tmp_dir / "bar").unlink()
+    (tmp_dir / "ipsum").unlink()
+    assert set(dvc.checkout(PIPELINE_FILE)["added"]) == {"bar", "ipsum"}
+
+    (tmp_dir / "bar").unlink()
+    (tmp_dir / "ipsum").unlink()
+    assert set(dvc.checkout(":")["added"]) == {"bar", "ipsum"}
+
+    (tmp_dir / "bar").unlink()
+    assert dvc.checkout("copy-foo-bar")["added"] == ["bar"]
+
+    (tmp_dir / "bar").unlink()
+    assert dvc.checkout("dvc.yaml:copy-foo-bar")["added"] == ["bar"]
+
+    (tmp_dir / "bar").unlink()
+    assert dvc.checkout(":copy-foo-bar")["added"] == ["bar"]
+
+    (tmp_dir / "bar").unlink()
+    (tmp_dir / "data").mkdir()
+    with (tmp_dir / "data").chdir():
+        assert dvc.checkout(relpath(tmp_dir / "dvc.yaml") + ":copy-foo-bar")[
+            "added"
+        ] == [relpath(tmp_dir / "bar")]
+
+    (tmp_dir / "bar").unlink()
+    assert dvc.checkout("bar")["added"] == ["bar"]
+
+
+def test_checkouts_on_same_stage_name_and_output_name(tmp_dir, dvc, run_copy):
+    tmp_dir.gen("foo", "foo")
+    run_copy("foo", "bar", name="copy-foo-bar")
+    run_copy("foo", "copy-foo-bar", name="make_collision")
+
+    (tmp_dir / "bar").unlink()
+    (tmp_dir / "copy-foo-bar").unlink()
+    assert dvc.checkout("copy-foo-bar")["added"] == ["bar"]
+    assert dvc.checkout("./copy-foo-bar")["added"] == ["copy-foo-bar"]
+
+
+def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
     tmp_dir.gen("foo", "foo")
     stage1 = run_copy("foo", "bar", name="copy-foo-bar")
     tmp_dir.gen("lorem", "lorem")

tests/func/test_commit.py

@@ -1,7 +1,8 @@
 import pytest
 
+from dvc.dvcfile import PIPELINE_FILE
 from dvc.stage.exceptions import StageCommitError
-from dvc.utils.stage import dump_stage_file, load_stage_file
+from dvc.utils.yaml import dump_yaml, load_yaml
 
 
 def test_commit_recursive(tmp_dir, dvc):
@@ -62,15 +63,15 @@ def test_commit_changed_md5(tmp_dir, dvc):
     tmp_dir.gen({"file": "file content"})
     (stage,) = dvc.add("file", no_commit=True)
 
-    stage_file_content = load_stage_file(stage.path)
+    stage_file_content = load_yaml(stage.path)
     stage_file_content["md5"] = "1111111111"
-    dump_stage_file(stage.path, stage_file_content)
+    dump_yaml(stage.path, stage_file_content)
 
     with pytest.raises(StageCommitError):
         dvc.commit(stage.path)
 
     dvc.commit(stage.path, force=True)
-    assert "md5" not in load_stage_file(stage.path)
+    assert "md5" not in load_yaml(stage.path)
 
 
 def test_commit_no_exec(tmp_dir, dvc):
@@ -81,3 +82,16 @@ def test_commit_no_exec(tmp_dir, dvc):
     assert dvc.status(stage.path)
     dvc.commit(stage.path, force=True)
     assert dvc.status(stage.path) == {}
+
+
+def test_commit_pipeline_stage(tmp_dir, dvc, run_copy):
+    tmp_dir.gen("foo", "foo")
+    stage = run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
+    assert dvc.status(stage.addressing)
+    assert dvc.commit(stage.addressing, force=True) == [stage]
+    assert not dvc.status(stage.addressing)
+
+    # just to confirm different variants work
+    assert dvc.commit(f":{stage.addressing}") == [stage]
+    assert dvc.commit(f"{PIPELINE_FILE}:{stage.addressing}") == [stage]
+    assert dvc.commit(PIPELINE_FILE) == [stage]

tests/func/test_data_cloud.py

@@ -27,7 +27,7 @@ from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
 from dvc.stage.exceptions import StageNotFound
 from dvc.utils import file_md5
 from dvc.utils.fs import remove
-from dvc.utils.stage import dump_stage_file, load_stage_file
+from dvc.utils.yaml import dump_yaml, load_yaml
 from tests.basic_env import TestDvc
 from tests.remotes import (
     GCP,
@@ -189,26 +189,22 @@ class TestS3Remote(S3, TestDataCloudBase):
         return S3Remote
 
 
-def setup_gdrive_cloud(remote_url, dvc):
-    config = copy.deepcopy(TEST_CONFIG)
-    config["remote"][TEST_REMOTE] = {
-        "url": remote_url,
-        "gdrive_service_account_email": "test",
-        "gdrive_service_account_p12_file_path": "test.p12",
-        "gdrive_use_service_account": True,
-    }
-
-    dvc.config = config
-    remote = DataCloud(dvc).get_remote()
-    remote._gdrive_create_dir("root", remote.path_info.path)
-
-
 class TestGDriveRemote(GDrive, TestDataCloudBase):
     def _setup_cloud(self):
         self._ensure_should_run()
 
-        setup_gdrive_cloud(self.get_url(), self.dvc)
+        url = self.get_url()
+        self.create_dir(self.dvc, url)
 
+        config = copy.deepcopy(TEST_CONFIG)
+        config["remote"][TEST_REMOTE] = {
+            "url": url,
+            "gdrive_service_account_email": "test",
+            "gdrive_service_account_p12_file_path": "test.p12",
+            "gdrive_use_service_account": True,
+        }
+
+        self.dvc.config = config
         self.cloud = DataCloud(self.dvc)
         remote = self.cloud.get_remote()
         self.assertIsInstance(remote, self._get_cloud_class())
@@ -421,12 +417,11 @@ class TestS3RemoteCLI(S3, TestDataCloudCLIBase):
 
 
 class TestGDriveRemoteCLI(GDrive, TestDataCloudCLIBase):
-    def _setup_cloud(self):
-        setup_gdrive_cloud(self.get_url(), self.dvc)
-
     def _test(self):
         url = self.get_url()
 
+        self.create_dir(self.dvc, url)
+
         self.main(["remote", "add", TEST_REMOTE, url])
         self.main(
             [
@@ -434,7 +429,7 @@ class TestGDriveRemoteCLI(GDrive, TestDataCloudCLIBase):
                 "modify",
                 TEST_REMOTE,
                 "gdrive_service_account_email",
-                "modified",
+                "test",
             ]
         )
         self.main(
@@ -443,7 +438,7 @@ class TestGDriveRemoteCLI(GDrive, TestDataCloudCLIBase):
                 "modify",
                 TEST_REMOTE,
                 "gdrive_service_account_p12_file_path",
-                "modified.p12",
+                "test.p12",
             ]
         )
         self.main(
@@ -523,9 +518,9 @@ class TestWarnOnOutdatedStage(TestDvc):
         self.main(["push"])
 
         stage_file_path = stage.relpath
-        content = load_stage_file(stage_file_path)
+        content = load_yaml(stage_file_path)
         del content["outs"][0]["md5"]
-        dump_stage_file(stage_file_path, content)
+        dump_yaml(stage_file_path, content)
 
         with self._caplog.at_level(logging.WARNING, logger="dvc"):
             self._caplog.clear()
@@ -912,3 +907,44 @@ def test_pull_stats(tmp_dir, dvc, caplog, setup_remote):
     with caplog.at_level(level=logging.INFO, logger="dvc"):
         main(["pull"])
     assert "Everything is up to date." in caplog.text
+
+
+@pytest.mark.parametrize(
+    "key,expected", [("all_tags", 2), ("all_branches", 3), ("all_commits", 3)]
+)
+def test_push_pull_all(tmp_dir, scm, dvc, setup_remote, key, expected):
+    setup_remote(dvc)
+    tmp_dir.dvc_gen({"foo": "foo"}, commit="first")
+    scm.tag("v1")
+    dvc.remove("foo.dvc")
+    tmp_dir.dvc_gen({"bar": "bar"}, commit="second")
+    scm.tag("v2")
+    with tmp_dir.branch("branch", new=True):
+        dvc.remove("bar.dvc")
+        tmp_dir.dvc_gen({"baz": "baz"}, commit="branch")
+
+    assert dvc.push(**{key: True}) == expected
+
+    clean(["foo", "bar", "baz"], dvc)
+    assert dvc.pull(**{key: True})["fetched"] == expected
+
+
+def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
+    remote_path = setup_remote(dvc)
+    tmp_dir.dvc_gen("foo", "foo")
+    run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
+
+    dvc.push("copy-foo-bar")
+    assert len(recurse_list_dir(remote_path)) == 1
+    # pushing everything so as we can check pull/fetch only downloads
+    # from specified targets
+    dvc.push()
+    clean(["foo", "bar"], dvc)
+
+    dvc.pull("copy-foo-bar")
+    assert (tmp_dir / "bar").exists()
+    assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1
+    clean(["bar"], dvc)
+
+    dvc.fetch("copy-foo-bar")
+    assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1

tests/func/test_dvcfile.py

@@ -1,11 +1,14 @@
+import textwrap
+
 import pytest
 
-from dvc.dvcfile import PIPELINE_FILE, Dvcfile
+from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK, Dvcfile
 from dvc.stage.exceptions import (
     StageFileDoesNotExistError,
-    StageNameUnspecified,
+    StageFileFormatError,
 )
 from dvc.stage.loader import StageNotFound
+from dvc.utils.yaml import dump_yaml
 
 
 def test_run_load_one_for_multistage(tmp_dir, dvc):
@@ -165,35 +168,125 @@ def test_stage_collection(tmp_dir, dvc):
     assert {s for s in dvc.stages} == {stage1, stage3, stage2}
 
 
-def test_stage_filter(tmp_dir, dvc, run_copy):
+def test_remove_stage(tmp_dir, dvc, run_copy):
     tmp_dir.gen("foo", "foo")
-    stage1 = run_copy("foo", "bar", name="copy-foo-bar")
-    stage2 = run_copy("bar", "bax", name="copy-bar-bax")
-    stage3 = run_copy("bax", "baz", name="copy-bax-baz")
-    stages = Dvcfile(dvc, PIPELINE_FILE).stages
-    assert set(stages.filter(None).values()) == {
-        stage1,
-        stage2,
-        stage3,
-    }
-    assert set(stages.filter("copy-bar-bax").values()) == {stage2}
-    assert stages.filter("copy-bar-bax").get("copy-bar-bax") == stage2
-    with pytest.raises(StageNameUnspecified):
-        stages.get(None)
+    stage = run_copy("foo", "bar", name="copy-foo-bar")
+    stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
 
-    with pytest.raises(StageNotFound):
-        assert stages["unknown"]
+    dvc_file = Dvcfile(dvc, PIPELINE_FILE)
+    assert dvc_file.exists()
+    assert {"copy-bar-foobar", "copy-foo-bar"} == set(
+        dvc_file._load()[0]["stages"].keys()
+    )
 
-    with pytest.raises(StageNotFound):
-        assert stages.filter("unknown")
+    dvc_file.remove_stage(stage)
+
+    assert ["copy-bar-foobar"] == list(dvc_file._load()[0]["stages"].keys())
+
+    # sanity check
+    stage2.reload()
+
+    # re-check to see if it fails if there's no stage entry
+    dvc_file.remove_stage(stage)
+    dvc_file.remove(force=True)
+    # should not fail when there's no file at all.
+    dvc_file.remove_stage(stage)
 
 
-def test_stage_filter_in_singlestage_file(tmp_dir, dvc, run_copy):
+def test_remove_stage_lockfile(tmp_dir, dvc, run_copy):
+    tmp_dir.gen("foo", "foo")
+    stage = run_copy("foo", "bar", name="copy-foo-bar")
+    stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
+
+    dvc_file = Dvcfile(dvc, PIPELINE_FILE)
+    lock_file = dvc_file._lockfile
+    assert dvc_file.exists()
+    assert lock_file.exists()
+    assert {"copy-bar-foobar", "copy-foo-bar"} == set(lock_file.load().keys())
+    lock_file.remove_stage(stage)
+
+    assert ["copy-bar-foobar"] == list(lock_file.load().keys())
+
+    # sanity check
+    stage2.reload()
+
+    # re-check to see if it fails if there's no stage entry
+    lock_file.remove_stage(stage)
+    lock_file.remove()
+    # should not fail when there's no file at all.
+    lock_file.remove_stage(stage)
+
+
+def test_remove_stage_dvcfiles(tmp_dir, dvc, run_copy):
     tmp_dir.gen("foo", "foo")
     stage = run_copy("foo", "bar", single_stage=True)
-    dvcfile = Dvcfile(dvc, stage.path)
-    assert set(dvcfile.stages.filter(None).values()) == {stage}
-    assert dvcfile.stages.filter(None).get(None) == stage
-    assert dvcfile.stages.filter("copy-bar-bax").get("copy-bar-bax") == stage
-    assert dvcfile.stages.filter("copy-bar-bax").get(None) == stage
-    assert set(dvcfile.stages.filter("unknown").values()) == {stage}
+
+    dvc_file = Dvcfile(dvc, stage.path)
+    assert dvc_file.exists()
+    dvc_file.remove_stage(stage)
+    assert not dvc_file.exists()
+
+    # re-check to see if it fails if there's no stage entry
+    dvc_file.remove_stage(stage)
+    dvc_file.remove(force=True)
+
+    # should not fail when there's no file at all.
+    dvc_file.remove_stage(stage)
+
+
+def test_remove_stage_on_lockfile_format_error(tmp_dir, dvc, run_copy):
+    tmp_dir.gen("foo", "foo")
+    stage = run_copy("foo", "bar", name="copy-foo-bar")
+    dvc_file = Dvcfile(dvc, stage.path)
+    lock_file = dvc_file._lockfile
+
+    data = dvc_file._load()[0]
+    lock_data = lock_file.load()
+    lock_data["gibberish"] = True
+    data["gibberish"] = True
+    dump_yaml(lock_file.relpath, lock_data)
+    with pytest.raises(StageFileFormatError):
+        dvc_file.remove_stage(stage)
+
+    lock_file.remove()
+    dvc_file.dump(stage)
+
+    dump_yaml(dvc_file.relpath, data)
+    with pytest.raises(StageFileFormatError):
+        dvc_file.remove_stage(stage)
+
+
+def test_remove_stage_preserves_comment(tmp_dir, dvc, run_copy):
+    tmp_dir.gen(
+        "dvc.yaml",
+        textwrap.dedent(
+            """\
+            stages:
+                generate-foo:
+                    cmd: "echo foo > foo"
+                    # This copies 'foo' text to 'foo' file.
+                    outs:
+                    - foo
+                copy-foo-bar:
+                    cmd: "python copy.py foo bar"
+                    deps:
+                    - foo
+                    outs:
+                    - bar"""
+        ),
+    )
+
+    dvc.reproduce(PIPELINE_FILE)
+
+    dvc_file = Dvcfile(dvc, PIPELINE_FILE)
+
+    assert dvc_file.exists()
+    assert (tmp_dir / PIPELINE_LOCK).exists()
+    assert (tmp_dir / "foo").exists()
+    assert (tmp_dir / "bar").exists()
+
+    dvc_file.remove_stage(dvc_file.stages["copy-foo-bar"])
+    assert (
+        "# This copies 'foo' text to 'foo' file."
+        in (tmp_dir / PIPELINE_FILE).read_text()
+    )

tests/func/test_gc.py

@@ -8,7 +8,7 @@ from git import Repo
 
 from dvc.exceptions import CollectCacheError
 from dvc.main import main
-from dvc.remote.local import LocalRemote
+from dvc.remote.local import LocalRemote, LocalRemoteTree
 from dvc.repo import Repo as DvcRepo
 from dvc.utils.fs import remove
 from tests.basic_env import TestDir, TestDvcGit
@@ -68,7 +68,7 @@ class TestGCBranchesTags(TestDvcGit):
         self.dvc.scm.tag("v1.0")
 
         self.dvc.scm.checkout("test", create_new=True)
-        self.dvc.remove(stages[0].relpath, dvc_only=True)
+        self.dvc.remove(stages[0].relpath)
         with open(fname, "w+") as fobj:
             fobj.write("test")
         stages = self.dvc.add(fname)
@@ -77,7 +77,7 @@ class TestGCBranchesTags(TestDvcGit):
         self.dvc.scm.commit("test")
 
         self.dvc.scm.checkout("master")
-        self.dvc.remove(stages[0].relpath, dvc_only=True)
+        self.dvc.remove(stages[0].relpath)
         with open(fname, "w+") as fobj:
             fobj.write("trash")
         stages = self.dvc.add(fname)
@@ -85,7 +85,7 @@ class TestGCBranchesTags(TestDvcGit):
         self.dvc.scm.add([".gitignore", stages[0].relpath])
         self.dvc.scm.commit("trash")
 
-        self.dvc.remove(stages[0].relpath, dvc_only=True)
+        self.dvc.remove(stages[0].relpath)
         with open(fname, "w+") as fobj:
             fobj.write("master")
         stages = self.dvc.add(fname)
@@ -153,7 +153,7 @@ class TestGCMultipleDvcRepos(TestDvcGit):
         cwd = os.getcwd()
         os.chdir(self.additional_path)
         # ADD FILE ONLY IN SECOND PROJECT
-        fname = os.path.join(self.additional_path, "only_in_second")
+        fname = "only_in_second"
         with open(fname, "w+") as fobj:
             fobj.write("only in additional repo")
 
@@ -161,7 +161,7 @@ class TestGCMultipleDvcRepos(TestDvcGit):
         self.assertEqual(len(stages), 1)
 
         # ADD FILE IN SECOND PROJECT THAT IS ALSO IN MAIN PROJECT
-        fname = os.path.join(self.additional_path, "in_both")
+        fname = "in_both"
         with open(fname, "w+") as fobj:
             fobj.write("in both repos")
 
@@ -321,7 +321,9 @@ def test_gc_cloud_remove_order(tmp_dir, scm, dvc, tmp_path_factory, mocker):
     dvc.remove(dir2.relpath)
     dvc.gc(workspace=True)
 
-    mocked_remove = mocker.patch.object(LocalRemote, "remove", autospec=True)
+    mocked_remove = mocker.patch.object(
+        LocalRemoteTree, "remove", autospec=True
+    )
     dvc.gc(workspace=True, cloud=True)
     assert len(mocked_remove.mock_calls) == 8
     # dir (and unpacked dir) should be first 4 checksums removed from

tests/func/test_get.py

@@ -101,7 +101,8 @@ def test_get_full_dvc_path(tmp_dir, erepo_dir, tmp_path_factory):
     external_data.write_text("ext_data")
 
     with erepo_dir.chdir():
-        erepo_dir.dvc_add(os.fspath(external_data), commit="add external data")
+        erepo_dir.dvc.add(os.fspath(external_data), external=True)
+        erepo_dir.scm_add("ext_data.dvc", commit="add external data")
 
     Repo.get(
         os.fspath(erepo_dir), os.fspath(external_data), "ext_data_imported"

tests/func/test_ignore.py

@@ -11,6 +11,7 @@ from dvc.ignore import (
     DvcIgnoreRepo,
 )
 from dvc.remote import LocalRemote
+from dvc.repo import Repo
 from dvc.scm.tree import WorkingTree
 from dvc.utils import relpath
 from dvc.utils.fs import get_mtime_and_size
@@ -139,7 +140,7 @@ def test_match_nested(tmp_dir, dvc):
     )
 
     remote = LocalRemote(dvc, {})
-    result = {os.fspath(f) for f in remote.walk_files(".")}
+    result = {os.fspath(f) for f in remote.tree.walk_files(".")}
     assert result == {".dvcignore", "foo"}
 
 
@@ -149,5 +150,22 @@ def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
     ext_dir.gen({"y.backup": "y", "tmp": "ext tmp"})
 
     remote = LocalRemote(dvc, {})
-    result = {relpath(f, ext_dir) for f in remote.walk_files(ext_dir)}
+    result = {relpath(f, ext_dir) for f in remote.tree.walk_files(ext_dir)}
     assert result == {"y.backup", "tmp"}
+
+
+def test_ignore_subrepo(tmp_dir, scm, dvc):
+    tmp_dir.gen({".dvcignore": "foo", "subdir": {"foo": "foo"}})
+    scm.add([".dvcignore"])
+    scm.commit("init parent dvcignore")
+
+    subrepo_dir = tmp_dir / "subdir"
+    assert not dvc.tree.exists(subrepo_dir / "foo")
+
+    with subrepo_dir.chdir():
+        subrepo = Repo.init(subdir=True)
+        scm.add(str(subrepo_dir / "foo"))
+        scm.commit("subrepo init")
+
+    for _ in subrepo.brancher(all_commits=True):
+        assert subrepo.tree.exists(subrepo_dir / "foo")

tests/func/test_lockfile.py

@@ -6,7 +6,7 @@ import yaml
 from dvc.dvcfile import PIPELINE_LOCK
 from dvc.serialize import get_params_deps
 from dvc.utils.fs import remove
-from dvc.utils.stage import parse_stage_for_update
+from dvc.utils.yaml import parse_yaml_for_update
 from tests.func.test_run_multistage import supported_params
 
 FS_STRUCTURE = {
@@ -20,7 +20,7 @@ FS_STRUCTURE = {
 
 def read_lock_file(file=PIPELINE_LOCK):
     with open(file) as f:
-        data = parse_stage_for_update(f.read(), file)
+        data = parse_yaml_for_update(f.read(), file)
     assert isinstance(data, OrderedDict)
     return data
 

tests/func/test_ls.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import textwrap
 
 import pytest
 
@@ -29,17 +30,19 @@ def match_files(files, expected_files):
 
 
 def create_dvc_pipeline(tmp_dir, dvc):
-    script = os.linesep.join(
-        [
-            "from pathlib import Path",
-            "Path({}).touch()".format(os.path.join("out", "file")),
-        ]
+    script = textwrap.dedent(
+        """\
+        import os, sys
+        f = sys.argv[1]
+        os.makedirs(os.path.dirname(f))
+        open(f, "w+").close()
+    """
     )
     tmp_dir.scm_gen({"script.py": script}, commit="init")
     tmp_dir.dvc_gen({"dep": "content"}, commit="init dvc")
     dvc.run(
         **{
-            "command": "python script.py",
+            "cmd": "python script.py {}".format(os.path.join("out", "file")),
             "outs": [os.path.join("out", "file")],
             "deps": ["dep"],
             "fname": "out.dvc",