dvc: bump to 1.0.0a9

This one doesn't allow us to release new pre-releases, because pypi
forbids that:

https://travis-ci.com/github/iterative/dvc/jobs/343343967#L2447

Let's install it separately for now, until there is a new official
mock-ssh-server release.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,10 +1,19 @@
 ---
-name: Bug report
-about: Create a report to help us improve
-
+name: "\U0001F41B Bug Report"
+about: Create a bug report to help us improve DVC
+labels: bug, triage
 ---
 
+## Bug Report
 
-**Please provide information about your setup**
-DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
+### Please provide information about your setup
 
+**Output of `dvc version`:**
+
+```console
+$ dvc version
+```
+
+**Additional Information (if any):**
+
+If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: "🤗 Need help?"
+    url: https://dvc.org/chat
+    about: If you have a question, ask us on Discord. Please join with this invite 👉

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,6 @@
 ---
-name: Feature request
+name: "\U0001F680 Feature Request"
 about: Suggest an idea for this project
-
+labels: feature request, triage
 ---
 

.pre-commit-config.yaml

@@ -27,18 +27,37 @@ repos:
   repo: https://github.com/lovesegfault/beautysh
   rev: master
 - hooks:
-  - id: dvc-pre-commit
-    language_version: python3
+  - args:
+    - git-hook
+    - pre-commit
+    entry: dvc
+    id: dvc-pre-commit
+    language: system
+    name: DVC pre-commit
     stages:
     - commit
-  - id: dvc-pre-push
-    language_version: python3
+    verbose: true
+    require_serial: true
+  - args:
+    - git-hook
+    - pre-push
+    entry: dvc
+    id: dvc-pre-push
+    language: system
+    name: DVC pre-push
     stages:
     - push
+    require_serial: true
   - always_run: true
+    args:
+    - git-hook
+    - post-checkout
+    entry: dvc
     id: dvc-post-checkout
-    language_version: python3
+    language: system
+    minimum_pre_commit_version: 2.2.0
+    name: DVC post-checkout
     stages:
     - post-checkout
-  repo: https://github.com/iterative/dvc
-  rev: master
+    require_serial: true
+  repo: local

.pre-commit-hooks.yaml

@@ -6,6 +6,7 @@
   language: python
   language_version: python3
   name: DVC pre-commit
+  require_serial: true
   stages:
   - commit
   verbose: true
@@ -17,6 +18,7 @@
   language: python
   language_version: python3
   name: DVC pre-push
+  require_serial: true
   stages:
   - push
 - always_run: true
@@ -29,5 +31,6 @@
   language_version: python3
   minimum_pre_commit_version: 2.2.0
   name: DVC post-checkout
+  require_serial: true
   stages:
   - post-checkout

README.rst

@@ -101,7 +101,8 @@ Snap (Snapcraft/Linux)
    snap install dvc --classic
 
 This corresponds to the latest tagged release.
-Add ``--edge`` for the latest ``master`` version.
+Add ``--beta`` for the latest tagged release candidate,
+or ``--edge`` for the latest ``master`` version.
 
 Choco (Chocolatey/Windows)
 --------------------------

dvc/cli.py

@@ -10,9 +10,11 @@ from .command import (
     commit,
     config,
     daemon,
+    dag,
     data_sync,
     destroy,
     diff,
+    freeze,
     gc,
     get,
     get_url,
@@ -21,12 +23,10 @@ from .command import (
     imp_url,
     init,
     install,
-    lock,
     ls,
     metrics,
     move,
     params,
-    pipeline,
     plots,
     remote,
     remove,
@@ -66,8 +66,8 @@ COMMANDS = [
     install,
     root,
     ls,
-    lock,
-    pipeline,
+    freeze,
+    dag,
     daemon,
     commit,
     diff,
@@ -116,14 +116,10 @@ def get_parent_parser():
 
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
-        "-q", "--quiet", action="store_true", default=False, help="Be quiet."
+        "-q", "--quiet", action="count", default=0, help="Be quiet."
     )
     log_level_group.add_argument(
-        "-v",
-        "--verbose",
-        action="store_true",
-        default=False,
-        help="Be verbose.",
+        "-v", "--verbose", action="count", default=0, help="Be verbose."
     )
 
     return parent_parser

dvc/command/add.py

@@ -18,6 +18,7 @@ class CmdAdd(CmdBase):
                 recursive=self.args.recursive,
                 no_commit=self.args.no_commit,
                 fname=self.args.file,
+                external=self.args.external,
             )
 
         except DvcException:
@@ -49,6 +50,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Don't put files/directories into cache.",
     )
+    parser.add_argument(
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow targets that are outside of the DVC repository.",
+    )
     parser.add_argument(
         "-f",
         "--file",

dvc/command/dag.py

@@ -0,0 +1,115 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _show_ascii(G):
+    from dvc.repo.graph import get_pipelines
+    from dvc.dagascii import draw
+
+    pipelines = get_pipelines(G)
+
+    ret = []
+    for pipeline in pipelines:
+        ret.append(draw(pipeline.nodes, pipeline.edges))
+
+    return "\n".join(ret)
+
+
+def _show_dot(G):
+    import io
+    from networkx.drawing.nx_pydot import write_dot
+
+    dot_file = io.StringIO()
+    write_dot(G, dot_file)
+    return dot_file.getvalue()
+
+
+def _build(G, target=None, full=False):
+    import networkx as nx
+    from dvc.repo.graph import get_pipeline, get_pipelines
+
+    if target:
+        H = get_pipeline(get_pipelines(G), target)
+        if not full:
+            descendants = nx.descendants(G, target)
+            descendants.add(target)
+            H.remove_nodes_from(set(G.nodes()) - descendants)
+    else:
+        H = G
+
+    def _relabel(stage):
+        return stage.addressing
+
+    return nx.relabel_nodes(H, _relabel, copy=False)
+
+
+class CmdDAG(CmdBase):
+    def run(self):
+        try:
+            target = None
+            if self.args.target:
+                stages = self.repo.collect(self.args.target)
+                if len(stages) > 1:
+                    logger.error(
+                        f"'{self.args.target}' contains more than one stage "
+                        "{stages}, please specify one stage"
+                    )
+                    return 1
+                target = stages[0]
+
+            G = _build(self.repo.graph, target=target, full=self.args.full,)
+
+            if self.args.dot:
+                logger.info(_show_dot(G))
+            else:
+                from dvc.utils.pager import pager
+
+                pager(_show_ascii(G))
+
+            return 0
+        except DvcException:
+            msg = "failed to show"
+            if self.args.target:
+                msg += f"a pipeline for '{target}'"
+            else:
+                msg += "pipelines"
+            logger.exception(msg)
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    DAG_HELP = "Visualize DVC project DAG."
+    dag_parser = subparsers.add_parser(
+        "dag",
+        parents=[parent_parser],
+        description=append_doc_link(DAG_HELP, "dag"),
+        help=DAG_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    dag_parser.add_argument(
+        "--dot",
+        action="store_true",
+        default=False,
+        help="Print DAG with .dot format.",
+    )
+    dag_parser.add_argument(
+        "--full",
+        action="store_true",
+        default=False,
+        help=(
+            "Show full DAG that the target belongs too, instead of "
+            "showing DAG consisting only of ancestors."
+        ),
+    )
+    dag_parser.add_argument(
+        "target",
+        nargs="?",
+        help="Stage or output to show pipeline for (optional). "
+        "Finds all stages in the workspace by default.",
+    )
+    dag_parser.set_defaults(func=CmdDAG)

dvc/command/freeze.py

@@ -0,0 +1,57 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+class CmdFreezeBase(CmdBase):
+    def _run(self, func, name):
+        ret = 0
+        for target in self.args.targets:
+            try:
+                func(target)
+            except DvcException:
+                logger.exception(f"failed to {name} '{target}'")
+                ret = 1
+        return ret
+
+
+class CmdFreeze(CmdFreezeBase):
+    def run(self):
+        return self._run(self.repo.freeze, "freeze")
+
+
+class CmdUnfreeze(CmdFreezeBase):
+    def run(self):
+        return self._run(self.repo.unfreeze, "unfreeze")
+
+
+def add_parser(subparsers, parent_parser):
+    FREEZE_HELP = "Freeze DVC-files."
+    freeze_parser = subparsers.add_parser(
+        "freeze",
+        parents=[parent_parser],
+        description=append_doc_link(FREEZE_HELP, "freeze"),
+        help=FREEZE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    freeze_parser.add_argument(
+        "targets", nargs="+", help="DVC-files to freeze."
+    )
+    freeze_parser.set_defaults(func=CmdFreeze)
+
+    UNFREEZE_HELP = "Unfreeze DVC-files."
+    unfreeze_parser = subparsers.add_parser(
+        "unfreeze",
+        parents=[parent_parser],
+        description=append_doc_link(UNFREEZE_HELP, "unfreeze"),
+        help=UNFREEZE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    unfreeze_parser.add_argument(
+        "targets", nargs="+", help="DVC-files to unfreeze."
+    )
+    unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/gc.py

@@ -21,7 +21,7 @@ class CmdGC(CmdBase):
 
         msg = "This will remove all cache except items used in "
 
-        msg += "the working tree"
+        msg += "the workspace"
         if self.args.all_commits:
             msg += " and all git commits"
         elif self.args.all_branches and self.args.all_tags:
@@ -77,7 +77,7 @@ def add_parser(subparsers, parent_parser):
         "--workspace",
         action="store_true",
         default=False,
-        help="Keep data files used in the current workspace (working tree).",
+        help="Keep data files used in the current workspace.",
     )
     gc_parser.add_argument(
         "-a",

dvc/command/lock.py

@@ -1,59 +0,0 @@
-import argparse
-import logging
-
-from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
-
-logger = logging.getLogger(__name__)
-
-
-class CmdLockBase(CmdBase):
-    def _run(self, unlock):
-        for target in self.args.targets:
-            try:
-                self.repo.lock_stage(target, unlock=unlock)
-            except DvcException:
-                logger.exception(
-                    "failed to {}lock '{}'".format(
-                        "un" if unlock else "", target
-                    )
-                )
-
-                return 1
-        return 0
-
-
-class CmdLock(CmdLockBase):
-    def run(self):
-        return self._run(False)
-
-
-class CmdUnlock(CmdLockBase):
-    def run(self):
-        return self._run(True)
-
-
-def add_parser(subparsers, parent_parser):
-    LOCK_HELP = "Lock DVC-files."
-    lock_parser = subparsers.add_parser(
-        "lock",
-        parents=[parent_parser],
-        description=append_doc_link(LOCK_HELP, "lock"),
-        help=LOCK_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    lock_parser.add_argument("targets", nargs="+", help="DVC-files to lock.")
-    lock_parser.set_defaults(func=CmdLock)
-
-    UNLOCK_HELP = "Unlock DVC-files."
-    unlock_parser = subparsers.add_parser(
-        "unlock",
-        parents=[parent_parser],
-        description=append_doc_link(UNLOCK_HELP, "unlock"),
-        help=UNLOCK_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    unlock_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to unlock."
-    )
-    unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/ls/__init__.py

@@ -31,7 +31,7 @@ class CmdList(CmdBaseNoRepo):
                 self.args.path,
                 rev=self.args.rev,
                 recursive=self.args.recursive,
-                outs_only=self.args.outs_only,
+                dvc_only=self.args.dvc_only,
             )
             if entries:
                 entries = _prettify(entries, sys.stdout.isatty())
@@ -62,7 +62,7 @@ def add_parser(subparsers, parent_parser):
         help="Recursively list files.",
     )
     list_parser.add_argument(
-        "--outs-only", action="store_true", help="Show only DVC outputs."
+        "--dvc-only", action="store_true", help="Show only DVC outputs."
     )
     list_parser.add_argument(
         "--rev",

dvc/command/metrics.py

@@ -63,31 +63,7 @@ class CmdMetricsShow(CmdBase):
         return 0
 
 
-class CmdMetricsAdd(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.add(self.args.path)
-        except DvcException:
-            msg = f"failed to add metric file '{self.args.path}'"
-            logger.exception(msg)
-            return 1
-
-        return 0
-
-
-class CmdMetricsRemove(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.remove(self.args.path)
-        except DvcException:
-            msg = f"failed to remove metric file '{self.args.path}'"
-            logger.exception(msg)
-            return 1
-
-        return 0
-
-
-def _show_diff(diff, markdown=False):
+def _show_diff(diff, markdown=False, no_path=False, old=False):
     from collections import OrderedDict
 
     from dvc.utils.diff import table
@@ -96,16 +72,23 @@ def _show_diff(diff, markdown=False):
     for fname, mdiff in diff.items():
         sorted_mdiff = OrderedDict(sorted(mdiff.items()))
         for metric, change in sorted_mdiff.items():
-            rows.append(
-                [
-                    fname,
-                    metric,
-                    change["new"],
-                    change.get("diff", "diff not supported"),
-                ]
-            )
+            row = [] if no_path else [fname]
+            row.append(metric)
+            if old:
+                row.append(change.get("old"))
+            row.append(change["new"])
+            row.append(change.get("diff", "diff not supported"))
+            rows.append(row)
 
-    return table(["Path", "Metric", "Value", "Change"], rows, markdown)
+    header = [] if no_path else ["Path"]
+    header.append("Metric")
+    if old:
+        header.extend(["Old", "New"])
+    else:
+        header.append("Value")
+    header.append("Change")
+
+    return table(header, rows, markdown)
 
 
 class CmdMetricsDiff(CmdBase):
@@ -124,7 +107,9 @@ class CmdMetricsDiff(CmdBase):
 
                 logger.info(json.dumps(diff))
             else:
-                table = _show_diff(diff, self.args.show_md)
+                table = _show_diff(
+                    diff, self.args.show_md, self.args.no_path, self.args.old
+                )
                 if table:
                     logger.info(table)
 
@@ -136,7 +121,7 @@ class CmdMetricsDiff(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    METRICS_HELP = "Commands to add, manage, collect, and display metrics."
+    METRICS_HELP = "Commands to display and compare metrics."
 
     metrics_parser = subparsers.add_parser(
         "metrics",
@@ -153,17 +138,6 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(metrics_subparsers)
 
-    METRICS_ADD_HELP = "Mark a DVC-tracked file as a metric."
-    metrics_add_parser = metrics_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
-        help=METRICS_ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_add_parser.add_argument("path", help="Path to a metric file.")
-    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
-
     METRICS_SHOW_HELP = "Print metrics, with optional formatting."
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
@@ -269,15 +243,16 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show tabulated output in the Markdown format (GFM).",
     )
-    metrics_diff_parser.set_defaults(func=CmdMetricsDiff)
-
-    METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
-    metrics_remove_parser = metrics_subparsers.add_parser(
-        "remove",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
-        help=METRICS_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
+    metrics_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show metric path.",
     )
-    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
-    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
+    metrics_diff_parser.add_argument(
+        "--old",
+        action="store_true",
+        default=False,
+        help="Show old metric value.",
+    )
+    metrics_diff_parser.set_defaults(func=CmdMetricsDiff)

dvc/command/params.py

@@ -8,16 +8,25 @@ from dvc.exceptions import DvcException
 logger = logging.getLogger(__name__)
 
 
-def _show_diff(diff, markdown=False):
+def _show_diff(diff, markdown=False, no_path=False):
     from dvc.utils.diff import table
 
     rows = []
     for fname, pdiff in diff.items():
         sorted_pdiff = OrderedDict(sorted(pdiff.items()))
         for param, change in sorted_pdiff.items():
-            rows.append([fname, param, change["old"], change["new"]])
+            row = [] if no_path else [fname]
+            row.append(param)
+            row.append(change["old"])
+            row.append(change["new"])
+            rows.append(row)
 
-    return table(["Path", "Param", "Old", "New"], rows, markdown)
+    header = [] if no_path else ["Path"]
+    header.append("Param")
+    header.append("Old")
+    header.append("New")
+
+    return table(header, rows, markdown)
 
 
 class CmdParamsDiff(CmdBase):
@@ -34,7 +43,7 @@ class CmdParamsDiff(CmdBase):
 
                 logger.info(json.dumps(diff))
             else:
-                table = _show_diff(diff, self.args.show_md)
+                table = _show_diff(diff, self.args.show_md, self.args.no_path)
                 if table:
                     logger.info(table)
 
@@ -100,4 +109,10 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show tabulated output in the Markdown format (GFM).",
     )
+    params_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show params path.",
+    )
     params_diff_parser.set_defaults(func=CmdParamsDiff)

dvc/command/pipeline.py

@@ -1,267 +0,0 @@
-import argparse
-import logging
-
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
-from dvc.exceptions import DvcException
-
-logger = logging.getLogger(__name__)
-
-
-class CmdPipelineShow(CmdBase):
-    def _show(self, target, commands, outs, locked):
-        import networkx
-        from dvc import dvcfile
-        from dvc.utils import parse_target
-
-        path, name = parse_target(target)
-        stage = dvcfile.Dvcfile(self.repo, path).stages[name]
-        G = self.repo.graph
-        stages = networkx.dfs_postorder_nodes(G, stage)
-        if locked:
-            stages = [s for s in stages if s.locked]
-
-        for stage in stages:
-            if commands:
-                if stage.cmd is None:
-                    continue
-                logger.info(stage.cmd)
-            elif outs:
-                for out in stage.outs:
-                    logger.info(str(out))
-            else:
-                logger.info(stage.addressing)
-
-    @staticmethod
-    def _build_output_graph(G, target_stage):
-        import networkx
-        from itertools import product
-
-        nodes = {str(out) for out in target_stage.outs}
-        edges = []
-
-        for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
-            from_stage_deps = {dep.path_info.parts for dep in from_stage.deps}
-            to_outs = {
-                to_out
-                for to_out in to_stage.outs
-                if to_out.path_info.parts in from_stage_deps
-            }
-            from_outs = {
-                from_out
-                for from_out in from_stage.outs
-                if str(from_out) in nodes
-            }
-            nodes |= {str(to_out) for to_out in to_outs}
-            for from_out, to_out in product(from_outs, to_outs):
-                edges.append((str(from_out), str(to_out)))
-        return nodes, edges
-
-    def _build_graph(self, target, commands=False, outs=False):
-        import networkx
-        from dvc import dvcfile
-        from dvc.repo.graph import get_pipeline
-        from dvc.utils import parse_target
-
-        path, name = parse_target(target)
-        target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
-        G = get_pipeline(self.repo.pipelines, target_stage)
-
-        nodes = set()
-        for stage in networkx.dfs_preorder_nodes(G, target_stage):
-            if commands:
-                if stage.cmd is None:
-                    continue
-                nodes.add(stage.cmd)
-            elif not outs:
-                nodes.add(stage.addressing)
-
-        edges = []
-        for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
-            if commands:
-                if to_stage.cmd is None:
-                    continue
-                edges.append((from_stage.cmd, to_stage.cmd))
-            elif not outs:
-                edges.append((from_stage.addressing, to_stage.addressing))
-
-        if outs:
-            nodes, edges = self._build_output_graph(G, target_stage)
-
-        return list(nodes), edges, networkx.is_tree(G)
-
-    def _show_ascii(self, target, commands, outs):
-        from dvc.dagascii import draw
-
-        nodes, edges, _ = self._build_graph(target, commands, outs)
-
-        if not nodes:
-            return
-
-        draw(nodes, edges)
-
-    def _show_dependencies_tree(self, target, commands, outs):
-        from treelib import Tree
-
-        nodes, edges, is_tree = self._build_graph(target, commands, outs)
-        if not nodes:
-            return
-        if not is_tree:
-            raise DvcException(
-                "DAG is not a tree, can not print it in tree-structure way, "
-                "please use --ascii instead"
-            )
-
-        tree = Tree()
-        tree.create_node(target, target)  # Root node
-        observe_list = [target]
-        while len(observe_list) > 0:
-            current_root = observe_list[0]
-            for edge in edges:
-                if edge[0] == current_root:
-                    tree.create_node(edge[1], edge[1], parent=current_root)
-                    observe_list.append(edge[1])
-            observe_list.pop(0)
-        tree.show()
-
-    def _write_dot(self, target, commands, outs):
-        import io
-        import networkx
-        from networkx.drawing.nx_pydot import write_dot
-
-        _, edges, _ = self._build_graph(target, commands, outs)
-        edges = [edge[::-1] for edge in edges]
-
-        simple_g = networkx.DiGraph()
-        simple_g.add_edges_from(edges)
-
-        dot_file = io.StringIO()
-        write_dot(simple_g, dot_file)
-        logger.info(dot_file.getvalue())
-
-    def run(self):
-        from dvc.dvcfile import DVC_FILE
-
-        if not self.args.targets:
-            self.args.targets = [DVC_FILE]
-
-        for target in self.args.targets:
-            try:
-                if self.args.ascii:
-                    self._show_ascii(
-                        target, self.args.commands, self.args.outs
-                    )
-                elif self.args.dot:
-                    self._write_dot(target, self.args.commands, self.args.outs)
-                elif self.args.tree:
-                    self._show_dependencies_tree(
-                        target, self.args.commands, self.args.outs
-                    )
-                else:
-                    self._show(
-                        target,
-                        self.args.commands,
-                        self.args.outs,
-                        self.args.locked,
-                    )
-            except DvcException:
-                msg = f"failed to show pipeline for '{target}'"
-                logger.exception(msg)
-                return 1
-        return 0
-
-
-class CmdPipelineList(CmdBase):
-    def run(self):
-        pipelines = self.repo.pipelines
-        for pipeline in pipelines:
-            for stage in pipeline:
-                logger.info(stage.addressing)
-            if len(pipeline) != 0:
-                logger.info("=" * 80)
-        logger.info("{} pipelines total".format(len(pipelines)))
-
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    PIPELINE_HELP = "Manage pipelines."
-    pipeline_parser = subparsers.add_parser(
-        "pipeline",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_HELP, "pipeline"),
-        help=PIPELINE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-
-    pipeline_subparsers = pipeline_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc pipeline CMD --help` for command-specific help.",
-    )
-
-    fix_subparsers(pipeline_subparsers)
-
-    PIPELINE_LIST_HELP = "List connected groups of stages (pipelines)."
-    pipeline_list_parser = pipeline_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
-        help=PIPELINE_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pipeline_list_parser.set_defaults(func=CmdPipelineList)
-
-    PIPELINE_SHOW_HELP = "Show stages in a pipeline."
-    pipeline_show_parser = pipeline_subparsers.add_parser(
-        "show",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline/show"),
-        help=PIPELINE_SHOW_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
-    pipeline_show_group.add_argument(
-        "-c",
-        "--commands",
-        action="store_true",
-        default=False,
-        help="Print commands instead of paths to DVC-files.",
-    )
-    pipeline_show_group.add_argument(
-        "-o",
-        "--outs",
-        action="store_true",
-        default=False,
-        help="Print output files instead of paths to DVC-files.",
-    )
-    pipeline_show_parser.add_argument(
-        "-l",
-        "--locked",
-        action="store_true",
-        default=False,
-        help="Print locked DVC stages",
-    )
-    pipeline_show_parser.add_argument(
-        "--ascii",
-        action="store_true",
-        default=False,
-        help="Output DAG as ASCII.",
-    )
-    pipeline_show_parser.add_argument(
-        "--dot",
-        action="store_true",
-        default=False,
-        help="Print DAG with .dot format.",
-    )
-    pipeline_show_parser.add_argument(
-        "--tree",
-        action="store_true",
-        default=False,
-        help="Output DAG as Dependencies Tree.",
-    )
-    pipeline_show_parser.add_argument(
-        "targets",
-        nargs="*",
-        help="DVC-files to show pipeline for. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
-    )
-    pipeline_show_parser.set_defaults(func=CmdPipelineShow)

dvc/command/plots.py

@@ -4,6 +4,8 @@ import os
 
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
+from dvc.schema import PLOT_PROPS
+from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
@@ -31,32 +33,28 @@ class CmdPlots(CmdBase):
     def _func(self, *args, **kwargs):
         raise NotImplementedError
 
+    def _props(self):
+        # Pass only props specified by user, to not shadow ones from plot def
+        props = {p: getattr(self.args, p) for p in PLOT_PROPS}
+        return {k: v for k, v in props.items() if v is not None}
+
     def run(self):
-        fields = None
-        jsonpath = None
-        if self.args.select:
-            if self.args.select.startswith("$"):
-                jsonpath = self.args.select
-            else:
-                fields = set(self.args.select.split(","))
+        if self.args.show_vega:
+            if not self.args.targets:
+                logger.error("please specify a target for `--show-vega`")
+                return 1
+            if len(self.args.targets) > 1:
+                logger.error(
+                    "you can only specify one target for `--show-vega`"
+                )
+                return 1
+
         try:
-            plots = self._func(
-                targets=self.args.targets,
-                template=self.args.template,
-                fields=fields,
-                x_field=self.args.x,
-                y_field=self.args.y,
-                path=jsonpath,
-                csv_header=not self.args.no_csv_header,
-                title=self.args.title,
-                x_title=self.args.xlab,
-                y_title=self.args.ylab,
-            )
+            plots = self._func(targets=self.args.targets, props=self._props())
 
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(plots))
+            if self.args.show_vega:
+                target = self.args.targets[0]
+                logger.info(plots[target])
                 return 0
 
             divs = [
@@ -90,6 +88,14 @@ class CmdPlotsDiff(CmdPlots):
         return self.repo.plots.diff(*args, revs=self.args.revisions, **kwargs)
 
 
+class CmdPlotsModify(CmdPlots):
+    def run(self):
+        self.repo.plots.modify(
+            self.args.target, props=self._props(), unset=self.args.unset,
+        )
+        return 0
+
+
 def add_parser(subparsers, parent_parser):
     PLOTS_HELP = (
         "Generating plots for metrics stored in structured files "
@@ -118,52 +124,13 @@ def add_parser(subparsers, parent_parser):
         help=SHOW_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    plots_show_parser.add_argument(
-        "-t",
-        "--template",
-        nargs="?",
-        default=None,
-        help="File to be injected with data.",
-    )
-    plots_show_parser.add_argument(
-        "-o", "--out", default=None, help="Destination path to save plots to.",
-    )
-    plots_show_parser.add_argument(
-        "-s",
-        "--select",
-        default=None,
-        help="Choose which field(s) or JSONPath to include in the plots.",
-    )
-    plots_show_parser.add_argument(
-        "-x", default=None, help="Field name for x axis."
-    )
-    plots_show_parser.add_argument(
-        "-y", default=None, help="Field name for y axis."
-    )
-    plots_show_parser.add_argument(
-        "--no-csv-header",
-        action="store_true",
-        default=False,
-        help="Required when CSV or TSV datafile does not have a header.",
-    )
-    plots_show_parser.add_argument(
-        "--show-json",
-        action="store_true",
-        default=False,
-        help="Show output in JSON format.",
-    )
-    plots_show_parser.add_argument("--title", default=None, help="Plot title.")
-    plots_show_parser.add_argument(
-        "--xlab", default=None, help="X axis title."
-    )
-    plots_show_parser.add_argument(
-        "--ylab", default=None, help="Y axis title."
-    )
     plots_show_parser.add_argument(
         "targets",
         nargs="*",
-        help="Metrics files to visualize. Shows all plots by default.",
+        help="Plots files to visualize. Shows all plots by default.",
     )
+    _add_props_arguments(plots_show_parser)
+    _add_output_arguments(plots_show_parser)
     plots_show_parser.set_defaults(func=CmdPlotsShow)
 
     PLOTS_DIFF_HELP = (
@@ -177,53 +144,70 @@ def add_parser(subparsers, parent_parser):
         help=PLOTS_DIFF_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    plots_diff_parser.add_argument(
-        "-t",
-        "--template",
-        nargs="?",
-        default=None,
-        help="File to be injected with data.",
-    )
     plots_diff_parser.add_argument(
         "--targets",
         nargs="*",
-        help="Metrics file to visualize. Shows all plots by default.",
-    )
-    plots_diff_parser.add_argument(
-        "-o", "--out", default=None, help="Destination path to save plots to.",
-    )
-    plots_diff_parser.add_argument(
-        "-s",
-        "--select",
-        default=None,
-        help="Choose which field(s) or JSONPath to include in the plot.",
-    )
-    plots_diff_parser.add_argument(
-        "-x", default=None, help="Field name for x axis."
-    )
-    plots_diff_parser.add_argument(
-        "-y", default=None, help="Field name for y axis."
-    )
-    plots_diff_parser.add_argument(
-        "--no-csv-header",
-        action="store_true",
-        default=False,
-        help="Provided CSV ot TSV datafile does not have a header.",
-    )
-    plots_diff_parser.add_argument(
-        "--show-json",
-        action="store_true",
-        default=False,
-        help="Show output in JSON format.",
-    )
-    plots_diff_parser.add_argument("--title", default=None, help="Plot title.")
-    plots_diff_parser.add_argument(
-        "--xlab", default=None, help="X axis title."
-    )
-    plots_diff_parser.add_argument(
-        "--ylab", default=None, help="Y axis title."
+        help="Plots file to visualize. Shows all plots by default.",
     )
     plots_diff_parser.add_argument(
         "revisions", nargs="*", default=None, help="Git commits to plot from",
     )
+    _add_props_arguments(plots_diff_parser)
+    _add_output_arguments(plots_diff_parser)
     plots_diff_parser.set_defaults(func=CmdPlotsDiff)
+
+    PLOTS_MODIFY_HELP = "Modify plot props associated with a target file."
+    plots_modify_parser = plots_subparsers.add_parser(
+        "modify",
+        parents=[parent_parser],
+        description=append_doc_link(PLOTS_MODIFY_HELP, "plots/modify"),
+        help=PLOTS_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_modify_parser.add_argument(
+        "target", help="Plot file to set props to.",
+    )
+    _add_props_arguments(plots_modify_parser)
+    plots_modify_parser.add_argument(
+        "--unset", nargs="*", help="Props to unset.",
+    )
+    plots_modify_parser.set_defaults(func=CmdPlotsModify)
+
+
+def _add_props_arguments(parser):
+    parser.add_argument(
+        "-t",
+        "--template",
+        nargs="?",
+        default=None,
+        help=(
+            "Special JSON or HTML schema file to inject with the data. "
+            "See {}".format(
+                format_link("https://man.dvc.org/plots#plot-templates")
+            )
+        ),
+    )
+    parser.add_argument("-x", default=None, help="Field name for x axis.")
+    parser.add_argument("-y", default=None, help="Field name for y axis.")
+    parser.add_argument(
+        "--no-csv-header",
+        action="store_false",
+        dest="csv_header",
+        default=None,  # Use default None to distinguish when it's not used
+        help="Provided CSV ot TSV datafile does not have a header.",
+    )
+    parser.add_argument("--title", default=None, help="Plot title.")
+    parser.add_argument("--x-label", default=None, help="X axis label.")
+    parser.add_argument("--y-label", default=None, help="Y axis lebel.")
+
+
+def _add_output_arguments(parser):
+    parser.add_argument(
+        "-o", "--out", default=None, help="Destination path to save plots to.",
+    )
+    parser.add_argument(
+        "--show-vega",
+        action="store_true",
+        default=False,
+        help="Show output in Vega format.",
+    )

dvc/command/remote.py

@@ -62,7 +62,11 @@ class CmdRemoteModify(CmdRemote):
     def run(self):
         with self.config.edit(self.args.level) as conf:
             self._check_exists(conf)
-            conf["remote"][self.args.name][self.args.option] = self.args.value
+            section = conf["remote"][self.args.name]
+            if self.args.unset:
+                section.pop(self.args.option, None)
+            else:
+                section[self.args.option] = self.args.value
         return 0
 
 

dvc/command/remove.py

@@ -1,7 +1,6 @@
 import argparse
 import logging
 
-import dvc.prompt as prompt
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -9,30 +8,10 @@ logger = logging.getLogger(__name__)
 
 
 class CmdRemove(CmdBase):
-    def _is_outs_only(self, target):
-        if not self.args.purge:
-            return True
-
-        if self.args.force:
-            return False
-
-        msg = "Are you sure you want to remove '{}' with its outputs?".format(
-            target
-        )
-
-        if prompt.confirm(msg):
-            return False
-
-        raise DvcException(
-            "Cannot purge without a confirmation from the user."
-            " Use `-f` to force."
-        )
-
     def run(self):
         for target in self.args.targets:
             try:
-                outs_only = self._is_outs_only(target)
-                self.repo.remove(target, outs_only=outs_only)
+                self.repo.remove(target, outs=self.args.outs)
             except DvcException:
                 logger.exception(f"failed to remove '{target}'")
                 return 1
@@ -40,7 +19,7 @@ class CmdRemove(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REMOVE_HELP = "Remove DVC-tracked files or directories."
+    REMOVE_HELP = "Remove stage entry and unprotect outputs"
     remove_parser = subparsers.add_parser(
         "remove",
         parents=[parent_parser],
@@ -48,27 +27,11 @@ def add_parser(subparsers, parent_parser):
         help=REMOVE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remove_parser_group = remove_parser.add_mutually_exclusive_group()
-    remove_parser_group.add_argument(
-        "-o",
+    remove_parser.add_argument(
         "--outs",
         action="store_true",
-        default=True,
-        help="Only remove DVC-file outputs. (Default)",
-    )
-    remove_parser_group.add_argument(
-        "-p",
-        "--purge",
-        action="store_true",
         default=False,
-        help="Remove DVC-file and all its outputs.",
-    )
-    remove_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Force purge.",
+        help="Remove outputs as well.",
     )
     remove_parser.add_argument(
         "targets", nargs="+", help="DVC-files to remove."

dvc/command/repro.py

@@ -5,6 +5,7 @@ import os
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
+from dvc.dvcfile import PIPELINE_FILE
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -69,7 +70,7 @@ def add_parser(subparsers, parent_parser):
     repro_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC-file to reproduce. 'Dvcfile' by default.",
+        help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
     )
     repro_parser.add_argument(
         "-f",

dvc/command/run.py

@@ -45,7 +45,7 @@ class CmdRun(CmdBase):
                 fname=self.args.file,
                 wdir=self.args.wdir,
                 no_exec=self.args.no_exec,
-                overwrite=self.args.overwrite_dvcfile,
+                overwrite=self.args.overwrite,
                 run_cache=not self.args.no_run_cache,
                 no_commit=self.args.no_commit,
                 outs_persist=self.args.outs_persist,
@@ -53,6 +53,7 @@ class CmdRun(CmdBase):
                 always_changed=self.args.always_changed,
                 name=self.args.name,
                 single_stage=self.args.single_stage,
+                external=self.args.external,
             )
         except DvcException:
             logger.exception("")
@@ -177,10 +178,10 @@ def add_parser(subparsers, parent_parser):
         help="Only create stage file without actually running it.",
     )
     run_parser.add_argument(
-        "--overwrite-dvcfile",
+        "--overwrite",
         action="store_true",
         default=False,
-        help="Overwrite existing DVC-file without asking for confirmation.",
+        help="Overwrite existing stage",
     )
     run_parser.add_argument(
         "--no-run-cache",
@@ -225,6 +226,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=argparse.SUPPRESS,
     )
+    run_parser.add_argument(
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow outputs that are outside of the DVC repository.",
+    )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."
     )

dvc/dagascii.py

@@ -96,10 +96,9 @@ class AsciiCanvas:
 
     def draw(self):
         """Draws ASCII canvas on the screen."""
-        pager = find_pager()
         lines = map("".join, self.canvas)
         joined_lines = os.linesep.join(lines)
-        pager(joined_lines)
+        return joined_lines
 
     def point(self, x, y, char):
         """Create a point on ASCII canvas.
@@ -316,4 +315,4 @@ def draw(vertexes, edges):
             int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
         )
 
-    canvas.draw()
+    return canvas.draw()

dvc/data_cloud.py

@@ -93,7 +93,7 @@ class DataCloud:
     def _save_pulled_checksums(self, cache):
         for checksum in cache.scheme_keys("local"):
             cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
-            if self.repo.cache.local.exists(cache_file):
+            if self.repo.cache.local.tree.exists(cache_file):
                 # We can safely save here, as existing corrupted files will
                 # be removed upon status, while files corrupted during
                 # download will not be moved from tmp_file

dvc/dependency/__init__.py

@@ -2,6 +2,7 @@ from collections import defaultdict
 from urllib.parse import urlparse
 
 import dvc.output as output
+from dvc.dependency.azure import AzureDependency
 from dvc.dependency.gs import GSDependency
 from dvc.dependency.hdfs import HDFSDependency
 from dvc.dependency.http import HTTPDependency
@@ -17,6 +18,7 @@ from dvc.scheme import Schemes
 from .repo import RepoDependency
 
 DEPS = [
+    AzureDependency,
     GSDependency,
     HDFSDependency,
     HTTPDependency,
@@ -30,6 +32,7 @@ DEP_MAP = {
     Schemes.LOCAL: LocalDependency,
     Schemes.SSH: SSHDependency,
     Schemes.S3: S3Dependency,
+    Schemes.AZURE: AzureDependency,
     Schemes.GS: GSDependency,
     Schemes.HDFS: HDFSDependency,
     Schemes.HTTP: HTTPDependency,

dvc/dependency/azure.py

@@ -0,0 +1,7 @@
+from dvc.dependency.base import BaseDependency
+from dvc.output.base import BaseOutput
+from dvc.remote.azure import AzureRemote
+
+
+class AzureDependency(BaseDependency, BaseOutput):
+    REMOTE = AzureRemote

dvc/dependency/repo.py

@@ -50,13 +50,24 @@ class RepoDependency(LocalDependency):
         return external_repo(d["url"], rev=rev)
 
     def _get_checksum(self, locked=True):
+        from dvc.repo.tree import RepoTree
+
         with self._make_repo(locked=locked) as repo:
             try:
                 return repo.find_out_by_relpath(self.def_path).info["md5"]
             except OutputNotFoundError:
                 path = PathInfo(os.path.join(repo.root_dir, self.def_path))
+
+                # we want stream but not fetch, so DVC out directories are
+                # walked, but dir contents is not fetched
+                tree = RepoTree(repo, stream=True)
+
                 # We are polluting our repo cache with some dir listing here
-                return self.repo.cache.local.get_checksum(path)
+                if tree.isdir(path):
+                    return self.repo.cache.local.get_dir_checksum(
+                        path, tree=tree
+                    )
+                return tree.get_file_checksum(path)
 
     def status(self):
         current_checksum = self._get_checksum(locked=True)
@@ -76,16 +87,16 @@ class RepoDependency(LocalDependency):
     def download(self, to):
         with self._make_repo() as repo:
             if self.def_repo.get(self.PARAM_REV_LOCK) is None:
-                self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+                self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
 
-            if hasattr(repo, "cache"):
-                repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
-
-            repo.pull_to(self.def_path, to.path_info)
+            cache = self.repo.cache.local
+            with repo.use_cache(cache):
+                _, _, cache_infos = repo.fetch_external([self.def_path])
+            cache.checkout(to.path_info, cache_infos[0])
 
     def update(self, rev=None):
         if rev:
             self.def_repo[self.PARAM_REV] = rev
 
         with self._make_repo(locked=False) as repo:
-            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+            self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()

dvc/dvcfile.py

@@ -17,11 +17,7 @@ from dvc.stage.exceptions import (
 from dvc.stage.loader import SingleStageLoader, StageLoader
 from dvc.utils import relpath
 from dvc.utils.collections import apply_diff
-from dvc.utils.stage import (
-    dump_stage_file,
-    parse_stage,
-    parse_stage_for_update,
-)
+from dvc.utils.yaml import dump_yaml, parse_yaml, parse_yaml_for_update
 
 logger = logging.getLogger(__name__)
 
@@ -101,7 +97,7 @@ class FileMixin:
 
         with self.repo.tree.open(self.path) as fd:
             stage_text = fd.read()
-        d = parse_stage(stage_text, self.path)
+        d = parse_yaml(stage_text, self.path)
         self.validate(d, self.relpath)
         return d, stage_text
 
@@ -149,8 +145,8 @@ class SingleStageFile(FileMixin):
         logger.debug(
             "Saving information to '{file}'.".format(file=relpath(self.path))
         )
-        dump_stage_file(self.path, serialize.to_single_stage_file(stage))
-        self.repo.scm.track_file(relpath(self.path))
+        dump_yaml(self.path, serialize.to_single_stage_file(stage))
+        self.repo.scm.track_file(self.relpath)
 
     def remove_with_prompt(self, force=False):
         if not self.exists():
@@ -165,6 +161,9 @@ class SingleStageFile(FileMixin):
 
         self.remove()
 
+    def remove_stage(self, stage):
+        self.remove()
+
 
 class PipelineFile(FileMixin):
     """Abstraction for pipelines file, .yaml + .lock combined."""
@@ -175,16 +174,19 @@ class PipelineFile(FileMixin):
     def _lockfile(self):
         return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
 
-    def dump(self, stage, update_pipeline=False, **kwargs):
+    def dump(self, stage, update_pipeline=False, no_lock=False):
         """Dumps given stage appropriately in the dvcfile."""
         from dvc.stage import PipelineStage
 
         assert isinstance(stage, PipelineStage)
         check_dvc_filename(self.path)
-        self._dump_lockfile(stage)
+
         if update_pipeline and not stage.is_data_source:
             self._dump_pipeline_file(stage)
 
+        if not no_lock:
+            self._dump_lockfile(stage)
+
     def _dump_lockfile(self, stage):
         self._lockfile.dump(stage)
 
@@ -192,8 +194,9 @@ class PipelineFile(FileMixin):
         data = {}
         if self.exists():
             with open(self.path) as fd:
-                data = parse_stage_for_update(fd.read(), self.path)
+                data = parse_yaml_for_update(fd.read(), self.path)
         else:
+            logger.info("Creating '%s'", self.relpath)
             open(self.path, "w+").close()
 
         data["stages"] = data.get("stages", {})
@@ -204,8 +207,11 @@ class PipelineFile(FileMixin):
         else:
             data["stages"].update(stage_data)
 
-        dump_stage_file(self.path, data)
-        self.repo.scm.track_file(relpath(self.path))
+        logger.info(
+            "Adding stage '%s' to '%s'", stage.name, self.relpath,
+        )
+        dump_yaml(self.path, data)
+        self.repo.scm.track_file(self.relpath)
 
     @property
     def stage(self):
@@ -227,6 +233,22 @@ class PipelineFile(FileMixin):
         super().remove()
         self._lockfile.remove()
 
+    def remove_stage(self, stage):
+        self._lockfile.remove_stage(stage)
+        if not self.exists():
+            return
+
+        with open(self.path, "r") as f:
+            d = parse_yaml_for_update(f.read(), self.path)
+
+        self.validate(d, self.path)
+        if stage.name not in d.get("stages", {}):
+            return
+
+        logger.debug("Removing '%s' from '%s'", stage.name, self.path)
+        del d["stages"][stage.name]
+        dump_yaml(self.path, d)
+
 
 class Lockfile(FileMixin):
     from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
@@ -235,7 +257,7 @@ class Lockfile(FileMixin):
         if not self.exists():
             return {}
         with self.repo.tree.open(self.path) as fd:
-            data = parse_stage(fd.read(), self.path)
+            data = parse_yaml(fd.read(), self.path)
         try:
             self.validate(data, fname=self.relpath)
         except StageFileFormatError:
@@ -247,15 +269,38 @@ class Lockfile(FileMixin):
     def dump(self, stage, **kwargs):
         stage_data = serialize.to_lockfile(stage)
         if not self.exists():
+            modified = True
+            logger.info("Generating lock file '%s'", self.relpath)
             data = stage_data
             open(self.path, "w+").close()
         else:
             with self.repo.tree.open(self.path, "r") as fd:
-                data = parse_stage_for_update(fd.read(), self.path)
+                data = parse_yaml_for_update(fd.read(), self.path)
+            modified = data.get(stage.name, {}) != stage_data.get(
+                stage.name, {}
+            )
+            if modified:
+                logger.info("Updating lock file '%s'", self.relpath)
             data.update(stage_data)
+        dump_yaml(self.path, data)
+        if modified:
+            self.repo.scm.track_file(self.relpath)
 
-        dump_stage_file(self.path, data)
-        self.repo.scm.track_file(relpath(self.path))
+    def remove_stage(self, stage):
+        if not self.exists():
+            return
+
+        with open(self.path) as f:
+            d = parse_yaml_for_update(f.read(), self.path)
+        self.validate(d, self.path)
+
+        if stage.name not in d:
+            return
+
+        logger.debug("Removing '%s' from '%s'", stage.name, self.path)
+        del d[stage.name]
+
+        dump_yaml(self.path, d)
 
 
 class Dvcfile:

dvc/exceptions.py

@@ -173,17 +173,24 @@ class BadMetricError(DvcException):
 class NoMetricsError(DvcException):
     def __init__(self):
         super().__init__(
-            "no metric files in this repository. "
-            "Use `dvc metrics add` to add a metric file to track."
+            "no metric files in this repository. Use `-m/-M` options for "
+            "`dvc run` to mark stage outputs as  metrics."
         )
 
 
-class StageFileCorruptedError(DvcException):
+class NoPlotsError(DvcException):
+    def __init__(self):
+        super().__init__(
+            "no plots in this repository. Use `--plots/--plots-no-cache` "
+            "options for `dvc run` to mark stage outputs as plots."
+        )
+
+
+class YAMLFileCorruptedError(DvcException):
     def __init__(self, path):
         path = relpath(path)
         super().__init__(
-            "unable to read DVC-file: {} "
-            "YAML file structure is corrupted".format(path)
+            f"unable to read: '{path}', YAML file structure is corrupted"
         )
 
 
@@ -294,19 +301,15 @@ class PathMissingError(DvcException):
         "The path '{}' does not exist in the target repository '{}'"
         " neither as a DVC output nor as a Git-tracked file."
     )
-    default_msg_output_only = (
+    default_msg_dvc_only = (
         "The path '{}' does not exist in the target repository '{}'"
         " as an DVC output."
     )
 
-    def __init__(self, path, repo, output_only=False):
-        msg = (
-            self.default_msg
-            if not output_only
-            else self.default_msg_output_only
-        )
+    def __init__(self, path, repo, dvc_only=False):
+        msg = self.default_msg if not dvc_only else self.default_msg_dvc_only
         super().__init__(msg.format(path, repo))
-        self.output_only = output_only
+        self.dvc_only = dvc_only
 
 
 class RemoteCacheRequiredError(DvcException):
@@ -326,3 +329,15 @@ class RemoteCacheRequiredError(DvcException):
 
 class IsADirectoryError(DvcException):
     """Raised when a file operation is requested on a directory."""
+
+
+class NoOutputOrStageError(DvcException):
+    """
+    Raised when the target is neither an output nor a stage name in dvc.yaml
+    """
+
+    def __init__(self, target, file):
+        super().__init__(
+            f"'{target}' "
+            f"does not exist as an output or a stage name in '{file}'"
+        )

dvc/external_repo.py

@@ -4,12 +4,12 @@ import tempfile
 import threading
 from contextlib import contextmanager
 from distutils.dir_util import copy_tree
+from typing import Iterable
 
-from funcy import cached_property, retry, suppress, wrap_with
+from funcy import cached_property, retry, wrap_with
 
 from dvc.config import NoRemoteError, NotDvcRepoError
 from dvc.exceptions import (
-    CheckoutError,
     FileMissingError,
     NoOutputInExternalRepoError,
     NoRemoteInExternalRepoError,
@@ -18,9 +18,10 @@ from dvc.exceptions import (
 )
 from dvc.path_info import PathInfo
 from dvc.repo import Repo
+from dvc.repo.tree import RepoTree
 from dvc.scm.git import Git
-from dvc.utils import tmp_fname
-from dvc.utils.fs import fs_copy, move, remove
+from dvc.scm.tree import is_working_tree
+from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
@@ -29,10 +30,12 @@ logger = logging.getLogger(__name__)
 def external_repo(url, rev=None, for_write=False):
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
+    if not rev:
+        rev = "HEAD"
     try:
-        repo = ExternalRepo(path, url)
+        repo = ExternalRepo(path, url, rev, for_write=for_write)
     except NotDvcRepoError:
-        repo = ExternalGitRepo(path, url)
+        repo = ExternalGitRepo(path, url, rev)
 
     try:
         yield repo
@@ -64,59 +67,99 @@ def clean_repos():
         _remove(path)
 
 
-class ExternalRepo(Repo):
-    def __init__(self, root_dir, url):
-        super().__init__(root_dir)
+class BaseExternalRepo:
+
+    _local_cache = None
+
+    @property
+    def local_cache(self):
+        if hasattr(self, "cache"):
+            return self.cache.local
+        return self._local_cache
+
+    @contextmanager
+    def use_cache(self, cache):
+        """Use the specified cache in place of default tmpdir cache for
+        download operations.
+        """
+        if hasattr(self, "cache"):
+            save_cache = self.cache.local
+            self.cache.local = cache
+        self._local_cache = cache
+
+        yield
+
+        if hasattr(self, "cache"):
+            self.cache.local = save_cache
+        self._local_cache = None
+
+    @cached_property
+    def repo_tree(self):
+        return RepoTree(self, fetch=True)
+
+    def get_rev(self):
+        if is_working_tree(self.tree):
+            return self.scm.get_rev()
+        if hasattr(self.tree, "tree"):
+            return self.tree.tree.rev
+        return self.tree.rev
+
+    def fetch_external(self, paths: Iterable, **kwargs):
+        """Fetch specified external repo paths into cache.
+
+        Returns 3-tuple in the form
+            (downloaded, failed, list(cache_infos))
+        where cache_infos can be used as checkout targets for the
+        fetched paths.
+        """
+        download_results = []
+        failed = 0
+
+        paths = [PathInfo(self.root_dir) / path for path in paths]
+
+        def download_update(result):
+            download_results.append(result)
+
+        save_infos = []
+        for path in paths:
+            if not self.repo_tree.exists(path):
+                raise PathMissingError(path, self.url)
+            save_info = self.local_cache.save(
+                path,
+                None,
+                tree=self.repo_tree,
+                download_callback=download_update,
+            )
+            save_infos.append(save_info)
+
+        return sum(download_results), failed, save_infos
+
+    def get_external(self, path, dest):
+        """Convenience wrapper for fetch_external and checkout."""
+        if self.local_cache:
+            # fetch DVC and git files to tmpdir cache, then checkout
+            _, _, save_infos = self.fetch_external([path])
+            self.local_cache.checkout(PathInfo(dest), save_infos[0])
+        else:
+            # git-only erepo with no cache, just copy files directly
+            # to dest
+            path = PathInfo(self.root_dir) / path
+            if not self.repo_tree.exists(path):
+                raise PathMissingError(path, self.url)
+            self.repo_tree.copytree(path, dest)
+
+
+class ExternalRepo(Repo, BaseExternalRepo):
+    def __init__(self, root_dir, url, rev, for_write=False):
+        if for_write:
+            super().__init__(root_dir)
+        else:
+            root_dir = os.path.realpath(root_dir)
+            super().__init__(root_dir, scm=Git(root_dir), rev=rev)
         self.url = url
         self._set_cache_dir()
         self._fix_upstream()
 
-    def pull_to(self, path, to_info):
-        """
-        Pull the corresponding file or directory specified by `path` and
-        checkout it into `to_info`.
-
-        It works with files tracked by Git and DVC, and also local files
-        outside the repository.
-        """
-        out = None
-        path_info = PathInfo(self.root_dir) / path
-
-        with suppress(OutputNotFoundError):
-            (out,) = self.find_outs_by_path(path_info, strict=False)
-
-        try:
-            if out and out.use_cache:
-                self._pull_cached(out, path_info, to_info)
-                return
-
-            # Check if it is handled by Git (it can't have an absolute path)
-            if os.path.isabs(path):
-                raise FileNotFoundError
-
-            fs_copy(path_info, to_info)
-        except FileNotFoundError:
-            raise PathMissingError(path, self.url)
-
-    def _pull_cached(self, out, path_info, dest):
-        with self.state:
-            tmp = PathInfo(tmp_fname(dest))
-            src = tmp / path_info.relative_to(out.path_info)
-
-            out.path_info = tmp
-
-            # Only pull unless all needed cache is present
-            if out.changed_cache(filter_info=src):
-                self.cloud.pull(out.get_used_cache(filter_info=src))
-
-            try:
-                out.checkout(filter_info=src)
-            except CheckoutError:
-                raise FileNotFoundError
-
-            move(src, dest)
-            remove(tmp)
-
     @wrap_with(threading.Lock())
     def _set_cache_dir(self):
         try:
@@ -125,6 +168,7 @@ class ExternalRepo(Repo):
             cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
 
         self.cache.local.cache_dir = cache_dir
+        self._local_cache = self.cache.local
 
     def _fix_upstream(self):
         if not os.path.isdir(self.url):
@@ -165,10 +209,11 @@ class ExternalRepo(Repo):
         self.config["core"]["remote"] = "auto-generated-upstream"
 
 
-class ExternalGitRepo:
-    def __init__(self, root_dir, url):
-        self.root_dir = root_dir
+class ExternalGitRepo(BaseExternalRepo):
+    def __init__(self, root_dir, url, rev):
+        self.root_dir = os.path.realpath(root_dir)
         self.url = url
+        self.tree = self.scm.get_tree(rev)
 
     @cached_property
     def scm(self):
@@ -181,23 +226,15 @@ class ExternalGitRepo:
     def find_out_by_relpath(self, path):
         raise OutputNotFoundError(path, self)
 
-    def pull_to(self, path, to_info):
-        try:
-            # Git handled files can't have absolute path
-            if os.path.isabs(path):
-                raise FileNotFoundError
-
-            fs_copy(os.path.join(self.root_dir, path), to_info)
-        except FileNotFoundError:
-            raise PathMissingError(path, self.url)
-
     @contextmanager
     def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
         """Opens a specified resource as a file object."""
+        tree = RepoTree(self)
         try:
-            abs_path = os.path.join(self.root_dir, path)
-            with open(abs_path, mode, encoding=encoding) as fd:
-                yield fd
+            with tree.open(
+                path, mode=mode, encoding=encoding, **kwargs
+            ) as fobj:
+                yield fobj
         except FileNotFoundError:
             raise PathMissingError(path, self.url)
 
@@ -209,14 +246,12 @@ def _cached_clone(url, rev, for_write=False):
     revision checked out. If for_write is set prevents reusing this dir via
     cache.
     """
-    if not for_write and Git.is_sha(rev) and (url, rev) in CLONES:
-        return CLONES[url, rev]
-
+    # even if we have already cloned this repo, we may need to
+    # fetch/fast-forward to get specified rev
     clone_path = _clone_default_branch(url, rev)
-    rev_sha = Git(clone_path).resolve_rev(rev or "HEAD")
 
-    if not for_write and (url, rev_sha) in CLONES:
-        return CLONES[url, rev_sha]
+    if not for_write and (url) in CLONES:
+        return CLONES[url]
 
     # Copy to a new dir to keep the clone clean
     repo_path = tempfile.mkdtemp("dvc-erepo")
@@ -224,11 +259,10 @@ def _cached_clone(url, rev, for_write=False):
     copy_tree(clone_path, repo_path)
 
     # Check out the specified revision
-    if rev is not None:
+    if for_write:
         _git_checkout(repo_path, rev)
-
-    if not for_write:
-        CLONES[url, rev_sha] = repo_path
+    else:
+        CLONES[url] = repo_path
     return repo_path
 
 

dvc/ignore.py

@@ -5,6 +5,7 @@ from funcy import cached_property
 from pathspec import PathSpec
 from pathspec.patterns import GitWildMatchPattern
 
+from dvc.path_info import PathInfo
 from dvc.scm.tree import BaseTree
 from dvc.utils import relpath
 
@@ -35,12 +36,19 @@ class DvcIgnorePatterns(DvcIgnore):
         return dirs, files
 
     def matches(self, dirname, basename):
-        abs_path = os.path.join(dirname, basename)
-        rel_path = relpath(abs_path, self.dirname)
-
-        if os.pardir + os.sep in rel_path:
+        # NOTE: `relpath` is too slow, so we have to assume that both
+        # `dirname` and `self.dirname` are relative or absolute together.
+        prefix = self.dirname + os.sep
+        if dirname == self.dirname:
+            path = basename
+        elif dirname.startswith(prefix):
+            rel = dirname[len(prefix) :]
+            # NOTE: `os.path.join` is ~x5.5 slower
+            path = f"{rel}{os.sep}{basename}"
+        else:
             return False
-        return self.ignore_spec.match_file(rel_path)
+
+        return self.ignore_spec.match_file(path)
 
     def __hash__(self):
         return hash(self.ignore_file_path)
@@ -76,7 +84,7 @@ class DvcIgnoreRepo(DvcIgnore):
         def is_dvc_repo(directory):
             from dvc.repo import Repo
 
-            return os.path.isdir(os.path.join(directory, Repo.DVC_DIR))
+            return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
 
         dirs = [d for d in dirs if not is_dvc_repo(d)]
 
@@ -84,13 +92,14 @@ class DvcIgnoreRepo(DvcIgnore):
 
 
 class DvcIgnoreFilter:
-    def __init__(self, tree):
+    def __init__(self, tree, root_dir):
         self.tree = tree
+        self.root_dir = root_dir
         self.ignores = {
             DvcIgnoreDirs([".git", ".hg", ".dvc"]),
             DvcIgnoreRepo(),
         }
-        for root, dirs, files in self.tree.walk(self.tree.tree_root):
+        for root, dirs, files in self.tree.walk(self.root_dir):
             self._update(root)
             dirs[:], files[:] = self(root, dirs, files)
 
@@ -107,37 +116,108 @@ class DvcIgnoreFilter:
 
 
 class CleanTree(BaseTree):
-    def __init__(self, tree):
+    def __init__(self, tree, tree_root=None):
         self.tree = tree
+        if tree_root:
+            self._tree_root = tree_root
+        else:
+            self._tree_root = self.tree.tree_root
 
     @cached_property
     def dvcignore(self):
-        return DvcIgnoreFilter(self.tree)
+        return DvcIgnoreFilter(self.tree, self.tree_root)
 
     @property
     def tree_root(self):
-        return self.tree.tree_root
+        return self._tree_root
 
     def open(self, path, mode="r", encoding="utf-8"):
-        return self.tree.open(path, mode, encoding)
+        if self.isfile(path):
+            return self.tree.open(path, mode, encoding)
+        raise FileNotFoundError
 
     def exists(self, path):
-        return self.tree.exists(path)
+        if self.tree.exists(path) and self._parents_exist(path):
+            if self.tree.isdir(path):
+                return self._valid_dirname(path)
+            return self._valid_filename(path)
+        return False
 
     def isdir(self, path):
-        return self.tree.isdir(path)
+        return (
+            self.tree.isdir(path)
+            and self._parents_exist(path)
+            and self._valid_dirname(path)
+        )
+
+    def _valid_dirname(self, path):
+        path = os.path.abspath(path)
+        if path == self.tree_root:
+            return True
+        dirname, basename = os.path.split(path)
+        dirs, _ = self.dvcignore(dirname, [basename], [])
+        if dirs:
+            return True
+        return False
 
     def isfile(self, path):
-        return self.tree.isfile(path)
+        return (
+            self.tree.isfile(path)
+            and self._parents_exist(path)
+            and self._valid_filename(path)
+        )
+
+    def _valid_filename(self, path):
+        dirname, basename = os.path.split(os.path.normpath(path))
+        _, files = self.dvcignore(os.path.abspath(dirname), [], [basename])
+        if files:
+            return True
+        return False
 
     def isexec(self, path):
-        return self.tree.isexec(path)
+        return self.exists(path) and self.tree.isexec(path)
+
+    def _parents_exist(self, path):
+        from dvc.repo import Repo
+
+        path = PathInfo(path)
+
+        # if parent is tree_root or inside a .dvc dir we can skip this check
+        if path.parent == self.tree_root or Repo.DVC_DIR in path.parts:
+            return True
+
+        # if path is outside of tree, assume this is a local remote/local cache
+        # link/move operation where we do not need to filter ignores
+        path = relpath(path, self.tree_root)
+        if path.startswith("..") or (
+            os.name == "nt"
+            and not os.path.commonprefix(
+                [os.path.abspath(path), self.tree_root]
+            )
+        ):
+            return True
+
+        # check if parent directories are in our ignores, starting from
+        # tree_root
+        for parent_dir in reversed(PathInfo(path).parents):
+            dirname, basename = os.path.split(parent_dir)
+            if basename == ".":
+                # parent_dir == tree_root
+                continue
+            dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
+            if not dirs:
+                return False
+        return True
 
     def walk(self, top, topdown=True):
         for root, dirs, files in self.tree.walk(top, topdown):
-            dirs[:], files[:] = self.dvcignore(root, dirs, files)
+            dirs[:], files[:] = self.dvcignore(
+                os.path.abspath(root), dirs, files
+            )
 
             yield root, dirs, files
 
     def stat(self, path):
-        return self.tree.stat(path)
+        if self.exists(path):
+            return self.tree.stat(path)
+        raise FileNotFoundError

dvc/istextfile.py

@@ -7,13 +7,17 @@
 TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
 
 
-def istextfile(fname, blocksize=512):
+def istextfile(fname, blocksize=512, tree=None):
     """ Uses heuristics to guess whether the given file is text or binary,
         by reading a single block of bytes from the file.
         If more than 30% of the chars in the block are non-text, or there
         are NUL ('\x00') bytes in the block, assume this is a binary file.
     """
-    with open(fname, "rb") as fobj:
+    if tree:
+        open_func = tree.open
+    else:
+        open_func = open
+    with open_func(fname, "rb") as fobj:
         block = fobj.read(blocksize)
 
     if not block:

dvc/logger.py

@@ -18,20 +18,45 @@ FOOTER = (
 )
 
 
+def addLoggingLevel(levelName, levelNum, methodName=None):
+    """
+    Adds a new logging level to the `logging` module and the
+    currently configured logging class.
+
+    Based on https://stackoverflow.com/questions/2183233
+    """
+    if methodName is None:
+        methodName = levelName.lower()
+
+    assert not hasattr(logging, levelName)
+    assert not hasattr(logging, methodName)
+    assert not hasattr(logging.getLoggerClass(), methodName)
+
+    def logForLevel(self, message, *args, **kwargs):
+        if self.isEnabledFor(levelNum):
+            self._log(levelNum, message, args, **kwargs)
+
+    def logToRoot(message, *args, **kwargs):
+        logging.log(levelNum, message, *args, **kwargs)
+
+    logging.addLevelName(levelNum, levelName)
+    setattr(logging, levelName, levelNum)
+    setattr(logging.getLoggerClass(), methodName, logForLevel)
+    setattr(logging, methodName, logToRoot)
+
+
 class LoggingException(Exception):
     def __init__(self, record):
         msg = "failed to log {}".format(str(record))
         super().__init__(msg)
 
 
-class ExcludeErrorsFilter(logging.Filter):
-    def filter(self, record):
-        return record.levelno < logging.WARNING
+def excludeFilter(level):
+    class ExcludeLevelFilter(logging.Filter):
+        def filter(self, record):
+            return record.levelno < level
 
-
-class ExcludeInfoFilter(logging.Filter):
-    def filter(self, record):
-        return record.levelno < logging.INFO
+    return ExcludeLevelFilter
 
 
 class ColorFormatter(logging.Formatter):
@@ -47,6 +72,7 @@ class ColorFormatter(logging.Formatter):
     """
 
     color_code = {
+        "TRACE": colorama.Fore.GREEN,
         "DEBUG": colorama.Fore.BLUE,
         "WARNING": colorama.Fore.YELLOW,
         "ERROR": colorama.Fore.RED,
@@ -116,7 +142,11 @@ class LoggerHandler(logging.StreamHandler):
 
 
 def _is_verbose():
-    return logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG
+    return (
+        logging.NOTSET
+        < logging.getLogger("dvc").getEffectiveLevel()
+        <= logging.DEBUG
+    )
 
 
 def _iter_causes(exc):
@@ -152,12 +182,14 @@ def disable_other_loggers():
 def setup(level=logging.INFO):
     colorama.init()
 
+    addLoggingLevel("TRACE", logging.DEBUG - 5)
     logging.config.dictConfig(
         {
             "version": 1,
             "filters": {
-                "exclude_errors": {"()": ExcludeErrorsFilter},
-                "exclude_info": {"()": ExcludeInfoFilter},
+                "exclude_errors": {"()": excludeFilter(logging.WARNING)},
+                "exclude_info": {"()": excludeFilter(logging.INFO)},
+                "exclude_debug": {"()": excludeFilter(logging.DEBUG)},
             },
             "formatters": {"color": {"()": ColorFormatter}},
             "handlers": {
@@ -175,6 +207,13 @@ def setup(level=logging.INFO):
                     "stream": "ext://sys.stdout",
                     "filters": ["exclude_info"],
                 },
+                "console_trace": {
+                    "class": "dvc.logger.LoggerHandler",
+                    "level": "TRACE",
+                    "formatter": "color",
+                    "stream": "ext://sys.stdout",
+                    "filters": ["exclude_debug"],
+                },
                 "console_errors": {
                     "class": "dvc.logger.LoggerHandler",
                     "level": "WARNING",
@@ -188,6 +227,7 @@ def setup(level=logging.INFO):
                     "handlers": [
                         "console_info",
                         "console_debug",
+                        "console_trace",
                         "console_errors",
                     ],
                 },

dvc/main.py

@@ -18,7 +18,6 @@ from dvc.utils import format_link
 
 "".encode("idna")
 
-
 logger = logging.getLogger("dvc")
 
 
@@ -38,11 +37,17 @@ def main(argv=None):
     try:
         args = parse_args(argv)
 
-        if args.quiet:
-            logger.setLevel(logging.CRITICAL)
-
-        elif args.verbose:
-            logger.setLevel(logging.DEBUG)
+        verbosity = args.verbose - args.quiet
+        if verbosity:
+            logger.setLevel(
+                {
+                    -2: logging.CRITICAL,
+                    -1: logging.ERROR,
+                    1: logging.DEBUG,
+                    2: logging.TRACE,
+                }[max(-2, min(verbosity, 2))]
+            )
+        logger.trace(args)
 
         cmd = args.func(args)
         ret = cmd.run()

dvc/output/base.py

@@ -31,7 +31,11 @@ class OutputIsNotFileOrDirError(DvcException):
 
 class OutputAlreadyTrackedError(DvcException):
     def __init__(self, path):
-        msg = f"output '{path}' is already tracked by SCM (e.g. Git)"
+        msg = f""" output '{path}' is already tracked by SCM (e.g. Git).
+    You can remove it from Git, then add to DVC.
+        To stop tracking from Git:
+            git rm -r --cached '{path}'
+            git commit -m "stop tracking {path}" """
         super().__init__(msg)
 
 
@@ -52,6 +56,12 @@ class BaseOutput:
     PARAM_METRIC_XPATH = "xpath"
     PARAM_PLOT = "plot"
     PARAM_PLOT_TEMPLATE = "template"
+    PARAM_PLOT_X = "x"
+    PARAM_PLOT_Y = "y"
+    PARAM_PLOT_X_LABEL = "x_label"
+    PARAM_PLOT_Y_LABEL = "y_label"
+    PARAM_PLOT_TITLE = "title"
+    PARAM_PLOT_CSV_HEADER = "csv_header"
     PARAM_PERSIST = "persist"
 
     METRIC_SCHEMA = Any(
@@ -171,7 +181,7 @@ class BaseOutput:
 
     @property
     def exists(self):
-        return self.remote.exists(self.path_info)
+        return self.remote.tree.exists(self.path_info)
 
     def save_info(self):
         return self.remote.save_info(self.path_info)
@@ -207,13 +217,13 @@ class BaseOutput:
 
     @property
     def is_empty(self):
-        return self.remote.is_empty(self.path_info)
+        return self.remote.tree.is_empty(self.path_info)
 
     def isdir(self):
-        return self.remote.isdir(self.path_info)
+        return self.remote.tree.isdir(self.path_info)
 
     def isfile(self):
-        return self.remote.isfile(self.path_info)
+        return self.remote.tree.isfile(self.path_info)
 
     def ignore(self):
         if not self.use_scm_ignore:
@@ -241,17 +251,15 @@ class BaseOutput:
             if self.metric or self.plot:
                 self.verify_metric()
             if not self.IS_DEPENDENCY:
-                logger.info(
-                    "Output '{}' doesn't use cache. Skipping saving.".format(
-                        self
-                    )
+                logger.debug(
+                    "Output '%s' doesn't use cache. Skipping saving.", self
                 )
             return
 
         assert not self.IS_DEPENDENCY
 
         if not self.changed():
-            logger.info(f"Output '{self}' didn't change. Skipping saving.")
+            logger.debug("Output '%s' didn't change. Skipping saving.", self)
             return
 
         self.info = self.save_info()
@@ -267,7 +275,8 @@ class BaseOutput:
         if self.IS_DEPENDENCY:
             return ret
 
-        ret[self.PARAM_CACHE] = self.use_cache
+        if not self.use_cache:
+            ret[self.PARAM_CACHE] = self.use_cache
 
         if isinstance(self.metric, dict):
             if (
@@ -276,12 +285,14 @@ class BaseOutput:
             ):
                 del self.metric[self.PARAM_METRIC_XPATH]
 
-        ret[self.PARAM_METRIC] = self.metric
+        if self.metric:
+            ret[self.PARAM_METRIC] = self.metric
 
         if self.plot:
             ret[self.PARAM_PLOT] = self.plot
 
-        ret[self.PARAM_PERSIST] = self.persist
+        if self.persist:
+            ret[self.PARAM_PERSIST] = self.persist
 
         return ret
 
@@ -315,7 +326,7 @@ class BaseOutput:
         )
 
     def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+        self.remote.tree.remove(self.path_info)
         if self.scheme != "local":
             return
 
@@ -326,7 +337,7 @@ class BaseOutput:
         if self.scheme == "local" and self.use_scm_ignore:
             self.repo.scm.ignore_remove(self.fspath)
 
-        self.remote.move(self.path_info, out.path_info)
+        self.remote.tree.move(self.path_info, out.path_info)
         self.def_path = out.def_path
         self.path_info = out.path_info
         self.save()
@@ -347,6 +358,17 @@ class BaseOutput:
         if self.exists:
             self.remote.unprotect(self.path_info)
 
+    def get_dir_cache(self, **kwargs):
+        if not self.is_dir_checksum:
+            raise DvcException("cannot get dir cache for file checksum")
+        if self.cache.changed_cache_file(self.checksum):
+            self.repo.cloud.pull(
+                NamedCache.make("local", self.checksum, str(self)),
+                show_checksums=False,
+                **kwargs,
+            )
+        return self.dir_cache
+
     def collect_used_dir_cache(
         self, remote=None, force=False, jobs=None, filter_info=None
     ):
@@ -371,16 +393,10 @@ class BaseOutput:
 
         cache = NamedCache()
 
-        if self.cache.changed_cache_file(self.checksum):
-            try:
-                self.repo.cloud.pull(
-                    NamedCache.make("local", self.checksum, str(self)),
-                    jobs=jobs,
-                    remote=remote,
-                    show_checksums=False,
-                )
-            except DvcException:
-                logger.debug(f"failed to pull cache for '{self}'")
+        try:
+            self.get_dir_cache(jobs=jobs, remote=remote)
+        except DvcException:
+            logger.debug(f"failed to pull cache for '{self}'")
 
         if self.cache.changed_cache_file(self.checksum):
             msg = (

dvc/remote/azure.py

@@ -1,64 +1,73 @@
 import logging
 import os
 import posixpath
-import re
 import threading
 from datetime import datetime, timedelta
-from urllib.parse import urlparse
 
 from funcy import cached_property, wrap_prop
 
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
 
 
+class AzureRemoteTree(BaseRemoteTree):
+    @property
+    def blob_service(self):
+        return self.remote.blob_service
+
+    def _generate_download_url(self, path_info, expires=3600):
+        from azure.storage.blob import BlobPermissions
+
+        expires_at = datetime.utcnow() + timedelta(seconds=expires)
+
+        sas_token = self.blob_service.generate_blob_shared_access_signature(
+            path_info.bucket,
+            path_info.path,
+            permission=BlobPermissions.READ,
+            expiry=expires_at,
+        )
+        download_url = self.blob_service.make_blob_url(
+            path_info.bucket, path_info.path, sas_token=sas_token
+        )
+        return download_url
+
+    def exists(self, path_info):
+        paths = self.remote.list_paths(path_info.bucket, path_info.path)
+        return any(path_info.path == path for path in paths)
+
+    def remove(self, path_info):
+        if path_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        logger.debug(f"Removing {path_info}")
+        self.blob_service.delete_blob(path_info.bucket, path_info.path)
+
+
 class AzureRemote(BaseRemote):
     scheme = Schemes.AZURE
     path_cls = CloudURLInfo
-    REGEX = (
-        r"azure://((?P<path>[^=;]*)?|("
-        # backward compatibility
-        r"(ContainerName=(?P<container_name>[^;]+);?)?"
-        r"(?P<connection_string>.+)?)?)$"
-    )
     REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
     LIST_OBJECT_PAGE_SIZE = 5000
+    TREE_CLS = AzureRemoteTree
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
 
         url = config.get("url", "azure://")
+        self.path_info = self.path_cls(url)
 
-        match = re.match(self.REGEX, url)  # backward compatibility
-        path = match.group("path")
-        bucket = (
-            urlparse(url if path else "").netloc
-            or match.group("container_name")  # backward compatibility
-            or os.getenv("AZURE_STORAGE_CONTAINER_NAME")
-        )
+        if not self.path_info.bucket:
+            container = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
+            self.path_info = self.path_cls(f"azure://{container}")
 
-        self.connection_string = (
-            config.get("connection_string")
-            or match.group("connection_string")  # backward compatibility
-            or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
-        )
-
-        if not bucket:
-            raise ValueError("azure storage container name missing")
-
-        if not self.connection_string:
-            raise ValueError("azure storage connection string missing")
-
-        self.path_info = (
-            self.path_cls(url)
-            if path
-            else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
+        self.connection_string = config.get("connection_string") or os.getenv(
+            "AZURE_STORAGE_CONNECTION_STRING"
         )
 
     @wrap_prop(threading.Lock())
@@ -81,14 +90,16 @@ class AzureRemote(BaseRemote):
             blob_service.create_container(self.path_info.bucket)
         return blob_service
 
-    def remove(self, path_info):
-        if path_info.scheme != self.scheme:
-            raise NotImplementedError
+    def get_etag(self, path_info):
+        etag = self.blob_service.get_blob_properties(
+            path_info.bucket, path_info.path
+        ).properties.etag
+        return etag.strip('"')
 
-        logger.debug(f"Removing {path_info}")
-        self.blob_service.delete_blob(path_info.bucket, path_info.path)
+    def get_file_checksum(self, path_info):
+        return self.get_etag(path_info)
 
-    def _list_paths(self, bucket, prefix, progress_callback=None):
+    def list_paths(self, bucket, prefix, progress_callback=None):
         blob_service = self.blob_service
         next_marker = None
         while True:
@@ -113,7 +124,7 @@ class AzureRemote(BaseRemote):
             )
         else:
             prefix = self.path_info.path
-        return self._list_paths(
+        return self.list_paths(
             self.path_info.bucket, prefix, progress_callback
         )
 
@@ -138,23 +149,3 @@ class AzureRemote(BaseRemote):
                 to_file,
                 progress_callback=pbar.update_to,
             )
-
-    def exists(self, path_info):
-        paths = self._list_paths(path_info.bucket, path_info.path)
-        return any(path_info.path == path for path in paths)
-
-    def _generate_download_url(self, path_info, expires=3600):
-        from azure.storage.blob import BlobPermissions
-
-        expires_at = datetime.utcnow() + timedelta(seconds=expires)
-
-        sas_token = self.blob_service.generate_blob_shared_access_signature(
-            path_info.bucket,
-            path_info.path,
-            permission=BlobPermissions.READ,
-            expiry=expires_at,
-        )
-        download_url = self.blob_service.make_blob_url(
-            path_info.bucket, path_info.path, sas_token=sas_token
-        )
-        return download_url

dvc/remote/base.py

@@ -26,6 +26,7 @@ from dvc.path_info import PathInfo, URLInfo, WindowsPathInfo
 from dvc.progress import Tqdm
 from dvc.remote.index import RemoteIndex, RemoteIndexNoop
 from dvc.remote.slow_link_detection import slow_link_guard
+from dvc.scm.tree import is_working_tree
 from dvc.state import StateNoop
 from dvc.utils import tmp_fname
 from dvc.utils.fs import makedirs, move
@@ -82,12 +83,99 @@ def index_locked(f):
     return wrapper
 
 
+class BaseRemoteTree:
+    SHARED_MODE_MAP = {None: (None, None), "group": (None, None)}
+
+    def __init__(self, remote, config):
+        self.remote = remote
+        shared = config.get("shared")
+        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
+
+    @property
+    def file_mode(self):
+        return self._file_mode
+
+    @property
+    def dir_mode(self):
+        return self._dir_mode
+
+    @property
+    def scheme(self):
+        return self.remote.scheme
+
+    @property
+    def path_cls(self):
+        return self.remote.path_cls
+
+    def open(self, path_info, mode="r", encoding=None):
+        if hasattr(self, "_generate_download_url"):
+            get_url = partial(self._generate_download_url, path_info)
+            return open_url(get_url, mode=mode, encoding=encoding)
+
+        raise RemoteActionNotImplemented("open", self.scheme)
+
+    def exists(self, path_info):
+        raise NotImplementedError
+
+    def isdir(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return False
+
+    def isfile(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return True
+
+    def iscopy(self, path_info):
+        """Check if this file is an independent copy."""
+        return False  # We can't be sure by default
+
+    def walk_files(self, path_info):
+        """Return a generator with `PathInfo`s to all the files"""
+        raise NotImplementedError
+
+    def is_empty(self, path_info):
+        return False
+
+    def remove(self, path_info):
+        raise RemoteActionNotImplemented("remove", self.scheme)
+
+    def makedirs(self, path_info):
+        """Optional: Implement only if the remote needs to create
+        directories before copying/linking/moving data
+        """
+
+    def move(self, from_info, to_info, mode=None):
+        assert mode is None
+        self.copy(from_info, to_info)
+        self.remove(from_info)
+
+    def copy(self, from_info, to_info):
+        raise RemoteActionNotImplemented("copy", self.scheme)
+
+    def copy_fobj(self, fobj, to_info):
+        raise RemoteActionNotImplemented("copy_fobj", self.scheme)
+
+    def symlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("symlink", self.scheme)
+
+    def hardlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("hardlink", self.scheme)
+
+    def reflink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("reflink", self.scheme)
+
+
 class BaseRemote:
     scheme = "base"
     path_cls = URLInfo
     REQUIRES = {}
     JOBS = 4 * cpu_count()
     INDEX_CLS = RemoteIndex
+    TREE_CLS = BaseRemoteTree
 
     PARAM_RELPATH = "relpath"
     CHECKSUM_DIR_SUFFIX = ".dir"
@@ -101,7 +189,6 @@ class BaseRemote:
     CAN_TRAVERSE = True
 
     CACHE_MODE = None
-    SHARED_MODE_MAP = {None: (None, None), "group": (None, None)}
 
     state = StateNoop()
 
@@ -110,9 +197,6 @@ class BaseRemote:
 
         self._check_requires(config)
 
-        shared = config.get("shared")
-        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
-
         self.checksum_jobs = (
             config.get("checksum_jobs")
             or (self.repo and self.repo.config["core"].get("checksum_jobs"))
@@ -133,6 +217,8 @@ class BaseRemote:
         else:
             self.index = RemoteIndexNoop()
 
+        self.tree = self.TREE_CLS(self, config)
+
     @classmethod
     def get_missing_deps(cls):
         import importlib
@@ -211,23 +297,33 @@ class BaseRemote:
                 checksums = dict(zip(file_infos, tasks))
         return checksums
 
-    def _collect_dir(self, path_info):
+    def _collect_dir(self, path_info, tree=None, save_tree=False, **kwargs):
         file_infos = set()
 
-        for fname in self.walk_files(path_info):
+        if tree:
+            walk_files = tree.walk_files
+        else:
+            walk_files = self.tree.walk_files
+
+        for fname in walk_files(path_info, **kwargs):
             if DvcIgnore.DVCIGNORE_FILE == fname.name:
                 raise DvcIgnoreInCollectedDirError(fname.parent)
 
             file_infos.add(fname)
 
-        checksums = {fi: self.state.get(fi) for fi in file_infos}
-        not_in_state = {
-            fi for fi, checksum in checksums.items() if checksum is None
-        }
+        if tree:
+            checksums = {fi: tree.get_file_checksum(fi) for fi in file_infos}
+            if save_tree:
+                for fi, checksum in checksums.items():
+                    self._save_file(fi, checksum, tree=tree, **kwargs)
+        else:
+            checksums = {fi: self.state.get(fi) for fi in file_infos}
+            not_in_state = {
+                fi for fi, checksum in checksums.items() if checksum is None
+            }
 
-        new_checksums = self._calculate_checksums(not_in_state)
-
-        checksums.update(new_checksums)
+            new_checksums = self._calculate_checksums(not_in_state)
+            checksums.update(new_checksums)
 
         result = [
             {
@@ -248,18 +344,25 @@ class BaseRemote:
         # Sorting the list by path to ensure reproducibility
         return sorted(result, key=itemgetter(self.PARAM_RELPATH))
 
-    def get_dir_checksum(self, path_info):
+    def get_dir_checksum(self, path_info, tree=None):
         if not self.cache:
             raise RemoteCacheRequiredError(path_info)
 
-        dir_info = self._collect_dir(path_info)
+        dir_info = self._collect_dir(path_info, tree=None)
+        if tree:
+            # don't save state entry for path_info if it is a tree path
+            path_info = None
+        return self._save_dir_info(dir_info, path_info)
+
+    def _save_dir_info(self, dir_info, path_info=None):
         checksum, tmp_info = self._get_dir_info_checksum(dir_info)
         new_info = self.cache.checksum_to_path_info(checksum)
         if self.cache.changed_cache_file(checksum):
-            self.cache.makedirs(new_info.parent)
-            self.cache.move(tmp_info, new_info, mode=self.CACHE_MODE)
+            self.cache.tree.makedirs(new_info.parent)
+            self.cache.tree.move(tmp_info, new_info, mode=self.CACHE_MODE)
 
-        self.state.save(path_info, checksum)
+        if path_info:
+            self.state.save(path_info, checksum)
         self.state.save(new_info, checksum)
 
         return checksum
@@ -326,7 +429,7 @@ class BaseRemote:
     def get_checksum(self, path_info):
         assert isinstance(path_info, str) or path_info.scheme == self.scheme
 
-        if not self.exists(path_info):
+        if not self.tree.exists(path_info):
             return None
 
         checksum = self.state.get(path_info)
@@ -337,14 +440,16 @@ class BaseRemote:
         if (
             checksum
             and self.is_dir_checksum(checksum)
-            and not self.exists(self.cache.checksum_to_path_info(checksum))
+            and not self.tree.exists(
+                self.cache.checksum_to_path_info(checksum)
+            )
         ):
             checksum = None
 
         if checksum:
             return checksum
 
-        if self.isdir(path_info):
+        if self.tree.isdir(path_info):
             checksum = self.get_dir_checksum(path_info)
         else:
             checksum = self.get_file_checksum(path_info)
@@ -378,7 +483,7 @@ class BaseRemote:
             "checking if '%s'('%s') has changed.", path_info, checksum_info
         )
 
-        if not self.exists(path_info):
+        if not self.tree.exists(path_info):
             logger.debug("'%s' doesn't exist.", path_info)
             return True
 
@@ -410,9 +515,9 @@ class BaseRemote:
         self._link(from_info, to_info, self.cache_types)
 
     def _link(self, from_info, to_info, link_types):
-        assert self.isfile(from_info)
+        assert self.tree.isfile(from_info)
 
-        self.makedirs(to_info.parent)
+        self.tree.makedirs(to_info.parent)
 
         self._try_links(from_info, to_info, link_types)
 
@@ -420,9 +525,9 @@ class BaseRemote:
         if self.cache_type_confirmed:
             return
 
-        is_link = getattr(self, f"is_{link_type}", None)
+        is_link = getattr(self.tree, f"is_{link_type}", None)
         if is_link and not is_link(path_info):
-            self.remove(path_info)
+            self.tree.remove(path_info)
             raise DvcException(f"failed to verify {link_type}")
 
         self.cache_type_confirmed = True
@@ -430,7 +535,7 @@ class BaseRemote:
     @slow_link_guard
     def _try_links(self, from_info, to_info, link_types):
         while link_types:
-            link_method = getattr(self, link_types[0])
+            link_method = getattr(self.tree, link_types[0])
             try:
                 self._do_link(from_info, to_info, link_method)
                 self._verify_link(to_info, link_types[0])
@@ -445,7 +550,7 @@ class BaseRemote:
         raise DvcException("no possible cache types left to try out.")
 
     def _do_link(self, from_info, to_info, link_method):
-        if self.exists(to_info):
+        if self.tree.exists(to_info):
             raise DvcException(f"Link '{to_info}' already exists!")
 
         link_method(from_info, to_info)
@@ -454,28 +559,47 @@ class BaseRemote:
             "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
         )
 
-    def _save_file(self, path_info, checksum, save_link=True):
+    def _save_file(
+        self, path_info, checksum, save_link=True, tree=None, **kwargs
+    ):
         assert checksum
 
         cache_info = self.checksum_to_path_info(checksum)
-        if self.changed_cache(checksum):
-            self.move(path_info, cache_info, mode=self.CACHE_MODE)
-            self.link(cache_info, path_info)
-        elif self.iscopy(path_info) and self._cache_is_copy(path_info):
-            # Default relink procedure involves unneeded copy
-            self.unprotect(path_info)
+        if tree:
+            if self.changed_cache(checksum):
+                with tree.open(path_info, mode="rb") as fobj:
+                    # if tree has fetch enabled, DVC out will be fetched on
+                    # open and we do not need to read/copy any data
+                    if not (
+                        tree.isdvc(path_info, strict=False) and tree.fetch
+                    ):
+                        self.tree.copy_fobj(fobj, cache_info)
+                callback = kwargs.get("download_callback")
+                if callback:
+                    callback(1)
         else:
-            self.remove(path_info)
-            self.link(cache_info, path_info)
+            if self.changed_cache(checksum):
+                self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
+                self.link(cache_info, path_info)
+            elif self.tree.iscopy(path_info) and self._cache_is_copy(
+                path_info
+            ):
+                # Default relink procedure involves unneeded copy
+                self.unprotect(path_info)
+            else:
+                self.tree.remove(path_info)
+                self.link(cache_info, path_info)
 
-        if save_link:
-            self.state.save_link(path_info)
+            if save_link:
+                self.state.save_link(path_info)
 
         # we need to update path and cache, since in case of reflink,
         # or copy cache type moving original file results in updates on
         # next executed command, which causes md5 recalculation
-        self.state.save(path_info, checksum)
+        if not tree or is_working_tree(tree):
+            self.state.save(path_info, checksum)
         self.state.save(cache_info, checksum)
+        return {self.PARAM_CHECKSUM: checksum}
 
     def _cache_is_copy(self, path_info):
         """Checks whether cache uses copies."""
@@ -487,78 +611,88 @@ class BaseRemote:
 
         workspace_file = path_info.with_name("." + uuid())
         test_cache_file = self.path_info / ".cache_type_test_file"
-        if not self.exists(test_cache_file):
-            with self.open(test_cache_file, "wb") as fobj:
+        if not self.tree.exists(test_cache_file):
+            with self.tree.open(test_cache_file, "wb") as fobj:
                 fobj.write(bytes(1))
         try:
             self.link(test_cache_file, workspace_file)
         finally:
-            self.remove(workspace_file)
-            self.remove(test_cache_file)
+            self.tree.remove(workspace_file)
+            self.tree.remove(test_cache_file)
 
         self.cache_type_confirmed = True
         return self.cache_types[0] == "copy"
 
-    def _save_dir(self, path_info, checksum, save_link=True):
+    def _save_dir(
+        self, path_info, checksum, save_link=True, tree=None, **kwargs
+    ):
+        if tree:
+            dir_info = self._collect_dir(
+                path_info, tree=tree, save_tree=True, **kwargs
+            )
+            checksum = self._save_dir_info(dir_info)
+        else:
+            dir_info = self.get_dir_cache(checksum)
+
+            for entry in Tqdm(
+                dir_info, desc="Saving " + path_info.name, unit="file"
+            ):
+                entry_info = path_info / entry[self.PARAM_RELPATH]
+                entry_checksum = entry[self.PARAM_CHECKSUM]
+                self._save_file(entry_info, entry_checksum, save_link=False)
+
+            if save_link:
+                self.state.save_link(path_info)
+
         cache_info = self.checksum_to_path_info(checksum)
-        dir_info = self.get_dir_cache(checksum)
-
-        for entry in Tqdm(
-            dir_info, desc="Saving " + path_info.name, unit="file"
-        ):
-            entry_info = path_info / entry[self.PARAM_RELPATH]
-            entry_checksum = entry[self.PARAM_CHECKSUM]
-            self._save_file(entry_info, entry_checksum, save_link=False)
-
-        if save_link:
-            self.state.save_link(path_info)
-
         self.state.save(cache_info, checksum)
-        self.state.save(path_info, checksum)
-
-    def is_empty(self, path_info):
-        return False
-
-    def isfile(self, path_info):
-        """Optional: Overwrite only if the remote has a way to distinguish
-        between a directory and a file.
-        """
-        return True
-
-    def isdir(self, path_info):
-        """Optional: Overwrite only if the remote has a way to distinguish
-        between a directory and a file.
-        """
-        return False
-
-    def iscopy(self, path_info):
-        """Check if this file is an independent copy."""
-        return False  # We can't be sure by default
-
-    def walk_files(self, path_info):
-        """Return a generator with `PathInfo`s to all the files"""
-        raise NotImplementedError
+        if not tree or is_working_tree(tree):
+            self.state.save(path_info, checksum)
+        return {self.PARAM_CHECKSUM: checksum}
 
     @staticmethod
     def protect(path_info):
         pass
 
-    def save(self, path_info, checksum_info, save_link=True):
+    def save(
+        self, path_info, checksum_info, save_link=True, tree=None, **kwargs
+    ):
         if path_info.scheme != self.scheme:
             raise RemoteActionNotImplemented(
                 f"save {path_info.scheme} -> {self.scheme}", self.scheme,
             )
 
-        checksum = checksum_info[self.PARAM_CHECKSUM]
-        self._save(path_info, checksum, save_link)
+        if tree:
+            if tree.isdir(path_info):
+                # save checksum will be computed during tree walk
+                checksum = None
+            else:
+                checksum = tree.get_file_checksum(path_info)
+        else:
+            checksum = checksum_info[self.PARAM_CHECKSUM]
+        return self._save(path_info, checksum, save_link, tree, **kwargs)
 
-    def _save(self, path_info, checksum, save_link=True):
-        to_info = self.checksum_to_path_info(checksum)
-        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
-        if self.isdir(path_info):
-            self._save_dir(path_info, checksum, save_link)
-            return
-        self._save_file(path_info, checksum, save_link)
+    def _save(self, path_info, checksum, save_link=True, tree=None, **kwargs):
+        if tree:
+            logger.debug("Saving tree path '%s' to cache.", path_info)
+        else:
+            to_info = self.checksum_to_path_info(checksum)
+            logger.debug("Saving '%s' to '%s'.", path_info, to_info)
+
+        if tree:
+            isdir = tree.isdir
+            save_link = False
+        else:
+            isdir = self.tree.isdir
+
+        if isdir(path_info):
+            return self._save_dir(
+                path_info, checksum, save_link, tree, **kwargs
+            )
+        return self._save_file(path_info, checksum, save_link, tree, **kwargs)
+
+    def open(self, *args, **kwargs):
+        return self.tree.open(*args, **kwargs)
 
     def _handle_transfer_exception(
         self, from_info, to_info, exception, operation
@@ -615,13 +749,13 @@ class BaseRemote:
             raise NotImplementedError
 
         if to_info.scheme == self.scheme != "local":
-            self.copy(from_info, to_info)
+            self.tree.copy(from_info, to_info)
             return 0
 
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        if self.isdir(from_info):
+        if self.tree.isdir(from_info):
             return self._download_dir(
                 from_info, to_info, name, no_progress_bar, file_mode, dir_mode
             )
@@ -632,7 +766,7 @@ class BaseRemote:
     def _download_dir(
         self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
     ):
-        from_infos = list(self.walk_files(from_info))
+        from_infos = list(self.tree.walk_files(from_info))
         to_infos = (
             to_info / info.relative_to(from_info) for info in from_infos
         )
@@ -679,36 +813,6 @@ class BaseRemote:
 
         return 0
 
-    def open(self, path_info, mode="r", encoding=None):
-        if hasattr(self, "_generate_download_url"):
-            get_url = partial(self._generate_download_url, path_info)
-            return open_url(get_url, mode=mode, encoding=encoding)
-
-        raise RemoteActionNotImplemented("open", self.scheme)
-
-    def remove(self, path_info):
-        raise RemoteActionNotImplemented("remove", self.scheme)
-
-    def move(self, from_info, to_info, mode=None):
-        assert mode is None
-        self.copy(from_info, to_info)
-        self.remove(from_info)
-
-    def copy(self, from_info, to_info):
-        raise RemoteActionNotImplemented("copy", self.scheme)
-
-    def symlink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("symlink", self.scheme)
-
-    def hardlink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("hardlink", self.scheme)
-
-    def reflink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("reflink", self.scheme)
-
-    def exists(self, path_info):
-        raise NotImplementedError
-
     def path_to_checksum(self, path):
         parts = self.path_cls(path).parts[-2:]
 
@@ -763,7 +867,7 @@ class BaseRemote:
 
     @index_locked
     def gc(self, named_cache, jobs=None):
-        used = self.extract_used_local_checksums(named_cache)
+        used = set(named_cache.scheme_keys("local"))
 
         if self.scheme != "":
             used.update(named_cache.scheme_keys(self.scheme))
@@ -779,8 +883,9 @@ class BaseRemote:
                 continue
             path_info = self.checksum_to_path_info(checksum)
             if self.is_dir_checksum(checksum):
+                # backward compatibility
                 self._remove_unpacked_dir(checksum)
-            self.remove(path_info)
+            self.tree.remove(path_info)
             removed = True
         if removed:
             self.index.clear()
@@ -827,9 +932,9 @@ class BaseRemote:
             self.protect(cache_info)
             return False
 
-        if self.exists(cache_info):
+        if self.tree.exists(cache_info):
             logger.warning("corrupted cache file '%s'.", cache_info)
-            self.remove(cache_info)
+            self.tree.remove(cache_info)
 
         return True
 
@@ -837,11 +942,6 @@ class BaseRemote:
         if self.changed_cache_file(checksum):
             return True
 
-        if not (path_info and filter_info) and not self._changed_unpacked_dir(
-            checksum
-        ):
-            return False
-
         for entry in self.get_dir_cache(checksum):
             entry_checksum = entry[self.PARAM_CHECKSUM]
 
@@ -853,9 +953,6 @@ class BaseRemote:
             if self.changed_cache_file(entry_checksum):
                 return True
 
-        if not (path_info and filter_info):
-            self._update_unpacked_dir(checksum)
-
         return False
 
     def changed_cache(self, checksum, path_info=None, filter_info=None):
@@ -1081,7 +1178,7 @@ class BaseRemote:
         ) as pbar:
 
             def exists_with_progress(path_info):
-                ret = self.exists(path_info)
+                ret = self.tree.exists(path_info)
                 pbar.update_msg(str(path_info))
                 return ret
 
@@ -1100,7 +1197,7 @@ class BaseRemote:
         return not self.changed_cache(current)
 
     def safe_remove(self, path_info, force=False):
-        if not self.exists(path_info):
+        if not self.tree.exists(path_info):
             return
 
         if not force and not self.already_cached(path_info):
@@ -1112,7 +1209,7 @@ class BaseRemote:
             if not prompt.confirm(msg):
                 raise ConfirmRemoveError(str(path_info))
 
-        self.remove(path_info)
+        self.tree.remove(path_info)
 
     def _checkout_file(
         self, path_info, checksum, force, progress_callback=None, relink=False
@@ -1120,7 +1217,7 @@ class BaseRemote:
         """The file is changed we need to checkout a new copy"""
         added, modified = True, False
         cache_info = self.checksum_to_path_info(checksum)
-        if self.exists(path_info):
+        if self.tree.exists(path_info):
             logger.debug("data '%s' will be replaced.", path_info)
             self.safe_remove(path_info, force=force)
             added, modified = False, True
@@ -1133,11 +1230,6 @@ class BaseRemote:
 
         return added, modified and not relink
 
-    def makedirs(self, path_info):
-        """Optional: Implement only if the remote needs to create
-        directories before copying/linking/moving data
-        """
-
     def _checkout_dir(
         self,
         path_info,
@@ -1150,9 +1242,9 @@ class BaseRemote:
         added, modified = False, False
         # Create dir separately so that dir is created
         # even if there are no files in it
-        if not self.exists(path_info):
+        if not self.tree.exists(path_info):
             added = True
-            self.makedirs(path_info)
+            self.tree.makedirs(path_info)
 
         dir_info = self.get_dir_cache(checksum)
 
@@ -1188,7 +1280,7 @@ class BaseRemote:
         return added, not added and modified and not relink
 
     def _remove_redundant_files(self, path_info, dir_info, force):
-        existing_files = set(self.walk_files(path_info))
+        existing_files = set(self.tree.walk_files(path_info))
 
         needed_files = {
             path_info / entry[self.PARAM_RELPATH] for entry in dir_info
@@ -1294,19 +1386,5 @@ class BaseRemote:
     def unprotect(path_info):
         pass
 
-    def _get_unpacked_dir_names(self, checksums):
-        return set()
-
-    def extract_used_local_checksums(self, named_cache):
-        used = set(named_cache.scheme_keys("local"))
-        unpacked = self._get_unpacked_dir_names(used)
-        return used | unpacked
-
-    def _changed_unpacked_dir(self, checksum):
-        return True
-
-    def _update_unpacked_dir(self, checksum):
-        pass
-
     def _remove_unpacked_dir(self, checksum):
         pass

dvc/remote/gdrive.py

@@ -12,7 +12,7 @@ from funcy.py3 import cat
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.scheme import Schemes
 from dvc.utils import format_link, tmp_fname
 
@@ -87,6 +87,20 @@ class GDriveURLInfo(CloudURLInfo):
         self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
 
 
+class GDriveRemoteTree(BaseRemoteTree):
+    def exists(self, path_info):
+        try:
+            self.remote.get_item_id(path_info)
+        except GDrivePathNotFound:
+            return False
+        else:
+            return True
+
+    def remove(self, path_info):
+        item_id = self.remote.get_item_id(path_info)
+        self.remote.gdrive_delete_file(item_id)
+
+
 class GDriveRemote(BaseRemote):
     scheme = Schemes.GDRIVE
     path_cls = GDriveURLInfo
@@ -95,6 +109,7 @@ class GDriveRemote(BaseRemote):
     # Always prefer traverse for GDrive since API usage quotas are a concern.
     TRAVERSE_WEIGHT_MULTIPLIER = 1
     TRAVERSE_PREFIX_LEN = 2
+    TREE_CLS = GDriveRemoteTree
 
     GDRIVE_CREDENTIALS_DATA = "GDRIVE_CREDENTIALS_DATA"
     DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"
@@ -281,7 +296,7 @@ class GDriveRemote(BaseRemote):
         cache = {
             "dirs": defaultdict(list),
             "ids": {},
-            "root_id": self._get_item_id(
+            "root_id": self.get_item_id(
                 self.path_info,
                 use_cache=False,
                 hint="Confirm the directory exists and you can access it.",
@@ -340,7 +355,7 @@ class GDriveRemote(BaseRemote):
                     "use the `gdrive_user_credentials_file` config\n"
                     "   option if you use multiple GDrive remotes with "
                     "different email accounts.\n\nDetails".format(
-                        item_id, self.path_info, self._credentials_location
+                        item_id, self.path_info, self.credentials_location
                     )
                 ) from exc
             raise
@@ -394,7 +409,7 @@ class GDriveRemote(BaseRemote):
             gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
 
     @_gdrive_retry
-    def _gdrive_delete_file(self, item_id):
+    def gdrive_delete_file(self, item_id):
         from pydrive2.files import ApiRequestError
 
         param = {"id": item_id}
@@ -498,7 +513,7 @@ class GDriveRemote(BaseRemote):
             [self._create_dir(min(parent_ids), title, path)] if create else []
         )
 
-    def _get_item_id(self, path_info, create=False, use_cache=True, hint=None):
+    def get_item_id(self, path_info, create=False, use_cache=True, hint=None):
         assert path_info.bucket == self._bucket
 
         item_ids = self._path_to_item_ids(path_info.path, create, use_cache)
@@ -508,25 +523,17 @@ class GDriveRemote(BaseRemote):
         assert not create
         raise GDrivePathNotFound(path_info, hint)
 
-    def exists(self, path_info):
-        try:
-            self._get_item_id(path_info)
-        except GDrivePathNotFound:
-            return False
-        else:
-            return True
-
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
         dirname = to_info.parent
         assert dirname
-        parent_id = self._get_item_id(dirname, True)
+        parent_id = self.get_item_id(dirname, True)
 
         self._gdrive_upload_file(
             parent_id, to_info.name, no_progress_bar, from_file, name
         )
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        item_id = self._get_item_id(from_info)
+        item_id = self.get_item_id(from_info)
         self._gdrive_download_file(item_id, to_file, name, no_progress_bar)
 
     def list_cache_paths(self, prefix=None, progress_callback=None):
@@ -552,12 +559,5 @@ class GDriveRemote(BaseRemote):
                 self._ids_cache["ids"][parent_id], item["title"]
             )
 
-    def remove(self, path_info):
-        item_id = self._get_item_id(path_info)
-        self._gdrive_delete_file(item_id)
-
     def get_file_checksum(self, path_info):
         raise NotImplementedError
-
-    def walk_files(self, path_info):
-        raise NotImplementedError

dvc/remote/gs.py

@@ -10,7 +10,7 @@ from funcy import cached_property, wrap_prop
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
@@ -65,11 +65,82 @@ def _upload_to_bucket(
             blob.upload_from_file(wrapped)
 
 
+class GSRemoteTree(BaseRemoteTree):
+    @property
+    def gs(self):
+        return self.remote.gs
+
+    def _generate_download_url(self, path_info, expires=3600):
+        expiration = timedelta(seconds=int(expires))
+
+        bucket = self.gs.bucket(path_info.bucket)
+        blob = bucket.get_blob(path_info.path)
+        if blob is None:
+            raise FileNotFoundError
+        return blob.generate_signed_url(expiration=expiration)
+
+    def exists(self, path_info):
+        """Check if the blob exists. If it does not exist,
+        it could be a part of a directory path.
+
+        eg: if `data/file.txt` exists, check for `data` should return True
+        """
+        return self.isfile(path_info) or self.isdir(path_info)
+
+    def isdir(self, path_info):
+        dir_path = path_info / ""
+        return bool(list(self.remote.list_paths(dir_path, max_items=1)))
+
+    def isfile(self, path_info):
+        if path_info.path.endswith("/"):
+            return False
+
+        blob = self.gs.bucket(path_info.bucket).blob(path_info.path)
+        return blob.exists()
+
+    def walk_files(self, path_info):
+        for fname in self.remote.list_paths(path_info / ""):
+            # skip nested empty directories
+            if fname.endswith("/"):
+                continue
+            yield path_info.replace(fname)
+
+    def remove(self, path_info):
+        if path_info.scheme != "gs":
+            raise NotImplementedError
+
+        logger.debug(f"Removing gs://{path_info}")
+        blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
+        if not blob:
+            return
+
+        blob.delete()
+
+    def makedirs(self, path_info):
+        if not path_info.path:
+            return
+
+        self.gs.bucket(path_info.bucket).blob(
+            (path_info / "").path
+        ).upload_from_string("")
+
+    def copy(self, from_info, to_info):
+        from_bucket = self.gs.bucket(from_info.bucket)
+        blob = from_bucket.get_blob(from_info.path)
+        if not blob:
+            msg = f"'{from_info.path}' doesn't exist in the cloud"
+            raise DvcException(msg)
+
+        to_bucket = self.gs.bucket(to_info.bucket)
+        from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path)
+
+
 class GSRemote(BaseRemote):
     scheme = Schemes.GS
     path_cls = CloudURLInfo
     REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
     PARAM_CHECKSUM = "md5"
+    TREE_CLS = GSRemoteTree
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
@@ -105,28 +176,7 @@ class GSRemote(BaseRemote):
         md5 = base64.b64decode(b64_md5)
         return codecs.getencoder("hex")(md5)[0].decode("utf-8")
 
-    def copy(self, from_info, to_info):
-        from_bucket = self.gs.bucket(from_info.bucket)
-        blob = from_bucket.get_blob(from_info.path)
-        if not blob:
-            msg = f"'{from_info.path}' doesn't exist in the cloud"
-            raise DvcException(msg)
-
-        to_bucket = self.gs.bucket(to_info.bucket)
-        from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path)
-
-    def remove(self, path_info):
-        if path_info.scheme != "gs":
-            raise NotImplementedError
-
-        logger.debug(f"Removing gs://{path_info}")
-        blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
-        if not blob:
-            return
-
-        blob.delete()
-
-    def _list_paths(
+    def list_paths(
         self, path_info, max_items=None, prefix=None, progress_callback=None
     ):
         if prefix:
@@ -141,44 +191,10 @@ class GSRemote(BaseRemote):
             yield blob.name
 
     def list_cache_paths(self, prefix=None, progress_callback=None):
-        return self._list_paths(
+        return self.list_paths(
             self.path_info, prefix=prefix, progress_callback=progress_callback
         )
 
-    def walk_files(self, path_info):
-        for fname in self._list_paths(path_info / ""):
-            # skip nested empty directories
-            if fname.endswith("/"):
-                continue
-            yield path_info.replace(fname)
-
-    def makedirs(self, path_info):
-        if not path_info.path:
-            return
-
-        self.gs.bucket(path_info.bucket).blob(
-            (path_info / "").path
-        ).upload_from_string("")
-
-    def isdir(self, path_info):
-        dir_path = path_info / ""
-        return bool(list(self._list_paths(dir_path, max_items=1)))
-
-    def isfile(self, path_info):
-        if path_info.path.endswith("/"):
-            return False
-
-        blob = self.gs.bucket(path_info.bucket).blob(path_info.path)
-        return blob.exists()
-
-    def exists(self, path_info):
-        """Check if the blob exists. If it does not exist,
-        it could be a part of a directory path.
-
-        eg: if `data/file.txt` exists, check for `data` should return True
-        """
-        return self.isfile(path_info) or self.isdir(path_info)
-
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
         bucket = self.gs.bucket(to_info.bucket)
         _upload_to_bucket(
@@ -201,12 +217,3 @@ class GSRemote(BaseRemote):
                 disable=no_progress_bar,
             ) as wrapped:
                 blob.download_to_file(wrapped)
-
-    def _generate_download_url(self, path_info, expires=3600):
-        expiration = timedelta(seconds=int(expires))
-
-        bucket = self.gs.bucket(path_info.bucket)
-        blob = bucket.get_blob(path_info.path)
-        if blob is None:
-            raise FileNotFoundError
-        return blob.generate_signed_url(expiration=expiration)

dvc/remote/hdfs.py

@@ -11,18 +11,75 @@ from urllib.parse import urlparse
 from dvc.scheme import Schemes
 from dvc.utils import fix_env, tmp_fname
 
-from .base import BaseRemote, RemoteCmdError
+from .base import BaseRemote, BaseRemoteTree, RemoteCmdError
 from .pool import get_connection
 
 logger = logging.getLogger(__name__)
 
 
+class HDFSRemoteTree(BaseRemoteTree):
+    @property
+    def hdfs(self):
+        return self.remote.hdfs
+
+    @contextmanager
+    def open(self, path_info, mode="r", encoding=None):
+        assert mode in {"r", "rt", "rb"}
+
+        try:
+            with self.hdfs(path_info) as hdfs, closing(
+                hdfs.open(path_info.path, mode="rb")
+            ) as fd:
+                if mode == "rb":
+                    yield fd
+                else:
+                    yield io.TextIOWrapper(fd, encoding=encoding)
+        except OSError as e:
+            # Empty .errno and not specific enough error class in pyarrow,
+            # see https://issues.apache.org/jira/browse/ARROW-6248
+            if "file does not exist" in str(e):
+                raise FileNotFoundError(*e.args)
+            raise
+
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info.scheme == "hdfs"
+        with self.hdfs(path_info) as hdfs:
+            return hdfs.exists(path_info.path)
+
+    def remove(self, path_info):
+        if path_info.scheme != "hdfs":
+            raise NotImplementedError
+
+        if self.exists(path_info):
+            logger.debug(f"Removing {path_info.path}")
+            with self.hdfs(path_info) as hdfs:
+                hdfs.rm(path_info.path)
+
+    def copy(self, from_info, to_info, **_kwargs):
+        dname = posixpath.dirname(to_info.path)
+        with self.hdfs(to_info) as hdfs:
+            hdfs.mkdir(dname)
+            # NOTE: this is how `hadoop fs -cp` works too: it copies through
+            # your local machine.
+            with hdfs.open(from_info.path, "rb") as from_fobj:
+                tmp_info = to_info.parent / tmp_fname(to_info.name)
+                try:
+                    with hdfs.open(tmp_info.path, "wb") as tmp_fobj:
+                        tmp_fobj.upload(from_fobj)
+                    hdfs.rename(tmp_info.path, to_info.path)
+                except Exception:
+                    self.remove(tmp_info)
+                    raise
+
+
 class HDFSRemote(BaseRemote):
     scheme = Schemes.HDFS
     REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
     PARAM_CHECKSUM = "checksum"
     REQUIRES = {"pyarrow": "pyarrow"}
     TRAVERSE_PREFIX_LEN = 2
+    TREE_CLS = HDFSRemoteTree
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
@@ -91,37 +148,6 @@ class HDFSRemote(BaseRemote):
         )
         return self._group(regex, stdout, "checksum")
 
-    def copy(self, from_info, to_info, **_kwargs):
-        dname = posixpath.dirname(to_info.path)
-        with self.hdfs(to_info) as hdfs:
-            hdfs.mkdir(dname)
-            # NOTE: this is how `hadoop fs -cp` works too: it copies through
-            # your local machine.
-            with hdfs.open(from_info.path, "rb") as from_fobj:
-                tmp_info = to_info.parent / tmp_fname(to_info.name)
-                try:
-                    with hdfs.open(tmp_info.path, "wb") as tmp_fobj:
-                        tmp_fobj.upload(from_fobj)
-                    hdfs.rename(tmp_info.path, to_info.path)
-                except Exception:
-                    self.remove(tmp_info)
-                    raise
-
-    def remove(self, path_info):
-        if path_info.scheme != "hdfs":
-            raise NotImplementedError
-
-        if self.exists(path_info):
-            logger.debug(f"Removing {path_info.path}")
-            with self.hdfs(path_info) as hdfs:
-                hdfs.rm(path_info.path)
-
-    def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info.scheme == "hdfs"
-        with self.hdfs(path_info) as hdfs:
-            return hdfs.exists(path_info.path)
-
     def _upload(self, from_file, to_info, **_kwargs):
         with self.hdfs(to_info) as hdfs:
             hdfs.mkdir(posixpath.dirname(to_info.path))
@@ -135,27 +161,8 @@ class HDFSRemote(BaseRemote):
             with open(to_file, "wb+") as fobj:
                 hdfs.download(from_info.path, fobj)
 
-    @contextmanager
-    def open(self, path_info, mode="r", encoding=None):
-        assert mode in {"r", "rt", "rb"}
-
-        try:
-            with self.hdfs(path_info) as hdfs, closing(
-                hdfs.open(path_info.path, mode="rb")
-            ) as fd:
-                if mode == "rb":
-                    yield fd
-                else:
-                    yield io.TextIOWrapper(fd, encoding=encoding)
-        except OSError as e:
-            # Empty .errno and not specific enough error class in pyarrow,
-            # see https://issues.apache.org/jira/browse/ARROW-6248
-            if "file does not exist" in str(e):
-                raise FileNotFoundError(*e.args)
-            raise
-
     def list_cache_paths(self, prefix=None, progress_callback=None):
-        if not self.exists(self.path_info):
+        if not self.tree.exists(self.path_info):
             return
 
         if prefix:
@@ -165,6 +172,8 @@ class HDFSRemote(BaseRemote):
         dirs = deque([root])
 
         with self.hdfs(self.path_info) as hdfs:
+            if prefix and not hdfs.exists(root):
+                return
             while dirs:
                 try:
                     entries = hdfs.ls(dirs.pop(), detail=True)

dvc/remote/http.py

@@ -8,7 +8,7 @@ import dvc.prompt as prompt
 from dvc.exceptions import DvcException, HTTPError
 from dvc.path_info import HTTPURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
@@ -23,6 +23,11 @@ def ask_password(host, user):
     )
 
 
+class HTTPRemoteTree(BaseRemoteTree):
+    def exists(self, path_info):
+        return bool(self.remote.request("HEAD", path_info.url))
+
+
 class HTTPRemote(BaseRemote):
     scheme = Schemes.HTTP
     path_cls = HTTPURLInfo
@@ -32,6 +37,7 @@ class HTTPRemote(BaseRemote):
     CHUNK_SIZE = 2 ** 16
     PARAM_CHECKSUM = "etag"
     CAN_TRAVERSE = False
+    TREE_CLS = HTTPRemoteTree
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
@@ -52,7 +58,7 @@ class HTTPRemote(BaseRemote):
         self.headers = {}
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        response = self._request("GET", from_info.url, stream=True)
+        response = self.request("GET", from_info.url, stream=True)
         if response.status_code != 200:
             raise HTTPError(response.status_code, response.reason)
         with open(to_file, "wb") as fd:
@@ -88,20 +94,17 @@ class HTTPRemote(BaseRemote):
                             break
                         yield chunk
 
-        response = self._request("POST", to_info.url, data=chunks())
+        response = self.request("POST", to_info.url, data=chunks())
         if response.status_code not in (200, 201):
             raise HTTPError(response.status_code, response.reason)
 
-    def exists(self, path_info):
-        return bool(self._request("HEAD", path_info.url))
-
     def _content_length(self, response):
         res = response.headers.get("Content-Length")
         return int(res) if res else None
 
     def get_file_checksum(self, path_info):
         url = path_info.url
-        headers = self._request("HEAD", url).headers
+        headers = self.request("HEAD", url).headers
         etag = headers.get("ETag") or headers.get("Content-MD5")
 
         if not etag:
@@ -155,7 +158,7 @@ class HTTPRemote(BaseRemote):
 
         return session
 
-    def _request(self, method, url, **kwargs):
+    def request(self, method, url, **kwargs):
         import requests
 
         kwargs.setdefault("allow_redirects", True)

dvc/remote/local.py

@@ -17,103 +17,86 @@ from dvc.remote.base import (
     STATUS_MISSING,
     STATUS_NEW,
     BaseRemote,
+    BaseRemoteTree,
     index_locked,
 )
 from dvc.remote.index import RemoteIndexNoop
 from dvc.scheme import Schemes
-from dvc.scm.tree import is_working_tree
+from dvc.scm.tree import WorkingTree, is_working_tree
 from dvc.system import System
 from dvc.utils import file_md5, relpath, tmp_fname
-from dvc.utils.fs import copyfile, makedirs, move, remove, walk_files
+from dvc.utils.fs import (
+    copy_fobj_to_file,
+    copyfile,
+    makedirs,
+    move,
+    remove,
+    walk_files,
+)
 
 logger = logging.getLogger(__name__)
 
 
-class LocalRemote(BaseRemote):
-    scheme = Schemes.LOCAL
-    path_cls = PathInfo
-    PARAM_CHECKSUM = "md5"
-    PARAM_PATH = "path"
-    TRAVERSE_PREFIX_LEN = 2
-    INDEX_CLS = RemoteIndexNoop
-
-    UNPACKED_DIR_SUFFIX = ".unpacked"
-
-    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
-
-    CACHE_MODE = 0o444
+class LocalRemoteTree(BaseRemoteTree):
     SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-        self.cache_dir = config.get("url")
-        self._dir_info = {}
-
     @property
-    def state(self):
-        return self.repo.state
-
-    @property
-    def cache_dir(self):
-        return self.path_info.fspath if self.path_info else None
-
-    @cache_dir.setter
-    def cache_dir(self, value):
-        self.path_info = PathInfo(value) if value else None
-
-    @classmethod
-    def supported(cls, config):
-        return True
+    def repo(self):
+        return self.remote.repo
 
     @cached_property
-    def cache_path(self):
-        return os.path.abspath(self.cache_dir)
+    def _work_tree(self):
+        if self.repo:
+            return WorkingTree(self.repo.root_dir)
+        return None
 
-    def checksum_to_path(self, checksum):
-        return os.path.join(self.cache_path, checksum[0:2], checksum[2:])
+    @property
+    def work_tree(self):
+        # When using repo.brancher, repo.tree may change to/from WorkingTree to
+        # GitTree arbitarily. When repo.tree is GitTree, local cache needs to
+        # use its own WorkingTree instance.
+        if self.repo and not is_working_tree(self.repo.tree):
+            return self._work_tree
+        return None
 
-    def list_cache_paths(self, prefix=None, progress_callback=None):
-        assert self.path_info is not None
-        if prefix:
-            path_info = self.path_info / prefix[:2]
-        else:
-            path_info = self.path_info
-        if progress_callback:
-            for path in walk_files(path_info):
-                progress_callback()
-                yield path
-        else:
-            yield from walk_files(path_info)
-
-    def get(self, md5):
-        if not md5:
-            return None
-
-        return self.checksum_to_path_info(md5).url
+    @staticmethod
+    def open(path_info, mode="r", encoding=None):
+        return open(path_info, mode=mode, encoding=encoding)
 
     def exists(self, path_info):
-        assert is_working_tree(self.repo.tree)
         assert isinstance(path_info, str) or path_info.scheme == "local"
+        if not self.repo:
+            return os.path.exists(path_info)
+        if self.work_tree and self.work_tree.exists(path_info):
+            return True
         return self.repo.tree.exists(path_info)
 
-    def makedirs(self, path_info):
-        makedirs(path_info, exist_ok=True, mode=self._dir_mode)
+    def isfile(self, path_info):
+        if not self.repo:
+            return os.path.isfile(path_info)
+        if self.work_tree and self.work_tree.isfile(path_info):
+            return True
+        return self.repo.tree.isfile(path_info)
 
-    def already_cached(self, path_info):
-        assert path_info.scheme in ["", "local"]
+    def isdir(self, path_info):
+        if not self.repo:
+            return os.path.isdir(path_info)
+        if self.work_tree and self.work_tree.isdir(path_info):
+            return True
+        return self.repo.tree.isdir(path_info)
 
-        current_md5 = self.get_checksum(path_info)
+    def iscopy(self, path_info):
+        return not (
+            System.is_symlink(path_info) or System.is_hardlink(path_info)
+        )
 
-        if not current_md5:
-            return False
-
-        return not self.changed_cache(current_md5)
-
-    def _verify_link(self, path_info, link_type):
-        if link_type == "hardlink" and self.getsize(path_info) == 0:
-            return
-
-        super()._verify_link(path_info, link_type)
+    def walk_files(self, path_info):
+        if self.work_tree:
+            tree = self.work_tree
+        else:
+            tree = self.repo.tree
+        for fname in tree.walk_files(path_info):
+            yield PathInfo(fname)
 
     def is_empty(self, path_info):
         path = path_info.fspath
@@ -126,32 +109,6 @@ class LocalRemote(BaseRemote):
 
         return False
 
-    @staticmethod
-    def isfile(path_info):
-        return os.path.isfile(path_info)
-
-    @staticmethod
-    def isdir(path_info):
-        return os.path.isdir(path_info)
-
-    def iscopy(self, path_info):
-        return not (
-            System.is_symlink(path_info) or System.is_hardlink(path_info)
-        )
-
-    @staticmethod
-    def getsize(path_info):
-        return os.path.getsize(path_info)
-
-    def walk_files(self, path_info):
-        assert is_working_tree(self.repo.tree)
-
-        for fname in self.repo.tree.walk_files(path_info):
-            yield PathInfo(fname)
-
-    def get_file_checksum(self, path_info):
-        return file_md5(path_info)[0]
-
     def remove(self, path_info):
         if isinstance(path_info, PathInfo):
             if path_info.scheme != "local":
@@ -163,6 +120,9 @@ class LocalRemote(BaseRemote):
         if self.exists(path):
             remove(path)
 
+    def makedirs(self, path_info):
+        makedirs(path_info, exist_ok=True, mode=self.dir_mode)
+
     def move(self, from_info, to_info, mode=None):
         if from_info.scheme != "local" or to_info.scheme != "local":
             raise NotImplementedError
@@ -171,9 +131,9 @@ class LocalRemote(BaseRemote):
 
         if mode is None:
             if self.isfile(from_info):
-                mode = self._file_mode
+                mode = self.file_mode
             else:
-                mode = self._dir_mode
+                mode = self.dir_mode
 
         move(from_info, to_info, mode=mode)
 
@@ -181,7 +141,18 @@ class LocalRemote(BaseRemote):
         tmp_info = to_info.parent / tmp_fname(to_info.name)
         try:
             System.copy(from_info, tmp_info)
-            os.chmod(tmp_info, self._file_mode)
+            os.chmod(tmp_info, self.file_mode)
+            os.rename(tmp_info, to_info)
+        except Exception:
+            self.remove(tmp_info)
+            raise
+
+    def copy_fobj(self, fobj, to_info):
+        self.makedirs(to_info.parent)
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        try:
+            copy_fobj_to_file(fobj, tmp_info)
+            os.chmod(tmp_info, self.file_mode)
             os.rename(tmp_info, to_info)
         except Exception:
             self.remove(tmp_info)
@@ -230,9 +201,102 @@ class LocalRemote(BaseRemote):
         System.reflink(from_info, tmp_info)
         # NOTE: reflink has its own separate inode, so you can set permissions
         # that are different from the source.
-        os.chmod(tmp_info, self._file_mode)
+        os.chmod(tmp_info, self.file_mode)
         os.rename(tmp_info, to_info)
 
+    @staticmethod
+    def getsize(path_info):
+        return os.path.getsize(path_info)
+
+
+class LocalRemote(BaseRemote):
+    scheme = Schemes.LOCAL
+    path_cls = PathInfo
+    PARAM_CHECKSUM = "md5"
+    PARAM_PATH = "path"
+    TRAVERSE_PREFIX_LEN = 2
+    INDEX_CLS = RemoteIndexNoop
+    TREE_CLS = LocalRemoteTree
+
+    UNPACKED_DIR_SUFFIX = ".unpacked"
+
+    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
+
+    CACHE_MODE = 0o444
+
+    def __init__(self, repo, config):
+        super().__init__(repo, config)
+        self.cache_dir = config.get("url")
+        self._dir_info = {}
+
+    @property
+    def state(self):
+        return self.repo.state
+
+    @property
+    def cache_dir(self):
+        return self.path_info.fspath if self.path_info else None
+
+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.path_info = PathInfo(value) if value else None
+
+    @classmethod
+    def supported(cls, config):
+        return True
+
+    @cached_property
+    def cache_path(self):
+        return os.path.abspath(self.cache_dir)
+
+    def checksum_to_path(self, checksum):
+        # NOTE: `self.cache_path` is already normalized so we can simply use
+        # `os.sep` instead of `os.path.join`. This results in this helper
+        # being ~5.5 times faster.
+        return (
+            f"{self.cache_path}{os.sep}{checksum[0:2]}{os.sep}{checksum[2:]}"
+        )
+
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        assert self.path_info is not None
+        if prefix:
+            path_info = self.path_info / prefix[:2]
+            if not self.tree.exists(path_info):
+                return
+        else:
+            path_info = self.path_info
+        if progress_callback:
+            for path in walk_files(path_info):
+                progress_callback()
+                yield path
+        else:
+            yield from walk_files(path_info)
+
+    def get(self, md5):
+        if not md5:
+            return None
+
+        return self.checksum_to_path_info(md5).url
+
+    def already_cached(self, path_info):
+        assert path_info.scheme in ["", "local"]
+
+        current_md5 = self.get_checksum(path_info)
+
+        if not current_md5:
+            return False
+
+        return not self.changed_cache(current_md5)
+
+    def _verify_link(self, path_info, link_type):
+        if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
+            return
+
+        super()._verify_link(path_info, link_type)
+
+    def get_file_checksum(self, path_info):
+        return file_md5(path_info)[0]
+
     def cache_exists(self, checksums, jobs=None, name=None):
         return [
             checksum
@@ -254,6 +318,8 @@ class LocalRemote(BaseRemote):
         copyfile(
             from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
         )
+
+        self.protect(tmp_file)
         os.rename(tmp_file, to_info)
 
     def _download(
@@ -263,10 +329,6 @@ class LocalRemote(BaseRemote):
             from_info, to_file, no_progress_bar=no_progress_bar, name=name
         )
 
-    @staticmethod
-    def open(path_info, mode="r", encoding=None):
-        return open(path_info, mode=mode, encoding=encoding)
-
     @index_locked
     def status(
         self,
@@ -450,8 +512,8 @@ class LocalRemote(BaseRemote):
         if download:
             func = partial(
                 remote.download,
-                dir_mode=self._dir_mode,
-                file_mode=self._file_mode,
+                dir_mode=self.tree.dir_mode,
+                file_mode=self.tree.file_mode,
             )
             status = STATUS_DELETED
             desc = "Downloading"
@@ -609,7 +671,7 @@ class LocalRemote(BaseRemote):
                 "a symlink or a hardlink.".format(path)
             )
 
-        os.chmod(path, self._file_mode)
+        os.chmod(path, self.tree.file_mode)
 
     def _unprotect_dir(self, path):
         assert is_working_tree(self.repo.tree)
@@ -647,70 +709,15 @@ class LocalRemote(BaseRemote):
             if actual != mode:
                 raise
 
-    def _get_unpacked_dir_path_info(self, checksum):
-        info = self.checksum_to_path_info(checksum)
-        return info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
-
     def _remove_unpacked_dir(self, checksum):
-        path_info = self._get_unpacked_dir_path_info(checksum)
-        self.remove(path_info)
-
-    def _path_info_changed(self, path_info):
-        if self.exists(path_info) and self.state.get(path_info):
-            return False
-        return True
-
-    def _update_unpacked_dir(self, checksum):
-        unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
-
-        if not self._path_info_changed(unpacked_dir_info):
-            return
-
-        self.remove(unpacked_dir_info)
-
-        try:
-            dir_info = self.get_dir_cache(checksum)
-            self._create_unpacked_dir(checksum, dir_info, unpacked_dir_info)
-        except DvcException:
-            logger.warning(f"Could not create '{unpacked_dir_info}'")
-
-            self.remove(unpacked_dir_info)
-
-    def _create_unpacked_dir(self, checksum, dir_info, unpacked_dir_info):
-        self.makedirs(unpacked_dir_info)
-
-        for entry in Tqdm(dir_info, desc="Creating unpacked dir", unit="file"):
-            entry_cache_info = self.checksum_to_path_info(
-                entry[self.PARAM_CHECKSUM]
-            )
-            relative_path = entry[self.PARAM_RELPATH]
-            # In shared cache mode some cache files might not be owned by the
-            # user, so we need to use symlinks because, unless
-            # /proc/sys/fs/protected_hardlinks is disabled, the user is not
-            # allowed to create hardlinks to files that he doesn't own.
-            link_types = ["hardlink", "symlink"]
-            self._link(
-                entry_cache_info, unpacked_dir_info / relative_path, link_types
-            )
-
-        self.state.save(unpacked_dir_info, checksum)
-
-    def _changed_unpacked_dir(self, checksum):
-        status_unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
-
-        return not self.state.get(status_unpacked_dir_info)
-
-    def _get_unpacked_dir_names(self, checksums):
-        unpacked = set()
-        for c in checksums:
-            if self.is_dir_checksum(c):
-                unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
-        return unpacked
+        info = self.checksum_to_path_info(checksum)
+        path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
+        self.tree.remove(path_info)
 
     def is_protected(self, path_info):
-        if not self.exists(path_info):
+        try:
+            mode = os.stat(path_info).st_mode
+        except FileNotFoundError:
             return False
 
-        mode = os.stat(path_info).st_mode
-
         return stat.S_IMODE(mode) == self.CACHE_MODE

dvc/remote/oss.py

@@ -7,12 +7,34 @@ from funcy import cached_property, wrap_prop
 
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
 
 
+class OSSRemoteTree(BaseRemoteTree):
+    @property
+    def oss_service(self):
+        return self.remote.oss_service
+
+    def _generate_download_url(self, path_info, expires=3600):
+        assert path_info.bucket == self.remote.path_info.bucket
+
+        return self.oss_service.sign_url("GET", path_info.path, expires)
+
+    def exists(self, path_info):
+        paths = self.remote.list_paths(path_info.path)
+        return any(path_info.path == path for path in paths)
+
+    def remove(self, path_info):
+        if path_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        logger.debug(f"Removing oss://{path_info}")
+        self.oss_service.delete_object(path_info.path)
+
+
 class OSSRemote(BaseRemote):
     """
     oss2 document:
@@ -38,6 +60,7 @@ class OSSRemote(BaseRemote):
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
     LIST_OBJECT_PAGE_SIZE = 100
+    TREE_CLS = OSSRemoteTree
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
@@ -83,14 +106,7 @@ class OSSRemote(BaseRemote):
             )
         return bucket
 
-    def remove(self, path_info):
-        if path_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        logger.debug(f"Removing oss://{path_info}")
-        self.oss_service.delete_object(path_info.path)
-
-    def _list_paths(self, prefix, progress_callback=None):
+    def list_paths(self, prefix, progress_callback=None):
         import oss2
 
         for blob in oss2.ObjectIterator(self.oss_service, prefix=prefix):
@@ -105,7 +121,7 @@ class OSSRemote(BaseRemote):
             )
         else:
             prefix = self.path_info.path
-        return self._list_paths(prefix, progress_callback)
+        return self.list_paths(prefix, progress_callback)
 
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
@@ -122,12 +138,3 @@ class OSSRemote(BaseRemote):
             self.oss_service.get_object_to_file(
                 from_info.path, to_file, progress_callback=pbar.update_to
             )
-
-    def _generate_download_url(self, path_info, expires=3600):
-        assert path_info.bucket == self.path_info.bucket
-
-        return self.oss_service.sign_url("GET", path_info.path, expires)
-
-    def exists(self, path_info):
-        paths = self._list_paths(path_info.path)
-        return any(path_info.path == path for path in paths)

dvc/remote/s3.py

@@ -9,17 +9,197 @@ from dvc.config import ConfigError
 from dvc.exceptions import DvcException, ETagMismatchError
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
 
 
+class S3RemoteTree(BaseRemoteTree):
+    @property
+    def s3(self):
+        return self.remote.s3
+
+    def _generate_download_url(self, path_info, expires=3600):
+        params = {"Bucket": path_info.bucket, "Key": path_info.path}
+        return self.s3.generate_presigned_url(
+            ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
+        )
+
+    def exists(self, path_info):
+        """Check if the blob exists. If it does not exist,
+        it could be a part of a directory path.
+
+        eg: if `data/file.txt` exists, check for `data` should return True
+        """
+        return self.isfile(path_info) or self.isdir(path_info)
+
+    def isdir(self, path_info):
+        # S3 doesn't have a concept for directories.
+        #
+        # Using `head_object` with a path pointing to a directory
+        # will throw a 404 error.
+        #
+        # A reliable way to know if a given path is a directory is by
+        # checking if there are more files sharing the same prefix
+        # with a `list_objects` call.
+        #
+        # We need to make sure that the path ends with a forward slash,
+        # since we can end with false-positives like the following example:
+        #
+        #       bucket
+        #       └── data
+        #          ├── alice
+        #          └── alpha
+        #
+        # Using `data/al` as prefix will return `[data/alice, data/alpha]`,
+        # While `data/al/` will return nothing.
+        #
+        dir_path = path_info / ""
+        return bool(list(self.remote.list_paths(dir_path, max_items=1)))
+
+    def isfile(self, path_info):
+        from botocore.exceptions import ClientError
+
+        if path_info.path.endswith("/"):
+            return False
+
+        try:
+            self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
+        except ClientError as exc:
+            if exc.response["Error"]["Code"] != "404":
+                raise
+            return False
+
+        return True
+
+    def walk_files(self, path_info, max_items=None):
+        for fname in self.remote.list_paths(path_info / "", max_items):
+            if fname.endswith("/"):
+                continue
+
+            yield path_info.replace(path=fname)
+
+    def remove(self, path_info):
+        if path_info.scheme != "s3":
+            raise NotImplementedError
+
+        logger.debug(f"Removing {path_info}")
+        self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
+
+    def makedirs(self, path_info):
+        # We need to support creating empty directories, which means
+        # creating an object with an empty body and a trailing slash `/`.
+        #
+        # We are not creating directory objects for every parent prefix,
+        # as it is not required.
+        if not path_info.path:
+            return
+
+        dir_path = path_info / ""
+        self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
+
+    def copy(self, from_info, to_info):
+        self._copy(self.s3, from_info, to_info, self.remote.extra_args)
+
+    @classmethod
+    def _copy_multipart(
+        cls, s3, from_info, to_info, size, n_parts, extra_args
+    ):
+        mpu = s3.create_multipart_upload(
+            Bucket=to_info.bucket, Key=to_info.path, **extra_args
+        )
+        mpu_id = mpu["UploadId"]
+
+        parts = []
+        byte_position = 0
+        for i in range(1, n_parts + 1):
+            obj = S3Remote.get_head_object(
+                s3, from_info.bucket, from_info.path, PartNumber=i
+            )
+            part_size = obj["ContentLength"]
+            lastbyte = byte_position + part_size - 1
+            if lastbyte > size:
+                lastbyte = size - 1
+
+            srange = f"bytes={byte_position}-{lastbyte}"
+
+            part = s3.upload_part_copy(
+                Bucket=to_info.bucket,
+                Key=to_info.path,
+                PartNumber=i,
+                UploadId=mpu_id,
+                CopySourceRange=srange,
+                CopySource={"Bucket": from_info.bucket, "Key": from_info.path},
+            )
+            parts.append(
+                {"PartNumber": i, "ETag": part["CopyPartResult"]["ETag"]}
+            )
+            byte_position += part_size
+
+        assert n_parts == len(parts)
+
+        s3.complete_multipart_upload(
+            Bucket=to_info.bucket,
+            Key=to_info.path,
+            UploadId=mpu_id,
+            MultipartUpload={"Parts": parts},
+        )
+
+    @classmethod
+    def _copy(cls, s3, from_info, to_info, extra_args):
+        # NOTE: object's etag depends on the way it was uploaded to s3 or the
+        # way it was copied within the s3. More specifically, it depends on
+        # the chunk size that was used to transfer it, which would affect
+        # whether an object would be uploaded as a single part or as a
+        # multipart.
+        #
+        # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
+        # then it was transferred as a single part, which means that the chunk
+        # size used to transfer it was greater or equal to the ContentLength
+        # of that object. So to preserve that tag over the next transfer, we
+        # could use any value >= ContentLength.
+        #
+        # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
+        # then it was transferred as a multipart, which means that the chunk
+        # size used to transfer it was less than ContentLength of that object.
+        # Unfortunately, in general, it doesn't mean that the chunk size was
+        # the same throughout the transfer, so it means that in order to
+        # preserve etag, we need to transfer each part separately, so the
+        # object is transfered in the same chunks as it was originally.
+        from boto3.s3.transfer import TransferConfig
+
+        obj = S3Remote.get_head_object(s3, from_info.bucket, from_info.path)
+        etag = obj["ETag"].strip('"')
+        size = obj["ContentLength"]
+
+        _, _, parts_suffix = etag.partition("-")
+        if parts_suffix:
+            n_parts = int(parts_suffix)
+            cls._copy_multipart(
+                s3, from_info, to_info, size, n_parts, extra_args=extra_args
+            )
+        else:
+            source = {"Bucket": from_info.bucket, "Key": from_info.path}
+            s3.copy(
+                source,
+                to_info.bucket,
+                to_info.path,
+                ExtraArgs=extra_args,
+                Config=TransferConfig(multipart_threshold=size + 1),
+            )
+
+        cached_etag = S3Remote.get_etag(s3, to_info.bucket, to_info.path)
+        if etag != cached_etag:
+            raise ETagMismatchError(etag, cached_etag)
+
+
 class S3Remote(BaseRemote):
     scheme = Schemes.S3
     path_cls = CloudURLInfo
     REQUIRES = {"boto3": "boto3"}
     PARAM_CHECKSUM = "etag"
+    TREE_CLS = S3RemoteTree
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
@@ -89,107 +269,6 @@ class S3Remote(BaseRemote):
             raise DvcException(f"s3://{bucket}/{path} does not exist") from exc
         return obj
 
-    @classmethod
-    def _copy_multipart(
-        cls, s3, from_info, to_info, size, n_parts, extra_args
-    ):
-        mpu = s3.create_multipart_upload(
-            Bucket=to_info.bucket, Key=to_info.path, **extra_args
-        )
-        mpu_id = mpu["UploadId"]
-
-        parts = []
-        byte_position = 0
-        for i in range(1, n_parts + 1):
-            obj = cls.get_head_object(
-                s3, from_info.bucket, from_info.path, PartNumber=i
-            )
-            part_size = obj["ContentLength"]
-            lastbyte = byte_position + part_size - 1
-            if lastbyte > size:
-                lastbyte = size - 1
-
-            srange = f"bytes={byte_position}-{lastbyte}"
-
-            part = s3.upload_part_copy(
-                Bucket=to_info.bucket,
-                Key=to_info.path,
-                PartNumber=i,
-                UploadId=mpu_id,
-                CopySourceRange=srange,
-                CopySource={"Bucket": from_info.bucket, "Key": from_info.path},
-            )
-            parts.append(
-                {"PartNumber": i, "ETag": part["CopyPartResult"]["ETag"]}
-            )
-            byte_position += part_size
-
-        assert n_parts == len(parts)
-
-        s3.complete_multipart_upload(
-            Bucket=to_info.bucket,
-            Key=to_info.path,
-            UploadId=mpu_id,
-            MultipartUpload={"Parts": parts},
-        )
-
-    @classmethod
-    def _copy(cls, s3, from_info, to_info, extra_args):
-        # NOTE: object's etag depends on the way it was uploaded to s3 or the
-        # way it was copied within the s3. More specifically, it depends on
-        # the chunk size that was used to transfer it, which would affect
-        # whether an object would be uploaded as a single part or as a
-        # multipart.
-        #
-        # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
-        # then it was transferred as a single part, which means that the chunk
-        # size used to transfer it was greater or equal to the ContentLength
-        # of that object. So to preserve that tag over the next transfer, we
-        # could use any value >= ContentLength.
-        #
-        # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
-        # then it was transferred as a multipart, which means that the chunk
-        # size used to transfer it was less than ContentLength of that object.
-        # Unfortunately, in general, it doesn't mean that the chunk size was
-        # the same throughout the transfer, so it means that in order to
-        # preserve etag, we need to transfer each part separately, so the
-        # object is transfered in the same chunks as it was originally.
-        from boto3.s3.transfer import TransferConfig
-
-        obj = cls.get_head_object(s3, from_info.bucket, from_info.path)
-        etag = obj["ETag"].strip('"')
-        size = obj["ContentLength"]
-
-        _, _, parts_suffix = etag.partition("-")
-        if parts_suffix:
-            n_parts = int(parts_suffix)
-            cls._copy_multipart(
-                s3, from_info, to_info, size, n_parts, extra_args=extra_args
-            )
-        else:
-            source = {"Bucket": from_info.bucket, "Key": from_info.path}
-            s3.copy(
-                source,
-                to_info.bucket,
-                to_info.path,
-                ExtraArgs=extra_args,
-                Config=TransferConfig(multipart_threshold=size + 1),
-            )
-
-        cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
-        if etag != cached_etag:
-            raise ETagMismatchError(etag, cached_etag)
-
-    def copy(self, from_info, to_info):
-        self._copy(self.s3, from_info, to_info, self.extra_args)
-
-    def remove(self, path_info):
-        if path_info.scheme != "s3":
-            raise NotImplementedError
-
-        logger.debug(f"Removing {path_info}")
-        self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
-
     def _list_objects(
         self, path_info, max_items=None, prefix=None, progress_callback=None
     ):
@@ -211,7 +290,7 @@ class S3Remote(BaseRemote):
             else:
                 yield from contents
 
-    def _list_paths(
+    def list_paths(
         self, path_info, max_items=None, prefix=None, progress_callback=None
     ):
         return (
@@ -222,69 +301,10 @@ class S3Remote(BaseRemote):
         )
 
     def list_cache_paths(self, prefix=None, progress_callback=None):
-        return self._list_paths(
+        return self.list_paths(
             self.path_info, prefix=prefix, progress_callback=progress_callback
         )
 
-    def isfile(self, path_info):
-        from botocore.exceptions import ClientError
-
-        if path_info.path.endswith("/"):
-            return False
-
-        try:
-            self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
-        except ClientError as exc:
-            if exc.response["Error"]["Code"] != "404":
-                raise
-            return False
-
-        return True
-
-    def exists(self, path_info):
-        """Check if the blob exists. If it does not exist,
-        it could be a part of a directory path.
-
-        eg: if `data/file.txt` exists, check for `data` should return True
-        """
-        return self.isfile(path_info) or self.isdir(path_info)
-
-    def makedirs(self, path_info):
-        # We need to support creating empty directories, which means
-        # creating an object with an empty body and a trailing slash `/`.
-        #
-        # We are not creating directory objects for every parent prefix,
-        # as it is not required.
-        if not path_info.path:
-            return
-
-        dir_path = path_info / ""
-        self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
-
-    def isdir(self, path_info):
-        # S3 doesn't have a concept for directories.
-        #
-        # Using `head_object` with a path pointing to a directory
-        # will throw a 404 error.
-        #
-        # A reliable way to know if a given path is a directory is by
-        # checking if there are more files sharing the same prefix
-        # with a `list_objects` call.
-        #
-        # We need to make sure that the path ends with a forward slash,
-        # since we can end with false-positives like the following example:
-        #
-        #       bucket
-        #       └── data
-        #          ├── alice
-        #          └── alpha
-        #
-        # Using `data/al` as prefix will return `[data/alice, data/alpha]`,
-        # While `data/al/` will return nothing.
-        #
-        dir_path = path_info / ""
-        return bool(list(self._list_paths(dir_path, max_items=1)))
-
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
         total = os.path.getsize(from_file)
         with Tqdm(
@@ -312,19 +332,6 @@ class S3Remote(BaseRemote):
                 from_info.bucket, from_info.path, to_file, Callback=pbar.update
             )
 
-    def _generate_download_url(self, path_info, expires=3600):
-        params = {"Bucket": path_info.bucket, "Key": path_info.path}
-        return self.s3.generate_presigned_url(
-            ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
-        )
-
-    def walk_files(self, path_info, max_items=None):
-        for fname in self._list_paths(path_info / "", max_items):
-            if fname.endswith("/"):
-                continue
-
-            yield path_info.replace(path=fname)
-
     def _append_aws_grants_to_extra_args(self, config):
         # Keys for extra_args can be one of the following list:
         # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS

dvc/remote/ssh/__init__.py

@@ -14,7 +14,7 @@ from funcy import first, memoize, silent, wrap_with
 
 import dvc.prompt as prompt
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemote
+from dvc.remote.base import BaseRemote, BaseRemoteTree
 from dvc.remote.pool import get_connection
 from dvc.scheme import Schemes
 from dvc.utils import to_chunks
@@ -33,6 +33,105 @@ def ask_password(host, user, port):
     )
 
 
+class SSHRemoteTree(BaseRemoteTree):
+    @property
+    def ssh(self):
+        return self.remote.ssh
+
+    @contextmanager
+    def open(self, path_info, mode="r", encoding=None):
+        assert mode in {"r", "rt", "rb", "wb"}
+
+        with self.ssh(path_info) as ssh, closing(
+            ssh.sftp.open(path_info.path, mode)
+        ) as fd:
+            if "b" in mode:
+                yield fd
+            else:
+                yield io.TextIOWrapper(fd, encoding=encoding)
+
+    def exists(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.exists(path_info.path)
+
+    def isdir(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.isdir(path_info.path)
+
+    def isfile(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.isfile(path_info.path)
+
+    def walk_files(self, path_info):
+        with self.ssh(path_info) as ssh:
+            for fname in ssh.walk_files(path_info.path):
+                yield path_info.replace(path=fname)
+
+    def remove(self, path_info):
+        if path_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(path_info) as ssh:
+            ssh.remove(path_info.path)
+
+    def makedirs(self, path_info):
+        with self.ssh(path_info) as ssh:
+            ssh.makedirs(path_info.path)
+
+    def move(self, from_info, to_info, mode=None):
+        assert mode is None
+        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.move(from_info.path, to_info.path)
+
+    def copy(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.atomic_copy(from_info.path, to_info.path)
+
+    def symlink(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.symlink(from_info.path, to_info.path)
+
+    def hardlink(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        # See dvc/remote/local/__init__.py - hardlink()
+        if self.getsize(from_info) == 0:
+
+            with self.ssh(to_info) as ssh:
+                ssh.sftp.open(to_info.path, "w").close()
+
+            logger.debug(
+                "Created empty file: {src} -> {dest}".format(
+                    src=str(from_info), dest=str(to_info)
+                )
+            )
+            return
+
+        with self.ssh(from_info) as ssh:
+            ssh.hardlink(from_info.path, to_info.path)
+
+    def reflink(self, from_info, to_info):
+        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.reflink(from_info.path, to_info.path)
+
+    def getsize(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.getsize(path_info.path)
+
+
 class SSHRemote(BaseRemote):
     scheme = Schemes.SSH
     REQUIRES = {"paramiko": "paramiko"}
@@ -46,6 +145,7 @@ class SSHRemote(BaseRemote):
     # We use conservative setting of 4 instead to not exhaust max sessions.
     CHECKSUM_JOBS = 4
     TRAVERSE_PREFIX_LEN = 2
+    TREE_CLS = SSHRemoteTree
 
     DEFAULT_CACHE_TYPES = ["copy"]
 
@@ -148,10 +248,6 @@ class SSHRemote(BaseRemote):
             sock=self.sock,
         )
 
-    def exists(self, path_info):
-        with self.ssh(path_info) as ssh:
-            return ssh.exists(path_info.path)
-
     def get_file_checksum(self, path_info):
         if path_info.scheme != self.scheme:
             raise NotImplementedError
@@ -159,74 +255,6 @@ class SSHRemote(BaseRemote):
         with self.ssh(path_info) as ssh:
             return ssh.md5(path_info.path)
 
-    def isdir(self, path_info):
-        with self.ssh(path_info) as ssh:
-            return ssh.isdir(path_info.path)
-
-    def isfile(self, path_info):
-        with self.ssh(path_info) as ssh:
-            return ssh.isfile(path_info.path)
-
-    def getsize(self, path_info):
-        with self.ssh(path_info) as ssh:
-            return ssh.getsize(path_info.path)
-
-    def copy(self, from_info, to_info):
-        if not from_info.scheme == to_info.scheme == self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(from_info) as ssh:
-            ssh.atomic_copy(from_info.path, to_info.path)
-
-    def symlink(self, from_info, to_info):
-        if not from_info.scheme == to_info.scheme == self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(from_info) as ssh:
-            ssh.symlink(from_info.path, to_info.path)
-
-    def hardlink(self, from_info, to_info):
-        if not from_info.scheme == to_info.scheme == self.scheme:
-            raise NotImplementedError
-
-        # See dvc/remote/local/__init__.py - hardlink()
-        if self.getsize(from_info) == 0:
-
-            with self.ssh(to_info) as ssh:
-                ssh.sftp.open(to_info.path, "w").close()
-
-            logger.debug(
-                "Created empty file: {src} -> {dest}".format(
-                    src=str(from_info), dest=str(to_info)
-                )
-            )
-            return
-
-        with self.ssh(from_info) as ssh:
-            ssh.hardlink(from_info.path, to_info.path)
-
-    def reflink(self, from_info, to_info):
-        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(from_info) as ssh:
-            ssh.reflink(from_info.path, to_info.path)
-
-    def remove(self, path_info):
-        if path_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(path_info) as ssh:
-            ssh.remove(path_info.path)
-
-    def move(self, from_info, to_info, mode=None):
-        assert mode is None
-        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(from_info) as ssh:
-            ssh.move(from_info.path, to_info.path)
-
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
         assert from_info.isin(self.path_info)
         with self.ssh(self.path_info) as ssh:
@@ -247,24 +275,14 @@ class SSHRemote(BaseRemote):
                 no_progress_bar=no_progress_bar,
             )
 
-    @contextmanager
-    def open(self, path_info, mode="r", encoding=None):
-        assert mode in {"r", "rt", "rb", "wb"}
-
-        with self.ssh(path_info) as ssh, closing(
-            ssh.sftp.open(path_info.path, mode)
-        ) as fd:
-            if "b" in mode:
-                yield fd
-            else:
-                yield io.TextIOWrapper(fd, encoding=encoding)
-
     def list_cache_paths(self, prefix=None, progress_callback=None):
         if prefix:
             root = posixpath.join(self.path_info.path, prefix[:2])
         else:
             root = self.path_info.path
         with self.ssh(self.path_info) as ssh:
+            if prefix and not ssh.exists(root):
+                return
             # If we simply return an iterator then with above closes instantly
             if progress_callback:
                 for path in ssh.walk_files(root):
@@ -273,15 +291,6 @@ class SSHRemote(BaseRemote):
             else:
                 yield from ssh.walk_files(root)
 
-    def walk_files(self, path_info):
-        with self.ssh(path_info) as ssh:
-            for fname in ssh.walk_files(path_info.path):
-                yield path_info.replace(path=fname)
-
-    def makedirs(self, path_info):
-        with self.ssh(path_info) as ssh:
-            ssh.makedirs(path_info.path)
-
     def batch_exists(self, path_infos, callback):
         def _exists(chunk_and_channel):
             chunk, channel = chunk_and_channel

dvc/remote/ssh/connection.py

@@ -237,11 +237,15 @@ class SSHConnection:
             or channel.recv_ready()
             or channel.recv_stderr_ready()
         ):
-            import select
+            import selectors
+
+            selector = selectors.DefaultSelector()
+            selector.register(stdout.channel, selectors.EVENT_READ)
 
             got_chunk = False
-            readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
-            for c in readq:
+            events = selector.select(self.timeout)
+            for key, _ in events:
+                c = key.fileobj
                 if c.recv_ready():
                     stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
                     got_chunk = True

dvc/repo/__init__.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from contextlib import contextmanager
 from functools import wraps
@@ -5,17 +6,26 @@ from functools import wraps
 from funcy import cached_property, cat, first
 
 from dvc.config import Config
+from dvc.dvcfile import PIPELINE_FILE, Dvcfile, is_valid_filename
 from dvc.exceptions import FileMissingError
 from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
-from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
+from dvc.exceptions import (
+    NoOutputOrStageError,
+    NotDvcRepoError,
+    OutputNotFoundError,
+)
 from dvc.ignore import CleanTree
 from dvc.path_info import PathInfo
 from dvc.repo.tree import RepoTree
+from dvc.scm.tree import is_working_tree
 from dvc.utils.fs import path_isin
 
+from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
 from ..utils import parse_target
 from .graph import check_acyclic, get_pipeline, get_pipelines
 
+logger = logging.getLogger(__name__)
+
 
 def locked(f):
     @wraps(f)
@@ -38,7 +48,7 @@ class Repo:
     from dvc.repo.add import add
     from dvc.repo.remove import remove
     from dvc.repo.ls import ls
-    from dvc.repo.lock import lock as lock_stage
+    from dvc.repo.freeze import freeze, unfreeze
     from dvc.repo.move import move
     from dvc.repo.run import run
     from dvc.repo.imp import imp
@@ -57,8 +67,8 @@ class Repo:
     from dvc.repo.get_url import get_url
     from dvc.repo.update import update
 
-    def __init__(self, root_dir=None):
-        from dvc.state import State
+    def __init__(self, root_dir=None, scm=None, rev=None):
+        from dvc.state import State, StateNoop
         from dvc.lock import make_lock
         from dvc.scm import SCM
         from dvc.cache import Cache
@@ -70,17 +80,24 @@ class Repo:
         from dvc.utils.fs import makedirs
         from dvc.stage.cache import StageCache
 
-        root_dir = self.find_root(root_dir)
+        if scm:
+            # use GitTree instead of WorkingTree as default repo tree instance
+            tree = scm.get_tree(rev)
+            self.root_dir = self.find_root(root_dir, tree)
+            self.scm = scm
+            self.tree = tree
+            self.state = StateNoop()
+        else:
+            root_dir = self.find_root(root_dir)
+            self.root_dir = os.path.abspath(os.path.realpath(root_dir))
 
-        self.root_dir = os.path.abspath(os.path.realpath(root_dir))
         self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
-
         self.config = Config(self.dvc_dir)
 
-        no_scm = self.config["core"].get("no_scm", False)
-        self.scm = SCM(self.root_dir, no_scm=no_scm)
-
-        self.tree = WorkingTree(self.root_dir)
+        if not scm:
+            no_scm = self.config["core"].get("no_scm", False)
+            self.scm = SCM(self.root_dir, no_scm=no_scm)
+            self.tree = WorkingTree(self.root_dir)
 
         self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
         self.index_dir = os.path.join(self.tmp_dir, "index")
@@ -94,9 +111,11 @@ class Repo:
             friendly=True,
         )
 
-        # NOTE: storing state and link_state in the repository itself to avoid
-        # any possible state corruption in 'shared cache dir' scenario.
-        self.state = State(self)
+        if not scm:
+            # NOTE: storing state and link_state in the repository itself to
+            # avoid any possible state corruption in 'shared cache dir'
+            # scenario.
+            self.state = State(self)
 
         self.cache = Cache(self)
         self.cloud = DataCloud(self)
@@ -115,7 +134,13 @@ class Repo:
 
     @tree.setter
     def tree(self, tree):
-        self._tree = tree if isinstance(tree, CleanTree) else CleanTree(tree)
+        if is_working_tree(tree) or tree.tree_root == self.root_dir:
+            root = None
+        else:
+            root = self.root_dir
+        self._tree = (
+            tree if isinstance(tree, CleanTree) else CleanTree(tree, root)
+        )
         # Our graph cache is no longer valid, as it was based on the previous
         # tree.
         self._reset()
@@ -124,9 +149,14 @@ class Repo:
         return f"{self.__class__.__name__}: '{self.root_dir}'"
 
     @classmethod
-    def find_root(cls, root=None):
+    def find_root(cls, root=None, tree=None):
         root_dir = os.path.realpath(root or os.curdir)
 
+        if tree:
+            if tree.isdir(os.path.join(root_dir, cls.DVC_DIR)):
+                return root_dir
+            raise NotDvcRepoError(f"'{root}' does not contain DVC directory")
+
         if not os.path.isdir(root_dir):
             raise NotDvcRepoError(f"directory '{root}' does not exist")
 
@@ -167,6 +197,25 @@ class Repo:
 
         self.scm.ignore_list(flist)
 
+    def get_stage(self, path=None, name=None):
+        if not path:
+            path = PIPELINE_FILE
+            logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
+
+        dvcfile = Dvcfile(self, path)
+        return dvcfile.stages[name]
+
+    def get_stages(self, path=None, name=None):
+        if not path:
+            path = PIPELINE_FILE
+            logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
+
+        if name:
+            return [self.get_stage(path, name)]
+
+        dvcfile = Dvcfile(self, path)
+        return list(dvcfile.stages.values())
+
     def check_modified_graph(self, new_stages):
         """Generate graph including the new stage to check for errors"""
         # Building graph might be costly for the ones with many DVC-files,
@@ -190,10 +239,9 @@ class Repo:
         stages = nx.dfs_postorder_nodes(graph)
         return [stage for stage in stages if path_isin(stage.path, path)]
 
-    def collect(self, target, with_deps=False, recursive=False, graph=None):
-        import networkx as nx
-        from ..dvcfile import Dvcfile
-
+    def collect(
+        self, target=None, with_deps=False, recursive=False, graph=None
+    ):
         if not target:
             return list(graph) if graph else self.stages
 
@@ -203,36 +251,81 @@ class Repo:
             )
 
         path, name = parse_target(target)
-        dvcfile = Dvcfile(self, path)
-        stages = list(dvcfile.stages.filter(name).values())
+        stages = self.get_stages(path, name)
         if not with_deps:
             return stages
 
         res = set()
         for stage in stages:
-            pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
-            res.update(nx.dfs_postorder_nodes(pipeline, stage))
+            res.update(self._collect_pipeline(stage, graph=graph))
         return res
 
-    def collect_granular(self, target, *args, **kwargs):
-        from ..dvcfile import Dvcfile, is_valid_filename
+    def _collect_pipeline(self, stage, graph=None):
+        import networkx as nx
 
+        pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
+        return nx.dfs_postorder_nodes(pipeline, stage)
+
+    def _collect_from_default_dvcfile(self, target):
+        dvcfile = Dvcfile(self, PIPELINE_FILE)
+        if dvcfile.exists():
+            return dvcfile.stages.get(target)
+
+    def collect_granular(
+        self, target=None, with_deps=False, recursive=False, graph=None
+    ):
+        """
+        Priority is in the order of following in case of ambiguity:
+            - .dvc file or .yaml file
+            - dir if recursive and directory exists
+            - stage_name
+            - output file
+        """
         if not target:
             return [(stage, None) for stage in self.stages]
 
         file, name = parse_target(target)
-        if is_valid_filename(file) and not kwargs.get("with_deps"):
-            # Optimization: do not collect the graph for a specific .dvc target
-            stages = Dvcfile(self, file).stages.filter(name)
-            return [(stage, None) for stage in stages.values()]
+        stages = []
 
-        try:
-            (out,) = self.find_outs_by_path(file, strict=False)
-            filter_info = PathInfo(os.path.abspath(file))
-            return [(out.stage, filter_info)]
-        except OutputNotFoundError:
-            stages = self.collect(target, *args, **kwargs)
-            return [(stage, None) for stage in stages]
+        # Optimization: do not collect the graph for a specific target
+        if not file:
+            # parsing is ambiguous when it does not have a colon
+            # or if it's not a dvcfile, as it can be a stage name
+            # in `dvc.yaml` or, an output in a stage.
+            logger.debug(
+                "Checking if stage '%s' is in '%s'", target, PIPELINE_FILE
+            )
+            if not (recursive and os.path.isdir(target)):
+                stage = self._collect_from_default_dvcfile(target)
+                if stage:
+                    stages = (
+                        self._collect_pipeline(stage) if with_deps else [stage]
+                    )
+        elif not with_deps and is_valid_filename(file):
+            stages = self.get_stages(file, name)
+
+        if not stages:
+            if not (recursive and os.path.isdir(target)):
+                try:
+                    (out,) = self.find_outs_by_path(target, strict=False)
+                    filter_info = PathInfo(os.path.abspath(target))
+                    return [(out.stage, filter_info)]
+                except OutputNotFoundError:
+                    pass
+
+            try:
+                stages = self.collect(target, with_deps, recursive, graph)
+            except StageFileDoesNotExistError as exc:
+                # collect() might try to use `target` as a stage name
+                # and throw error that dvc.yaml does not exist, whereas it
+                # should say that both stage name and file does not exist.
+                if file and is_valid_filename(file):
+                    raise
+                raise NoOutputOrStageError(target, exc.file) from exc
+            except StageNotFound as exc:
+                raise NoOutputOrStageError(target, exc.file) from exc
+
+        return [(stage, None) for stage in stages]
 
     def used_cache(
         self,
@@ -348,7 +441,7 @@ class Repo:
         stages = stages or self.stages
         outs = Trie()  # Use trie to efficiently find overlapping outs and deps
 
-        for stage in filter(bool, stages):
+        for stage in filter(bool, stages):  # bug? not using it later
             for out in stage.outs:
                 out_key = out.path_info.parts
 
@@ -429,22 +522,18 @@ class Repo:
         return PlotTemplates(self.dvc_dir)
 
     def _collect_stages(self):
-        from dvc.dvcfile import Dvcfile, is_valid_filename
-
         stages = []
         outs = set()
 
         for root, dirs, files in self.tree.walk(self.root_dir):
             for file_name in filter(is_valid_filename, files):
-                path = os.path.join(root, file_name)
-                stage_loader = Dvcfile(self, path).stages
-                stages.extend(stage_loader.values())
+                new_stages = self.get_stages(os.path.join(root, file_name))
+                stages.extend(new_stages)
                 outs.update(
                     out.fspath
-                    for stage in stages
-                    for out in (
-                        out for out in stage.outs if out.scheme == "local"
-                    )
+                    for stage in new_stages
+                    for out in stage.outs
+                    if out.scheme == "local"
                 )
             dirs[:] = [d for d in dirs if os.path.join(root, d) not in outs]
         return stages
@@ -488,17 +577,18 @@ class Repo:
         tree = RepoTree(self, stream=True)
         path = os.path.join(self.root_dir, path)
         try:
-            with tree.open(
-                os.path.join(self.root_dir, path),
-                mode=mode,
-                encoding=encoding,
-                remote=remote,
-            ) as fobj:
-                yield fobj
+            with self.state:
+                with tree.open(
+                    os.path.join(self.root_dir, path),
+                    mode=mode,
+                    encoding=encoding,
+                    remote=remote,
+                ) as fobj:
+                    yield fobj
         except FileNotFoundError as exc:
             raise FileMissingError(path) from exc
         except IsADirectoryError as exc:
-            raise DvcIsADirectoryError from exc
+            raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
 
     def close(self):
         self.scm.close()

dvc/repo/add.py

@@ -21,7 +21,9 @@ logger = logging.getLogger(__name__)
 
 @locked
 @scm_context
-def add(repo, targets, recursive=False, no_commit=False, fname=None):
+def add(
+    repo, targets, recursive=False, no_commit=False, fname=None, external=False
+):
     if recursive and fname:
         raise RecursiveAddingWhileUsingFilename()
 
@@ -52,7 +54,9 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
                     )
                 )
 
-            stages = _create_stages(repo, sub_targets, fname, pbar=pbar)
+            stages = _create_stages(
+                repo, sub_targets, fname, pbar=pbar, external=external
+            )
 
             try:
                 repo.check_modified_graph(stages)
@@ -120,7 +124,7 @@ def _find_all_targets(repo, target, recursive):
     return [target]
 
 
-def _create_stages(repo, targets, fname, pbar=None):
+def _create_stages(repo, targets, fname, pbar=None, external=False):
     from dvc.stage import Stage, create_stage
 
     stages = []
@@ -132,7 +136,14 @@ def _create_stages(repo, targets, fname, pbar=None):
         unit="file",
     ):
         path, wdir, out = resolve_paths(repo, out)
-        stage = create_stage(Stage, repo, fname or path, wdir=wdir, outs=[out])
+        stage = create_stage(
+            Stage,
+            repo,
+            fname or path,
+            wdir=wdir,
+            outs=[out],
+            external=external,
+        )
         if stage:
             Dvcfile(repo, stage.path).remove_with_prompt(force=True)
 

dvc/repo/brancher.py

@@ -18,7 +18,7 @@ def brancher(  # noqa: E302
         str: the display name for the currently selected tree, it could be:
             - a git revision identifier
             - empty string it there is no branches to iterate over
-            - "Working Tree" if there are uncommitted changes in the SCM repo
+            - "workspace" if there are uncommitted changes in the SCM repo
     """
     if not any([revs, all_branches, all_tags, all_commits]):
         yield ""
@@ -30,10 +30,10 @@ def brancher(  # noqa: E302
     scm = self.scm
 
     self.tree = WorkingTree(self.root_dir)
-    yield "working tree"
+    yield "workspace"
 
-    if revs and "working tree" in revs:
-        revs.remove("working tree")
+    if revs and "workspace" in revs:
+        revs.remove("workspace")
 
     if all_commits:
         revs = scm.list_all_commits()
@@ -48,6 +48,9 @@ def brancher(  # noqa: E302
         if revs:
             for sha, names in group_by(scm.resolve_rev, revs).items():
                 self.tree = scm.get_tree(sha)
-                yield ", ".join(names)
+                # ignore revs that don't contain repo root
+                # (i.e. revs from before a subdir=True repo was init'ed)
+                if self.tree.exists(self.root_dir):
+                    yield ", ".join(names)
     finally:
         self.tree = saved_tree

dvc/repo/checkout.py

@@ -1,7 +1,11 @@
 import logging
 import os
 
-from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit
+from dvc.exceptions import (
+    CheckoutError,
+    CheckoutErrorSuggestGit,
+    NoOutputOrStageError,
+)
 from dvc.progress import Tqdm
 from dvc.utils import relpath
 
@@ -18,11 +22,11 @@ def _get_unused_links(repo):
     return repo.state.get_unused_links(used)
 
 
-def _fspath_dir(path, root):
+def _fspath_dir(path):
     if not os.path.exists(str(path)):
         return str(path)
 
-    path = relpath(path, root)
+    path = relpath(path)
     return os.path.join(path, "") if os.path.isdir(path) else path
 
 
@@ -56,7 +60,7 @@ def _checkout(
         targets = [None]
         unused = _get_unused_links(self)
 
-    stats["deleted"] = [_fspath_dir(u, self.root_dir) for u in unused]
+    stats["deleted"] = [_fspath_dir(u) for u in unused]
     self.state.remove_links(unused)
 
     if isinstance(targets, str):
@@ -70,7 +74,11 @@ def _checkout(
                     target, with_deps=with_deps, recursive=recursive
                 )
             )
-        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+        except (
+            StageFileDoesNotExistError,
+            StageFileBadNameError,
+            NoOutputOrStageError,
+        ) as exc:
             if not target:
                 raise
             raise CheckoutErrorSuggestGit(target) from exc
@@ -87,9 +95,7 @@ def _checkout(
                 filter_info=filter_info,
             )
             for key, items in result.items():
-                stats[key].extend(
-                    _fspath_dir(path, self.root_dir) for path in items
-                )
+                stats[key].extend(_fspath_dir(path) for path in items)
 
     if stats.get("failed"):
         raise CheckoutError(stats["failed"], stats)

dvc/repo/commit.py

@@ -46,3 +46,4 @@ def commit(self, target, with_deps=False, recursive=False, force=False):
         stage.commit()
 
         Dvcfile(self, stage.path).dump(stage)
+    return stages

dvc/repo/diff.py

@@ -7,7 +7,7 @@ from dvc.scm.tree import is_working_tree
 @locked
 def diff(self, a_rev="HEAD", b_rev=None):
     """
-    By default, it compares the working tree with the last commit's tree.
+    By default, it compares the workspace with the last commit's tree.
 
     This implementation differs from `git diff` since DVC doesn't have
     the concept of `index`, but it keeps the same interface, thus,

dvc/repo/fetch.py

@@ -1,9 +1,7 @@
 import logging
 
-from dvc.cache import NamedCache
 from dvc.config import NoRemoteError
-from dvc.exceptions import DownloadError, OutputNotFoundError
-from dvc.path_info import PathInfo
+from dvc.exceptions import DownloadError
 from dvc.scm.base import CloneError
 
 logger = logging.getLogger(__name__)
@@ -78,35 +76,15 @@ def _fetch(
 
 
 def _fetch_external(self, repo_url, repo_rev, files, jobs):
-    from dvc.external_repo import external_repo, ExternalRepo
+    from dvc.external_repo import external_repo
 
     failed, downloaded = 0, 0
     try:
         with external_repo(repo_url, repo_rev) as repo:
-            is_dvc_repo = isinstance(repo, ExternalRepo)
-            # gather git-only tracked files if dvc repo
-            git_files = [] if is_dvc_repo else files
-            if is_dvc_repo:
-                repo.cache.local.cache_dir = self.cache.local.cache_dir
-                with repo.state:
-                    cache = NamedCache()
-                    for name in files:
-                        try:
-                            out = repo.find_out_by_relpath(name)
-                        except OutputNotFoundError:
-                            # try to add to cache if they are git-tracked files
-                            git_files.append(name)
-                        else:
-                            cache.update(out.get_used_cache())
-
-                        try:
-                            downloaded += repo.cloud.pull(cache, jobs=jobs)
-                        except DownloadError as exc:
-                            failed += exc.amount
-
-            d, f = _git_to_cache(self.cache.local, repo.root_dir, git_files)
-            downloaded += d
-            failed += f
+            with repo.use_cache(self.cache.local):
+                d, f, _ = repo.fetch_external(files, jobs=jobs)
+                downloaded += d
+                failed += f
     except CloneError:
         failed += 1
         logger.exception(
@@ -114,27 +92,3 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs):
         )
 
     return downloaded, failed
-
-
-def _git_to_cache(cache, repo_root, files):
-    """Save files from a git repo directly to the cache."""
-    failed = set()
-    num_downloads = 0
-    repo_root = PathInfo(repo_root)
-    for file in files:
-        info = cache.save_info(repo_root / file)
-        if info.get(cache.PARAM_CHECKSUM) is None:
-            failed.add(file)
-            continue
-
-        if cache.changed_cache(info[cache.PARAM_CHECKSUM]):
-            logger.debug("fetched '%s' from '%s' repo", file, repo_root)
-            num_downloads += 1
-            cache.save(repo_root / file, info, save_link=False)
-
-    if failed:
-        logger.exception(
-            "failed to fetch data for {}".format(", ".join(failed))
-        )
-
-    return num_downloads, len(failed)

dvc/repo/freeze.py

@@ -0,0 +1,21 @@
+from . import locked
+
+
+@locked
+def _set(repo, target, frozen):
+    from dvc.utils import parse_target
+
+    path, name = parse_target(target)
+    stage = repo.get_stage(path, name)
+    stage.frozen = frozen
+    stage.dvcfile.dump(stage, update_pipeline=True)
+
+    return stage
+
+
+def freeze(repo, target):
+    return _set(repo, target, True)
+
+
+def unfreeze(repo, target):
+    return _set(repo, target, False)

dvc/repo/get.py

@@ -4,7 +4,6 @@ import os
 import shortuuid
 
 from dvc.exceptions import DvcException
-from dvc.path_info import PathInfo
 from dvc.utils import resolve_output
 from dvc.utils.fs import remove
 
@@ -51,7 +50,6 @@ def get(url, path, out=None, rev=None):
                 # Also, we can't use theoretical "move" link type here, because
                 # the same cache file might be used a few times in a directory.
                 repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]
-
-            repo.pull_to(path, PathInfo(out))
+            repo.get_external(path, out)
     finally:
         remove(tmp_dir)

dvc/repo/imp.py

@@ -3,4 +3,4 @@ def imp(self, url, path, out=None, rev=None):
     if rev is not None:
         erepo["rev"] = rev
 
-    return self.imp_url(path, out=out, erepo=erepo, locked=True)
+    return self.imp_url(path, out=out, erepo=erepo, frozen=True)

dvc/repo/imp_url.py

@@ -5,12 +5,12 @@ from dvc.utils import relpath, resolve_output, resolve_paths
 from dvc.utils.fs import path_isin
 
 from ..exceptions import OutputDuplicationError
-from . import locked as locked_repo
+from . import locked
 
 
-@locked_repo
+@locked
 @scm_context
-def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
+def imp_url(self, url, out=None, fname=None, erepo=None, frozen=True):
     from dvc.dvcfile import Dvcfile
     from dvc.stage import Stage, create_stage
 
@@ -48,7 +48,7 @@ def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
 
     stage.run()
 
-    stage.locked = locked
+    stage.frozen = frozen
 
     dvcfile.dump(stage)
 

dvc/repo/lock.py

@@ -1,15 +0,0 @@
-from . import locked
-
-
-@locked
-def lock(self, target, unlock=False):
-    from .. import dvcfile
-    from dvc.utils import parse_target
-
-    path, name = parse_target(target)
-    dvcfile = dvcfile.Dvcfile(self, path)
-    stage = dvcfile.stages[name]
-    stage.locked = False if unlock else True
-    dvcfile.dump(stage, update_pipeline=True)
-
-    return stage

dvc/repo/ls.py

@@ -4,7 +4,7 @@ from dvc.path_info import PathInfo
 
 @staticmethod
 def ls(
-    url, path=None, rev=None, recursive=None, outs_only=False,
+    url, path=None, rev=None, recursive=None, dvc_only=False,
 ):
     """Methods for getting files and outputs for the repo.
 
@@ -13,7 +13,7 @@ def ls(
         path (str, optional): relative path into the repo
         rev (str, optional): SHA commit, branch or tag name
         recursive (bool, optional): recursively walk the repo
-        outs_only (bool, optional): show only DVC-artifacts
+        dvc_only (bool, optional): show only DVC-artifacts
 
     Returns:
         list of `entry`
@@ -28,25 +28,16 @@ def ls(
         }
     """
     from dvc.external_repo import external_repo
-    from dvc.repo import Repo
 
     with external_repo(url, rev) as repo:
         path_info = PathInfo(repo.root_dir)
         if path:
             path_info /= path
 
-        ret = {}
-        if isinstance(repo, Repo):
-            ret = _ls(repo, path_info, recursive, True)
-
-        nondvc = {}
-        if not outs_only:
-            nondvc = _ls(repo, path_info, recursive, False)
-
-        ret.update(nondvc)
+        ret = _ls(repo, path_info, recursive, dvc_only)
 
         if path and not ret:
-            raise PathMissingError(path, repo, output_only=outs_only)
+            raise PathMissingError(path, repo, dvc_only=dvc_only)
 
         ret_list = []
         for path, info in ret.items():
@@ -56,46 +47,52 @@ def ls(
         return ret_list
 
 
-def _ls(repo, path_info, recursive=None, dvc=False):
-    from dvc.ignore import CleanTree
-    from dvc.repo.tree import DvcTree
-    from dvc.scm.tree import WorkingTree
+def _ls(repo, path_info, recursive=None, dvc_only=False):
+    from dvc.repo.tree import RepoTree
 
-    if dvc:
-        tree = DvcTree(repo)
-    else:
-        tree = CleanTree(WorkingTree(repo.root_dir))
+    # use our own RepoTree instance instead of repo.repo_tree since we do not
+    # want fetch/stream enabled for ls
+    tree = RepoTree(repo)
 
     ret = {}
     try:
-        for root, dirs, files in tree.walk(path_info.fspath):
+        for root, dirs, files in tree.walk(path_info.fspath, dvcfiles=True):
             for fname in files:
                 info = PathInfo(root) / fname
-                path = str(info.relative_to(path_info))
-                ret[path] = {
-                    "isout": dvc,
-                    "isdir": False,
-                    "isexec": False if dvc else tree.isexec(info.fspath),
-                }
+                dvc = tree.isdvc(info)
+                if dvc or not dvc_only:
+                    path = str(info.relative_to(path_info))
+                    ret[path] = {
+                        "isout": dvc,
+                        "isdir": False,
+                        "isexec": False if dvc else tree.isexec(info),
+                    }
 
             if not recursive:
                 for dname in dirs:
                     info = PathInfo(root) / dname
-                    path = str(info.relative_to(path_info))
-                    ret[path] = {
-                        "isout": tree.isdvc(info.fspath) if dvc else False,
-                        "isdir": True,
-                        "isexec": False if dvc else tree.isexec(info.fspath),
-                    }
+                    if not dvc_only or (
+                        tree.dvctree and tree.dvctree.exists(info)
+                    ):
+                        dvc = tree.isdvc(info)
+                        path = str(info.relative_to(path_info))
+                        ret[path] = {
+                            "isout": dvc,
+                            "isdir": True,
+                            "isexec": False if dvc else tree.isexec(info),
+                        }
                 break
     except NotADirectoryError:
-        return {
-            path_info.name: {
-                "isout": dvc,
-                "isdir": False,
-                "isexec": False if dvc else tree.isexec(path_info.fspath),
+        dvc = tree.isdvc(path_info)
+        if dvc or not dvc_only:
+            return {
+                path_info.name: {
+                    "isout": dvc,
+                    "isdir": False,
+                    "isexec": False if dvc else tree.isexec(path_info),
+                }
             }
-        }
+        return {}
     except FileNotFoundError:
         return {}
 

dvc/repo/metrics/__init__.py

@@ -2,21 +2,11 @@ class Metrics:
     def __init__(self, repo):
         self.repo = repo
 
-    def add(self, *args, **kwargs):
-        from dvc.repo.metrics.add import add
-
-        return add(self.repo, *args, **kwargs)
-
     def show(self, *args, **kwargs):
         from dvc.repo.metrics.show import show
 
         return show(self.repo, *args, **kwargs)
 
-    def remove(self, *args, **kwargs):
-        from dvc.repo.metrics.remove import remove
-
-        return remove(self.repo, *args, **kwargs)
-
     def diff(self, *args, **kwargs):
         from .diff import diff
 

dvc/repo/metrics/add.py

@@ -1,5 +0,0 @@
-from dvc.repo.metrics.modify import modify
-
-
-def add(repo, path):
-    modify(repo, path)

dvc/repo/metrics/modify.py

@@ -1,22 +0,0 @@
-from dvc.dvcfile import Dvcfile
-from dvc.exceptions import DvcException
-from dvc.repo import locked
-
-
-@locked
-def modify(repo, path, delete=False):
-    outs = repo.find_outs_by_path(path)
-    assert len(outs) == 1
-    out = outs[0]
-
-    if out.scheme != "local":
-        msg = "output '{}' scheme '{}' is not supported for metrics"
-        raise DvcException(msg.format(out.path, out.path_info.scheme))
-
-    if delete:
-        out.metric = None
-
-    out.verify_metric()
-
-    dvcfile = Dvcfile(repo, out.stage.path)
-    dvcfile.dump(out.stage)

dvc/repo/metrics/remove.py

@@ -1,5 +0,0 @@
-from dvc.repo.metrics.modify import modify
-
-
-def remove(repo, path):
-    modify(repo, path, delete=True)

dvc/repo/metrics/show.py

@@ -104,13 +104,13 @@ def show(
     if not res:
         raise NoMetricsError()
 
-    # Hide working tree metrics if they are the same as in the active branch
+    # Hide workspace metrics if they are the same as in the active branch
     try:
         active_branch = repo.scm.active_branch()
     except TypeError:
         pass  # Detached head
     else:
-        if res.get("working tree") == res.get(active_branch):
-            res.pop("working tree", None)
+        if res.get("workspace") == res.get(active_branch):
+            res.pop("workspace", None)
 
     return res

dvc/repo/params/show.py

@@ -58,13 +58,13 @@ def show(repo, revs=None):
     if not res:
         raise NoParamsError("no parameter configs files in this repository")
 
-    # Hide working tree params if they are the same as in the active branch
+    # Hide workspace params if they are the same as in the active branch
     try:
         active_branch = repo.scm.active_branch()
     except TypeError:
         pass  # Detached head
     else:
-        if res.get("working tree") == res.get(active_branch):
-            res.pop("working tree", None)
+        if res.get("workspace") == res.get(active_branch):
+            res.pop("workspace", None)
 
     return res

dvc/repo/plots/__init__.py

@@ -11,3 +11,25 @@ class Plots:
         from .diff import diff
 
         return diff(self.repo, *args, **kwargs)
+
+    def modify(self, path, props=None, unset=None):
+        from dvc.dvcfile import Dvcfile
+
+        (out,) = self.repo.find_outs_by_path(path)
+
+        # This out will become a plot unless it is one already
+        if not isinstance(out.plot, dict):
+            out.plot = {}
+
+        for field in unset or ():
+            out.plot.pop(field, None)
+        out.plot.update(props or {})
+
+        # Empty dict will move it to non-plots
+        if not out.plot:
+            out.plot = True
+
+        out.verify_metric()
+
+        dvcfile = Dvcfile(self.repo, out.stage.path)
+        dvcfile.dump(out.stage, update_pipeline=True)

dvc/repo/plots/data.py

@@ -258,7 +258,7 @@ def _load_from_revisions(repo, datafile, revisions):
     exceptions = []
 
     for rev in repo.brancher(revs=revisions):
-        if rev == "working tree" and rev not in revisions:
+        if rev == "workspace" and rev not in revisions:
             continue
 
         try:

dvc/repo/plots/diff.py

@@ -3,7 +3,7 @@ def _revisions(revs, is_dirty):
     if len(revisions) <= 1:
         if len(revisions) == 0 and is_dirty:
             revisions.append("HEAD")
-        revisions.append("working tree")
+        revisions.append("workspace")
     return revisions
 
 

dvc/repo/plots/show.py

@@ -1,13 +1,14 @@
+import copy
 import logging
 import os
-from collections import defaultdict
 
-from funcy import first, last
+from funcy import first, last, project
 
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, NoPlotsError
 from dvc.repo import locked
+from dvc.schema import PLOT_PROPS
 
-from .data import PlotData
+from .data import NoMetricInHistoryError, PlotData
 from .template import NoDataForTemplateError, Template
 
 logger = logging.getLogger(__name__)
@@ -43,29 +44,24 @@ def _evaluate_templatepath(repo, template=None):
 
 @locked
 def fill_template(
-    repo,
-    datafile,
-    template_path,
-    revisions,
-    fields=None,
-    path=None,
-    csv_header=True,
-    x_field=None,
-    y_field=None,
-    **kwargs,
+    repo, datafile, template_path, revisions, props,
 ):
-    if x_field and fields:
-        fields.add(x_field)
+    # Copy things to not modify passed values
+    props = props.copy()
+    fields = copy.copy(props.get("fields"))
 
-    if y_field and fields:
-        fields.add(y_field)
+    if props.get("x") and fields:
+        fields.add(props.get("x"))
+
+    if props.get("y") and fields:
+        fields.add(props.get("y"))
 
     template_datafiles, x_anchor, y_anchor = _parse_template(
         template_path, datafile
     )
-    append_index = x_anchor and not x_field
+    append_index = x_anchor and not props.get("x")
     if append_index:
-        x_field = PlotData.INDEX_FIELD
+        props["x"] = PlotData.INDEX_FIELD
 
     template_data = {}
     for template_datafile in template_datafiles:
@@ -76,13 +72,13 @@ def fill_template(
         for pd in plot_datas:
             rev_data_points = pd.to_datapoints(
                 fields=fields,
-                path=path,
-                csv_header=csv_header,
+                path=props.get("path"),
+                csv_header=props.get("csv_header", True),
                 append_index=append_index,
             )
 
-            if y_anchor and not y_field:
-                y_field = _infer_y_field(rev_data_points, x_field)
+            if y_anchor and not props.get("y"):
+                props["y"] = _infer_y_field(rev_data_points, props.get("x"))
             tmp_data.extend(rev_data_points)
 
         template_data[template_datafile] = tmp_data
@@ -91,12 +87,7 @@ def fill_template(
         raise NoDataForTemplateError(template_path)
 
     content = Template.fill(
-        template_path,
-        template_data,
-        priority_datafile=datafile,
-        x_field=x_field,
-        y_field=y_field,
-        **kwargs,
+        template_path, template_data, priority_datafile=datafile, props=props,
     )
 
     path = datafile or ",".join(template_datafiles)
@@ -113,17 +104,20 @@ def _infer_y_field(rev_data_points, x_field):
     return y_field
 
 
-def _show(repo, datafile=None, template=None, revs=None, **kwargs):
+def _show(repo, datafile=None, revs=None, props=None):
     if revs is None:
-        revs = ["working tree"]
+        revs = ["workspace"]
 
-    if not datafile and not template:
+    if props is None:
+        props = {}
+
+    if not datafile and not props.get("template"):
         raise NoDataOrTemplateProvided()
 
-    template_path = _evaluate_templatepath(repo, template)
+    template_path = _evaluate_templatepath(repo, props.get("template"))
 
     plot_datafile, plot_content = fill_template(
-        repo, datafile, template_path, revs, **kwargs
+        repo, datafile, template_path, revs, props
     )
 
     return plot_datafile, plot_content
@@ -148,67 +142,76 @@ def _parse_template(template_path, priority_datafile):
     )
 
 
-def _collect_plots(repo):
-    plots = defaultdict(set)
+def _collect_plots(repo, targets=None):
+    from dvc.exceptions import OutputNotFoundError
+    from contextlib import suppress
 
-    for stage in repo.stages:
-        for out in stage.outs:
-            if not out.plot:
-                continue
-
-            if isinstance(out.plot, dict):
-                template = out.plot[out.PARAM_PLOT_TEMPLATE]
-            else:
-                template = None
-
-            plots[str(out)].add(template)
-
-    return plots
-
-
-def show(repo, targets=None, template=None, revs=None, **kwargs) -> dict:
-    if isinstance(targets, str):
-        targets = [targets]
+    def _targets_to_outs(targets):
+        for t in targets:
+            with suppress(OutputNotFoundError):
+                (out,) = repo.find_outs_by_path(t)
+                yield out
 
     if targets:
-        for target in targets:
-            return {
-                target: _show(
-                    repo,
-                    datafile=target,
-                    template=template,
-                    revs=revs,
-                    **kwargs,
-                )[1]
-            }
-
-    if not revs:
-        plots = _collect_plots(repo)
+        outs = _targets_to_outs(targets)
     else:
-        plots = defaultdict(set)
-        for rev in repo.brancher(revs=revs):
-            for plot, templates in _collect_plots(repo).items():
-                plots[plot].update(templates)
+        outs = (out for stage in repo.stages for out in stage.outs if out.plot)
+
+    return {str(out): _plot_props(out) for out in outs}
+
+
+def _plot_props(out):
+    if not out.plot:
+        raise DvcException(
+            f"'{out}' is not a plot. Use `dvc plots modify` to change that."
+        )
+    if isinstance(out.plot, list):
+        raise DvcException("Multiple plots per data file not supported yet.")
+    if isinstance(out.plot, bool):
+        return {}
+
+    return project(out.plot, PLOT_PROPS)
+
+
+def show(repo, targets=None, revs=None, props=None) -> dict:
+    if isinstance(targets, str):
+        targets = [targets]
+    if props is None:
+        props = {}
+
+    # Collect plot data files with associated props
+    plots = {}
+    for rev in repo.brancher(revs=revs):
+        if revs is not None and rev not in revs:
+            continue
+
+        for datafile, file_props in _collect_plots(repo, targets).items():
+            # props from command line overwrite plot props from out definition
+            full_props = {**file_props, **props}
+
+            if datafile in plots:
+                saved_rev, saved_props = plots[datafile]
+                if saved_props != props:
+                    logger.warning(
+                        f"Inconsistent plot props for '{datafile}' in "
+                        f"'{saved_rev}' and '{rev}'. "
+                        f"Going to use ones from '{saved_rev}'"
+                    )
+            else:
+                plots[datafile] = rev, full_props
 
     if not plots:
-        datafile, plot = _show(
-            repo, datafile=None, template=template, revs=revs, **kwargs
-        )
+        if targets:
+            raise NoMetricInHistoryError(", ".join(targets))
+
+        try:
+            datafile, plot = _show(repo, datafile=None, revs=revs, props=props)
+        except NoDataOrTemplateProvided:
+            raise NoPlotsError()
+
         return {datafile: plot}
 
-    ret = {}
-    for plot, templates in plots.items():
-        tmplt = template
-        if len(templates) == 1:
-            tmplt = list(templates)[0]
-        elif not template:
-            raise TooManyTemplatesError(
-                f"'{plot}' uses multiple templates '{templates}'. "
-                "Use `-t|--template` to specify the template to use. "
-            )
-
-        ret[plot] = _show(
-            repo, datafile=plot, template=tmplt, revs=revs, **kwargs
-        )[1]
-
-    return ret
+    return {
+        datafile: _show(repo, datafile=datafile, revs=revs, props=props)[1]
+        for datafile, (_, props) in plots.items()
+    }

dvc/repo/plots/template.py

@@ -38,8 +38,8 @@ class Template:
     X_ANCHOR = "<DVC_METRIC_X>"
     Y_ANCHOR = "<DVC_METRIC_Y>"
     TITLE_ANCHOR = "<DVC_METRIC_TITLE>"
-    X_TITLE_ANCHOR = "<DVC_METRIC_X_TITLE>"
-    Y_TITLE_ANCHOR = "<DVC_METRIC_Y_TITLE>"
+    X_LABEL_ANCHOR = "<DVC_METRIC_X_LABEL>"
+    Y_LABEL_ANCHOR = "<DVC_METRIC_Y_LABEL>"
 
     def __init__(self, templates_dir):
         self.plot_templates_dir = templates_dir
@@ -86,29 +86,24 @@ class Template:
 
     @staticmethod
     def fill(
-        template_path,
-        data,
-        priority_datafile=None,
-        x_field=None,
-        y_field=None,
-        title=None,
-        x_title=None,
-        y_title=None,
+        template_path, data, priority_datafile=None, props=None,
     ):
+        props = props or {}
+
         with open(template_path) as fobj:
             result_content = fobj.read()
 
-        if x_field:
-            Template._check_field_exists(data, x_field)
-        if y_field:
-            Template._check_field_exists(data, y_field)
+        if props.get("x"):
+            Template._check_field_exists(data, props.get("x"))
+        if props.get("y"):
+            Template._check_field_exists(data, props.get("y"))
 
         result_content = Template._replace_data_anchors(
             result_content, data, priority_datafile
         )
 
         result_content = Template._replace_metadata_anchors(
-            result_content, title, x_field, x_title, y_field, y_title
+            result_content, props
         )
 
         return result_content
@@ -122,34 +117,23 @@ class Template:
                 raise NoFieldInDataError(field)
 
     @staticmethod
-    def _replace_metadata_anchors(
-        result_content, title, x_field, x_title, y_field, y_title
-    ):
-        if Template.TITLE_ANCHOR in result_content:
-            if title:
-                result_content = result_content.replace(
-                    Template.TITLE_ANCHOR, title
-                )
-            else:
-                result_content = result_content.replace(
-                    Template.TITLE_ANCHOR, ""
-                )
-        if Template.X_ANCHOR in result_content and x_field:
-            result_content = result_content.replace(Template.X_ANCHOR, x_field)
-        if Template.Y_ANCHOR in result_content and y_field:
-            result_content = result_content.replace(Template.Y_ANCHOR, y_field)
-        if Template.X_TITLE_ANCHOR in result_content:
-            if not x_title and x_field:
-                x_title = x_field
-            result_content = result_content.replace(
-                Template.X_TITLE_ANCHOR, x_title
-            )
-        if Template.Y_TITLE_ANCHOR in result_content:
-            if not y_title and y_field:
-                y_title = y_field
-            result_content = result_content.replace(
-                Template.Y_TITLE_ANCHOR, y_title
-            )
+    def _replace_metadata_anchors(result_content, props):
+        props.setdefault("title", "")
+        props.setdefault("x_label", props.get("x"))
+        props.setdefault("y_label", props.get("y"))
+
+        replace_pairs = [
+            (Template.TITLE_ANCHOR, "title"),
+            (Template.X_ANCHOR, "x"),
+            (Template.Y_ANCHOR, "y"),
+            (Template.X_LABEL_ANCHOR, "x_label"),
+            (Template.Y_LABEL_ANCHOR, "y_label"),
+        ]
+        for anchor, key in replace_pairs:
+            value = props.get(key)
+            if anchor in result_content and value is not None:
+                result_content = result_content.replace(anchor, value)
+
         return result_content
 
     @staticmethod
@@ -186,12 +170,12 @@ class DefaultLinearTemplate(Template):
             "x": {
                 "field": Template.X_ANCHOR,
                 "type": "quantitative",
-                "title": Template.X_TITLE_ANCHOR,
+                "title": Template.X_LABEL_ANCHOR,
             },
             "y": {
                 "field": Template.Y_ANCHOR,
                 "type": "quantitative",
-                "title": Template.Y_TITLE_ANCHOR,
+                "title": Template.Y_LABEL_ANCHOR,
                 "scale": {"zero": False},
             },
             "color": {"field": "rev", "type": "nominal"},
@@ -211,13 +195,13 @@ class DefaultConfusionTemplate(Template):
                 "field": Template.X_ANCHOR,
                 "type": "nominal",
                 "sort": "ascending",
-                "title": Template.X_TITLE_ANCHOR,
+                "title": Template.X_LABEL_ANCHOR,
             },
             "y": {
                 "field": Template.Y_ANCHOR,
                 "type": "nominal",
                 "sort": "ascending",
-                "title": Template.Y_TITLE_ANCHOR,
+                "title": Template.Y_LABEL_ANCHOR,
             },
             "color": {"aggregate": "count", "type": "quantitative"},
             "facet": {"field": "rev", "type": "nominal"},
@@ -236,12 +220,12 @@ class DefaultScatterTemplate(Template):
             "x": {
                 "field": Template.X_ANCHOR,
                 "type": "quantitative",
-                "title": Template.X_TITLE_ANCHOR,
+                "title": Template.X_LABEL_ANCHOR,
             },
             "y": {
                 "field": Template.Y_ANCHOR,
                 "type": "quantitative",
-                "title": Template.Y_TITLE_ANCHOR,
+                "title": Template.Y_LABEL_ANCHOR,
                 "scale": {"zero": False},
             },
             "color": {"field": "rev", "type": "nominal"},

dvc/repo/remove.py

@@ -7,16 +7,11 @@ logger = logging.getLogger(__name__)
 
 
 @locked
-def remove(self, target, outs_only=False):
-    from ..dvcfile import Dvcfile
-
+def remove(self, target, outs=False):
     path, name = parse_target(target)
-    dvcfile = Dvcfile(self, path)
-    stages = list(dvcfile.stages.filter(name).values())
-    for stage in stages:
-        stage.remove_outs(force=True)
+    stages = self.get_stages(path, name)
 
-    if not outs_only:
-        dvcfile.remove()
+    for stage in stages:
+        stage.remove(remove_outs=outs, force=outs)
 
     return stages

dvc/repo/reproduce.py

@@ -10,9 +10,9 @@ logger = logging.getLogger(__name__)
 
 
 def _reproduce_stage(stage, **kwargs):
-    if stage.locked and not stage.is_import:
+    if stage.frozen and not stage.is_import:
         logger.warning(
-            "{} is locked. Its dependencies are"
+            "{} is frozen. Its dependencies are"
             " not going to be reproduced.".format(stage)
         )
 
@@ -34,7 +34,7 @@ def _get_active_graph(G):
 
     active = G.copy()
     for stage in G:
-        if not stage.locked:
+        if not stage.frozen:
             continue
         active.remove_edges_from(G.out_edges(stage))
         for edge in G.out_edges(stage):
@@ -61,7 +61,6 @@ def reproduce(
     all_pipelines=False,
     **kwargs
 ):
-    from ..dvcfile import Dvcfile
     from dvc.utils import parse_target
 
     if not target and not all_pipelines:
@@ -81,8 +80,7 @@ def reproduce(
         if all_pipelines:
             pipelines = active_pipelines
         else:
-            dvcfile = Dvcfile(self, path)
-            stage = dvcfile.stages[name]
+            stage = self.get_stage(path, name)
             pipelines = [get_pipeline(active_pipelines, stage)]
 
         targets = []
@@ -164,7 +162,12 @@ def _reproduce_stages(
 
     force_downstream = kwargs.pop("force_downstream", False)
     result = []
+    # `ret` is used to add a cosmetic newline.
+    ret = []
     for stage in pipeline:
+        if ret:
+            logger.info("")
+
         try:
             ret = _reproduce_stage(stage, **kwargs)
 

dvc/repo/run.py

@@ -3,7 +3,11 @@ import os
 from funcy import concat, first
 
 from dvc.exceptions import InvalidArgumentError
-from dvc.stage.exceptions import DuplicateStageName, InvalidStageName
+from dvc.stage.exceptions import (
+    DuplicateStageName,
+    InvalidStageName,
+    StageFileAlreadyExistsError,
+)
 
 from ..exceptions import OutputDuplicationError
 from . import locked
@@ -45,6 +49,9 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
     from dvc.stage import PipelineStage, Stage, create_stage
     from dvc.dvcfile import Dvcfile, PIPELINE_FILE
 
+    if not kwargs.get("cmd"):
+        raise InvalidArgumentError("command is not specified")
+
     stage_cls = PipelineStage
     path = PIPELINE_FILE
     stage_name = kwargs.get("name")
@@ -71,10 +78,12 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
 
     dvcfile = Dvcfile(self, stage.path)
     if dvcfile.exists():
-        if stage_name and stage_name in dvcfile.stages:
+        if kwargs.get("overwrite", True):
+            dvcfile.remove_stage(stage)
+        elif stage_cls != PipelineStage:
+            raise StageFileAlreadyExistsError(dvcfile.relpath)
+        elif stage_name and stage_name in dvcfile.stages:
             raise DuplicateStageName(stage_name, dvcfile)
-        if stage_cls != PipelineStage:
-            dvcfile.remove_with_prompt(force=kwargs.get("overwrite", True))
 
     try:
         self.check_modified_graph([stage])
@@ -82,13 +91,12 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
         raise OutputDuplicationError(exc.output, set(exc.stages) - {stage})
 
     if no_exec:
-        for out in stage.outs:
-            out.ignore()
+        stage.ignore_outs()
     else:
         stage.run(
             no_commit=kwargs.get("no_commit", False),
             run_cache=kwargs.get("run_cache", True),
         )
 
-    dvcfile.dump(stage, update_pipeline=True)
+    dvcfile.dump(stage, update_pipeline=True, no_lock=no_exec)
     return stage

dvc/repo/status.py

@@ -14,9 +14,9 @@ def _joint_status(stages):
     status = {}
 
     for stage in stages:
-        if stage.locked and not stage.is_repo_import:
+        if stage.frozen and not stage.is_repo_import:
             logger.warning(
-                "{} is locked. Its dependencies are"
+                "{} is frozen. Its dependencies are"
                 " not going to be shown in the status output.".format(stage)
             )
 

dvc/repo/tree.py

@@ -1,10 +1,13 @@
 import logging
 import os
 
+from dvc.dvcfile import is_valid_filename
 from dvc.exceptions import OutputNotFoundError
 from dvc.path_info import PathInfo
 from dvc.remote.base import RemoteActionNotImplemented
-from dvc.scm.tree import BaseTree, WorkingTree
+from dvc.scm.tree import BaseTree
+from dvc.utils import file_md5
+from dvc.utils.fs import copy_fobj_to_file, makedirs
 
 logger = logging.getLogger(__name__)
 
@@ -40,43 +43,56 @@ class DvcTree(BaseTree):
 
         return outs
 
+    def _get_granular_checksum(self, path, out, remote=None):
+        if not self.fetch and not self.stream:
+            raise FileNotFoundError
+        dir_cache = out.get_dir_cache(remote=remote)
+        for entry in dir_cache:
+            entry_relpath = entry[out.remote.PARAM_RELPATH]
+            if os.name == "nt":
+                entry_relpath = entry_relpath.replace("/", os.sep)
+            if path == out.path_info / entry_relpath:
+                return entry[out.remote.PARAM_CHECKSUM]
+        raise FileNotFoundError
+
     def open(self, path, mode="r", encoding="utf-8", remote=None):
         try:
             outs = self._find_outs(path, strict=False)
         except OutputNotFoundError as exc:
             raise FileNotFoundError from exc
 
-        if len(outs) != 1 or outs[0].is_dir_checksum:
+        if len(outs) != 1 or (
+            outs[0].is_dir_checksum and path == outs[0].path_info
+        ):
             raise IsADirectoryError
 
         out = outs[0]
-        # temporary hack to make cache use WorkingTree and not GitTree, because
-        # cache dir doesn't exist in the latter.
-        saved_tree = self.repo.tree
-        self.repo.tree = WorkingTree(self.repo.root_dir)
-        try:
-            if out.changed_cache():
-                if not self.fetch and not self.stream:
-                    raise FileNotFoundError
+        if out.changed_cache(filter_info=path):
+            if not self.fetch and not self.stream:
+                raise FileNotFoundError
 
-                remote_obj = self.repo.cloud.get_remote(remote)
-                if self.stream:
-                    try:
-                        remote_info = remote_obj.checksum_to_path_info(
-                            out.checksum
-                        )
-                        return remote_obj.open(
-                            remote_info, mode=mode, encoding=encoding
-                        )
-                    except RemoteActionNotImplemented:
-                        pass
-                with self.repo.state:
-                    cache_info = out.get_used_cache(remote=remote)
-                    self.repo.cloud.pull(cache_info, remote=remote)
-        finally:
-            self.repo.tree = saved_tree
+            remote_obj = self.repo.cloud.get_remote(remote)
+            if self.stream:
+                if out.is_dir_checksum:
+                    checksum = self._get_granular_checksum(path, out)
+                else:
+                    checksum = out.checksum
+                try:
+                    remote_info = remote_obj.checksum_to_path_info(checksum)
+                    return remote_obj.open(
+                        remote_info, mode=mode, encoding=encoding
+                    )
+                except RemoteActionNotImplemented:
+                    pass
+            cache_info = out.get_used_cache(filter_info=path, remote=remote)
+            self.repo.cloud.pull(cache_info, remote=remote)
 
-        return open(out.cache_path, mode=mode, encoding=encoding)
+        if out.is_dir_checksum:
+            checksum = self._get_granular_checksum(path, out)
+            cache_path = out.cache.checksum_to_path_info(checksum).url
+        else:
+            cache_path = out.cache_path
+        return open(cache_path, mode=mode, encoding=encoding)
 
     def exists(self, path):
         try:
@@ -91,10 +107,25 @@ class DvcTree(BaseTree):
 
         path_info = PathInfo(os.path.abspath(path))
         outs = self._find_outs(path, strict=False, recursive=True)
-        if len(outs) != 1 or outs[0].path_info != path_info:
+        if len(outs) != 1:
             return True
 
-        return outs[0].is_dir_checksum
+        out = outs[0]
+        if not out.is_dir_checksum:
+            if out.path_info != path_info:
+                return True
+            return False
+
+        if out.path_info == path_info:
+            return True
+
+        # for dir checksum, we need to check if this is a file inside the
+        # directory
+        try:
+            self._get_granular_checksum(path, out)
+            return False
+        except FileNotFoundError:
+            return True
 
     def isfile(self, path):
         if not self.exists(path):
@@ -127,7 +158,7 @@ class DvcTree(BaseTree):
         else:
             assert False
 
-    def walk(self, top, topdown=True):
+    def walk(self, top, topdown=True, download_callback=None, **kwargs):
         from pygtrie import Trie
 
         assert topdown
@@ -147,19 +178,29 @@ class DvcTree(BaseTree):
             trie[out.path_info.parts] = out
 
             if out.is_dir_checksum and (self.fetch or self.stream):
-                # will pull dir cache if needed
-                with self.repo.state:
-                    cache = out.collect_used_dir_cache()
-                for _, names in cache.scheme_names(out.scheme):
-                    for name in names:
-                        path_info = out.path_info.parent / name
-                        trie[path_info.parts] = None
+                # pull dir cache if needed
+                dir_cache = out.get_dir_cache(**kwargs)
+
+                # pull dir contents if needed
+                if self.fetch:
+                    if out.changed_cache(filter_info=top):
+                        used_cache = out.get_used_cache(filter_info=top)
+                        downloaded = self.repo.cloud.pull(used_cache, **kwargs)
+                        if download_callback:
+                            download_callback(downloaded)
+
+                for entry in dir_cache:
+                    entry_relpath = entry[out.remote.PARAM_RELPATH]
+                    if os.name == "nt":
+                        entry_relpath = entry_relpath.replace("/", os.sep)
+                    path_info = out.path_info / entry_relpath
+                    trie[path_info.parts] = None
 
         yield from self._walk(root, trie, topdown=topdown)
 
-    def isdvc(self, path):
+    def isdvc(self, path, **kwargs):
         try:
-            return len(self._find_outs(path)) == 1
+            return len(self._find_outs(path, **kwargs)) == 1
         except OutputNotFoundError:
             pass
         return False
@@ -167,6 +208,15 @@ class DvcTree(BaseTree):
     def isexec(self, path):
         return False
 
+    def get_file_checksum(self, path_info):
+        outs = self._find_outs(path_info, strict=False)
+        if len(outs) != 1:
+            raise OutputNotFoundError
+        out = outs[0]
+        if out.is_dir_checksum:
+            return self._get_granular_checksum(path_info, out)
+        return out.checksum
+
 
 class RepoTree(BaseTree):
     """DVC + git-tracked files tree.
@@ -185,7 +235,22 @@ class RepoTree(BaseTree):
             # git-only erepo's do not need dvctree
             self.dvctree = None
 
+    @property
+    def fetch(self):
+        if self.dvctree:
+            return self.dvctree.fetch
+        return False
+
+    @property
+    def stream(self):
+        if self.dvctree:
+            return self.dvctree.stream
+        return False
+
     def open(self, path, mode="r", encoding="utf-8", **kwargs):
+        if "b" in mode:
+            encoding = None
+
         if self.dvctree and self.dvctree.exists(path):
             try:
                 return self.dvctree.open(
@@ -206,8 +271,8 @@ class RepoTree(BaseTree):
             self.dvctree and self.dvctree.isdir(path)
         )
 
-    def isdvc(self, path):
-        return self.dvctree is not None and self.dvctree.isdvc(path)
+    def isdvc(self, path, **kwargs):
+        return self.dvctree is not None and self.dvctree.isdvc(path, **kwargs)
 
     def isfile(self, path):
         return self.repo.tree.isfile(path) or (
@@ -219,6 +284,9 @@ class RepoTree(BaseTree):
             return self.dvctree.isexec(path)
         return self.repo.tree.isexec(path)
 
+    def stat(self, path):
+        return self.repo.tree.stat(path)
+
     def _walk_one(self, walk):
         try:
             root, dirs, files = next(walk)
@@ -228,7 +296,7 @@ class RepoTree(BaseTree):
         for _ in dirs:
             yield from self._walk_one(walk)
 
-    def _walk(self, dvc_walk, repo_walk):
+    def _walk(self, dvc_walk, repo_walk, dvcfiles=False):
         try:
             _, dvc_dirs, dvc_fnames = next(dvc_walk)
             repo_root, repo_dirs, repo_fnames = next(repo_walk)
@@ -244,9 +312,11 @@ class RepoTree(BaseTree):
         dirs = shared + dvc_only + repo_only
 
         # merge file lists
-        files = set(dvc_fnames)
-        for filename in repo_fnames:
-            files.add(filename)
+        files = {
+            fname
+            for fname in dvc_fnames + repo_fnames
+            if dvcfiles or not is_valid_filename(fname)
+        }
 
         yield repo_root, dirs, list(files)
 
@@ -257,14 +327,24 @@ class RepoTree(BaseTree):
 
         for dirname in dirs:
             if dirname in shared:
-                yield from self._walk(dvc_walk, repo_walk)
+                yield from self._walk(dvc_walk, repo_walk, dvcfiles=dvcfiles)
             elif dirname in dvc_set:
                 yield from self._walk_one(dvc_walk)
             elif dirname in repo_set:
                 yield from self._walk_one(repo_walk)
 
-    def walk(self, top, topdown=True):
-        """Walk and merge both DVC and repo trees."""
+    def walk(self, top, topdown=True, dvcfiles=False, **kwargs):
+        """Walk and merge both DVC and repo trees.
+
+        Args:
+            top: path to walk from
+            topdown: if True, tree will be walked from top down.
+            dvcfiles: if True, dvcfiles will be included in the files list
+                for walked directories.
+
+        Any kwargs will be passed into methods used for fetching and/or
+        streaming DVC outs from remotes.
+        """
         assert topdown
 
         if not self.exists(top):
@@ -276,7 +356,7 @@ class RepoTree(BaseTree):
         dvc_exists = self.dvctree and self.dvctree.exists(top)
         repo_exists = self.repo.tree.exists(top)
         if dvc_exists and not repo_exists:
-            yield from self.dvctree.walk(top, topdown=topdown)
+            yield from self.dvctree.walk(top, topdown=topdown, **kwargs)
             return
         if repo_exists and not dvc_exists:
             yield from self.repo.tree.walk(top, topdown=topdown)
@@ -284,6 +364,50 @@ class RepoTree(BaseTree):
         if not dvc_exists and not repo_exists:
             raise FileNotFoundError
 
-        dvc_walk = self.dvctree.walk(top, topdown=topdown)
+        dvc_walk = self.dvctree.walk(top, topdown=topdown, **kwargs)
         repo_walk = self.repo.tree.walk(top, topdown=topdown)
-        yield from self._walk(dvc_walk, repo_walk)
+        yield from self._walk(dvc_walk, repo_walk, dvcfiles=dvcfiles)
+
+    def walk_files(self, top, **kwargs):
+        for root, _, files in self.walk(top, **kwargs):
+            for fname in files:
+                yield PathInfo(root) / fname
+
+    def get_file_checksum(self, path_info):
+        """Return file checksum for specified path.
+
+        If path_info is a DVC out, the pre-computed checksum for the file
+        will be used. If path_info is a git file, MD5 will be computed for
+        the git object.
+        """
+        if not self.exists(path_info):
+            raise FileNotFoundError
+        if self.dvctree and self.dvctree.exists(path_info):
+            try:
+                return self.dvctree.get_file_checksum(path_info)
+            except OutputNotFoundError:
+                pass
+        return file_md5(path_info, self)[0]
+
+    def copytree(self, top, dest):
+        top = PathInfo(top)
+        dest = PathInfo(dest)
+
+        if not self.exists(top):
+            raise FileNotFoundError
+
+        if self.isfile(top):
+            makedirs(dest.parent, exist_ok=True)
+            with self.open(top, mode="rb") as fobj:
+                copy_fobj_to_file(fobj, dest)
+            return
+
+        for root, _, files in self.walk(top):
+            root = PathInfo(root)
+            dest_dir = root.relative_to(top)
+            makedirs(dest_dir, exist_ok=True)
+            for fname in files:
+                src = root / fname
+                dest = dest_dir / fname
+                with self.open(src, mode="rb") as fobj:
+                    copy_fobj_to_file(fobj, dest)

dvc/schema.py

@@ -11,7 +11,8 @@ SINGLE_STAGE_SCHEMA = {
     StageParams.PARAM_WDIR: Any(str, None),
     StageParams.PARAM_DEPS: Any([dependency.SCHEMA], None),
     StageParams.PARAM_OUTS: Any([output.SCHEMA], None),
-    StageParams.PARAM_LOCKED: bool,
+    StageParams.PARAM_LOCKED: bool,  # backard compatibility
+    StageParams.PARAM_FROZEN: bool,
     StageParams.PARAM_META: object,
     StageParams.PARAM_ALWAYS_CHANGED: bool,
 }
@@ -29,8 +30,20 @@ OUT_PSTAGE_DETAILED_SCHEMA = {
     str: {BaseOutput.PARAM_CACHE: bool, BaseOutput.PARAM_PERSIST: bool}
 }
 
-PLOT_PSTAGE_SCHEMA = OUT_PSTAGE_DETAILED_SCHEMA.copy()
-PLOT_PSTAGE_SCHEMA[str][BaseOutput.PARAM_PLOT_TEMPLATE] = str
+PLOT_PROPS = {
+    BaseOutput.PARAM_PLOT_TEMPLATE: str,
+    BaseOutput.PARAM_PLOT_X: str,
+    BaseOutput.PARAM_PLOT_Y: str,
+    BaseOutput.PARAM_PLOT_X_LABEL: str,
+    BaseOutput.PARAM_PLOT_Y_LABEL: str,
+    BaseOutput.PARAM_PLOT_TITLE: str,
+    BaseOutput.PARAM_PLOT_CSV_HEADER: bool,
+}
+PLOT_PROPS_SCHEMA = {
+    **OUT_PSTAGE_DETAILED_SCHEMA[str],
+    **PLOT_PROPS,
+}
+PLOT_PSTAGE_SCHEMA = {str: Any(PLOT_PROPS_SCHEMA, [PLOT_PROPS_SCHEMA])}
 
 PARAM_PSTAGE_NON_DEFAULT_SCHEMA = {str: [str]}
 
@@ -42,7 +55,7 @@ SINGLE_PIPELINE_STAGE_SCHEMA = {
         Optional(StageParams.PARAM_PARAMS): [
             Any(str, PARAM_PSTAGE_NON_DEFAULT_SCHEMA)
         ],
-        Optional(StageParams.PARAM_LOCKED): bool,
+        Optional(StageParams.PARAM_FROZEN): bool,
         Optional(StageParams.PARAM_META): object,
         Optional(StageParams.PARAM_ALWAYS_CHANGED): bool,
         Optional(StageParams.PARAM_OUTS): [

dvc/scm/tree.py

@@ -32,7 +32,8 @@ class BaseTree:
     def walk_files(self, top):
         for root, _, files in self.walk(top):
             for file in files:
-                yield os.path.join(root, file)
+                # NOTE: os.path.join is ~5.5 times slower
+                yield f"{root}{os.sep}{file}"
 
 
 class WorkingTree(BaseTree):

dvc/serialize.py

@@ -10,7 +10,7 @@ from dvc.output import BaseOutput
 from dvc.stage.params import StageParams
 from dvc.stage.utils import resolve_wdir
 from dvc.utils.collections import apply_diff
-from dvc.utils.stage import parse_stage_for_update
+from dvc.utils.yaml import parse_yaml_for_update
 
 if TYPE_CHECKING:
     from dvc.stage import PipelineStage, Stage
@@ -69,14 +69,22 @@ def _serialize_params(params: List[ParamsDependency]):
     for param_dep in sort_by_path(params):
         dump = param_dep.dumpd()
         path, params = dump[PARAM_PATH], dump[PARAM_PARAMS]
-        k = list(params.keys())
-        if not k:
-            continue
-        key_vals[path] = OrderedDict([(key, params[key]) for key in sorted(k)])
+        if isinstance(params, dict):
+            k = list(params.keys())
+            if not k:
+                continue
+            key_vals[path] = OrderedDict(
+                [(key, params[key]) for key in sorted(k)]
+            )
+        else:
+            assert isinstance(params, list)
+            k = params
+            key_vals = OrderedDict()
         # params from default file is always kept at the start of the `params:`
         if path == DEFAULT_PARAMS_FILE:
             keys = k + keys
-            key_vals.move_to_end(path, last=False)
+            if key_vals:
+                key_vals.move_to_end(path, last=False)
         else:
             # if it's not a default file, change the shape
             # to: {path: k}
@@ -107,7 +115,7 @@ def to_pipeline_file(stage: "PipelineStage"):
             stage.PARAM_PLOTS,
             _get_outs([out for out in stage.outs if out.plot]),
         ),
-        (stage.PARAM_LOCKED, stage.locked),
+        (stage.PARAM_FROZEN, stage.frozen),
         (stage.PARAM_ALWAYS_CHANGED, stage.always_changed),
     ]
     return {
@@ -157,7 +165,7 @@ def to_single_stage_file(stage: "Stage"):
     # - apply changes to a returned structure
     # - serialize it
     if stage._stage_text is not None:
-        saved_state = parse_stage_for_update(stage._stage_text, stage.path)
+        saved_state = parse_yaml_for_update(stage._stage_text, stage.path)
         # Stage doesn't work with meta in any way, so .dumpd() doesn't
         # have it. We simply copy it over.
         if "meta" in saved_state:

dvc/stage/__init__.py

@@ -19,6 +19,7 @@ from .utils import (
     check_circular_dependency,
     check_duplicated_arguments,
     check_missing_outputs,
+    check_no_externals,
     check_stage_path,
     compute_md5,
     fill_stage_dependencies,
@@ -42,6 +43,7 @@ def loads_from(cls, repo, path, wdir, data):
             [
                 Stage.PARAM_CMD,
                 Stage.PARAM_LOCKED,
+                Stage.PARAM_FROZEN,
                 Stage.PARAM_ALWAYS_CHANGED,
                 Stage.PARAM_MD5,
                 "name",
@@ -51,7 +53,7 @@ def loads_from(cls, repo, path, wdir, data):
     return cls(**kw)
 
 
-def create_stage(cls, repo, path, **kwargs):
+def create_stage(cls, repo, path, external=False, **kwargs):
     from dvc.dvcfile import check_dvc_filename
 
     wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
@@ -62,6 +64,8 @@ def create_stage(cls, repo, path, **kwargs):
 
     stage = loads_from(cls, repo, path, wdir, kwargs)
     fill_stage_outputs(stage, **kwargs)
+    if not external:
+        check_no_externals(stage)
     fill_stage_dependencies(
         stage, **project(kwargs, ["deps", "erepo", "params"])
     )
@@ -77,7 +81,7 @@ def create_stage(cls, repo, path, **kwargs):
             logger.warning("Build cache is ignored when persisting outputs.")
 
         if not ignore_run_cache and stage.can_be_skipped:
-            logger.info("Stage is cached, skipping.")
+            logger.info("Stage is cached, skipping")
             return None
 
     return stage
@@ -93,7 +97,8 @@ class Stage(params.StageParams):
         deps=None,
         outs=None,
         md5=None,
-        locked=False,
+        locked=False,  # backward compatibility
+        frozen=False,
         always_changed=False,
         stage_text=None,
         dvcfile=None,
@@ -110,7 +115,7 @@ class Stage(params.StageParams):
         self.outs = outs
         self.deps = deps
         self.md5 = md5
-        self.locked = locked
+        self.frozen = locked or frozen
         self.always_changed = always_changed
         self._stage_text = stage_text
         self._dvcfile = dvcfile
@@ -144,14 +149,10 @@ class Stage(params.StageParams):
         self._dvcfile = dvcfile
 
     def __repr__(self):
-        return "Stage: '{path}'".format(
-            path=self.path_in_repo if self.path else "No path"
-        )
+        return f"Stage: '{self.addressing}'"
 
     def __str__(self):
-        return "stage: '{path}'".format(
-            path=self.relpath if self.path else "No path"
-        )
+        return f"stage: '{self.addressing}'"
 
     @property
     def addressing(self):
@@ -159,7 +160,7 @@ class Stage(params.StageParams):
         Useful for alternative presentations where we don't need
         `Stage:` prefix.
         """
-        return self.relpath
+        return self.relpath if self.path else "No path"
 
     def __hash__(self):
         return hash(self.path_in_repo)
@@ -205,14 +206,15 @@ class Stage(params.StageParams):
         return isinstance(self.deps[0], dependency.RepoDependency)
 
     def changed_deps(self):
-        if self.locked:
+        if self.frozen:
             return False
 
         if self.is_callback:
-            logger.warning(
-                '{stage} is a "callback" stage '
+            logger.debug(
+                '%s is a "callback" stage '
                 "(has a command and no dependencies) and thus always "
-                "considered as changed.".format(stage=self)
+                "considered as changed.",
+                self,
             )
             return True
 
@@ -248,9 +250,9 @@ class Stage(params.StageParams):
 
         return False
 
-    def changed_stage(self, warn=False):
+    def changed_stage(self):
         changed = self.md5 != self.compute_md5()
-        if changed and warn:
+        if changed:
             logger.debug(self._changed_stage_entry())
         return changed
 
@@ -259,14 +261,12 @@ class Stage(params.StageParams):
         is_changed = (
             # Short-circuit order: stage md5 is fast,
             # deps are expected to change
-            self.changed_stage(warn=True)
+            self.changed_stage()
             or self.changed_deps()
             or self.changed_outs()
         )
         if is_changed:
-            logger.info("%s changed.", self)
-        else:
-            logger.info("%s didn't change.", self)
+            logger.debug("%s changed.", self)
         return is_changed
 
     @rwlocked(write=["outs"])
@@ -285,17 +285,18 @@ class Stage(params.StageParams):
             out.unprotect()
 
     @rwlocked(write=["outs"])
-    def remove(self, force=False, remove_outs=True):
+    def remove(self, force=False, remove_outs=True, purge=True):
         if remove_outs:
             self.remove_outs(ignore_remove=True, force=force)
         else:
             self.unprotect_outs()
-        self.dvcfile.remove()
+        if purge:
+            self.dvcfile.remove_stage(self)
 
     @rwlocked(read=["deps"], write=["outs"])
     def reproduce(self, interactive=False, **kwargs):
-
         if not (kwargs.get("force", False) or self.changed()):
+            logger.info("Stage '%s' didn't change, skipping", self.addressing)
             return None
 
         msg = (
@@ -356,7 +357,11 @@ class Stage(params.StageParams):
         return get_dump(self)
 
     def compute_md5(self):
-        m = compute_md5(self)
+        # `dvc add`ed files don't need stage md5
+        if self.is_data_source and not (self.is_import or self.is_repo_import):
+            m = None
+        else:
+            m = compute_md5(self)
         logger.debug(f"Computed {self} md5: '{m}'")
         return m
 
@@ -375,6 +380,10 @@ class Stage(params.StageParams):
         for out in self.outs:
             out.save()
 
+    def ignore_outs(self):
+        for out in self.outs:
+            out.ignore()
+
     @staticmethod
     def _changed_entries(entries):
         return [str(entry) for entry in entries if entry.changed_checksum()]
@@ -398,16 +407,16 @@ class Stage(params.StageParams):
 
     @rwlocked(read=["deps"], write=["outs"])
     def run(self, dry=False, no_commit=False, force=False, run_cache=True):
-        if (self.cmd or self.is_import) and not self.locked and not dry:
+        if (self.cmd or self.is_import) and not self.frozen and not dry:
             self.remove_outs(ignore_remove=False, force=False)
 
-        if not self.locked and self.is_import:
+        if not self.frozen and self.is_import:
             sync_import(self, dry, force)
-        elif not self.locked and self.cmd:
+        elif not self.frozen and self.cmd:
             run_stage(self, dry, force, run_cache)
         else:
             args = (
-                ("outputs", "locked ") if self.locked else ("data sources", "")
+                ("outputs", "frozen ") if self.frozen else ("data sources", "")
             )
             logger.info("Verifying %s in %s%s", *args, self)
             if not dry:
@@ -449,7 +458,7 @@ class Stage(params.StageParams):
         ret = []
         show_import = self.is_repo_import and check_updates
 
-        if not self.locked or show_import:
+        if not self.frozen or show_import:
             self._status_deps(ret)
         self._status_outs(ret)
         self._status_always_changed(ret)
@@ -523,19 +532,13 @@ class PipelineStage(Stage):
     def __hash__(self):
         return hash((self.path_in_repo, self.name))
 
-    def __repr__(self):
-        return "Stage: '{path}:{name}'".format(
-            path=self.relpath if self.path else "No path", name=self.name
-        )
-
-    def __str__(self):
-        return "stage: '{path}:{name}'".format(
-            path=self.relpath if self.path else "No path", name=self.name
-        )
-
     @property
     def addressing(self):
-        return super().addressing + ":" + self.name
+        from dvc.dvcfile import PIPELINE_FILE
+
+        if self.path and self.relpath == PIPELINE_FILE:
+            return self.name
+        return f"{super().addressing}:{self.name}"
 
     def reload(self):
         return self.dvcfile.stages[self.name]
@@ -548,8 +551,8 @@ class PipelineStage(Stage):
         if self.cmd_changed:
             ret.append("changed command")
 
-    def changed_stage(self, warn=False):
-        if self.cmd_changed and warn:
+    def changed_stage(self):
+        if self.cmd_changed:
             logger.debug(self._changed_stage_entry())
         return self.cmd_changed
 

dvc/stage/cache.py

@@ -11,7 +11,7 @@ from dvc.serialize import to_single_stage_lockfile
 from dvc.stage.loader import StageLoader
 from dvc.utils import dict_sha256, relpath
 from dvc.utils.fs import makedirs
-from dvc.utils.stage import dump_stage_file
+from dvc.utils.yaml import dump_yaml
 
 logger = logging.getLogger(__name__)
 
@@ -77,16 +77,32 @@ class StageCache:
 
         return None
 
-    def _create_stage(self, cache):
+    def _create_stage(self, cache, wdir=None):
         from dvc.stage import create_stage, PipelineStage
 
+        params = []
+        for param in cache.get("params", []):
+            if isinstance(param, str):
+                params.append(param)
+                continue
+
+            assert isinstance(param, dict)
+            assert len(param) == 1
+            path = list(param.keys())[0]
+            params_list = param[path]
+            assert isinstance(params_list, list)
+            params.append(f"{path}:" + ",".join(params_list))
+
         stage = create_stage(
             PipelineStage,
             repo=self.repo,
             path="dvc.yaml",
             cmd=cache["cmd"],
-            deps=[dep["path"] for dep in cache["deps"]],
+            wdir=wdir,
+            params=params,
+            deps=[dep["path"] for dep in cache.get("deps", [])],
             outs=[out["path"] for out in cache["outs"]],
+            external=True,
         )
         StageLoader.fill_from_lock(stage, cache)
         return stage
@@ -104,7 +120,7 @@ class StageCache:
         # NOTE: using temporary stage to avoid accidentally modifying original
         # stage and to workaround `commit/checkout` not working for uncached
         # outputs.
-        cached_stage = self._create_stage(cache)
+        cached_stage = self._create_stage(cache, wdir=stage.wdir)
 
         outs_no_cache = [
             out.def_path for out in stage.outs if not out.use_cache
@@ -139,7 +155,7 @@ class StageCache:
         path = self._get_cache_path(cache_key, cache_value)
         dpath = os.path.dirname(path)
         makedirs(dpath, exist_ok=True)
-        dump_stage_file(path, cache)
+        dump_yaml(path, cache)
 
     def is_cached(self, stage):
         return bool(self._load(stage))

dvc/stage/exceptions.py

@@ -15,15 +15,8 @@ class StageFileFormatError(DvcException):
 
 class StageFileDoesNotExistError(DvcException):
     def __init__(self, fname):
-        from dvc.dvcfile import DVC_FILE_SUFFIX, is_dvc_file
-
-        msg = f"'{fname}' does not exist."
-
-        sname = fname + DVC_FILE_SUFFIX
-        if is_dvc_file(sname):
-            msg += f" Do you mean '{sname}'?"
-
-        super().__init__(msg)
+        self.file = fname
+        super().__init__(f"'{self.file}' does not exist.")
 
 
 class StageFileAlreadyExistsError(DvcException):
@@ -65,6 +58,10 @@ class StageCommitError(DvcException):
     pass
 
 
+class StageExternalOutputsError(DvcException):
+    pass
+
+
 class StageUpdateError(DvcException):
     def __init__(self, path):
         super().__init__(
@@ -87,8 +84,10 @@ class MissingDataSource(DvcException):
 
 class StageNotFound(KeyError, DvcException):
     def __init__(self, file, name):
+        self.file = file.relpath
+        self.name = name
         super().__init__(
-            f"Stage '{name}' not found inside '{file.relpath}' file"
+            f"Stage '{self.name}' not found inside '{self.file}' file"
         )
 
 

dvc/stage/imports.py

@@ -5,12 +5,12 @@ logger = logging.getLogger(__name__)
 
 def update_import(stage, rev=None):
     stage.deps[0].update(rev=rev)
-    locked = stage.locked
-    stage.locked = False
+    frozen = stage.frozen
+    stage.frozen = False
     try:
         stage.reproduce()
     finally:
-        stage.locked = locked
+        stage.frozen = frozen
 
 
 def sync_import(stage, dry=False, force=False):
@@ -23,11 +23,7 @@ def sync_import(stage, dry=False, force=False):
     if dry:
         return
 
-    if (
-        not force
-        and not stage.changed_stage(warn=True)
-        and stage.already_cached()
-    ):
+    if not force and not stage.changed_stage() and stage.already_cached():
         stage.outs[0].checkout()
     else:
         stage.deps[0].download(stage.outs[0])

dvc/stage/loader.py

@@ -30,16 +30,6 @@ class StageLoader(Mapping):
         self.stages_data = stages_data or {}
         self.lockfile_data = lockfile_data or {}
 
-    def filter(self, item=None):
-        if not item:
-            return self
-
-        if item not in self:
-            raise StageNotFound(self.dvcfile, item)
-
-        data = {item: self.stages_data[item]} if item in self else {}
-        return self.__class__(self.dvcfile, data, self.lockfile_data)
-
     @staticmethod
     def fill_from_lock(stage, lock_data):
         from .params import StageParams
@@ -224,9 +214,6 @@ class SingleStageLoader(Mapping):
         self.stage_data = stage_data or {}
         self.stage_text = stage_text
 
-    def filter(self, item=None):
-        return self
-
     def __getitem__(self, item):
         if item:
             logger.warning(

dvc/stage/params.py

@@ -4,7 +4,8 @@ class StageParams:
     PARAM_WDIR = "wdir"
     PARAM_DEPS = "deps"
     PARAM_OUTS = "outs"
-    PARAM_LOCKED = "locked"
+    PARAM_LOCKED = "locked"  # backward compatibility
+    PARAM_FROZEN = "frozen"
     PARAM_META = "meta"
     PARAM_ALWAYS_CHANGED = "always_changed"
     PARAM_PARAMS = "params"

dvc/stage/run.py

@@ -79,11 +79,15 @@ def cmd_run(stage, *args, **kwargs):
 
 
 def _is_cached(stage):
-    return (
+    cached = (
         not stage.is_callback
         and not stage.always_changed
+        and not stage.changed_stage()
         and stage.already_cached()
     )
+    if cached:
+        logger.info("Stage '%s' is cached", stage.addressing)
+    return cached
 
 
 def restored_from_cache(stage):
@@ -93,7 +97,10 @@ def restored_from_cache(stage):
         return False
     # restore stage from build cache
     stage_cache.restore(stage)
-    return stage.outs_cached()
+    restored = stage.outs_cached()
+    if restored:
+        logger.info("Restored stage '%s' from run-cache", stage.addressing)
+    return restored
 
 
 def run_stage(stage, dry=False, force=False, run_cache=False):
@@ -102,10 +109,16 @@ def run_stage(stage, dry=False, force=False, run_cache=False):
             run_cache and restored_from_cache(stage)
         )
         if stage_cached:
-            logger.info("Stage is cached, skipping.")
+            logger.info("Skipping run, checking out outputs")
             stage.checkout()
             return
 
-    logger.info("Running command:\n\t%s", stage.cmd)
+    callback_str = "callback " if stage.is_callback else ""
+    logger.info(
+        "Running %s" "stage '%s' with command:",
+        callback_str,
+        stage.addressing,
+    )
+    logger.info("\t%s", stage.cmd)
     if not dry:
         cmd_run(stage)

dvc/stage/utils.py

@@ -6,9 +6,10 @@ from dvc import dependency, output
 from dvc.utils.fs import path_isin
 
 from ..remote import LocalRemote, S3Remote
-from ..utils import dict_md5, relpath
+from ..utils import dict_md5, format_link, relpath
 from .exceptions import (
     MissingDataSource,
+    StageExternalOutputsError,
     StagePathNotDirectoryError,
     StagePathNotFoundError,
     StagePathOutsideError,
@@ -68,6 +69,31 @@ def fill_stage_dependencies(stage, deps=None, erepo=None, params=None):
     stage.deps += dependency.loads_params(stage, params or [])
 
 
+def check_no_externals(stage):
+    from urllib.parse import urlparse
+
+    # NOTE: preventing users from accidentally using external outputs. See
+    # https://github.com/iterative/dvc/issues/1545 for more details.
+
+    def _is_external(out):
+        # NOTE: in case of `remote://` notation, the user clearly knows that
+        # this is an advanced feature and so we shouldn't error-out.
+        if out.is_in_repo or urlparse(out.def_path).scheme == "remote":
+            return False
+        return True
+
+    outs = [str(out) for out in stage.outs if _is_external(out)]
+    if not outs:
+        return
+
+    str_outs = ", ".join(outs)
+    link = format_link("https://dvc.org/doc/user-guide/managing-external-data")
+    raise StageExternalOutputsError(
+        f"Output(s) outside of DVC project: {str_outs}. "
+        f"See {link} for more info."
+    )
+
+
 def check_circular_dependency(stage):
     from dvc.exceptions import CircularDependencyError
 
@@ -136,7 +162,8 @@ def compute_md5(stage):
     return dict_md5(
         d,
         exclude=[
-            stage.PARAM_LOCKED,
+            stage.PARAM_LOCKED,  # backward compatibility
+            stage.PARAM_FROZEN,
             BaseOutput.PARAM_METRIC,
             BaseOutput.PARAM_PERSIST,
         ],
@@ -155,7 +182,7 @@ def get_dump(stage):
             stage.PARAM_MD5: stage.md5,
             stage.PARAM_CMD: stage.cmd,
             stage.PARAM_WDIR: resolve_wdir(stage.wdir, stage.path),
-            stage.PARAM_LOCKED: stage.locked,
+            stage.PARAM_FROZEN: stage.frozen,
             stage.PARAM_DEPS: [d.dumpd() for d in stage.deps],
             stage.PARAM_OUTS: [o.dumpd() for o in stage.outs],
             stage.PARAM_ALWAYS_CHANGED: stage.always_changed,

dvc/state.py

@@ -7,6 +7,7 @@ import sqlite3
 from urllib.parse import urlencode, urlunparse
 
 from dvc.exceptions import DvcException
+from dvc.scm.tree import WorkingTree
 from dvc.utils import current_timestamp, relpath, to_chunks
 from dvc.utils.fs import get_inode, get_mtime_and_size, remove
 
@@ -91,6 +92,7 @@ class State:  # pylint: disable=too-many-instance-attributes
     def __init__(self, repo):
         self.repo = repo
         self.root_dir = repo.root_dir
+        self.tree = WorkingTree(self.root_dir)
 
         state_config = repo.config.get("state", {})
         self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
@@ -125,7 +127,7 @@ class State:  # pylint: disable=too-many-instance-attributes
         self.dump()
 
     def _execute(self, cmd, parameters=()):
-        logger.debug(cmd)
+        logger.trace(cmd)
         return self.cursor.execute(cmd, parameters)
 
     def _fetchall(self):
@@ -364,9 +366,7 @@ class State:  # pylint: disable=too-many-instance-attributes
         assert checksum is not None
         assert os.path.exists(path_info)
 
-        actual_mtime, actual_size = get_mtime_and_size(
-            path_info, self.repo.tree
-        )
+        actual_mtime, actual_size = get_mtime_and_size(path_info, self.tree)
         actual_inode = get_inode(path_info)
 
         existing_record = self.get_state_record_for_inode(actual_inode)
@@ -397,7 +397,7 @@ class State:  # pylint: disable=too-many-instance-attributes
         if not os.path.exists(path):
             return None
 
-        actual_mtime, actual_size = get_mtime_and_size(path, self.repo.tree)
+        actual_mtime, actual_size = get_mtime_and_size(path, self.tree)
         actual_inode = get_inode(path)
 
         existing_record = self.get_state_record_for_inode(actual_inode)
@@ -423,7 +423,7 @@ class State:  # pylint: disable=too-many-instance-attributes
         if not os.path.exists(path_info):
             return
 
-        mtime, _ = get_mtime_and_size(path_info, self.repo.tree)
+        mtime, _ = get_mtime_and_size(path_info, self.tree)
         inode = get_inode(path_info)
         relative_path = relpath(path_info, self.root_dir)
 
@@ -450,7 +450,7 @@ class State:  # pylint: disable=too-many-instance-attributes
                 continue
 
             actual_inode = get_inode(path)
-            actual_mtime, _ = get_mtime_and_size(path, self.repo.tree)
+            actual_mtime, _ = get_mtime_and_size(path, self.tree)
 
             if (inode, mtime) == (actual_inode, actual_mtime):
                 logger.debug("Removing '%s' as unused link.", path)

dvc/utils/__init__.py

@@ -52,18 +52,14 @@ def file_md5(fname, tree=None):
         exists_func = tree.exists
         stat_func = tree.stat
         open_func = tree.open
-        # assume we don't need to run dos2unix when comparing git blobs
-        binary = True
     else:
         exists_func = os.path.exists
         stat_func = os.stat
         open_func = open
-        binary = False
 
     if exists_func(fname):
         hash_md5 = hashlib.md5()
-        if not binary:
-            binary = not istextfile(fname)
+        binary = not istextfile(fname, tree=tree)
         size = stat_func(fname).st_size
         no_progress_bar = True
         if size >= LARGE_FILE_SIZE:
@@ -399,7 +395,7 @@ def format_link(link):
 
 
 def parse_target(target, default=None):
-    from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
+    from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK, is_valid_filename
     from dvc.exceptions import DvcException
 
     if not target:
@@ -413,13 +409,19 @@ def parse_target(target, default=None):
         match.group("path"),
         match.group("name"),
     )
+    if path:
+        if os.path.basename(path) == PIPELINE_LOCK:
+            raise DvcException(
+                "Did you mean: `{}`?".format(
+                    target.replace(".lock", ".yaml", 1)
+                )
+            )
+        if not name:
+            ret = (target, None)
+            return ret if is_valid_filename(target) else ret[::-1]
+
     if not path:
         path = default or PIPELINE_FILE
         logger.debug("Assuming file to be '%s'", path)
 
-    if os.path.basename(path) == PIPELINE_LOCK:
-        raise DvcException(
-            "Did you mean: `{}`?".format(target.replace(".lock", ".yaml", 1))
-        )
-
     return path, name

dvc/utils/pager.py

@@ -0,0 +1,48 @@
+"""Draws DAG in ASCII."""
+
+import logging
+import os
+import pydoc
+import sys
+
+from dvc.env import DVC_PAGER
+from dvc.utils import format_link
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_PAGER = "less"
+DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
+    DEFAULT_PAGER
+)
+
+
+def make_pager(cmd):
+    def _pager(text):
+        return pydoc.tempfilepager(pydoc.plain(text), cmd)
+
+    return _pager
+
+
+def find_pager():
+    if not sys.stdout.isatty():
+        return pydoc.plainpager
+
+    env_pager = os.getenv(DVC_PAGER)
+    if env_pager:
+        return make_pager(env_pager)
+
+    if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
+        return make_pager(DEFAULT_PAGER_FORMATTED)
+
+    logger.warning(
+        "Unable to find `less` in the PATH. Check out "
+        "{} for more info.".format(
+            format_link("https://man.dvc.org/pipeline/show")
+        )
+    )
+    return pydoc.plainpager
+
+
+def pager(text):
+    find_pager()(text)

dvc/utils/yaml.py

@@ -4,7 +4,7 @@ import yaml
 from ruamel.yaml import YAML
 from ruamel.yaml.error import YAMLError
 
-from dvc.exceptions import StageFileCorruptedError
+from dvc.exceptions import YAMLFileCorruptedError
 
 try:
     from yaml import CSafeLoader as SafeLoader
@@ -12,35 +12,35 @@ except ImportError:
     from yaml import SafeLoader
 
 
-def load_stage_file(path):
+def load_yaml(path):
     with open(path, encoding="utf-8") as fd:
-        return parse_stage(fd.read(), path)
+        return parse_yaml(fd.read(), path)
 
 
-def parse_stage(text, path):
+def parse_yaml(text, path):
     try:
         return yaml.load(text, Loader=SafeLoader) or {}
     except yaml.error.YAMLError as exc:
-        raise StageFileCorruptedError(path) from exc
+        raise YAMLFileCorruptedError(path) from exc
 
 
-def parse_stage_for_update(text, path):
+def parse_yaml_for_update(text, path):
     """Parses text into Python structure.
 
-    Unlike `parse_stage()` this returns ordered dicts, values have special
+    Unlike `parse_yaml()` this returns ordered dicts, values have special
     attributes to store comments and line breaks. This allows us to preserve
     all of those upon dump.
 
-    This one is, however, several times slower than simple `parse_stage()`.
+    This one is, however, several times slower than simple `parse_yaml()`.
     """
     try:
         yaml = YAML()
         return yaml.load(text) or {}
     except YAMLError as exc:
-        raise StageFileCorruptedError(path) from exc
+        raise YAMLFileCorruptedError(path) from exc
 
 
-def dump_stage_file(path, data):
+def dump_yaml(path, data):
     with open(path, "w", encoding="utf-8") as fd:
         yaml = YAML()
         yaml.default_flow_style = False

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "1.0.0a3"
+_BASE_VERSION = "1.0.0a9"
 
 
 def _generate_version(base_version):

scripts/ci/before_install.sh

@@ -15,7 +15,7 @@ fi
 
 echo >env.sh
 
-if [[ "$TRAVIS_BUILD_STAGE_NAME" == "Test" ]]; then
+if [[ "$TRAVIS_BUILD_STAGE_NAME" == "test" ]]; then
   if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then
     # NOTE: ssh keys for ssh test to be able to ssh to the localhost
     ssh-keygen -t rsa -N "" -f mykey
@@ -52,7 +52,11 @@ elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
 fi
 
 if [[ -n "$TRAVIS_TAG" ]]; then
-  echo "export SNAP_CHANNEL=stable" >>env.sh
+  if [[ $(echo "$TRAVIS_TAG" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$') ]]; then
+    echo "export SNAP_CHANNEL=stable" >>env.sh
+  else
+    echo "export SNAP_CHANNEL=beta" >>env.sh
+  fi
 else
   echo "export SNAP_CHANNEL=edge" >>env.sh
 fi

scripts/ci/check_patch.sh

@@ -3,7 +3,7 @@
 set -x
 set -e
 
-pip install Pygments collective.checkdocs pre-commit
+pip install Pygments collective.checkdocs pre-commit .
 
 # stop the build if there are any readme formatting errors
 python setup.py checkdocs

scripts/ci/install.sh

@@ -10,6 +10,8 @@ scriptdir="$(dirname $0)"
 # work or not.
 $scriptdir/retry.sh pip install .[all,tests]
 
+$scriptdir/retry.sh pip install git+https://github.com/iterative/mock-ssh-server.git
+
 git config --global user.email "[email protected]"
 git config --global user.name "DVC Tester"
 

scripts/ci/install_azurite.sh

@@ -3,13 +3,6 @@
 set -e
 set -x
 
-# install docker
-export DEBIAN_FRONTEND=noninteractive
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-sudo apt-get update
-sudo apt-get install -y docker-ce
-
 # run azurite
 sudo docker run -d --restart always -e executable=blob -p 10000:10000 \
   --tmpfs /opt/azurite/folder \

scripts/completion/dvc.bash

@@ -4,14 +4,15 @@
 #   - https://opensource.com/article/18/3/creating-bash-completion-script
 #   - https://stackoverflow.com/questions/12933362
 
-_dvc_commands='add cache checkout commit config destroy diff fetch get-url get gc \
-  import-url import init install lock list metrics move params pipeline plots pull push \
-  remote remove repro root run status unlock unprotect update version'
+_dvc_commands='add cache checkout commit config dag destroy diff fetch freeze \
+  get-url get gc import-url import init install list metrics move params \
+  plots pull push remote remove repro root run status unfreeze unprotect \
+  update version'
 
 _dvc_options='-h --help -V --version'
 _dvc_global_options='-h --help -q --quiet -v --verbose'
 
-_dvc_add='-R --recursive -f --file --no-commit'
+_dvc_add='-R --recursive -f --file --no-commit --external'
 _dvc_add_COMPGEN=_dvc_compgen_files
 _dvc_cache='dir'
 _dvc_cache_dir=' --global --system --local -u --unset'
@@ -20,6 +21,8 @@ _dvc_checkout_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_commit='-f --force -d --with-deps -R --recursive'
 _dvc_commit_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_config='-u --unset --local --system --global'
+_dvc_dag='--dot --full'
+_dvc_dag_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_destroy='-f --force'
 _dvc_diff='-t --show-json --show-hash --show-md'
 _dvc_fetch='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
@@ -31,51 +34,43 @@ _dvc_import='-o --out --rev'
 _dvc_import_url='-f --file'
 _dvc_init='--no-scm -f --force'
 _dvc_install=''
-_dvc_list='-R --recursive --outs-only --rev'
+_dvc_list='-R --recursive --dvc-only --rev'
 _dvc_list_COMPGEN=_dvc_compgen_files
-_dvc_lock=''
-_dvc_lock_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_metrics='add diff modify remove show'
-_dvc_metrics_add='-t --type -x --xpath'
-_dvc_metrics_add_COMPGEN=_dvc_compgen_files
-_dvc_metrics_diff='--targets -t --type -x --xpath -R --show-json --show-md'
-_dvc_metrics_modify='-t --type -x --xpath'
-_dvc_metrics_modify_COMPGEN=_dvc_compgen_files
-_dvc_metrics_remove=''
-_dvc_metrics_remove_COMPGEN=_dvc_compgen_files
+_dvc_freeze=''
+_dvc_freeze_COMPGEN=_dvc_compgen_DVCFiles
+_dvc_metrics='diff show'
+_dvc_metrics_diff='--targets -t --type -x --xpath -R --show-json --show-md --no-path --old'
 _dvc_metrics_show='-t --type -x --xpath -a --all-branches -T --all-tags -R --recursive'
 _dvc_metrics_show_COMPGEN=_dvc_compgen_files
 _dvc_move=''
 _dvc_move_COMPGEN=_dvc_compgen_files
 _dvc_params='diff'
-_dvc_params_diff='--all --show-json --show-md'
-_dvc_pipeline='list show'
-_dvc_pipeline_list=''
-_dvc_pipeline_show='-c --commands -o --outs --ascii --dot --tree -l --locked'
-_dvc_pipeline_show_COMPGEN=_dvc_compgen_DVCFiles
+_dvc_params_diff='--all --show-json --show-md --no-path'
 _dvc_plots='show diff'
-_dvc_plots_show='-t --template -f --file -s --select -x -y --stdout --no-csv-header --no-html --title --xlab --ylab'
-_dvc_plots_diff='-t --template -d --datafile -f --file -s --select -x -y --stdout --no-csv-header --no-html --title --xlab --ylab'
+_dvc_plots_show='-t --template -o --out -x -y --show-vega --no-csv-header --title --x-label --y-label'
+_dvc_plots_diff='-t --template --targets -o --out -x -y --show-vega --no-csv-header --title --x-label --y-label'
+_dvc_plots_modify='-t --template -x -y --no-csv-header --title --x-label --y-label'
 _dvc_pull='-j --jobs -r --remote -a --all-branches -T --all-tags -f --force -d --with-deps -R --recursive'
 _dvc_pull_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_push='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
 _dvc_push_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_remote='add default list modify remove'
+_dvc_remote='add default list modify remove rename'
 _dvc_remote_add='--global --system --local -d --default -f --force'
 _dvc_remote_default='--global --system --local -u --unset'
 _dvc_remote_list='--global --system --local'
 _dvc_remote_modify='--global --system --local -u --unset'
 _dvc_remote_remove='--global --system --local'
+_dvc_remote_rename='--global --system --local'
 _dvc_remove='-o --outs -p --purge -f --force'
 _dvc_remove_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --no-run-cache --force-downstream --no-commit -R --recursive --downstream'
 _dvc_repro_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_root=''
-_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --no-run-cache --no-commit -w --wdir'
+_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --no-run-cache --no-commit -w --wdir --external'
 _dvc_run_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud'
 _dvc_status_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_unlock_COMPGEN=_dvc_compgen_DVCFiles
+_dvc_unfreeze_COMPGEN=_dvc_compgen_DVCFiles
 _dvc_unprotect_COMPGEN=_dvc_compgen_files
 _dvc_update='--rev'
 _dvc_update_COMPGEN=_dvc_compgen_DVCFiles

scripts/completion/dvc.zsh

@@ -17,6 +17,7 @@ _dvc_commands() {
     "checkout:Checkout data files from cache."
     "commit:Save changed data to cache and update DVC-files."
     "config:Get or set config settings."
+    "dag:Visualize DVC project DAG."
     "destroy:Remove DVC-files, local DVC config and data cache."
     "diff:Show added, modified, or deleted data between commits in the DVC repository, or between a commit and the workspace."
     "fetch:Get files or directories tracked by DVC from remote storage into the cache."
@@ -29,10 +30,9 @@ _dvc_commands() {
     "install:Install DVC git hooks into the repository."
     "list:List repository contents, including files and directories tracked by DVC and by Git."
     "lock:Lock DVC-file."
-    "metrics:Commands to add, manage, collect and display metrics."
+    "metrics:Commands to display and compare metrics."
     "move:Rename or move a DVC controlled data file or a directory."
     "params:Commands to display params."
-    "pipeline:Manage pipelines."
     "pull:Pull data files from a DVC remote storage."
     "push:Push data files to a DVC remote storage."
     "plots:Generate plot for metrics structured as JSON, CSV or TSV."
@@ -66,6 +66,7 @@ _dvc_add=(
   {-R,--recursive}"[Recursively add each file under the directory.]"
   "--no-commit[Don't put files/directories into cache.]"
   {-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
+  "--external[Allow targets that are outside of the DVC project.]"
   "1:File:_files"
 )
 
@@ -96,6 +97,12 @@ _dvc_config=(
   {-u,--unset}"[Unset option.]"
 )
 
+_dvc_dag=(
+  "--dot[Print DAG in DOT format.]"
+  "--full[Show full DAG that the target belongs too, instead of showing DAG consisting only of ancestors.]"
+  "1:Stage:"
+)
+
 _dvc_destroy=(
   {-f,--force}"[Force destruction.]"
 )
@@ -132,7 +139,7 @@ _dvc_get=(
 )
 
 _dvc_gc=(
-  {-w,--workspace}"[Keep data files used in the current workspace (working tree).]"
+  {-w,--workspace}"[Keep data files used in the current workspace.]"
   {-a,--all-branches}"[Keep data files for the tips of all Git branches.]"
   "--all-commits[Keep data files for all Git commits.]"
   {-T,--all-tags}"[Keep data files for all Git tags.]"
@@ -163,20 +170,20 @@ _dvc_init=(
 
 _dvc_install=()
 
-_dvc_lock=(
+_dvc_freeze=(
   "*:Stages:_files -g '(*.dvc|Dvcfile)'"
 )
 
 _dvc_list=(
   "--rev[Git revision (e.g. branch, tag, SHA)]:Revision:"
   {-R,--recursive}"[Recursively add each file under the directory.]"
-  "--outs-only[Only outputs DVC-outs.]"
+  "--dvc-only[Only outputs DVC-outs.]"
   "1:URL:"
   "2:Path:"
 )
 
 _dvc_metrics=(
-  "1:Sub command:(add show diff modify remove)"
+  "1:Sub command:(show diff)"
 )
 
 _dvc_move=(
@@ -188,10 +195,6 @@ _dvc_params=(
   "1:Sub command:(diff)"
 )
 
-_dvc_pipeline=(
-  "1:Sub command:(show list)"
-)
-
 _dvc_pull=(
   {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
   {-r,--remote}"[Remote repository to pull from.]:Remote repository:"
@@ -214,11 +217,11 @@ _dvc_push=(
 )
 
 _dvc_plots=(
-  "1:Sub command:(show diff)"
+  "1:Sub command:(show diff modify)"
 )
 
 _dvc_remote=(
-  "1:Sub command:(add default remove modify list)"
+  "1:Sub command:(add default remove modify list rename)"
 )
 
 _dvc_remove=(
@@ -265,6 +268,7 @@ _dvc_run=(
   "--no-commit[Don't put files/directories into cache.]"
   "--outs-persist[Declare output file or directory that will not be removed upon repro.]:Output persistent:_files"
   "--outs-persist-no-cache[Declare output file or directory that will not be removed upon repro (do not put into DVC cache).]:Output persistent regular:_files"
+  "--external[Allow outputs that are outside of the DVC project.]"
 )
 
 _dvc_status=(
@@ -278,7 +282,7 @@ _dvc_status=(
   "*:Stages:_files -g '(*.dvc|Dvcfile)'"
 )
 
-_dvc_unlock=(
+_dvc_unfreeze=(
   "*:Stages:_files -g '(*.dvc|Dvcfile)'"
 )
 
@@ -305,9 +309,11 @@ case $words[1] in
   checkout) _arguments $_dvc_global_options $_dvc_checkout ;;
   commit) _arguments $_dvc_global_options $_dvc_commit ;;
   config) _arguments $_dvc_global_options $_dvc_config ;;
+  dag) _arguments $_dvc_global_options $_dvc_dag ;;
   destroy) _arguments $_dvc_global_options $_dvc_destroy ;;
   diff) _arguments $_dvc_global_options $_dvc_diff ;;
   fetch) _arguments $_dvc_global_options $_dvc_fetch ;;
+  freeze) _arguments $_dvc_global_options $_dvc_freeze ;;
   get-url) _arguments $_dvc_global_options $_dvc_geturl ;;
   get) _arguments $_dvc_global_options $_dvc_get ;;
   gc) _arguments $_dvc_global_options $_dvc_gc ;;
@@ -315,12 +321,10 @@ case $words[1] in
   import) _arguments $_dvc_global_options $_dvc_import ;;
   init) _arguments $_dvc_global_options $_dvc_init ;;
   install) _arguments $_dvc_global_options $_dvc_install ;;
-  lock) _arguments $_dvc_global_options $_dvc_lock ;;
   list) _arguments $_dvc_global_options $_dvc_list ;;
   metrics) _arguments $_dvc_global_options $_dvc_metrics ;;
   move) _arguments $_dvc_global_options $_dvc_move ;;
   params) _arguments $_dvc_global_options $_dvc_params ;;
-  pipeline) _arguments $_dvc_global_options $_dvc_pipeline ;;
   pull) _arguments $_dvc_global_options $_dvc_pull ;;
   push) _arguments $_dvc_global_options $_dvc_push ;;
   plots) _arguments $_dvc_global_options $_dvc_plots ;;
@@ -330,7 +334,7 @@ case $words[1] in
   root) _arguments $_dvc_global_options $_dvc_root ;;
   run) _arguments $_dvc_global_options $_dvc_run ;;
   status) _arguments $_dvc_global_options $_dvc_status ;;
-  unlock) _arguments $_dvc_global_options $_dvc_unlock ;;
+  unfreeze) _arguments $_dvc_global_options $_dvc_unfreeze ;;
   unprotect) _arguments $_dvc_global_options $_dvc_unprotect ;;
   update) _arguments $_dvc_global_options $_dvc_update ;;
 esac

setup.cfg

@@ -12,6 +12,6 @@ select=B,C,E,F,W,T4,B9
 [isort]
 include_trailing_comma=true
 known_first_party=dvc,tests
-known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,mockssh,moto,nanotime,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,tqdm,voluptuous,yaml,zc
+known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,mockssh,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,tqdm,voluptuous,yaml,zc
 line_length=79
 multi_line_output=3

setup.py

@@ -92,12 +92,7 @@ s3 = ["boto3>=1.9.201"]
 azure = ["azure-storage-blob==2.1.0"]
 oss = ["oss2==2.6.1"]
 ssh = ["paramiko>=2.5.0"]
-hdfs = [
-    # pyarrow-0.16.0 import fails on 3.7 (works on 3.6 though)
-    # due to: https://issues.apache.org/jira/browse/ARROW-7852
-    "pyarrow==0.15.1; python_version < '3.8'",
-    "pyarrow==0.16.0; python_version == '3.8'",
-]
+hdfs = ["pyarrow>=0.17.0"]
 # gssapi should not be included in all_remotes, because it doesn't have wheels
 # for linux and mac, so it will fail to compile if user doesn't have all the
 # requirements, including kerberos itself. Once all the wheels are available,
@@ -126,7 +121,7 @@ tests_requirements = [
     "flake8-docstrings",
     "pydocstyle<4.0",
     "jaraco.windows==3.9.2",
-    "mock-ssh-server>=0.6.0",
+    "mock-ssh-server",
     "moto==1.3.14.dev464",
     "rangehttpserver==1.2.0",
     "beautifulsoup4==4.4.0",

tests/conftest.py

@@ -34,7 +34,7 @@ user = "user"
 key_path = os.path.join(here, f"{user}.key")
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def ssh_server():
     users = {user: key_path}
     with mockssh.Server(users) as s:

tests/func/metrics/conftest.py

@@ -0,0 +1 @@
+from tests.func.plots import run_copy_metrics  # noqa: F401