exp pull/push: better result handling in exp sharing(#7448)

fix: #7448
Current we skip on those experiments which already existed on both sides, but
do not exclude them from the result list. And if we have diverged experiment,
pulling/pushing them will stop the current progress and raise an exception, but
didn't give any information on the condition of other experiments.

1. Change the exp pull and push to match the new API in `scmrepo`.
2. Handle sync status after `exp pull/push` finished.
3. Bump scmrepo to 0.0.19

Co-authored-by:

.dvcignore

@@ -0,0 +1,4 @@
+/tests
+/dvc
+/.github
+/bin

.github/workflows/benchmarks.yaml

@@ -14,7 +14,7 @@ jobs:
         uses: styfle/[email protected]
         with:
           access_token: ${{ github.token }}
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v3
         with:
             python-version: 3.7
       - uses: iterative/dvc-bench@main

.github/workflows/packages.yaml

@@ -10,11 +10,11 @@ jobs:
   deb:
     runs-on: ubuntu-18.04
     steps:
-    - uses: actions/checkout@v2.4.0
+    - uses: actions/checkout@v3
       with:
           fetch-depth: 0
     - name: Set up Python 3.9
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v3
       with:
         python-version: 3.9
     - name: Set up Ruby 2.4
@@ -41,11 +41,11 @@ jobs:
   rpm:
     runs-on: ubuntu-18.04
     steps:
-    - uses: actions/checkout@v2.4.0
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - name: Set up Python 3.9
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v3
       with:
         python-version: 3.9
     - name: Set up Ruby 2.4
@@ -72,11 +72,11 @@ jobs:
   osxpkg:
     runs-on: macos-10.15
     steps:
-    - uses: actions/checkout@v2.4.0
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - name: Set up Python 3.9
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v3
       with:
         python-version: 3.9
     - name: Set up Ruby 2.4
@@ -106,11 +106,11 @@ jobs:
   exe:
     runs-on: windows-2019
     steps:
-    - uses: actions/checkout@v2.4.0
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - name: Set up Python 3.9
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v3
       with:
         python-version: 3.9
     - name: Install deps
@@ -136,11 +136,11 @@ jobs:
   pip:
     runs-on: ubuntu-18.04
     steps:
-    - uses: actions/checkout@v2.4.0
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v3
       with:
         python-version: 3.7
     - name: Install

.github/workflows/tests.yaml

@@ -20,16 +20,17 @@ jobs:
       uses: styfle/[email protected]
       with:
         access_token: ${{ github.token }}
-    - uses: actions/checkout@v2.4.0
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v3
       with:
         python-version: 3.8
     - name: Install requirements
       run: |
         pip install --upgrade pip setuptools wheel
+        pip install collective.checkdocs==0.2
         pip install ".[dev]" pre-commit
     - name: Check README
       run: python setup.py checkdocs
@@ -43,11 +44,11 @@ jobs:
         os: [ubuntu-20.04, windows-latest, macos-latest]
         pyv: ["3.7", "3.8", "3.9", "3.10"]
     steps:
-    - uses: actions/checkout@v2.4.0
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - name: Set up Python
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.pyv }}
     - name: get pip cache dir
@@ -56,7 +57,7 @@ jobs:
         echo "::set-output name=dir::$(pip cache dir)"
     - name: set pip cache
       id: pip-cache
-      uses: actions/cache@v2.1.7
+      uses: actions/cache@v3
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.cfg') }}
@@ -67,10 +68,6 @@ jobs:
       run: |
         pip install --upgrade pip setuptools wheel
         pip install -e ".[dev]"
-    - name: setup git
-      run: |
-        git config --global user.email "[email protected]"
-        git config --global user.name "DVC Tester"
     - name: run tests
       timeout-minutes: 40
       run: >-
@@ -78,7 +75,7 @@ jobs:
         --cov-report=xml --cov-report=term
         ${{ env.extra_test_args }}
     - name: upload coverage report
-      uses: codecov/codecov-action@v2.1.0
+      uses: codecov/codecov-action@v3
       with:
         file: ./coverage.xml
         fail_ci_if_error: false

.gitignore

@@ -1,10 +1,13 @@
+__pycache__/
 neatlynx.conf
-neatlynx/__pycache__
 .idea
 /cache
 *.pyc
-.env/
-.env2.7/
+
+.env*/
+.venv
+env/
+venv/
 .python-version
 
 .dvc.conf.lock
@@ -26,6 +29,7 @@ innosetup/config.ini
 
 .coverage
 .coverage.*
+coverage.xml
 
 *.sw?
 
@@ -40,3 +44,9 @@ scripts/ci/gcp-creds.json
 
 *~
 /dvc/_dvc_version.py
+
+.mypy_cache/
+.pytest_cache/
+.nox/
+.tox/
+htmlcov/

.pre-commit-config.yaml

@@ -6,9 +6,9 @@ repos:
       - id: black
         language_version: python3
     repo: https://github.com/ambv/black
-    rev: 21.12b0
+    rev: 22.3.0
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.2.0
     hooks:
       - id: check-toml
       - id: check-yaml

README.rst

@@ -183,8 +183,8 @@ Fedora / CentOS (rpm)
 Comparison to related technologies
 ==================================
 
-#. Data Engineering tools such as `AirFlow <https://airflow.apache.org/>`,
-   `Luigi <https://github.com/spotify/luigi>`, and others - in DVC data,
+#. Data Engineering tools such as `AirFlow <https://airflow.apache.org/>`_,
+   `Luigi <https://github.com/spotify/luigi>`_, and others - in DVC data,
    model and ML pipelines represent a single ML project focused on data
    scientists' experience.  Data engineering tools orchestrate multiple data
    projects and focus on efficient execution. A DVC project can be used from
@@ -212,7 +212,7 @@ Comparison to related technologies
 #. `DVC Studio <https://studio.iterative.ai/>`_ - official online
    platform for DVC projects.  It can be used to manage data and models, run
    and track experiments, and visualize and share results.  Also, it
-   integrates with `CML (CI/CD for ML) <https://cml.dev/>` for training
+   integrates with `CML (CI/CD for ML) <https://cml.dev/>`__ for training
    models in the cloud or Kubernetes.
 
 

dvc/__main__.py

@@ -1,7 +1,7 @@
 """Main entry point for DVC command line tool."""
 import sys
 
-from dvc.main import main
+from dvc.cli import main
 
 if __name__ == "__main__":
     sys.exit(main(sys.argv[1:]))

dvc/_debug.py

@@ -1,10 +1,32 @@
 from contextlib import ExitStack, contextmanager
+from datetime import datetime
 from typing import TYPE_CHECKING, Callable, Union
 
 if TYPE_CHECKING:
     from argparse import Namespace
 
 
+@contextmanager
+def viztracer_profile(
+    path: Union[Callable[[], str], str],
+    max_stack_depth: int = -1,
+):
+    try:
+        import viztracer  # pylint: disable=import-error
+    except ImportError:
+        print("Failed to run profiler, viztracer is not installed")
+        yield
+        return
+
+    tracer = viztracer.VizTracer(max_stack_depth=max_stack_depth)
+
+    tracer.start()
+    yield
+    tracer.stop()
+
+    tracer.save(path() if callable(path) else path)
+
+
 @contextmanager
 def yappi_profile(
     path: Union[Callable[[], str], str] = None, wall_clock: bool = True
@@ -100,12 +122,18 @@ def debugtools(args: "Namespace" = None, **kwargs):
         if kw.get("instrument") or kw.get("instrument_open"):
             stack.enter_context(instrument(kw.get("instrument_open", False)))
         if kw.get("yappi"):
-            from datetime import datetime
-
             output = "callgrind.dvc-{0:%Y%m%d}_{0:%H%M%S}.out"
             stack.enter_context(
                 yappi_profile(path=lambda: output.format(datetime.now()))
             )
+        if kw.get("viztracer") or kw.get("viztracer_depth"):
+            output = "viztracer.dvc-{0:%Y%m%d}_{0:%H%M%S}.json"
+            stack.enter_context(
+                viztracer_profile(
+                    path=lambda: output.format(datetime.now()),
+                    max_stack_depth=kw.get("viztracer_depth") or -1,
+                )
+            )
         yield
 
 
@@ -118,6 +146,10 @@ def add_debugging_flags(parser):
     parser.add_argument(
         "--yappi", action="store_true", default=False, help=SUPPRESS
     )
+    parser.add_argument(
+        "--viztracer", action="store_true", default=False, help=SUPPRESS
+    )
+    parser.add_argument("--viztracer-depth", type=int, help=SUPPRESS)
     parser.add_argument("--cprofile-dump", help=SUPPRESS)
     parser.add_argument(
         "--pdb", action="store_true", default=False, help=SUPPRESS

dvc/api.py

@@ -19,16 +19,17 @@ def get_url(path, repo=None, rev=None, remote=None):
     directory in the remote storage.
     """
     with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
-        fs_path = _repo.fs.path.join(_repo.root_dir, path)
         with reraise(FileNotFoundError, PathMissingError(path, repo)):
-            metadata = _repo.repo_fs.metadata(fs_path)
+            info = _repo.repo_fs.info(path)
 
-        if not metadata.is_dvc:
+        dvc_info = info.get("dvc_info")
+        if not dvc_info:
             raise OutputNotFoundError(path, repo)
 
-        cloud = metadata.repo.cloud
-        md5 = metadata.repo.dvcfs.info(fs_path)["md5"]
-        return cloud.get_url_for(remote, checksum=md5)
+        dvc_repo = info["repo"]
+        md5 = dvc_info["md5"]
+
+        return dvc_repo.cloud.get_url_for(remote, checksum=md5)
 
 
 def open(  # noqa, pylint: disable=redefined-builtin

dvc/cli/__init__.py

@@ -0,0 +1,143 @@
+"""This module provides an entrypoint to the dvc cli and parsing utils."""
+
+import logging
+import sys
+
+# Workaround for CPython bug. See [1] and [2] for more info.
+# [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
+# [2] https://bugs.python.org/issue29288
+
+"".encode("idna")
+
+logger = logging.getLogger("dvc")
+
+
+class DvcParserError(Exception):
+    """Base class for CLI parser errors."""
+
+    def __init__(self):
+        super().__init__("parser error")
+
+
+def parse_args(argv=None):
+    """Parses CLI arguments.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Raises:
+        DvcParserError: raised for argument parsing errors.
+    """
+    from .parser import get_main_parser
+
+    parser = get_main_parser()
+    args = parser.parse_args(argv)
+    args.parser = parser
+    return args
+
+
+def main(argv=None):  # noqa: C901
+    """Main entry point for dvc CLI.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Returns:
+        int: command's return code.
+    """
+    from dvc._debug import debugtools
+    from dvc.config import ConfigError
+    from dvc.exceptions import DvcException, NotDvcRepoError
+    from dvc.logger import FOOTER, disable_other_loggers
+
+    # NOTE: stderr/stdout may be closed if we are running from dvc.daemon.
+    # On Linux we directly call cli.main after double forking and closing
+    # the copied parent's standard file descriptors. If we make any logging
+    # calls in this state it will cause an exception due to writing to a closed
+    # file descriptor.
+    if sys.stderr.closed:  # pylint: disable=using-constant-test
+        logging.disable()
+    elif sys.stdout.closed:  # pylint: disable=using-constant-test
+        logging.disable(logging.INFO)
+
+    args = None
+    disable_other_loggers()
+
+    outerLogLevel = logger.level
+    try:
+        args = parse_args(argv)
+
+        level = None
+        if args.quiet:
+            level = logging.CRITICAL
+        elif args.verbose == 1:
+            level = logging.DEBUG
+        elif args.verbose > 1:
+            level = logging.TRACE
+
+        if level is not None:
+            logger.setLevel(level)
+
+        logger.trace(args)
+
+        if not sys.stdout.closed and not args.quiet:
+            from dvc.ui import ui
+
+            ui.enable()
+
+        with debugtools(args):
+            cmd = args.func(args)
+            ret = cmd.do_run()
+    except ConfigError:
+        logger.exception("configuration error")
+        ret = 251
+    except KeyboardInterrupt:
+        logger.exception("interrupted by the user")
+        ret = 252
+    except NotDvcRepoError:
+        logger.exception("")
+        ret = 253
+    except DvcException:
+        ret = 255
+        logger.exception("")
+    except DvcParserError:
+        ret = 254
+    except Exception as exc:  # noqa, pylint: disable=broad-except
+        # pylint: disable=no-member
+        import errno
+
+        if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+            from dvc.utils import error_link
+
+            logger.exception(
+                "too many open files, please visit "
+                "{} to see how to handle this "
+                "problem".format(error_link("many-files")),
+                extra={"tb_only": True},
+            )
+        else:
+            from dvc.info import get_dvc_info
+
+            logger.exception("unexpected error")
+
+            dvc_info = get_dvc_info()
+            logger.debug("Version info for developers:\n%s", dvc_info)
+
+            logger.info(FOOTER)
+        ret = 255
+
+    try:
+        from dvc import analytics
+
+        if analytics.is_enabled():
+            analytics.collect_and_send_report(args, ret)
+
+        return ret
+    finally:
+        logger.setLevel(outerLogLevel)
+
+        from dvc.external_repo import clean_repos
+
+        # Remove cached repos in the end of the call, these are anonymous
+        # so won't be reused by any other subsequent run anyway.
+        clean_repos()

dvc/cli/command.py

@@ -5,29 +5,6 @@ from abc import ABC, abstractmethod
 logger = logging.getLogger(__name__)
 
 
-def fix_subparsers(subparsers):
-    """Workaround for bug in Python 3. See more info at:
-    https://bugs.python.org/issue16308
-    https://github.com/iterative/dvc/issues/769
-
-    Args:
-        subparsers: subparsers to fix.
-    """
-    subparsers.required = True
-    subparsers.dest = "cmd"
-
-
-def append_doc_link(help_message, path):
-    from dvc.utils import format_link
-
-    if not path:
-        return help_message
-    doc_base = "https://man.dvc.org/"
-    return "{message}\nDocumentation: {link}".format(
-        message=help_message, link=format_link(doc_base + path)
-    )
-
-
 class CmdBase(ABC):
     UNINITIALIZED = False
 

dvc/cli/completion.py

@@ -1,17 +1,6 @@
-import argparse
-import logging
-
 import shtab
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.ui import ui
-
-logger = logging.getLogger(__name__)
-FILE = shtab.FILE
-DIR = shtab.DIRECTORY
-
-PREAMBLE = {
-    "bash": """
+BASH_PREAMBLE = """
 # $1=COMP_WORDS[1]
 _dvc_compgen_DVCFiles() {
   compgen -d -S '/' -- $1  # recurse into subdirs
@@ -30,11 +19,12 @@ _dvc_compgen_stages_and_files() {
 }
 
 _dvc_compgen_exps() {
-    local _dvc_exps=($(dvc exp list -q --all --names-only))
+    local _dvc_exps=($(dvc exp list -q --all-commits --names-only))
     compgen -W "${_dvc_exps[*]}" -- $1
 }
-    """,
-    "zsh": """
+"""
+
+ZSH_PREAMBLE = """
 _dvc_compadd_DVCFiles() {
     _files -g '(*?.dvc|Dvcfile|dvc.yaml)'
 }
@@ -63,49 +53,21 @@ _dvc_compadd_stages_and_files() {
 }
 
 _dvc_compadd_exps() {
-    _describe 'experiments' "($(dvc exp list -q -a --names-only))"
+    _describe 'experiments' "($(dvc exp list -q --all-commits --names-only))"
 }
-    """,
+"""
+
+PREAMBLE = {
+    "bash": BASH_PREAMBLE,
+    "zsh": ZSH_PREAMBLE,
 }
 
+FILE = shtab.FILE
+DIR = shtab.DIRECTORY
 DVC_FILE = {"bash": "_dvc_compgen_DVCFiles", "zsh": "_dvc_compadd_DVCFiles"}
-
 STAGE = {"bash": "_dvc_compgen_stages", "zsh": "_dvc_compadd_stages"}
-
 DVCFILES_AND_STAGE = {
     "bash": "_dvc_compgen_stages_and_files",
     "zsh": "_dvc_compadd_stages_and_files",
 }
-
 EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
-
-
-class CmdCompletion(CmdBaseNoRepo):
-    def run(self):
-        from dvc.cli import get_main_parser
-
-        parser = get_main_parser()
-        shell = self.args.shell
-        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
-        ui.write(script, force=True)
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    COMPLETION_HELP = "Generate shell tab completion."
-    COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
-    completion_parser = subparsers.add_parser(
-        "completion",
-        parents=[parent_parser],
-        description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
-        help=COMPLETION_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    completion_parser.add_argument(
-        "-s",
-        "--shell",
-        help="Shell syntax for completions.",
-        default="bash",
-        choices=["bash", "zsh"],
-    )
-    completion_parser.set_defaults(func=CmdCompletion)

dvc/cli/parser.py

@@ -1,11 +1,10 @@
-"""DVC command line interface"""
+"""Main parser for the dvc cli."""
 import argparse
 import logging
 import os
 import sys
 
-from ._debug import add_debugging_flags
-from .command import (
+from dvc.commands import (
     add,
     cache,
     check_ignore,
@@ -28,7 +27,6 @@ from .command import (
     imp_url,
     init,
     install,
-    live,
     ls,
     machine,
     metrics,
@@ -45,8 +43,8 @@ from .command import (
     update,
     version,
 )
-from .command.base import fix_subparsers
-from .exceptions import DvcParserError
+
+from . import DvcParserError
 
 logger = logging.getLogger(__name__)
 
@@ -87,7 +85,6 @@ COMMANDS = [
     stage,
     experiments,
     check_ignore,
-    live,
     machine,
 ]
 
@@ -96,7 +93,7 @@ def _find_parser(parser, cmd_cls):
     defaults = parser._defaults  # pylint: disable=protected-access
     if not cmd_cls or cmd_cls == defaults.get("func"):
         parser.print_help()
-        raise DvcParserError()
+        raise DvcParserError
 
     actions = parser._actions  # pylint: disable=protected-access
     for action in actions:
@@ -143,6 +140,8 @@ def get_parent_parser():
     When overwriting `-q` or `-v`, you need to instantiate a new object
     in order to prevent some weird behavior.
     """
+    from dvc._debug import add_debugging_flags
+
     parent_parser = argparse.ArgumentParser(add_help=False)
     add_debugging_flags(parent_parser)
     log_level_group = parent_parser.add_mutually_exclusive_group()
@@ -207,23 +206,11 @@ def get_main_parser():
         help="Use `dvc COMMAND --help` for command-specific help.",
     )
 
+    from .utils import fix_subparsers
+
     fix_subparsers(subparsers)
 
     for cmd in COMMANDS:
         cmd.add_parser(subparsers, parent_parser)
 
     return parser
-
-
-def parse_args(argv=None):
-    """Parses CLI arguments.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Raises:
-        dvc.exceptions.DvcParserError: raised for argument parsing errors.
-    """
-    parser = get_main_parser()
-    args = parser.parse_args(argv)
-    return args

dvc/cli/utils.py

@@ -0,0 +1,31 @@
+def fix_subparsers(subparsers):
+    """Workaround for bug in Python 3. See more info at:
+    https://bugs.python.org/issue16308
+    https://github.com/iterative/dvc/issues/769
+
+    Args:
+        subparsers: subparsers to fix.
+    """
+    subparsers.required = True
+    subparsers.dest = "cmd"
+
+
+def append_doc_link(help_message, path):
+    from dvc.utils import format_link
+
+    if not path:
+        return help_message
+    doc_base = "https://man.dvc.org/"
+    return "{message}\nDocumentation: {link}".format(
+        message=help_message, link=format_link(doc_base + path)
+    )
+
+
+def fix_plumbing_subparsers(subparsers):
+    # metavar needs to be explicitly set in order to hide plumbing subcommands
+    # from the 'positional arguments' choices list
+    # see: https://bugs.python.org/issue22848
+    cmds = [
+        cmd for cmd, parser in subparsers.choices.items() if parser.add_help
+    ]
+    subparsers.metavar = "{{{}}}".format(",".join(cmds))

dvc/command/experiments/__init__.py

@@ -1,53 +0,0 @@
-import argparse
-
-from dvc.command.base import append_doc_link, fix_subparsers
-from dvc.command.experiments import (
-    apply,
-    branch,
-    diff,
-    gc,
-    init,
-    ls,
-    pull,
-    push,
-    remove,
-    run,
-    show,
-)
-
-SUB_COMMANDS = [
-    apply,
-    branch,
-    diff,
-    gc,
-    init,
-    ls,
-    pull,
-    push,
-    remove,
-    run,
-    show,
-]
-
-
-def add_parser(subparsers, parent_parser):
-    EXPERIMENTS_HELP = "Commands to run and compare experiments."
-
-    experiments_parser = subparsers.add_parser(
-        "experiments",
-        parents=[parent_parser],
-        aliases=["exp"],
-        description=append_doc_link(EXPERIMENTS_HELP, "exp"),
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        help=EXPERIMENTS_HELP,
-    )
-
-    experiments_subparsers = experiments_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc experiments CMD --help` to display "
-        "command-specific help.",
-    )
-
-    fix_subparsers(experiments_subparsers)
-    for cmd in SUB_COMMANDS:
-        cmd.add_parser(experiments_subparsers, parent_parser)

dvc/command/live.py

@@ -1,95 +0,0 @@
-import argparse
-from pathlib import Path
-
-from dvc.command import completion
-from dvc.command.base import CmdBase, fix_subparsers
-from dvc.render.utils import match_renderers
-from dvc.ui import ui
-
-
-class CmdLive(CmdBase):
-    UNINITIALIZED = True
-
-    def _run(self, target, revs=None):
-        metrics, plots = self.repo.live.show(target=target, revs=revs)
-
-        if plots:
-            html_path = Path.cwd() / (self.args.target + "_html")
-            from dvc.render.utils import render
-
-            renderers = match_renderers(plots, self.repo.plots.templates)
-            index_path = render(self.repo, renderers, metrics, html_path)
-            ui.write(index_path.as_uri())
-            return 0
-        return 1
-
-
-class CmdLiveShow(CmdLive):
-    def run(self):
-        return self._run(self.args.target)
-
-
-class CmdLiveDiff(CmdLive):
-    def run(self):
-        return self._run(self.args.target, self.args.revs)
-
-
-def shared_parent_parser():
-    parent_parser = argparse.ArgumentParser(add_help=False)
-    parent_parser.add_argument(
-        "target", help="Logs dir to produce summary from"
-    ).complete = completion.DIR
-    parent_parser.add_argument(
-        "-o",
-        "--out",
-        default=None,
-        help="Destination path to save plots to",
-        metavar="<path>",
-    ).complete = completion.DIR
-    return parent_parser
-
-
-def add_parser(subparsers, parent_parser):
-    LIVE_DESCRIPTION = (
-        "Commands to visualize and compare dvclive-produced logs."
-    )
-    live_parser = subparsers.add_parser(
-        "live",
-        parents=[parent_parser],
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        description=LIVE_DESCRIPTION,
-    )
-    live_subparsers = live_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc live CMD --help` to display command-specific help.",
-    )
-
-    fix_subparsers(live_subparsers)
-
-    SHOW_HELP = "Visualize dvclive directory content."
-    live_show_parser = live_subparsers.add_parser(
-        "show",
-        parents=[parent_parser, shared_parent_parser()],
-        help=SHOW_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    live_show_parser.set_defaults(func=CmdLiveShow)
-
-    DIFF_HELP = (
-        "Show multiple versions of dvclive data, "
-        "by plotting it in single view."
-    )
-    live_diff_parser = live_subparsers.add_parser(
-        "diff",
-        parents=[parent_parser, shared_parent_parser()],
-        help=DIFF_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    live_diff_parser.add_argument(
-        "--revs",
-        nargs="*",
-        default=None,
-        help="Git revision (e.g. SHA, branch, tag)",
-        metavar="<commit>",
-    )
-    live_diff_parser.set_defaults(func=CmdLiveDiff)

dvc/commands/add.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -101,9 +102,10 @@ def add_parser(subparsers, parent_parser):
         "--jobs",
         type=int,
         help=(
-            "Number of jobs to run simultaneously. "
+            "Only used along with '--to-remote'. "
+            "Number of jobs to run simultaneously "
+            "when pushing data to remote."
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/cache.py

@@ -1,8 +1,8 @@
 import argparse
 
-from dvc.command import completion
-from dvc.command.base import append_doc_link, fix_subparsers
-from dvc.command.config import CmdConfig
+from dvc.cli import completion
+from dvc.cli.utils import append_doc_link, fix_subparsers
+from dvc.commands.config import CmdConfig
 from dvc.ui import ui
 
 
@@ -28,7 +28,7 @@ class CmdCacheDir(CmdConfig):
 
 
 def add_parser(subparsers, parent_parser):
-    from dvc.command.config import parent_config_parser
+    from dvc.commands.config import parent_config_parser
 
     CACHE_HELP = "Manage cache settings."
 

dvc/commands/check_ignore.py

@@ -1,7 +1,8 @@
 import argparse
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
 
 

dvc/commands/checkout.py

@@ -1,8 +1,9 @@
 import argparse
 import operator
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import CheckoutError
 from dvc.ui import ui
 

dvc/commands/commit.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 

dvc/commands/completion.py

@@ -0,0 +1,40 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.completion import PREAMBLE
+from dvc.cli.utils import append_doc_link
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdCompletion(CmdBaseNoRepo):
+    def run(self):
+        import shtab
+
+        shell = self.args.shell
+        parser = self.args.parser
+        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
+        ui.write(script, force=True)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    COMPLETION_HELP = "Generate shell tab completion."
+    COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
+    completion_parser = subparsers.add_parser(
+        "completion",
+        parents=[parent_parser],
+        description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
+        help=COMPLETION_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    completion_parser.add_argument(
+        "-s",
+        "--shell",
+        help="Shell syntax for completions.",
+        default="bash",
+        choices=["bash", "zsh"],
+    )
+    completion_parser.set_defaults(func=CmdCompletion)

dvc/commands/config.py

@@ -2,7 +2,8 @@ import argparse
 import logging
 import os
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)

dvc/commands/daemon.py

@@ -1,5 +1,6 @@
-from dvc.command import completion
-from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.cli import completion
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import fix_subparsers
 
 
 class CmdDaemonBase(CmdBaseNoRepo):

dvc/commands/dag.py

@@ -1,6 +1,7 @@
 import argparse
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
 
 
@@ -27,6 +28,32 @@ def _show_dot(G):
     return dot_file.getvalue()
 
 
+def _show_mermaid(G, markdown: bool = False):
+    from dvc.repo.graph import get_pipelines
+
+    pipelines = get_pipelines(G)
+
+    graph = "flowchart TD"
+
+    total_nodes = 0
+    for pipeline in pipelines:
+        node_ids = {}
+        nodes = sorted(str(x) for x in pipeline.nodes)
+        for node in nodes:
+            total_nodes += 1
+            node_id = f"node{total_nodes}"
+            graph += f"\n\t{node_id}[{node}]"
+            node_ids[node] = node_id
+        edges = sorted((str(a), str(b)) for b, a in pipeline.edges)
+        for a, b in edges:
+            graph += f"\n\t{node_ids[str(a)]}-->{node_ids[str(b)]}"
+
+    if markdown:
+        return f"```mermaid\n{graph}\n```"
+
+    return graph
+
+
 def _collect_targets(repo, target, outs):
     if not target:
         return []
@@ -104,6 +131,8 @@ class CmdDAG(CmdBase):
 
         if self.args.dot:
             ui.write(_show_dot(G))
+        elif self.args.mermaid or self.args.markdown:
+            ui.write(_show_mermaid(G, self.args.markdown))
         else:
             with ui.pager():
                 ui.write(_show_ascii(G))
@@ -126,6 +155,20 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Print DAG with .dot format.",
     )
+    dag_parser.add_argument(
+        "--mermaid",
+        action="store_true",
+        default=False,
+        help="Print DAG with mermaid format.",
+    )
+    dag_parser.add_argument(
+        "--md",
+        "--show-md",
+        action="store_true",
+        default=False,
+        dest="markdown",
+        help="Print DAG with mermaid format wrapped in Markdown block.",
+    )
     dag_parser.add_argument(
         "--full",
         action="store_true",

dvc/commands/data_sync.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -18,7 +19,7 @@ class CmdDataBase(CmdBase):
 
 class CmdDataPull(CmdDataBase):
     def log_summary(self, stats):
-        from dvc.command.checkout import log_changes
+        from dvc.commands.checkout import log_changes
 
         log_changes(stats)
         super().log_summary(stats)
@@ -97,7 +98,7 @@ class CmdDataFetch(CmdDataBase):
 
 
 def shared_parent_parser():
-    from dvc.cli import get_parent_parser
+    from dvc.cli.parser import get_parent_parser
 
     # Parent parser used in pull/push/status
     parent_parser = argparse.ArgumentParser(
@@ -110,7 +111,6 @@ def shared_parent_parser():
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )
@@ -127,7 +127,7 @@ def shared_parent_parser():
 
 
 def add_parser(subparsers, _parent_parser):
-    from dvc.command.status import CmdDataStatus
+    from dvc.commands.status import CmdDataStatus
 
     # Pull
     PULL_HELP = "Download tracked files or directories from remote storage."

dvc/commands/destroy.py

@@ -1,7 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 

dvc/commands/diff.py

@@ -2,8 +2,9 @@ import argparse
 import logging
 import os
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
@@ -127,7 +128,7 @@ class CmdDiff(CmdBase):
             show_hash = self.args.show_hash
             hide_missing = self.args.b_rev or self.args.hide_missing
             if hide_missing:
-                del diff["not in cache"]
+                diff.pop("not in cache", None)
 
             for key, entries in diff.items():
                 entries = sorted(

dvc/commands/experiments/__init__.py

@@ -0,0 +1,101 @@
+import argparse
+
+from dvc.cli.utils import (
+    append_doc_link,
+    fix_plumbing_subparsers,
+    fix_subparsers,
+)
+from dvc.commands.experiments import (
+    apply,
+    branch,
+    diff,
+    exec_run,
+    gc,
+    init,
+    ls,
+    pull,
+    push,
+    remove,
+    run,
+    show,
+)
+
+SUB_COMMANDS = [
+    apply,
+    branch,
+    diff,
+    exec_run,
+    gc,
+    init,
+    ls,
+    pull,
+    push,
+    remove,
+    run,
+    show,
+]
+
+
+def add_parser(subparsers, parent_parser):
+    EXPERIMENTS_HELP = "Commands to run and compare experiments."
+
+    experiments_parser = subparsers.add_parser(
+        "experiments",
+        parents=[parent_parser],
+        aliases=["exp"],
+        description=append_doc_link(EXPERIMENTS_HELP, "exp"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        help=EXPERIMENTS_HELP,
+    )
+
+    experiments_subparsers = experiments_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc experiments CMD --help` to display "
+        "command-specific help.",
+    )
+
+    fix_subparsers(experiments_subparsers)
+    for cmd in SUB_COMMANDS:
+        cmd.add_parser(experiments_subparsers, parent_parser)
+    fix_plumbing_subparsers(experiments_subparsers)
+
+
+def add_rev_selection_flags(
+    experiments_subcmd_parser, command: str, default: bool = True
+):
+    experiments_subcmd_parser.add_argument(
+        "-A",
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help=(
+            f"{command} all experiments in the repository "
+            "(overrides `--rev` and `--num`)."
+        ),
+    )
+    default_msg = " (HEAD by default)" if default else ""
+    msg = (
+        f"{command} experiments derived from the specified `<commit>` as "
+        f"baseline{default_msg}."
+    )
+    experiments_subcmd_parser.add_argument(
+        "--rev",
+        type=str,
+        default=None,
+        help=msg,
+        metavar="<commit>",
+    )
+    experiments_subcmd_parser.add_argument(
+        "-n",
+        "--num",
+        type=int,
+        default=1,
+        dest="num",
+        metavar="<num>",
+        help=(
+            f"{command} experiments from the last `num` commits "
+            "(first parents) starting from the `--rev` baseline. "
+            "Give a negative value to include all first-parent commits "
+            "(similar to `git log -n`)."
+        ),
+    )

dvc/commands/experiments/apply.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 

dvc/commands/experiments/branch.py

@@ -1,7 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 

dvc/commands/experiments/diff.py

@@ -1,9 +1,10 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.metrics import DEFAULT_PRECISION
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.commands.metrics import DEFAULT_PRECISION
 from dvc.exceptions import DvcException
 from dvc.ui import ui
 

dvc/commands/experiments/exec_run.py

@@ -0,0 +1,42 @@
+import logging
+
+from dvc.cli.command import CmdBaseNoRepo
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExecutorRun(CmdBaseNoRepo):
+    """Run an experiment executor."""
+
+    def run(self):
+        from dvc.repo.experiments.executor.base import (
+            BaseExecutor,
+            ExecutorInfo,
+        )
+        from dvc.utils.serialize import load_json
+
+        info = ExecutorInfo.from_dict(load_json(self.args.infofile))
+        BaseExecutor.reproduce(
+            info=info,
+            rev="",
+            queue=None,
+            log_level=logger.getEffectiveLevel(),
+            infofile=self.args.infofile,
+        )
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    EXEC_RUN_HELP = "Run an experiment executor."
+    exec_run_parser = experiments_subparsers.add_parser(
+        "exec-run",
+        parents=[parent_parser],
+        description=EXEC_RUN_HELP,
+        add_help=False,
+    )
+    exec_run_parser.add_argument(
+        "--infofile",
+        help="Path to executor info file",
+        default=None,
+    )
+    exec_run_parser.set_defaults(func=CmdExecutorRun)

dvc/commands/experiments/gc.py

@@ -1,7 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import InvalidArgumentError
 from dvc.ui import ui
 

dvc/commands/experiments/init.py

@@ -1,12 +1,19 @@
 import argparse
 import logging
+from typing import TYPE_CHECKING, List
 
 from funcy import compact
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import InvalidArgumentError
 from dvc.ui import ui
 
+if TYPE_CHECKING:
+    from dvc.dependency import Dependency
+    from dvc.stage import PipelineStage
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -28,10 +35,9 @@ class CmdExperimentsInit(CmdBase):
         "plots": PLOTS,
         "live": DVCLIVE,
     }
-    EXP_LINK = "https://s.dvc.org/g/exp/run"
 
     def run(self):
-        from dvc.command.stage import parse_cmd
+        from dvc.commands.stage import parse_cmd
 
         cmd = parse_cmd(self.args.command)
         if not self.args.interactive and not cmd:
@@ -57,7 +63,7 @@ class CmdExperimentsInit(CmdBase):
             }
         )
 
-        initialized_stage = init(
+        initialized_stage, initialized_deps = init(
             self.repo,
             name=self.args.name,
             type=self.args.type,
@@ -66,38 +72,47 @@ class CmdExperimentsInit(CmdBase):
             interactive=self.args.interactive,
             force=self.args.force,
         )
-
-        text = ui.rich_text.assemble(
-            "\n" if self.args.interactive else "",
-            "Created ",
-            (self.args.name, "bright_blue"),
-            " stage in ",
-            ("dvc.yaml", "green"),
-            ".",
-        )
-        if not self.args.run:
-            text.append_text(
-                ui.rich_text.assemble(
-                    " To run, use ",
-                    ('"dvc exp run"', "green"),
-                    ".\nSee ",
-                    (self.EXP_LINK, "repr.url"),
-                    ".",
-                )
-            )
-
-        ui.write(text, styled=True)
+        self._post_init_display(initialized_stage, initialized_deps)
         if self.args.run:
-            return self.repo.experiments.run(
-                targets=[initialized_stage.addressing]
-            )
-
+            self.repo.experiments.run(targets=[initialized_stage.addressing])
         return 0
 
+    def _post_init_display(
+        self, stage: "PipelineStage", new_deps: List["Dependency"]
+    ) -> None:
+        from dvc.utils import humanize
+
+        path_fmt = "[green]{0}[/green]".format
+        if new_deps:
+            deps_paths = humanize.join(map(path_fmt, new_deps))
+            ui.write(f"Creating dependencies: {deps_paths}", styled=True)
+
+        ui.write(
+            f"Creating [b]{self.args.name}[/b] stage in [green]dvc.yaml[/]",
+            styled=True,
+        )
+        if stage.outs or not self.args.run:
+            # separate the above status-like messages with help/tips section
+            ui.write(styled=True)
+
+        if stage.outs:
+            outs_paths = humanize.join(map(path_fmt, stage.outs))
+            tips = f"Ensure your experiment command creates {outs_paths}."
+            ui.write(tips, styled=True)
+
+        if not self.args.run:
+            ui.write(
+                'You can now run your experiment using [b]"dvc exp run"[/].',
+                styled=True,
+            )
+        else:
+            # separate between `exp.run` output and `dvc exp init` output
+            ui.write(styled=True)
+
 
 def add_parser(experiments_subparsers, parent_parser):
 
-    EXPERIMENTS_INIT_HELP = "Initialize experiments."
+    EXPERIMENTS_INIT_HELP = "Quickly setup any project to use experiments."
     experiments_init_parser = experiments_subparsers.add_parser(
         "init",
         parents=[parent_parser],

dvc/commands/experiments/ls.py

@@ -1,18 +1,20 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsList(CmdBase):
     def run(self):
-        names_only = self.args.names_only
+        name_only = self.args.name_only
         exps = self.repo.experiments.ls(
+            all_commits=self.args.all_commits,
             rev=self.args.rev,
+            num=self.args.num,
             git_remote=self.args.git_remote,
-            all_=self.args.all,
         )
         for baseline in exps:
             tag = self.repo.scm.describe(baseline)
@@ -21,16 +23,17 @@ class CmdExperimentsList(CmdBase):
                 if branch:
                     tag = branch.split("/")[-1]
             name = tag if tag else baseline[:7]
-            if not names_only:
+            if not name_only:
                 print(f"{name}:")
             for exp_name in exps[baseline]:
-                indent = "" if names_only else "\t"
+                indent = "" if name_only else "\t"
                 print(f"{indent}{exp_name}")
 
         return 0
 
 
 def add_parser(experiments_subparsers, parent_parser):
+    from . import add_rev_selection_flags
 
     EXPERIMENTS_LIST_HELP = "List local and remote experiments."
     experiments_list_parser = experiments_subparsers.add_parser(
@@ -40,20 +43,9 @@ def add_parser(experiments_subparsers, parent_parser):
         help=EXPERIMENTS_LIST_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    add_rev_selection_flags(experiments_list_parser, "List")
     experiments_list_parser.add_argument(
-        "--rev",
-        type=str,
-        default=None,
-        help=(
-            "List experiments derived from the specified revision. "
-            "Defaults to HEAD if neither `--rev` nor `--all` are specified."
-        ),
-        metavar="<rev>",
-    )
-    experiments_list_parser.add_argument(
-        "--all", action="store_true", help="List all experiments."
-    )
-    experiments_list_parser.add_argument(
+        "--name-only",
         "--names-only",
         action="store_true",
         help="Only output experiment names (without parent commits).",

dvc/commands/experiments/pull.py

@@ -1,18 +1,33 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import InvalidArgumentError
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsPull(CmdBase):
-    def run(self):
+    def raise_error_if_all_disabled(self):
+        if not any(
+            [self.args.experiment, self.args.all_commits, self.args.rev]
+        ):
+            raise InvalidArgumentError(
+                "Either provide an `experiment` argument, or use the "
+                "`--rev` or `--all-commits` flag."
+            )
 
-        self.repo.experiments.pull(
+    def run(self):
+        self.raise_error_if_all_disabled()
+
+        pulled_exps = self.repo.experiments.pull(
             self.args.git_remote,
             self.args.experiment,
+            all_commits=self.args.all_commits,
+            rev=self.args.rev,
+            num=self.args.num,
             force=self.args.force,
             pull_cache=self.args.pull_cache,
             dvc_remote=self.args.dvc_remote,
@@ -20,10 +35,13 @@ class CmdExperimentsPull(CmdBase):
             run_cache=self.args.run_cache,
         )
 
-        ui.write(
-            f"Pulled experiment '{self.args.experiment}'",
-            f"from Git remote '{self.args.git_remote}'.",
-        )
+        if pulled_exps:
+            ui.write(
+                f"Pulled experiment '{pulled_exps}'",
+                f"from Git remote '{self.args.git_remote}'.",
+            )
+        else:
+            ui.write("No experiments to pull.")
         if not self.args.pull_cache:
             ui.write(
                 "To pull cached outputs for this experiment"
@@ -35,6 +53,8 @@ class CmdExperimentsPull(CmdBase):
 
 
 def add_parser(experiments_subparsers, parent_parser):
+    from . import add_rev_selection_flags
+
     EXPERIMENTS_PULL_HELP = "Pull an experiment from a Git remote."
     experiments_pull_parser = experiments_subparsers.add_parser(
         "pull",
@@ -43,6 +63,7 @@ def add_parser(experiments_subparsers, parent_parser):
         help=EXPERIMENTS_PULL_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    add_rev_selection_flags(experiments_pull_parser, "Pull", False)
     experiments_pull_parser.add_argument(
         "-f",
         "--force",
@@ -87,6 +108,10 @@ def add_parser(experiments_subparsers, parent_parser):
         metavar="<git_remote>",
     )
     experiments_pull_parser.add_argument(
-        "experiment", help="Experiment to pull.", metavar="<experiment>"
+        "experiment",
+        nargs="*",
+        default=None,
+        help="Experiments to pull.",
+        metavar="<experiment>",
     )
     experiments_pull_parser.set_defaults(func=CmdExperimentsPull)

dvc/commands/experiments/push.py

@@ -1,19 +1,35 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import InvalidArgumentError
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsPush(CmdBase):
+    def raise_error_if_all_disabled(self):
+        if not any(
+            [self.args.experiment, self.args.all_commits, self.args.rev]
+        ):
+            raise InvalidArgumentError(
+                "Either provide an `experiment` argument, or use the "
+                "`--rev` or `--all-commits` flag."
+            )
+
     def run(self):
 
-        self.repo.experiments.push(
+        self.raise_error_if_all_disabled()
+
+        pushed_exps = self.repo.experiments.push(
             self.args.git_remote,
             self.args.experiment,
+            all_commits=self.args.all_commits,
+            rev=self.args.rev,
+            num=self.args.num,
             force=self.args.force,
             push_cache=self.args.push_cache,
             dvc_remote=self.args.dvc_remote,
@@ -21,10 +37,13 @@ class CmdExperimentsPush(CmdBase):
             run_cache=self.args.run_cache,
         )
 
-        ui.write(
-            f"Pushed experiment '{self.args.experiment}'"
-            f"to Git remote '{self.args.git_remote}'."
-        )
+        if pushed_exps:
+            ui.write(
+                f"Pushed experiment '{pushed_exps}'"
+                f"to Git remote '{self.args.git_remote}'."
+            )
+        else:
+            ui.write("No experiments to push.")
         if not self.args.push_cache:
             ui.write(
                 "To push cached outputs",
@@ -36,6 +55,8 @@ class CmdExperimentsPush(CmdBase):
 
 
 def add_parser(experiments_subparsers, parent_parser):
+    from . import add_rev_selection_flags
+
     EXPERIMENTS_PUSH_HELP = "Push a local experiment to a Git remote."
     experiments_push_parser = experiments_subparsers.add_parser(
         "push",
@@ -44,6 +65,7 @@ def add_parser(experiments_subparsers, parent_parser):
         help=EXPERIMENTS_PUSH_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    add_rev_selection_flags(experiments_push_parser, "Push", False)
     experiments_push_parser.add_argument(
         "-f",
         "--force",
@@ -88,6 +110,10 @@ def add_parser(experiments_subparsers, parent_parser):
         metavar="<git_remote>",
     )
     experiments_push_parser.add_argument(
-        "experiment", help="Experiment to push.", metavar="<experiment>"
+        "experiment",
+        nargs="*",
+        default=None,
+        help="Experiments to push.",
+        metavar="<experiment>",
     ).complete = completion.EXPERIMENT
     experiments_push_parser.set_defaults(func=CmdExperimentsPush)

dvc/commands/experiments/remove.py

@@ -1,25 +1,49 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsRemove(CmdBase):
+    def raise_error_if_all_disabled(self):
+        if not any(
+            [
+                self.args.experiment,
+                self.args.all_commits,
+                self.args.rev,
+                self.args.queue,
+            ]
+        ):
+            raise InvalidArgumentError(
+                "Either provide an `experiment` argument, or use the "
+                "`--rev` or `--all-commits` flag."
+            )
+
     def run(self):
 
-        self.repo.experiments.remove(
+        self.raise_error_if_all_disabled()
+
+        removed_list = self.repo.experiments.remove(
             exp_names=self.args.experiment,
+            all_commits=self.args.all_commits,
+            rev=self.args.rev,
+            num=self.args.num,
             queue=self.args.queue,
-            clear_all=self.args.all,
-            remote=self.args.git_remote,
+            git_remote=self.args.git_remote,
         )
+        removed = ",".join(removed_list)
+        ui.write(f"Removed experiments: {removed}")
 
         return 0
 
 
 def add_parser(experiments_subparsers, parent_parser):
+    from . import add_rev_selection_flags
 
     EXPERIMENTS_REMOVE_HELP = "Remove experiments."
     experiments_remove_parser = experiments_subparsers.add_parser(
@@ -30,15 +54,10 @@ def add_parser(experiments_subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remove_group = experiments_remove_parser.add_mutually_exclusive_group()
+    add_rev_selection_flags(experiments_remove_parser, "Remove", False)
     remove_group.add_argument(
         "--queue", action="store_true", help="Remove all queued experiments."
     )
-    remove_group.add_argument(
-        "-A",
-        "--all",
-        action="store_true",
-        help="Remove all committed experiments.",
-    )
     remove_group.add_argument(
         "-g",
         "--git-remote",

dvc/commands/experiments/run.py

@@ -1,10 +1,10 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import append_doc_link
-from dvc.command.repro import CmdRepro
-from dvc.command.repro import add_arguments as add_repro_arguments
+from dvc.cli import completion
+from dvc.cli.utils import append_doc_link
+from dvc.commands.repro import CmdRepro
+from dvc.commands.repro import add_arguments as add_repro_arguments
 from dvc.exceptions import InvalidArgumentError
 from dvc.ui import ui
 
@@ -38,7 +38,8 @@ class CmdExperimentsRun(CmdRepro):
             checkpoint_resume=self.args.checkpoint_resume,
             reset=self.args.reset,
             tmp_dir=self.args.tmp_dir,
-            **self._repro_kwargs,
+            machine=self.args.machine,
+            **self._common_kwargs,
         )
 
         if self.args.metrics and results:
@@ -80,7 +81,7 @@ def add_parser(experiments_subparsers, parent_parser):
 
 
 def _add_run_common(parser):
-    """Add common args for 'exp run' and 'exp resume'."""
+    """Add common args for 'exp run'."""
     # inherit arguments from `dvc repro`
     add_repro_arguments(parser)
     parser.add_argument(
@@ -130,3 +131,12 @@ def _add_run_common(parser):
             "your workspace."
         ),
     )
+    parser.add_argument(
+        "--machine",
+        default=None,
+        help=argparse.SUPPRESS,
+        # help=(
+        #     "Run this experiment on the specified 'dvc machine' instance."
+        # )
+        # metavar="<name>",
+    )

dvc/commands/experiments/show.py

@@ -1,16 +1,17 @@
 import argparse
 import logging
+import os
+import re
 from collections import Counter, OrderedDict, defaultdict
 from datetime import date, datetime
-from fnmatch import fnmatch
-from pathlib import Path
-from typing import TYPE_CHECKING, Dict, Iterable, Optional
+from typing import TYPE_CHECKING
 
 from funcy import lmap
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.metrics import DEFAULT_PRECISION
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.commands.metrics import DEFAULT_PRECISION
 from dvc.exceptions import DvcException, InvalidArgumentError
 from dvc.ui import ui
 from dvc.utils.flatten import flatten
@@ -27,64 +28,6 @@ FILL_VALUE_ERRORED = "!"
 logger = logging.getLogger(__name__)
 
 
-def _filter_name(names, label, filter_strs):
-    ret = defaultdict(dict)
-    path_filters = defaultdict(list)
-
-    for filter_s in filter_strs:
-        path, _, name = filter_s.rpartition(":")
-        path_filters[path].append(name)
-
-    for path, filters in path_filters.items():
-        if path:
-            match_paths = [path]
-        else:
-            match_paths = names.keys()
-        for match_path in match_paths:
-            for f in filters:
-                matches = [
-                    name for name in names[match_path] if fnmatch(name, f)
-                ]
-                if not matches:
-                    raise InvalidArgumentError(
-                        f"'{f}' does not match any known {label}"
-                    )
-                ret[match_path].update({match: None for match in matches})
-
-    return ret
-
-
-def _filter_names(
-    names: Dict[str, Dict[str, None]],
-    label: str,
-    include: Optional[Iterable],
-    exclude: Optional[Iterable],
-):
-    if include and exclude:
-        intersection = set(include) & set(exclude)
-        if intersection:
-            values = ", ".join(intersection)
-            raise InvalidArgumentError(
-                f"'{values}' specified in both --include-{label} and"
-                f" --exclude-{label}"
-            )
-
-    if include:
-        ret = _filter_name(names, label, include)
-    else:
-        ret = names
-
-    if exclude:
-        to_remove = _filter_name(names, label, exclude)
-        for path in to_remove:
-            if path in ret:
-                for key in to_remove[path]:
-                    if key in ret[path]:
-                        del ret[path][key]
-
-    return ret
-
-
 def _update_names(names, items):
     for name, item in items:
         item = item.get("data", {})
@@ -96,26 +39,17 @@ def _update_names(names, items):
 def _collect_names(all_experiments, **kwargs):
     metric_names = defaultdict(dict)
     param_names = defaultdict(dict)
+    deps_names = set()
 
     for _, experiments in all_experiments.items():
         for exp_data in experiments.values():
             exp = exp_data.get("data", {})
             _update_names(metric_names, exp.get("metrics", {}).items())
             _update_names(param_names, exp.get("params", {}).items())
-    metric_names = _filter_names(
-        metric_names,
-        "metrics",
-        kwargs.get("include_metrics"),
-        kwargs.get("exclude_metrics"),
-    )
-    param_names = _filter_names(
-        param_names,
-        "params",
-        kwargs.get("include_params"),
-        kwargs.get("exclude_params"),
-    )
+            for dep_name in exp.get("deps", {}):
+                deps_names.add(dep_name)
 
-    return metric_names, param_names
+    return metric_names, param_names, sorted(deps_names)
 
 
 experiment_types = {
@@ -131,8 +65,12 @@ experiment_types = {
 def _collect_rows(
     base_rev,
     experiments,
+    all_headers,
+    metric_headers,
+    param_headers,
     metric_names,
     param_names,
+    deps_names,
     precision=DEFAULT_PRECISION,
     sort_by=None,
     sort_order=None,
@@ -152,14 +90,18 @@ def _collect_rows(
 
     new_checkpoint = True
     for i, (rev, results) in enumerate(experiments.items()):
+        fill_value = FILL_VALUE_ERRORED if results.get("error") else fill_value
+        row_dict = {k: fill_value for k in all_headers}
+
         exp = results.get("data", {})
+
         if exp.get("running"):
             state = "Running"
         elif exp.get("queued"):
             state = "Queued"
         else:
             state = fill_value
-        executor = exp.get("executor", fill_value)
+
         is_baseline = rev == "baseline"
 
         if is_baseline:
@@ -167,9 +109,7 @@ def _collect_rows(
         else:
             name_rev = rev[:7]
 
-        exp_name = exp.get("name", "")
         tip = exp.get("checkpoint_tip")
-
         parent_rev = exp.get("checkpoint_parent", "")
         parent_exp = experiments.get(parent_rev, {}).get("data", {})
         parent_tip = parent_exp.get("checkpoint_tip")
@@ -195,32 +135,38 @@ def _collect_rows(
         if not is_baseline:
             new_checkpoint = not (tip and tip == parent_tip)
 
-        row = [
-            exp_name,
-            name_rev,
-            typ,
-            _format_time(exp.get("timestamp"), fill_value, iso),
-            parent,
-            state,
-            executor,
-        ]
-        fill_value = FILL_VALUE_ERRORED if results.get("error") else fill_value
+        row_dict["Experiment"] = exp.get("name", "")
+        row_dict["rev"] = name_rev
+        row_dict["typ"] = typ
+        row_dict["Created"] = _format_time(
+            exp.get("timestamp"), fill_value, iso
+        )
+        row_dict["parent"] = parent
+        row_dict["State"] = state
+        row_dict["Executor"] = exp.get("executor", fill_value)
+
         _extend_row(
-            row,
+            row_dict,
             metric_names,
+            metric_headers,
             exp.get("metrics", {}).items(),
             precision,
             fill_value=fill_value,
         )
         _extend_row(
-            row,
+            row_dict,
             param_names,
+            param_headers,
             exp.get("params", {}).items(),
             precision,
             fill_value=fill_value,
         )
-
-        yield row
+        for dep in deps_names:
+            hash_info = exp.get("deps", {}).get(dep, {}).get("hash")
+            if hash_info is not None:
+                hash_info = hash_info[:7]
+            row_dict[dep] = hash_info
+        yield list(row_dict.values())
 
 
 def _sort_column(sort_by, metric_names, param_names):
@@ -286,13 +232,9 @@ def _format_time(datetime_obj, fill_value=FILL_VALUE, iso=False):
     return datetime_obj.strftime(fmt)
 
 
-def _extend_row(row, names, items, precision, fill_value=FILL_VALUE):
+def _extend_row(row, names, headers, items, precision, fill_value=FILL_VALUE):
     from dvc.compare import _format_field, with_value
 
-    if not items:
-        row.extend(fill_value for keys in names.values() for _ in keys)
-        return
-
     for fname, data in items:
         item = data.get("data", {})
         item = flatten(item) if isinstance(item, dict) else {fname: item}
@@ -304,18 +246,11 @@ def _extend_row(row, names, items, precision, fill_value=FILL_VALUE):
             # wrap field data in ui.rich_text, otherwise rich may
             # interpret unescaped braces from list/dict types as rich
             # markup tags
-            row.append(ui.rich_text(str(_format_field(value, precision))))
-
-
-def _parse_filter_list(param_list):
-    ret = []
-    for param_str in param_list:
-        path, _, param_str = param_str.rpartition(":")
-        if path:
-            ret.extend(f"{path}:{param}" for param in param_str.split(","))
-        else:
-            ret.extend(param_str.split(","))
-    return ret
+            value = ui.rich_text(str(_format_field(value, precision)))
+            if name in headers:
+                row[name] = value
+            else:
+                row[f"{fname}:{name}"] = value
 
 
 def experiments_table(
@@ -325,6 +260,7 @@ def experiments_table(
     metric_names,
     param_headers,
     param_names,
+    deps_names,
     sort_by=None,
     sort_order=None,
     precision=DEFAULT_PRECISION,
@@ -335,15 +271,18 @@ def experiments_table(
 
     from dvc.compare import TabularData
 
-    td = TabularData(
-        lconcat(headers, metric_headers, param_headers), fill_value=fill_value
-    )
+    all_headers = lconcat(headers, metric_headers, param_headers, deps_names)
+    td = TabularData(all_headers, fill_value=fill_value)
     for base_rev, experiments in all_experiments.items():
         rows = _collect_rows(
             base_rev,
             experiments,
+            all_headers,
+            metric_headers,
+            param_headers,
             metric_names,
             param_names,
+            deps_names,
             sort_by=sort_by,
             sort_order=sort_order,
             precision=precision,
@@ -375,32 +314,22 @@ def prepare_exp_id(kwargs) -> "RichText":
 
 
 def baseline_styler(typ):
-    return {"style": "bold"} if typ == "baseline" else {"style": "default"}
+    return {"style": "bold"} if typ == "baseline" else {}
 
 
 def show_experiments(
     all_experiments,
+    keep=None,
+    drop=None,
     pager=True,
-    no_timestamp=False,
     csv=False,
     markdown=False,
-    html=False,
+    pcp=False,
     **kwargs,
 ):
     from funcy.seqs import flatten as flatten_list
 
-    include_metrics = _parse_filter_list(kwargs.pop("include_metrics", []))
-    exclude_metrics = _parse_filter_list(kwargs.pop("exclude_metrics", []))
-    include_params = _parse_filter_list(kwargs.pop("include_params", []))
-    exclude_params = _parse_filter_list(kwargs.pop("exclude_params", []))
-
-    metric_names, param_names = _collect_names(
-        all_experiments,
-        include_metrics=include_metrics,
-        exclude_metrics=exclude_metrics,
-        include_params=include_params,
-        exclude_params=exclude_params,
-    )
+    metric_names, param_names, deps_names = _collect_names(all_experiments)
 
     headers = [
         "Experiment",
@@ -425,24 +354,23 @@ def show_experiments(
         metric_names,
         param_headers,
         param_names,
+        deps_names,
         kwargs.get("sort_by"),
         kwargs.get("sort_order"),
         kwargs.get("precision"),
         kwargs.get("fill_value"),
         kwargs.get("iso"),
     )
-
-    if no_timestamp or html:
-        td.drop("Created")
+    if keep:
+        for col in td.keys():
+            if re.match(keep, col):
+                td.protect(col)
 
     for col in ("State", "Executor"):
         if td.is_empty(col):
             td.drop(col)
 
     row_styles = lmap(baseline_styler, td.column("typ"))
-    for n, row_style in enumerate(row_styles):
-        if n % 2 != 0:
-            row_style["style"] += " on grey23"
 
     if not csv:
         merge_headers = ["Experiment", "rev", "typ", "parent"]
@@ -451,14 +379,22 @@ def show_experiments(
         )
         td.drop(*merge_headers[1:])
 
-    headers = {"metrics": metric_headers, "params": param_headers}
+    headers = {
+        "metrics": metric_headers,
+        "params": param_headers,
+        "deps": deps_names,
+    }
     styles = {
         "Experiment": {"no_wrap": True, "header_style": "black on grey93"},
         "Created": {"header_style": "black on grey93"},
         "State": {"header_style": "black on grey93"},
         "Executor": {"header_style": "black on grey93"},
     }
-    header_bg_colors = {"metrics": "cornsilk1", "params": "light_cyan1"}
+    header_bg_colors = {
+        "metrics": "cornsilk1",
+        "params": "light_cyan1",
+        "deps": "plum2",
+    }
     styles.update(
         {
             header: {
@@ -472,35 +408,51 @@ def show_experiments(
         }
     )
 
-    if kwargs.get("only_changed", False) or html:
-        td.drop_duplicates("cols")
+    if kwargs.get("only_changed", False) or pcp:
+        td.drop_duplicates("cols", ignore_empty=False)
 
-    html_args = {}
-    if html:
-        td.dropna("rows", how="all")
-        td.column("Experiment")[:] = [
-            # remove tree characters
-            str(x).encode("ascii", "ignore").strip().decode()
-            for x in td.column("Experiment")
-        ]
+    cols_to_drop = set()
+    if drop is not None:
+        cols_to_drop = {col for col in td.keys() if re.match(drop, col)}
+    if pcp:
+        cols_to_drop.add("Created")
+    td.drop(*cols_to_drop)
+
+    if pcp:
+        subset = {x for x in td.keys() if x != "Experiment"}
+        td.dropna(
+            "rows",
+            how="all",
+            subset=subset,
+        )
+        td.drop_duplicates("rows", subset=subset)
+        if "Experiment" in td:
+            td.column("Experiment")[:] = [
+                # remove tree characters
+                str(x).encode("ascii", "ignore").strip().decode()
+                for x in td.column("Experiment")
+            ]
         out = kwargs.get("out") or "dvc_plots"
-        html_args["output_path"] = (Path.cwd() / out).resolve()
-        html_args["color_by"] = kwargs.get("sort_by") or "Experiment"
+        output_file = os.path.join(out, "index.html")
+        ui.write(
+            td.to_parallel_coordinates(
+                output_path=os.path.abspath(output_file),
+                color_by=kwargs.get("sort_by") or "Experiment",
+            )
+        )
+        if kwargs.get("open"):
+            return ui.open_browser(output_file)
 
-    td.render(
-        pager=pager,
-        borders="horizontals",
-        rich_table=True,
-        header_styles=styles,
-        row_styles=row_styles,
-        csv=csv,
-        markdown=markdown,
-        html=html,
-        **html_args,
-    )
-
-    if html and kwargs.get("open"):
-        return ui.open_browser(Path(out) / "index.html")
+    else:
+        td.render(
+            pager=pager,
+            borders="horizontals",
+            rich_table=True,
+            header_styles=styles,
+            row_styles=row_styles,
+            csv=csv,
+            markdown=markdown,
+        )
 
 
 def _normalize_headers(names, count):
@@ -517,14 +469,6 @@ def _format_json(item):
     return encode_exception(item)
 
 
-def _raise_error_if_all_disabled(**kwargs):
-    if not any(kwargs.values()):
-        raise InvalidArgumentError(
-            "Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags` "
-            "or `--all-commits` needs to be set."
-        )
-
-
 class CmdExperimentsShow(CmdBase):
     def run(self):
         try:
@@ -532,9 +476,11 @@ class CmdExperimentsShow(CmdBase):
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
                 all_commits=self.args.all_commits,
-                sha_only=self.args.sha,
+                revs=self.args.rev,
                 num=self.args.num,
+                sha_only=self.args.sha,
                 param_deps=self.args.param_deps,
+                fetch_running=self.args.fetch_running,
             )
         except DvcException:
             logger.exception("failed to show experiments")
@@ -553,11 +499,8 @@ class CmdExperimentsShow(CmdBase):
 
             show_experiments(
                 all_experiments,
-                include_metrics=self.args.include_metrics,
-                exclude_metrics=self.args.exclude_metrics,
-                include_params=self.args.include_params,
-                exclude_params=self.args.exclude_params,
-                no_timestamp=self.args.no_timestamp,
+                keep=self.args.keep,
+                drop=self.args.drop,
                 sort_by=self.args.sort_by,
                 sort_order=self.args.sort_order,
                 precision=precision,
@@ -567,7 +510,7 @@ class CmdExperimentsShow(CmdBase):
                 csv=self.args.csv,
                 markdown=self.args.markdown,
                 only_changed=self.args.only_changed,
-                html=self.args.html,
+                pcp=self.args.pcp,
                 out=self.args.out,
                 open=self.args.open,
             )
@@ -575,6 +518,8 @@ class CmdExperimentsShow(CmdBase):
 
 
 def add_parser(experiments_subparsers, parent_parser):
+    from . import add_rev_selection_flags
+
     EXPERIMENTS_SHOW_HELP = "Print experiments."
     experiments_show_parser = experiments_subparsers.add_parser(
         "show",
@@ -583,6 +528,7 @@ def add_parser(experiments_subparsers, parent_parser):
         help=EXPERIMENTS_SHOW_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    add_rev_selection_flags(experiments_show_parser, "Show")
     experiments_show_parser.add_argument(
         "-a",
         "--all-branches",
@@ -597,22 +543,6 @@ def add_parser(experiments_subparsers, parent_parser):
         default=False,
         help="Show experiments derived from all Git tags.",
     )
-    experiments_show_parser.add_argument(
-        "-A",
-        "--all-commits",
-        action="store_true",
-        default=False,
-        help="Show experiments derived from all Git commits.",
-    )
-    experiments_show_parser.add_argument(
-        "-n",
-        "--num",
-        type=int,
-        default=1,
-        dest="num",
-        metavar="<num>",
-        help="Show the last `num` commits from HEAD.",
-    )
     experiments_show_parser.add_argument(
         "--no-pager",
         action="store_true",
@@ -620,32 +550,23 @@ def add_parser(experiments_subparsers, parent_parser):
         help="Do not pipe output into a pager.",
     )
     experiments_show_parser.add_argument(
-        "--include-metrics",
-        action="append",
-        default=[],
-        help="Include the specified metrics in output table.",
-        metavar="<metrics_list>",
+        "--only-changed",
+        action="store_true",
+        default=False,
+        help=(
+            "Only show metrics/params with values varying "
+            "across the selected experiments."
+        ),
     )
     experiments_show_parser.add_argument(
-        "--exclude-metrics",
-        action="append",
-        default=[],
-        help="Exclude the specified metrics from output table.",
-        metavar="<metrics_list>",
+        "--drop",
+        help="Remove the columns matching the specified regex pattern.",
+        metavar="<regex_pattern>",
     )
     experiments_show_parser.add_argument(
-        "--include-params",
-        action="append",
-        default=[],
-        help="Include the specified params in output table.",
-        metavar="<params_list>",
-    )
-    experiments_show_parser.add_argument(
-        "--exclude-params",
-        action="append",
-        default=[],
-        help="Exclude the specified params from output table.",
-        metavar="<params_list>",
+        "--keep",
+        help="Preserve the columns matching the specified regex pattern.",
+        metavar="<regex_pattern>",
     )
     experiments_show_parser.add_argument(
         "--param-deps",
@@ -667,12 +588,6 @@ def add_parser(experiments_subparsers, parent_parser):
         choices=("asc", "desc"),
         default="asc",
     )
-    experiments_show_parser.add_argument(
-        "--no-timestamp",
-        action="store_true",
-        default=False,
-        help="Do not show experiment timestamps.",
-    )
     experiments_show_parser.add_argument(
         "--sha",
         action="store_true",
@@ -711,31 +626,30 @@ def add_parser(experiments_subparsers, parent_parser):
         metavar="<n>",
     )
     experiments_show_parser.add_argument(
-        "--only-changed",
+        "--parallel-coordinates-plot",
+        "--pcp",
+        dest="pcp",
         action="store_true",
         default=False,
-        help=(
-            "Only show metrics/params with values varying "
-            "across the selected experiments."
-        ),
-    )
-    experiments_show_parser.add_argument(
-        "--html",
-        action="store_true",
-        default=False,
-        help="Generate a parallel coordinates plot from the tabulated output.",
+        help="Generate a Parallel Coordinates Plot from the tabulated output.",
     )
     experiments_show_parser.add_argument(
         "-o",
         "--out",
         default=None,
-        help="Destination folder to save the HTML to",
+        help="Destination folder to save the Parallel Coordinates Plot to",
         metavar="<path>",
     ).complete = completion.DIR
     experiments_show_parser.add_argument(
         "--open",
         action="store_true",
         default=False,
-        help="Open the HTML directly in the browser.",
+        help="Open the Parallel Coordinates Plot directly in the browser.",
+    )
+    experiments_show_parser.add_argument(
+        "--no-fetch",
+        dest="fetch_running",
+        action="store_false",
+        help=argparse.SUPPRESS,
     )
     experiments_show_parser.set_defaults(func=CmdExperimentsShow)

dvc/commands/freeze.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)

dvc/commands/gc.py

@@ -2,7 +2,8 @@ import argparse
 import logging
 import os
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
@@ -136,7 +137,6 @@ def add_parser(subparsers, parent_parser):
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/get.py

@@ -1,10 +1,11 @@
 import argparse
 import logging
 
+from dvc.cli import completion
+from dvc.cli.command import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
-from . import completion
-from .base import CmdBaseNoRepo, append_doc_link
+from ..cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -93,7 +94,6 @@ def add_parser(subparsers, parent_parser):
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/get_url.py

@@ -1,10 +1,11 @@
 import argparse
 import logging
 
+from dvc.cli import completion
+from dvc.cli.command import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
-from . import completion
-from .base import CmdBaseNoRepo, append_doc_link
+from ..cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -43,7 +44,6 @@ def add_parser(subparsers, parent_parser):
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/git_hook.py

@@ -1,7 +1,8 @@
 import logging
 import os
 
-from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import fix_subparsers
 from dvc.exceptions import NotDvcRepoError
 
 logger = logging.getLogger(__name__)
@@ -25,7 +26,7 @@ class CmdHookBase(CmdBaseNoRepo):
 
 class CmdPreCommit(CmdHookBase):
     def _run(self):
-        from dvc.main import main
+        from dvc.cli import main
 
         return main(["status"])
 
@@ -49,14 +50,14 @@ class CmdPostCheckout(CmdHookBase):
         if os.path.isdir(os.path.join(".git", "rebase-merge")):
             return 0
 
-        from dvc.main import main
+        from dvc.cli import main
 
         return main(["checkout"])
 
 
 class CmdPrePush(CmdHookBase):
     def _run(self):
-        from dvc.main import main
+        from dvc.cli import main
 
         return main(["push"])
 

dvc/commands/imp.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -90,7 +91,6 @@ def add_parser(subparsers, parent_parser):
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/imp_url.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -90,7 +91,6 @@ def add_parser(subparsers, parent_parser):
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/init.py

@@ -4,7 +4,8 @@ import logging
 import colorama
 
 from dvc import analytics
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
 from dvc.utils import boxify
 from dvc.utils import format_link as fmt_link
 

dvc/commands/install.py

@@ -1,7 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)

dvc/commands/ls/__init__.py

@@ -1,9 +1,10 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.command.ls.ls_colors import LsColors
+from dvc.cli import completion
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
+from dvc.commands.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
 from dvc.ui import ui
 

dvc/commands/machine.py

@@ -1,7 +1,8 @@
 import argparse
 
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
-from dvc.command.config import CmdConfig
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link, fix_subparsers
+from dvc.commands.config import CmdConfig
 from dvc.compare import TabularData
 from dvc.config import ConfigError
 from dvc.exceptions import DvcException
@@ -241,6 +242,7 @@ class CmdMachineStatus(CmdBase):
         "instance_hdd_size",
         "instance_gpu",
     ]
+    FILL_VALUE = "-"
 
     def _add_row(
         self,
@@ -250,7 +252,11 @@ class CmdMachineStatus(CmdBase):
     ):
 
         if not all_status:
-            row = [name, None, "offline"]
+            row = [
+                name,
+                self.FILL_VALUE,
+                "offline",
+            ]  # back to `None` after #7167
             td.append(row)
         for i, status in enumerate(all_status, start=1):
             row = [name, f"num_{i}", "running" if status else "offline"]
@@ -264,7 +270,7 @@ class CmdMachineStatus(CmdBase):
             raise MachineDisabledError
 
         td = TabularData(
-            self.INSTANCE_FIELD + self.SHOWN_FIELD, fill_value="-"
+            self.INSTANCE_FIELD + self.SHOWN_FIELD, fill_value=self.FILL_VALUE
         )
 
         if self.args.name:
@@ -304,7 +310,7 @@ class CmdMachineSsh(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    from dvc.command.config import parent_config_parser
+    from dvc.commands.config import parent_config_parser
 
     machine_HELP = "Set up and manage cloud machines."
     machine_parser = subparsers.add_parser(

dvc/commands/metrics.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
 from dvc.ui import ui
 from dvc.utils.serialize import encode_exception

dvc/commands/move.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)

dvc/commands/params.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
 from dvc.ui import ui
 

dvc/commands/plots.py

@@ -1,30 +1,27 @@
 import argparse
 import json
 import logging
+import os
 
 from funcy import first
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.render.utils import match_renderers, render
-from dvc.render.vega import VegaRenderer
 from dvc.ui import ui
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
 
-def _show_json(renderers, path: None):
-    if any(lambda r: r.needs_output_path for r in renderers) and not path:
-        raise DvcException("Output path ('-o') is required!")
+def _show_json(renderers, split=False):
+    from dvc.render.convert import to_json
 
     result = {
-        renderer.filename: json.loads(renderer.as_json(path=path))
-        for renderer in renderers
+        renderer.name: to_json(renderer, split) for renderer in renderers
     }
-    if result:
-        ui.write_json(result)
+    ui.write_json(result)
 
 
 class CmdPlots(CmdBase):
@@ -41,6 +38,10 @@ class CmdPlots(CmdBase):
     def run(self):
         from pathlib import Path
 
+        from dvc_render import render_html
+
+        from dvc.render.match import match_renderers
+
         if self.args.show_vega:
             if not self.args.targets:
                 logger.error("please specify a target for `--show-vega`")
@@ -69,34 +70,54 @@ class CmdPlots(CmdBase):
                     "visualization file will not be created."
                 )
 
+            out: str = self.args.out or "dvc_plots"
+
+            renderers_out = (
+                out if self.args.json else os.path.join(out, "static")
+            )
             renderers = match_renderers(
-                plots_data=plots_data, templates=self.repo.plots.templates
+                plots_data=plots_data, out=renderers_out
             )
 
             if self.args.show_vega:
-                renderer = first(
-                    filter(lambda r: isinstance(r, VegaRenderer), renderers)
-                )
+                renderer = first(filter(lambda r: r.TYPE == "vega", renderers))
                 if renderer:
-                    ui.write_json(renderer.asdict())
+                    ui.write_json(json.loads(renderer.get_filled_template()))
                 return 0
             if self.args.json:
-                _show_json(renderers, self.args.out)
+                _show_json(renderers, self.args.split)
                 return 0
 
-            rel: str = self.args.out or "dvc_plots"
-            path: Path = (Path.cwd() / rel).resolve()
-            index_path = render(
-                self.repo,
-                renderers,
-                path=path,
-                html_template_path=self.args.html_template,
+            html_template_path = self.args.html_template
+            if not html_template_path:
+                html_template_path = self.repo.config.get("plots", {}).get(
+                    "html_template", None
+                )
+                if html_template_path and not os.path.isabs(
+                    html_template_path
+                ):
+                    html_template_path = os.path.join(
+                        self.repo.dvc_dir, html_template_path
+                    )
+
+            output_file: Path = (Path.cwd() / out).resolve() / "index.html"
+
+            render_html(
+                renderers=renderers,
+                output_file=output_file,
+                template_path=html_template_path,
             )
 
-            ui.write(index_path.as_uri())
-
-            if self.args.open:
-                return ui.open_browser(index_path)
+            ui.write(output_file.as_uri())
+            auto_open = self.repo.config["plots"].get("auto_open", False)
+            if self.args.open or auto_open:
+                if not auto_open:
+                    ui.write(
+                        "To enable auto opening, you can run:\n"
+                        "\n"
+                        "\tdvc config plots.auto_open true"
+                    )
+                return ui.open_browser(output_file)
 
             return 0
 
@@ -132,6 +153,37 @@ class CmdPlotsModify(CmdPlots):
         return 0
 
 
+class CmdPlotsTemplates(CmdBase):
+    TEMPLATES_CHOICES = [
+        "simple",
+        "linear",
+        "confusion",
+        "confusion_normalized",
+        "scatter",
+        "smooth",
+    ]
+
+    def run(self):
+        from dvc_render.vega_templates import dump_templates
+
+        try:
+            out = (
+                os.path.join(os.getcwd(), self.args.out)
+                if self.args.out
+                else self.repo.plots.templates_dir
+            )
+
+            targets = [self.args.target] if self.args.target else None
+            dump_templates(output=out, targets=targets)
+            templates_path = os.path.relpath(out, os.getcwd())
+            ui.write(f"Templates have been written into '{templates_path}'.")
+
+            return 0
+        except DvcException:
+            logger.exception("")
+            return 1
+
+
 def add_parser(subparsers, parent_parser):
     PLOTS_HELP = (
         "Commands to visualize and compare plot metrics in structured files "
@@ -168,7 +220,8 @@ def add_parser(subparsers, parent_parser):
         "Shows all plots by default.",
     ).complete = completion.FILE
     _add_props_arguments(plots_show_parser)
-    _add_output_arguments(plots_show_parser)
+    _add_output_argument(plots_show_parser)
+    _add_ui_arguments(plots_show_parser)
     plots_show_parser.set_defaults(func=CmdPlotsShow)
 
     PLOTS_DIFF_HELP = (
@@ -203,7 +256,8 @@ def add_parser(subparsers, parent_parser):
         "revisions", nargs="*", default=None, help="Git commits to plot from"
     )
     _add_props_arguments(plots_diff_parser)
-    _add_output_arguments(plots_diff_parser)
+    _add_output_argument(plots_diff_parser)
+    _add_ui_arguments(plots_diff_parser)
     plots_diff_parser.set_defaults(func=CmdPlotsDiff)
 
     PLOTS_MODIFY_HELP = (
@@ -229,6 +283,27 @@ def add_parser(subparsers, parent_parser):
     )
     plots_modify_parser.set_defaults(func=CmdPlotsModify)
 
+    TEMPLATES_HELP = (
+        "Write built-in plots templates to a directory "
+        "(.dvc/plots by default)."
+    )
+    plots_templates_parser = plots_subparsers.add_parser(
+        "templates",
+        parents=[parent_parser],
+        description=append_doc_link(TEMPLATES_HELP, "plots/templates"),
+        help=TEMPLATES_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_templates_parser.add_argument(
+        "target",
+        default=None,
+        nargs="?",
+        choices=CmdPlotsTemplates.TEMPLATES_CHOICES,
+        help="Template to write. Writes all templates by default.",
+    )
+    _add_output_argument(plots_templates_parser, typ="templates")
+    plots_templates_parser.set_defaults(func=CmdPlotsTemplates)
+
 
 def _add_props_arguments(parser):
     parser.add_argument(
@@ -268,14 +343,17 @@ def _add_props_arguments(parser):
     )
 
 
-def _add_output_arguments(parser):
+def _add_output_argument(parser, typ="plots"):
     parser.add_argument(
         "-o",
         "--out",
         default=None,
-        help="Destination path to save plots to",
+        help=f"Directory to save {typ} to.",
         metavar="<path>",
     ).complete = completion.DIR
+
+
+def _add_ui_arguments(parser):
     parser.add_argument(
         "--show-vega",
         action="store_true",
@@ -289,6 +367,9 @@ def _add_output_arguments(parser):
         default=False,
         help=argparse.SUPPRESS,
     )
+    parser.add_argument(
+        "--split", action="store_true", default=False, help=argparse.SUPPRESS
+    )
     parser.add_argument(
         "--open",
         action="store_true",

dvc/commands/remote.py

@@ -1,7 +1,7 @@
 import argparse
 
-from dvc.command.base import append_doc_link, fix_subparsers
-from dvc.command.config import CmdConfig
+from dvc.cli.utils import append_doc_link, fix_subparsers
+from dvc.commands.config import CmdConfig
 from dvc.ui import ui
 from dvc.utils import format_link
 
@@ -113,8 +113,8 @@ class CmdRemoteDefault(CmdRemote):
 class CmdRemoteList(CmdRemote):
     def run(self):
         conf = self.config.read(self.args.level)
-        for name, conf in conf["remote"].items():
-            ui.write(name, conf["url"], sep="\t")
+        for name, remote_conf in conf["remote"].items():
+            ui.write(name, remote_conf["url"], sep="\t")
         return 0
 
 
@@ -151,7 +151,7 @@ class CmdRemoteRename(CmdRemote):
 
 
 def add_parser(subparsers, parent_parser):
-    from dvc.command.config import parent_config_parser
+    from dvc.commands.config import parent_config_parser
 
     REMOTE_HELP = "Set up and manage data remotes."
     remote_parser = subparsers.add_parser(

dvc/commands/remove.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)

dvc/commands/repro.py

@@ -1,15 +1,18 @@
 import argparse
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.status import CmdDataStatus
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.commands.status import CmdDataStatus
 
 
 class CmdRepro(CmdBase):
     def run(self):
         from dvc.ui import ui
 
-        stages = self.repo.reproduce(**self._repro_kwargs)
+        stages = self.repo.reproduce(
+            **self._common_kwargs, **self._repro_kwargs
+        )
         if len(stages) == 0:
             ui.write(CmdDataStatus.UP_TO_DATE_MSG)
         else:
@@ -26,7 +29,7 @@ class CmdRepro(CmdBase):
         return 0
 
     @property
-    def _repro_kwargs(self):
+    def _common_kwargs(self):
         return {
             "targets": self.args.targets,
             "single_item": self.args.single_item,
@@ -35,12 +38,17 @@ class CmdRepro(CmdBase):
             "interactive": self.args.interactive,
             "pipeline": self.args.pipeline,
             "all_pipelines": self.args.all_pipelines,
-            "run_cache": not self.args.no_run_cache,
-            "no_commit": self.args.no_commit,
             "downstream": self.args.downstream,
             "recursive": self.args.recursive,
             "force_downstream": self.args.force_downstream,
             "pull": self.args.pull,
+        }
+
+    @property
+    def _repro_kwargs(self):
+        return {
+            "run_cache": not self.args.no_run_cache,
+            "no_commit": self.args.no_commit,
             "glob": self.args.glob,
         }
 
@@ -109,15 +117,6 @@ def add_arguments(repro_parser):
         default=False,
         help="Reproduce all stages in the specified directory.",
     )
-    repro_parser.add_argument(
-        "--no-run-cache",
-        action="store_true",
-        default=False,
-        help=(
-            "Execute stage commands even if they have already been run with "
-            "the same command/dependencies/outputs/etc before."
-        ),
-    )
     repro_parser.add_argument(
         "--force-downstream",
         action="store_true",
@@ -125,12 +124,6 @@ def add_arguments(repro_parser):
         help="Reproduce all descendants of a changed stage even if their "
         "direct dependencies didn't change.",
     )
-    repro_parser.add_argument(
-        "--no-commit",
-        action="store_true",
-        default=False,
-        help="Don't put files/directories into cache.",
-    )
     repro_parser.add_argument(
         "--downstream",
         action="store_true",
@@ -146,12 +139,6 @@ def add_arguments(repro_parser):
             "from the run-cache."
         ),
     )
-    repro_parser.add_argument(
-        "--glob",
-        action="store_true",
-        default=False,
-        help="Allows targets containing shell-style wildcards.",
-    )
 
 
 def add_parser(subparsers, parent_parser):
@@ -165,5 +152,28 @@ def add_parser(subparsers, parent_parser):
         help=REPRO_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    # repro/exp run shared args
     add_arguments(repro_parser)
+    # repro only args
+    repro_parser.add_argument(
+        "--no-run-cache",
+        action="store_true",
+        default=False,
+        help=(
+            "Execute stage commands even if they have already been run with "
+            "the same command/dependencies/outputs/etc before."
+        ),
+    )
+    repro_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    repro_parser.add_argument(
+        "--glob",
+        action="store_true",
+        default=False,
+        help="Allows targets containing shell-style wildcards.",
+    )
     repro_parser.set_defaults(func=CmdRepro)

dvc/commands/root.py

@@ -1,7 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)

dvc/commands/run.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.stage import parse_cmd
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.commands.stage import parse_cmd
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -52,7 +53,7 @@ class CmdRun(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    from dvc.command.stage import _add_common_args
+    from dvc.commands.stage import _add_common_args
 
     RUN_HELP = (
         "Generate a dvc.yaml file from a command and execute the command."

dvc/commands/stage.py

@@ -3,8 +3,9 @@ import logging
 from itertools import chain, filterfalse
 from typing import TYPE_CHECKING, Dict, Iterable, List
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link, fix_subparsers
 from dvc.utils.cli_parse import parse_params
 from dvc.utils.humanize import truncate_text
 
@@ -93,9 +94,7 @@ class CmdStageList(CmdBase):
         self.repo.stage_collection_error_handler = log_error
 
         stages = self._get_stages()
-        names_only = self.args.names_only
-
-        data = prepare_stages_data(stages, description=not names_only)
+        data = prepare_stages_data(stages, description=not self.args.name_only)
         ui.table(data.items())
 
         return 0
@@ -213,7 +212,7 @@ def _add_common_args(parser):
         "--live-no-summary",
         action="store_true",
         default=False,
-        help="Signal dvclive logger to not dump latest metrics file.",
+        help=argparse.SUPPRESS,
     )
     parser.add_argument(
         "--live-no-html",
@@ -357,6 +356,7 @@ def add_parser(subparsers, parent_parser):
         help="List all stages inside the specified directory.",
     )
     stage_list_parser.add_argument(
+        "--name-only",
         "--names-only",
         action="store_true",
         default=False,

dvc/commands/status.py

@@ -1,6 +1,6 @@
 import logging
 
-from dvc.command.data_sync import CmdDataBase
+from dvc.commands.data_sync import CmdDataBase
 from dvc.exceptions import DvcException
 from dvc.ui import ui
 from dvc.utils import format_link

dvc/commands/unprotect.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)

dvc/commands/update.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -70,7 +71,6 @@ def add_parser(subparsers, parent_parser):
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/version.py

@@ -1,7 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)

dvc/compare.py

@@ -11,6 +11,7 @@ from typing import (
     List,
     Mapping,
     MutableSequence,
+    Optional,
     Sequence,
     Set,
     Tuple,
@@ -21,6 +22,7 @@ from typing import (
 from funcy import reraise
 
 if TYPE_CHECKING:
+    from dvc.types import StrPath
     from dvc.ui.table import CellT
 
 
@@ -37,11 +39,21 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         self._columns: Dict[str, Column] = {name: Column() for name in columns}
         self._keys: List[str] = list(columns)
         self._fill_value = fill_value
+        self._protected: Set[str] = set()
 
     @property
     def columns(self) -> List[Column]:
         return list(map(self.column, self.keys()))
 
+    def is_protected(self, col_name) -> bool:
+        return col_name in self._protected
+
+    def protect(self, *col_names: str):
+        self._protected.update(col_names)
+
+    def unprotect(self, *col_names: str):
+        self._protected = self._protected.difference(col_names)
+
     def column(self, name: str) -> Column:
         return self._columns[name]
 
@@ -125,9 +137,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         return len(self.columns), len(self)
 
     def drop(self, *col_names: str) -> None:
-        for col_name in col_names:
-            self._keys.remove(col_name)
-            self._columns.pop(col_name)
+        for col in col_names:
+            if not self.is_protected(col):
+                self._keys.remove(col)
+                self._columns.pop(col)
 
     def rename(self, from_col_name: str, to_col_name: str) -> None:
         self._columns[to_col_name] = self._columns.pop(from_col_name)
@@ -153,17 +166,23 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             writer.writerow(row)
         return buff.getvalue()
 
-    def to_parallel_coordinates(self, output_path, color_by):
-        from dvc.render.html import write
-        from dvc.render.plotly import ParallelCoordinatesRenderer
+    def to_parallel_coordinates(
+        self, output_path: "StrPath", color_by: str = None
+    ) -> "StrPath":
+        from dvc_render.html import render_html
+        from dvc_render.plotly import ParallelCoordinatesRenderer
 
-        index_path = write(
-            output_path,
+        render_html(
             renderers=[
-                ParallelCoordinatesRenderer(self, color_by, self._fill_value)
+                ParallelCoordinatesRenderer(
+                    self.as_dict(),
+                    color_by=color_by,
+                    fill_value=self._fill_value,
+                )
             ],
+            output_file=output_path,
         )
-        return index_path.as_uri()
+        return output_path
 
     def add_column(self, name: str) -> None:
         self._columns[name] = Column([self._fill_value] * len(self))
@@ -185,14 +204,6 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
 
         if kwargs.pop("csv", False):
             ui.write(self.to_csv(), end="")
-
-        elif kwargs.pop("html", False):
-            ui.write(
-                self.to_parallel_coordinates(
-                    kwargs["output_path"], kwargs.get("color_by")
-                )
-            )
-
         else:
             ui.table(self, headers=self.keys(), **kwargs)
 
@@ -204,7 +215,12 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             {k: self._columns[k][i] for k in keys} for i in range(len(self))
         ]
 
-    def dropna(self, axis: str = "rows", how="any"):
+    def dropna(
+        self,
+        axis: str = "rows",
+        how="any",
+        subset: Optional[Iterable[str]] = None,
+    ):
         if axis not in ["rows", "cols"]:
             raise ValueError(
                 f"Invalid 'axis' value {axis}."
@@ -216,13 +232,15 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             )
 
         match_line: Set = set()
-        match = True
+        match_any = True
         if how == "all":
-            match = False
+            match_any = False
 
         for n_row, row in enumerate(self):
             for n_col, col in enumerate(row):
-                if (col == self._fill_value) is match:
+                if subset and self.keys()[n_col] not in subset:
+                    continue
+                if (col == self._fill_value) is match_any:
                     if axis == "rows":
                         match_line.add(n_row)
                         break
@@ -249,7 +267,12 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         else:
             self.drop(*to_drop)
 
-    def drop_duplicates(self, axis: str = "rows"):
+    def drop_duplicates(
+        self,
+        axis: str = "rows",
+        subset: Optional[Iterable[str]] = None,
+        ignore_empty: bool = True,
+    ):
         if axis not in ["rows", "cols"]:
             raise ValueError(
                 f"Invalid 'axis' value {axis}."
@@ -259,8 +282,12 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         if axis == "cols":
             cols_to_drop: List[str] = []
             for n_col, col in enumerate(self.columns):
+                if subset and self.keys()[n_col] not in subset:
+                    continue
                 # Cast to str because Text is not hashable error
-                unique_vals = {str(x) for x in col if x != self._fill_value}
+                unique_vals = {str(x) for x in col}
+                if ignore_empty and self._fill_value in unique_vals:
+                    unique_vals -= {self._fill_value}
                 if len(unique_vals) == 1:
                     cols_to_drop.append(self.keys()[n_col])
             self.drop(*cols_to_drop)
@@ -269,6 +296,13 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             unique_rows = []
             rows_to_drop: List[int] = []
             for n_row, row in enumerate(self):
+                if subset:
+                    row = [
+                        col
+                        for n_col, col in enumerate(row)
+                        if self.keys()[n_col] in subset
+                    ]
+
                 tuple_row = tuple(row)
                 if tuple_row in unique_rows:
                     rows_to_drop.append(n_row)

dvc/config.py

@@ -94,7 +94,7 @@ class Config(dict):
             try:
                 from dvc.repo import Repo
 
-                self.dvc_dir = os.path.join(Repo.find_dvc_dir())
+                self.dvc_dir = Repo.find_dvc_dir()
             except NotDvcRepoError:
                 self.dvc_dir = None
         else:
@@ -266,7 +266,12 @@ class Config(dict):
                     "key_path": func,
                 }
             },
-            "machine": {str: {"startup_script": func}},
+            "machine": {
+                str: {
+                    "startup_script": func,
+                    "setup_script": func,
+                }
+            },
         }
         return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
 

dvc/config_schema.py

@@ -79,13 +79,13 @@ REMOTE_COMMON = {
     "checksum_jobs": All(Coerce(int), Range(1)),
     "jobs": All(Coerce(int), Range(1)),
     Optional("no_traverse"): Bool,  # obsoleted
-    "verify": Bool,
 }
 LOCAL_COMMON = {
     "type": supported_cache_type,
     Optional("protected", default=False): Bool,  # obsoleted
     "shared": All(Lower, Choices("group")),
     Optional("slow_link_warning", default=True): Bool,
+    Optional("verify", default=False): Bool,
 }
 HTTP_COMMON = {
     "auth": All(Lower, Choices("basic", "digest", "custom")),
@@ -95,6 +95,7 @@ HTTP_COMMON = {
     "ask_password": Bool,
     "ssl_verify": Any(Bool, str),
     "method": str,
+    Optional("verify", default=False): Bool,
 }
 WEBDAV_COMMON = {
     "user": str,
@@ -105,6 +106,7 @@ WEBDAV_COMMON = {
     "key_path": str,
     "timeout": Coerce(int),
     "ssl_verify": Any(Bool, str),
+    Optional("verify", default=False): Bool,
 }
 
 SCHEMA = {
@@ -158,11 +160,13 @@ SCHEMA = {
                     "cache_regions": bool,
                     "read_timeout": Coerce(int),
                     "connect_timeout": Coerce(int),
+                    Optional("verify", default=False): Bool,
                     **REMOTE_COMMON,
                 },
                 "gs": {
                     "projectname": str,
                     "credentialpath": str,
+                    Optional("verify", default=False): Bool,
                     **REMOTE_COMMON,
                 },
                 "ssh": {
@@ -175,6 +179,7 @@ SCHEMA = {
                     "timeout": Coerce(int),
                     "gss_auth": Bool,
                     "allow_agent": Bool,
+                    Optional("verify", default=False): Bool,
                     **REMOTE_COMMON,
                 },
                 "hdfs": {"user": str, "kerb_ticket": str, **REMOTE_COMMON},
@@ -185,6 +190,7 @@ SCHEMA = {
                     "ssl_verify": Any(Bool, str),
                     "token": str,
                     "use_https": Bool,
+                    Optional("verify", default=False): Bool,
                     **REMOTE_COMMON,
                 },
                 "azure": {
@@ -200,12 +206,14 @@ SCHEMA = {
                     "exclude_visual_studio_code_credential": Bool,
                     "exclude_shared_token_cache_credential": Bool,
                     "exclude_managed_identity_credential": Bool,
+                    Optional("verify", default=False): Bool,
                     **REMOTE_COMMON,
                 },
                 "oss": {
                     "oss_key_id": str,
                     "oss_key_secret": str,
                     "oss_endpoint": str,
+                    Optional("verify", default=True): Bool,
                     **REMOTE_COMMON,
                 },
                 "gdrive": {
@@ -216,6 +224,7 @@ SCHEMA = {
                     "gdrive_service_account_user_email": str,
                     "gdrive_service_account_json_file_path": str,
                     Optional("gdrive_trash_only", default=False): Bool,
+                    Optional("verify", default=True): Bool,
                     **REMOTE_COMMON,
                 },
                 "http": {**HTTP_COMMON, **REMOTE_COMMON},
@@ -248,6 +257,7 @@ SCHEMA = {
             "instance_gpu": Lower,
             "ssh_private": str,
             "startup_script": str,
+            "setup_script": str,
         },
     },
     # section for experimental features
@@ -256,7 +266,10 @@ SCHEMA = {
         # enabled by default. It's of no use, kept for backward compatibility.
         Optional("parametrization", default=True): Bool,
     },
-    "plots": {"html_template": str},
+    "plots": {
+        "html_template": str,
+        Optional("auto_open", default=False): Bool,
+    },
     "exp": {
         "code": str,
         "data": str,

dvc/daemon.py

@@ -42,7 +42,7 @@ def _spawn_windows(cmd, env):
 
 
 def _spawn_posix(cmd, env):
-    from dvc.main import main
+    from dvc.cli import main
 
     # NOTE: using os._exit instead of sys.exit, because dvc built
     # with PyInstaller has trouble with SystemExit exception and throws
@@ -106,8 +106,9 @@ def daemon(args):
     cmd = ["daemon", "-q"] + args
 
     env = fix_env()
-    file_path = os.path.abspath(inspect.stack()[0][1])
-    env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
+    if not is_binary():
+        file_path = os.path.abspath(inspect.stack()[0][1])
+        env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
     env[DVC_DAEMON] = "1"
 
     _spawn(cmd, env)

dvc/data/__init__.py

@@ -0,0 +1,33 @@
+import logging
+from typing import TYPE_CHECKING, Iterator, Union
+
+from .tree import Tree
+
+if TYPE_CHECKING:
+    from dvc.hash_info import HashInfo
+    from dvc.objects.db import ObjectDB
+    from dvc.objects.file import HashFile
+
+logger = logging.getLogger(__name__)
+
+
+def check(odb: "ObjectDB", obj: "HashFile", **kwargs):
+    if isinstance(obj, Tree):
+        for _, _, oid in obj:
+            odb.check(oid, **kwargs)
+
+    odb.check(obj.hash_info, **kwargs)
+
+
+def load(odb: "ObjectDB", hash_info: "HashInfo") -> "HashFile":
+    if hash_info.isdir:
+        return Tree.load(odb, hash_info)
+    return odb.get(hash_info)
+
+
+def iterobjs(
+    obj: Union["Tree", "HashFile"]
+) -> Iterator[Union["Tree", "HashFile"]]:
+    if isinstance(obj, Tree):
+        yield from (entry_obj for _, entry_obj in obj)
+    yield obj

dvc/data/_progress.py

@@ -0,0 +1,33 @@
+from dvc.progress import Tqdm
+
+
+class QueryingProgress(Tqdm):
+    def __init__(self, iterable=None, total=None, name=None, phase="Querying"):
+        msg_part = "cache in " + f"'{name}'" if name else "remote cache"
+        msg_fmt = "{phase} " + msg_part
+
+        self._estimating_msg = msg_fmt.format(phase="Estimating size of")
+        self._listing_msg = msg_fmt.format(phase="Querying")
+        self.desc = desc = msg_fmt.format(phase=phase)
+        super().__init__(
+            iterable=iterable,
+            desc=desc,
+            total=total,
+            unit="files",
+            unit_scale=False,
+            bar_format=self.BAR_FMT_DEFAULT,
+        )
+
+    def callback(self, phase, *args):
+        total = args[0] if args else self.total
+        completed = args[1] if len(args) > 1 else self.n
+        desc = self.desc
+        if phase == "estimating":
+            desc = self._estimating_msg
+        elif phase == "querying":
+            desc = self._listing_msg
+
+        if desc != self.desc:
+            # let Tqdm take care of when to update
+            self.set_description_str(desc, refresh=False)
+        self.update_to(completed, total)

dvc/data/checkout.py

@@ -5,14 +5,13 @@ from dvc import prompt
 from dvc.exceptions import CacheLinkError, CheckoutError, ConfirmRemoveError
 from dvc.fs.utils import test_links, transfer
 from dvc.ignore import DvcIgnoreFilter
-from dvc.objects.db.slow_link_detection import (  # type: ignore[attr-defined]
-    slow_link_guard,
-)
-from dvc.objects.diff import ROOT
-from dvc.objects.diff import diff as odiff
-from dvc.objects.stage import stage
 from dvc.types import Optional
 
+from .diff import ROOT
+from .diff import diff as odiff
+from .slow_link_detection import slow_link_guard  # type: ignore[attr-defined]
+from .stage import stage
+
 logger = logging.getLogger(__name__)
 
 

dvc/data/db/__init__.py

@@ -7,20 +7,13 @@ if TYPE_CHECKING:
 
 
 def get_odb(fs, fs_path, **config):
-    from .base import ObjectDB
-    from .gdrive import GDriveObjectDB
+    from dvc.objects.db import ObjectDB
+
     from .local import LocalObjectDB
-    from .oss import OSSObjectDB
 
     if fs.scheme == Schemes.LOCAL:
         return LocalObjectDB(fs, fs_path, **config)
 
-    if fs.scheme == Schemes.GDRIVE:
-        return GDriveObjectDB(fs, fs_path, **config)
-
-    if fs.scheme == Schemes.OSS:
-        return OSSObjectDB(fs, fs_path, **config)
-
     return ObjectDB(fs, fs_path, **config)
 
 

dvc/data/db/index.py

@@ -3,7 +3,8 @@ import os
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Iterable, Set
 
-from ..errors import ObjectDBError
+from dvc.objects.errors import ObjectDBError
+from dvc.utils.decorators import with_diskcache
 
 if TYPE_CHECKING:
     from dvc.types import StrPath
@@ -88,7 +89,7 @@ class ObjectDBIndex(ObjectDBIndexBase):
         tmp_dir: "StrPath",
         name: str,
     ):  # pylint: disable=super-init-not-called
-        from diskcache import Index
+        from diskcache import Cache, Index
 
         from dvc.fs.local import LocalFileSystem
         from dvc.utils.fs import makedirs
@@ -96,18 +97,28 @@ class ObjectDBIndex(ObjectDBIndexBase):
         self.index_dir = os.path.join(tmp_dir, self.INDEX_DIR, name)
         makedirs(self.index_dir, exist_ok=True)
         self.fs = LocalFileSystem()
-        self.index = Index(self.index_dir)
+        self.index = Index.fromcache(
+            Cache(
+                self.index_dir,
+                disk_pickle_protocol=4,
+                eviction_policy="none",
+            )
+        )
 
+    @with_diskcache(name="index")
     def __iter__(self):
         return iter(self.index)
 
+    @with_diskcache(name="index")
     def __contains__(self, hash_):
         return hash_ in self.index
 
+    @with_diskcache(name="index")
     def dir_hashes(self):
         """Iterate over .dir hashes stored in the index."""
         yield from (hash_ for hash_, is_dir in self.index.items() if is_dir)
 
+    @with_diskcache(name="index")
     def clear(self):
         """Clear this index (to force re-indexing later)."""
         from diskcache import Timeout
@@ -117,6 +128,7 @@ class ObjectDBIndex(ObjectDBIndexBase):
         except Timeout as exc:
             raise ObjectDBError("Failed to clear ODB index") from exc
 
+    @with_diskcache(name="index")
     def update(self, dir_hashes: Iterable[str], file_hashes: Iterable[str]):
         """Update this index, adding the specified hashes."""
         from diskcache import Timeout
@@ -131,6 +143,7 @@ class ObjectDBIndex(ObjectDBIndexBase):
         except Timeout as exc:
             raise ObjectDBError("Failed to update ODB index") from exc
 
+    @with_diskcache(name="index")
     def intersection(self, hashes: Set[str]):
         """Iterate over values from `hashes` which exist in the index."""
         yield from hashes.intersection(self.index.keys())

dvc/data/db/local.py

@@ -1,18 +1,17 @@
 import logging
 import os
 import stat
+from functools import partial
 
 from funcy import cached_property
 from shortuuid import uuid
 
 from dvc.hash_info import HashInfo
+from dvc.objects.db import ObjectDB, noop, wrap_iter
 from dvc.objects.errors import ObjectFormatError
-from dvc.progress import Tqdm
 from dvc.utils import relpath
 from dvc.utils.fs import copyfile, remove, umask, walk_files
 
-from .base import ObjectDB
-
 logger = logging.getLogger(__name__)
 
 
@@ -61,15 +60,12 @@ class LocalObjectDB(ObjectDB):
         return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
 
     def hashes_exist(
-        self, hashes, jobs=None, name=None
+        self, hashes, jobs=None, progress=noop
     ):  # pylint: disable=unused-argument
         ret = []
+        progress = partial(progress, "querying", len(hashes))
 
-        for hash_ in Tqdm(
-            hashes,
-            unit="file",
-            desc="Querying " + ("cache in " + name if name else "local cache"),
-        ):
+        for hash_ in wrap_iter(hashes, progress):
             hash_info = HashInfo(self.fs.PARAM_CHECKSUM, hash_)
             try:
                 self.check(hash_info)
@@ -79,7 +75,7 @@ class LocalObjectDB(ObjectDB):
 
         return ret
 
-    def _list_paths(self, prefix=None, progress_callback=None):
+    def _list_paths(self, prefix=None):
         assert self.fs_path is not None
         if prefix:
             fs_path = self.fs.path.join(self.fs_path, prefix[:2])
@@ -90,12 +86,7 @@ class LocalObjectDB(ObjectDB):
 
         # NOTE: use utils.fs walk_files since fs.walk_files will not follow
         # symlinks
-        if progress_callback:
-            for path in walk_files(fs_path):
-                progress_callback()
-                yield path
-        else:
-            yield from walk_files(fs_path)
+        yield from walk_files(fs_path)
 
     def _remove_unpacked_dir(self, hash_):
         hash_fs_path = self.hash_to_path(hash_)

dvc/data/db/reference.py

@@ -1,18 +1,16 @@
-import io
 import logging
 import os
 from typing import TYPE_CHECKING, Dict
 
+from dvc.objects.db import ObjectDB
+from dvc.objects.errors import ObjectFormatError
 from dvc.scheme import Schemes
 
-from ..errors import ObjectFormatError
 from ..reference import ReferenceHashFile
-from .base import ObjectDB
 
 if TYPE_CHECKING:
-    from dvc.fs.base import FileSystem
+    from dvc.fs.base import AnyFSPath, FileSystem
     from dvc.hash_info import HashInfo
-    from dvc.types import AnyPath
 
 logger = logging.getLogger(__name__)
 
@@ -38,10 +36,10 @@ class ReferenceObjectDB(ObjectDB):
             pass
         fs_path = self.hash_to_path(hash_info.value)
         try:
-            with self.fs.open(fs_path, "rb") as fobj:
-                ref_file = ReferenceHashFile.from_bytes(
-                    fobj.read(), fs_cache=self._fs_cache
-                )
+            ref_file = ReferenceHashFile.from_bytes(
+                self.fs.cat_file(fs_path),
+                fs_cache=self._fs_cache,
+            )
         except OSError:
             raise FileNotFoundError
         try:
@@ -55,8 +53,8 @@ class ReferenceObjectDB(ObjectDB):
     def _add_file(
         self,
         from_fs: "FileSystem",
-        from_info: "AnyPath",
-        to_info: "AnyPath",
+        from_info: "AnyFSPath",
+        to_info: "AnyFSPath",
         hash_info: "HashInfo",
         hardlink: bool = False,
     ):
@@ -71,10 +69,8 @@ class ReferenceObjectDB(ObjectDB):
             )
         ref_file = ReferenceHashFile(from_info, from_fs, hash_info)
         self._obj_cache[hash_info] = ref_file
-        ref_fobj = io.BytesIO(ref_file.to_bytes())
-        ref_fobj.seek(0)
         try:
-            self.fs.upload(ref_fobj, to_info)
+            self.fs.pipe_file(to_info, ref_file.to_bytes())
         except OSError as exc:
             if isinstance(exc, FileExistsError) or (
                 os.name == "nt"

dvc/data/diff.py

@@ -3,8 +3,7 @@ from typing import TYPE_CHECKING, List, Optional, Tuple
 
 if TYPE_CHECKING:
     from dvc.hash_info import HashInfo
-
-    from .file import HashFile
+    from dvc.objects.file import HashFile
 
 ADD = "add"
 MODIFY = "modify"
@@ -96,7 +95,7 @@ def diff(
         return entry_obj.hash_info if entry_obj else None
 
     def _in_cache(oid, cache):
-        from .errors import ObjectFormatError
+        from dvc.objects.errors import ObjectFormatError
 
         if not oid:
             return False

dvc/data/gc.py

@@ -0,0 +1,40 @@
+def gc(odb, used, jobs=None, cache_odb=None, shallow=True):
+    from dvc.data.tree import Tree
+    from dvc.objects.errors import ObjectDBPermissionError
+
+    from ._progress import QueryingProgress
+
+    if odb.read_only:
+        raise ObjectDBPermissionError("Cannot gc read-only ODB")
+    if not cache_odb:
+        cache_odb = odb
+    used_hashes = set()
+    for hash_info in used:
+        used_hashes.add(hash_info.value)
+        if hash_info.isdir and not shallow:
+            tree = Tree.load(cache_odb, hash_info)
+            used_hashes.update(
+                entry_obj.hash_info.value for _, entry_obj in tree
+            )
+
+    def _is_dir_hash(_hash):
+        from dvc.hash_info import HASH_DIR_SUFFIX
+
+        return _hash.endswith(HASH_DIR_SUFFIX)
+
+    removed = False
+    # hashes must be sorted to ensure we always remove .dir files first
+
+    hashes = QueryingProgress(odb.all(jobs), name=odb.fs_path)
+    for hash_ in sorted(hashes, key=_is_dir_hash, reverse=True):
+        if hash_ in used_hashes:
+            continue
+        fs_path = odb.hash_to_path(hash_)
+        if _is_dir_hash(hash_):
+            # backward compatibility
+            # pylint: disable=protected-access
+            odb._remove_unpacked_dir(hash_)
+        odb.fs.remove(fs_path)
+        removed = True
+
+    return removed

dvc/data/meta.py

@@ -1,15 +1,25 @@
 from collections import OrderedDict
 from dataclasses import dataclass, field
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from dvc.objects.db import ObjectDB
+    from dvc.objects.file import HashFile
 
 
 @dataclass
 class Meta:
     PARAM_SIZE = "size"
     PARAM_NFILES = "nfiles"
+    PARAM_ISEXEC = "isexec"
 
     size: Optional[int] = field(default=None)
     nfiles: Optional[int] = field(default=None)
+    isexec: Optional[bool] = field(default=False)
+
+    obj: Optional["HashFile"] = field(default=None)
+    odb: Optional["ObjectDB"] = field(default=None)
+    remote: Optional[str] = field(default=None)
 
     @classmethod
     def from_dict(cls, d):
@@ -18,7 +28,9 @@ class Meta:
 
         size = d.pop(cls.PARAM_SIZE, None)
         nfiles = d.pop(cls.PARAM_NFILES, None)
-        return cls(size=size, nfiles=nfiles)
+        isexec = d.pop(cls.PARAM_ISEXEC, False)
+
+        return cls(size=size, nfiles=nfiles, isexec=isexec)
 
     def to_dict(self):
         ret = OrderedDict()
@@ -29,4 +41,7 @@ class Meta:
         if self.nfiles is not None:
             ret[self.PARAM_NFILES] = self.nfiles
 
+        if self.isexec:
+            ret[self.PARAM_ISEXEC] = self.isexec
+
         return ret

dvc/data/reference.py

@@ -4,15 +4,13 @@ import os
 import pickle
 from typing import TYPE_CHECKING, Optional
 
-from .errors import ObjectFormatError
-from .file import HashFile
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.file import HashFile
 
 if TYPE_CHECKING:
-    from dvc.fs.base import FileSystem
+    from dvc.fs.base import AnyFSPath, FileSystem
     from dvc.hash_info import HashInfo
-    from dvc.types import AnyPath
-
-    from .db.base import ObjectDB
+    from dvc.objects.db import ObjectDB
 
 logger = logging.getLogger(__name__)
 
@@ -25,7 +23,7 @@ class ReferenceHashFile(HashFile):
 
     def __init__(
         self,
-        fs_path: "AnyPath",
+        fs_path: "AnyFSPath",
         fs: "FileSystem",
         hash_info: "HashInfo",
         checksum: Optional[str] = None,
@@ -53,17 +51,12 @@ class ReferenceHashFile(HashFile):
         return self.fs.checksum(self.fs_path)
 
     def to_bytes(self):
-        from dvc.fs.repo import RepoFileSystem
-
         # NOTE: dumping reference FS's this way is insecure, as the
         # fully parsed remote FS config will include credentials
         #
         # ReferenceHashFiles should currently only be serialized in
         # memory and not to disk
         fs_path = self.fs_path
-        if isinstance(self.fs, RepoFileSystem):
-            fs_path = self.fs.path.relpath(fs_path, self.fs.root_dir)
-
         dict_ = {
             self.PARAM_PATH: fs_path,
             self.PARAM_HASH: self.hash_info,
@@ -78,7 +71,7 @@ class ReferenceHashFile(HashFile):
     @classmethod
     def from_bytes(cls, data: bytes, fs_cache: Optional[dict] = None):
         from dvc.fs import get_fs_cls
-        from dvc.fs.repo import RepoFileSystem
+        from dvc.fs.repo import RepoFileSystem, _RepoFileSystem
 
         try:
             dict_ = pickle.loads(data)
@@ -95,12 +88,11 @@ class ReferenceHashFile(HashFile):
         fs = fs_cache.get((scheme, config_pairs)) if fs_cache else None
         if not fs:
             config = dict(config_pairs)
-            if RepoFileSystem.PARAM_REPO_URL in config:
-                fs = RepoFileSystem(**config)
-                fs_path = fs.path.join(fs.root_dir, fs_path)
+            if _RepoFileSystem.PARAM_REPO_URL in config:
+                fs_cls = RepoFileSystem
             else:
                 fs_cls = get_fs_cls(config, scheme=scheme)
-                fs = fs_cls(**config)
+            fs = fs_cls(**config)
         return ReferenceHashFile(
             fs_path,
             fs,

dvc/data/stage.py

@@ -8,18 +8,16 @@ from typing import TYPE_CHECKING, Dict, Optional, Tuple
 from dvc.exceptions import DvcIgnoreInCollectedDirError
 from dvc.hash_info import HashInfo
 from dvc.ignore import DvcIgnore
+from dvc.objects.file import HashFile
 from dvc.progress import Tqdm
-from dvc.utils import file_md5
+from dvc.utils import file_md5, is_exec
 
 from .db.reference import ReferenceObjectDB
-from .file import HashFile
 from .meta import Meta
 
 if TYPE_CHECKING:
-    from dvc.fs.base import FileSystem
-    from dvc.types import AnyPath
-
-    from .db.base import ObjectDB
+    from dvc.fs.base import AnyFSPath, FileSystem
+    from dvc.objects.db import ObjectDB
 
 logger = logging.getLogger(__name__)
 
@@ -27,18 +25,23 @@ logger = logging.getLogger(__name__)
 _STAGING_MEMFS_PATH = "dvc-staging"
 
 
-def _upload_file(from_fs_path, fs, odb, upload_odb):
+def _upload_file(from_fs_path, fs, odb, upload_odb, callback=None):
+    from dvc.fs._callback import FsspecCallback
     from dvc.utils import tmp_fname
     from dvc.utils.stream import HashedStreamReader
 
     fs_path = upload_odb.fs.path
     tmp_info = fs_path.join(upload_odb.fs_path, tmp_fname())
-    with fs.open(from_fs_path, mode="rb", chunk_size=fs.CHUNK_SIZE) as stream:
+    with fs.open(from_fs_path, mode="rb") as stream:
         stream = HashedStreamReader(stream)
-        size = fs.getsize(from_fs_path)
-        upload_odb.fs.upload(
-            stream, tmp_info, desc=fs_path.name(from_fs_path), total=size
-        )
+        size = fs.size(from_fs_path)
+        with FsspecCallback.as_tqdm_callback(
+            callback,
+            desc=fs_path.name(from_fs_path),
+            bytes=True,
+            total=size,
+        ) as cb:
+            upload_odb.fs.upload(stream, tmp_info, size=size, callback=cb)
 
     odb.add(tmp_info, upload_odb.fs, stream.hash_info)
     meta = Meta(size=size)
@@ -73,7 +76,7 @@ def _get_file_hash(fs_path, fs, name):
     else:
         raise NotImplementedError
 
-    meta = Meta(size=info["size"])
+    meta = Meta(size=info["size"], isexec=is_exec(info.get("mode", 0)))
     hash_info = HashInfo(name, hash_value)
     return meta, hash_info
 
@@ -169,7 +172,7 @@ def _build_tree(fs_path, fs, name, **kwargs):
         assert key
         tree.add(key, meta, obj.hash_info)
 
-        tree_meta.size += meta.size
+        tree_meta.size += meta.size or 0
         tree_meta.nfiles += 1
 
     return tree_meta, tree
@@ -245,8 +248,9 @@ def _get_staging(odb: "ObjectDB") -> "ObjectDB":
 
 
 def _load_from_state(odb, staging, fs_path, fs, name):
+    from dvc.objects.errors import ObjectFormatError
+
     from . import check, load
-    from .errors import ObjectFormatError
     from .tree import Tree
 
     state = odb.state
@@ -287,7 +291,7 @@ def _stage_external_tree_info(odb, tree, name):
 
 def stage(
     odb: "ObjectDB",
-    fs_path: "AnyPath",
+    fs_path: "AnyFSPath",
     fs: "FileSystem",
     name: str,
     upload: bool = False,

dvc/data/status.py

@@ -7,9 +7,10 @@ from dvc.scheme import Schemes
 from .tree import Tree
 
 if TYPE_CHECKING:
-    from .db.base import ObjectDB
+    from dvc.objects.db import ObjectDB
+    from dvc.objects.file import HashFile
+
     from .db.index import ObjectDBIndexBase
-    from .file import HashFile
 
 logger = logging.getLogger(__name__)
 
@@ -26,14 +27,20 @@ class CompareStatusResult(NamedTuple):
     deleted: Set["HashInfo"]
 
 
-def _indexed_dir_hashes(odb, index, dir_objs, name, cache_odb):
+def _indexed_dir_hashes(odb, index, dir_objs, name, cache_odb, jobs=None):
     # Validate our index by verifying all indexed .dir hashes
     # still exist on the remote
+    from ._progress import QueryingProgress
+
     dir_hashes = set(dir_objs.keys())
     indexed_dirs = set(index.dir_hashes())
     indexed_dir_exists = set()
     if indexed_dirs:
-        indexed_dir_exists.update(odb.list_hashes_exists(indexed_dirs))
+        hashes = QueryingProgress(
+            odb.list_hashes_exists(indexed_dirs, jobs=jobs),
+            total=len(indexed_dirs),
+        )
+        indexed_dir_exists.update(hashes)
         missing_dirs = indexed_dirs.difference(indexed_dir_exists)
         if missing_dirs:
             logger.debug(
@@ -45,7 +52,13 @@ def _indexed_dir_hashes(odb, index, dir_objs, name, cache_odb):
 
     # Check if non-indexed (new) dir hashes exist on remote
     dir_exists = dir_hashes.intersection(indexed_dir_exists)
-    dir_exists.update(odb.list_hashes_exists(dir_hashes - dir_exists))
+    dir_missing = dir_hashes - dir_exists
+    dir_exists.update(
+        QueryingProgress(
+            odb.list_hashes_exists(dir_missing, jobs=jobs),
+            total=len(dir_missing),
+        )
+    )
 
     # If .dir hash exists in the ODB, assume directory contents
     # also exists
@@ -75,6 +88,7 @@ def status(
     index: Optional["ObjectDBIndexBase"] = None,
     cache_odb: Optional["ObjectDB"] = None,
     shallow: bool = True,
+    jobs: Optional[int] = None,
     **kwargs,
 ) -> "StatusResult":
     """Return status of whether or not the specified objects exist odb.
@@ -120,7 +134,9 @@ def status(
     if index and hashes:
         if dir_objs:
             exists = hashes.intersection(
-                _indexed_dir_hashes(odb, index, dir_objs, name, cache_odb)
+                _indexed_dir_hashes(
+                    odb, index, dir_objs, name, cache_odb, jobs=jobs
+                )
             )
             hashes.difference_update(exists)
         if hashes:
@@ -128,7 +144,12 @@ def status(
             hashes.difference_update(exists)
 
     if hashes:
-        exists.update(odb.hashes_exist(hashes, name=odb.fs_path, **kwargs))
+        from ._progress import QueryingProgress
+
+        with QueryingProgress(phase="Checking", name=odb.fs_path) as pbar:
+            exists.update(
+                odb.hashes_exist(hashes, jobs=jobs, progress=pbar.callback)
+            )
     return StatusResult(
         {hash_infos[hash_] for hash_ in exists},
         {hash_infos[hash_] for hash_ in (hashes - exists)},
@@ -143,6 +164,7 @@ def compare_status(
     check_deleted: bool = True,
     src_index: Optional["ObjectDBIndexBase"] = None,
     dest_index: Optional["ObjectDBIndexBase"] = None,
+    jobs: Optional[int] = None,
     **kwargs,
 ) -> "CompareStatusResult":
     """Compare status for the specified objects between two ODBs.
@@ -156,13 +178,13 @@ def compare_status(
     if "cache_odb" not in kwargs:
         kwargs["cache_odb"] = src
     dest_exists, dest_missing = status(
-        dest, obj_ids, index=dest_index, **kwargs
+        dest, obj_ids, index=dest_index, jobs=jobs, **kwargs
     )
     # for transfer operations we can skip src status check when all objects
     # already exist in dest
     if dest_missing or check_deleted:
         src_exists, src_missing = status(
-            src, obj_ids, index=src_index, **kwargs
+            src, obj_ids, index=src_index, jobs=jobs, **kwargs
         )
     else:
         src_exists = dest_exists

dvc/data/transfer.py

@@ -10,8 +10,8 @@ from dvc.utils.threadpool import ThreadPoolExecutor
 
 if TYPE_CHECKING:
     from dvc.hash_info import HashInfo
+    from dvc.objects.db import ObjectDB
 
-    from .db.base import ObjectDB
     from .db.index import ObjectDBIndexBase
     from .tree import Tree
 
@@ -40,7 +40,8 @@ def _log_exceptions(func):
 def find_tree_by_obj_id(
     odbs: Iterable[Optional["ObjectDB"]], obj_id: "HashInfo"
 ) -> Optional["Tree"]:
-    from .errors import ObjectFormatError
+    from dvc.objects.errors import ObjectFormatError
+
     from .tree import Tree
 
     for odb in odbs:
@@ -150,7 +151,9 @@ def transfer(
     if src == dest:
         return 0
 
-    status = compare_status(src, dest, obj_ids, check_deleted=False, **kwargs)
+    status = compare_status(
+        src, dest, obj_ids, check_deleted=False, jobs=jobs, **kwargs
+    )
     if not status.new:
         return 0
 

dvc/data/tree.py

@@ -1,22 +1,44 @@
 import json
 import logging
 import posixpath
-from typing import TYPE_CHECKING, Dict, Optional, Tuple
+from typing import TYPE_CHECKING, Dict, Iterable, Optional, Tuple
 
 from funcy import cached_property
 
-from .errors import ObjectFormatError
-from .file import HashFile
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.file import HashFile
+
 from .stage import get_file_hash
 
 if TYPE_CHECKING:
     from dvc.hash_info import HashInfo
+    from dvc.objects.db import ObjectDB
 
     from .meta import Meta
 
 logger = logging.getLogger(__name__)
 
 
+class TreeError(Exception):
+    pass
+
+
+def _try_load(
+    odbs: Iterable["ObjectDB"],
+    hash_info: "HashInfo",
+) -> Optional["HashFile"]:
+    for odb in odbs:
+        if not odb:
+            continue
+
+        try:
+            return Tree.load(odb, hash_info)
+        except (FileNotFoundError, ObjectFormatError):
+            pass
+
+    return None
+
+
 class Tree(HashFile):
     PARAM_RELPATH = "relpath"
 
@@ -25,7 +47,7 @@ class Tree(HashFile):
         self._dict: Dict[Tuple[str], Tuple["Meta", "HashInfo"]] = {}
 
     @cached_property
-    def trie(self):
+    def _trie(self):
         from pygtrie import Trie
 
         return Trie(self._dict)
@@ -40,8 +62,7 @@ class Tree(HashFile):
 
         memfs = MemoryFileSystem()
         fs_path = "memory://{}".format(tmp_fname(""))
-        with memfs.open(fs_path, "wb") as fobj:
-            fobj.write(self.as_bytes())
+        memfs.pipe_file(fs_path, self.as_bytes())
         self.fs = memfs
         self.fs_path = fs_path
         if hash_info:
@@ -51,6 +72,30 @@ class Tree(HashFile):
             assert self.hash_info.value
             self.hash_info.value += ".dir"
 
+    def _load(self, key, meta, hash_info):
+        if hash_info and hash_info.isdir and not meta.obj:
+            meta.obj = _try_load([meta.odb, meta.remote], hash_info)
+            if meta.obj:
+                for ikey, value in meta.obj.iteritems():
+                    self._trie[key + ikey] = value
+                    self._dict[key + ikey] = value
+
+    def iteritems(self, prefix=None):
+        kwargs = {}
+        if prefix:
+            kwargs = {"prefix": prefix}
+            item = self._trie.longest_prefix(prefix)
+            if item:
+                key, (meta, hash_info) = item
+                self._load(key, meta, hash_info)
+
+        for key, (meta, hash_info) in self._trie.iteritems(**kwargs):
+            self._load(key, meta, hash_info)
+            yield key, (meta, hash_info)
+
+    def shortest_prefix(self, *args, **kwargs):
+        return self._trie.shortest_prefix(*args, **kwargs)
+
     def __len__(self):
         return len(self._dict)
 
@@ -130,7 +175,7 @@ class Tree(HashFile):
         """
         tree = Tree(self.fs_path, self.fs, self.hash_info)
         try:
-            for key, (meta, oid) in self.trie.items(prefix):
+            for key, (meta, oid) in self._trie.items(prefix):
                 tree.add(key, meta, oid)
         except KeyError:
             pass
@@ -148,18 +193,39 @@ class Tree(HashFile):
         tree = Tree(None, None, None)
         depth = len(prefix)
         try:
-            for key, (meta, entry_oid) in self.trie.items(prefix):
+            for key, (meta, entry_oid) in self._trie.items(prefix):
                 tree.add(key[depth:], meta, entry_oid)
         except KeyError:
             return None
         tree.digest()
         return tree
 
+    def ls(self, prefix=None):
+        kwargs = {}
+        if prefix:
+            kwargs["prefix"] = prefix
+
+        meta, hash_info = self._trie.get(prefix, (None, None))
+        if hash_info and hash_info.isdir and meta and not meta.obj:
+            raise TreeError
+
+        ret = []
+
+        def node_factory(_, key, children, *args):
+            if key == prefix:
+                list(children)
+            else:
+                ret.append(key[-1])
+
+        self._trie.traverse(node_factory, **kwargs)
+
+        return ret
+
 
 def du(odb, tree):
     try:
         return sum(
-            odb.fs.getsize(odb.hash_to_path(oid.value)) for _, _, oid in tree
+            odb.fs.size(odb.hash_to_path(oid.value)) for _, _, oid in tree
         )
     except FileNotFoundError:
         return None

dvc/data_cloud.py

@@ -3,11 +3,11 @@
 import logging
 from typing import TYPE_CHECKING, Iterable, Optional
 
-from dvc.objects.db import get_index
+from dvc.data.db import get_index
 
 if TYPE_CHECKING:
     from dvc.hash_info import HashInfo
-    from dvc.objects.db.base import ObjectDB
+    from dvc.objects.db import ObjectDB
 
 logger = logging.getLogger(__name__)
 
@@ -55,8 +55,8 @@ class DataCloud:
         raise NoRemoteError(error_msg)
 
     def _init_odb(self, name):
+        from dvc.data.db import get_odb
         from dvc.fs import get_cloud_fs
-        from dvc.objects.db import get_odb
 
         cls, config, fs_path = get_cloud_fs(self.repo, name=name)
         config["tmp_dir"] = self.repo.index_db_dir
@@ -78,7 +78,7 @@ class DataCloud:
                 By default remote from core.remote config option is used.
             odb: optional ODB to push to. Overrides remote.
         """
-        from dvc.objects.transfer import transfer
+        from dvc.data.transfer import transfer
 
         if not odb:
             odb = self.get_remote_odb(remote, "push")
@@ -107,7 +107,7 @@ class DataCloud:
                 By default remote from core.remote config option is used.
             odb: optional ODB to pull from. Overrides remote.
         """
-        from dvc.objects.transfer import transfer
+        from dvc.data.transfer import transfer
 
         if not odb:
             odb = self.get_remote_odb(remote, "pull")
@@ -141,7 +141,7 @@ class DataCloud:
             log_missing: log warning messages if file doesn't exist
                 neither in cache, neither in cloud.
         """
-        from dvc.objects.status import compare_status
+        from dvc.data.status import compare_status
 
         if not odb:
             odb = self.get_remote_odb(remote, "status")

dvc/dependency/__init__.py

@@ -47,16 +47,29 @@ def loads_from(stage, s_list, erepo=None):
 def _merge_params(s_list):
     d = defaultdict(list)
     default_file = ParamsDependency.DEFAULT_PARAMS_FILE
+
+    # figure out completely tracked params file, and ignore specific keys
+    wholly_tracked = set()
+    for key in s_list:
+        if not isinstance(key, dict):
+            continue
+        wholly_tracked.update(k for k, params in key.items() if not params)
+
     for key in s_list:
         if isinstance(key, str):
-            d[default_file].append(key)
+            if default_file not in wholly_tracked:
+                d[default_file].append(key)
             continue
+
         if not isinstance(key, dict):
             msg = "Only list of str/dict is supported. Got: "
             msg += f"'{type(key).__name__}'."
             raise ValueError(msg)
 
         for k, params in key.items():
+            if k in wholly_tracked:
+                d[k] = []
+                continue
             if not isinstance(params, list):
                 msg = "Expected list of params for custom params file "
                 msg += f"'{k}', got '{type(params).__name__}'."

dvc/dependency/param.py

@@ -1,6 +1,8 @@
 import logging
 import os
+import typing
 from collections import defaultdict
+from typing import Dict
 
 import dpath.util
 from voluptuous import Any
@@ -18,6 +20,14 @@ class MissingParamsError(DvcException):
     pass
 
 
+class MissingParamsFile(DvcException):
+    pass
+
+
+class ParamsIsADirectoryError(DvcException):
+    pass
+
+
 class BadParamFileError(DvcException):
     pass
 
@@ -27,51 +37,83 @@ class ParamsDependency(Dependency):
     PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
     DEFAULT_PARAMS_FILE = "params.yaml"
 
-    def __init__(self, stage, path, params):
-        info = {}
-        self.params = []
-        if params:
-            if isinstance(params, list):
-                self.params = params
-            else:
-                assert isinstance(params, dict)
-                self.params = list(params.keys())
-                info = {self.PARAM_PARAMS: params}
-
-        super().__init__(
-            stage,
-            path
-            or os.path.join(stage.repo.root_dir, self.DEFAULT_PARAMS_FILE),
-            info=info,
+    def __init__(self, stage, path, params=None, repo=None):
+        self.params = list(params) if params else []
+        info = (
+            {self.PARAM_PARAMS: params} if isinstance(params, dict) else None
         )
+        repo = repo or stage.repo
+        path = path or os.path.join(repo.root_dir, self.DEFAULT_PARAMS_FILE)
+        super().__init__(stage, path, info=info, repo=repo)
 
     def dumpd(self):
         ret = super().dumpd()
         if not self.hash_info:
-            ret[self.PARAM_PARAMS] = self.params
+            ret[self.PARAM_PARAMS] = self.params or {}
         return ret
 
     def fill_values(self, values=None):
         """Load params values dynamically."""
-        if not values:
+        if values is None:
             return
+
         info = {}
+        if not self.params:
+            info.update(values)
         for param in self.params:
             if param in values:
                 info[param] = values[param]
         self.hash_info = HashInfo(self.PARAM_PARAMS, info)
 
-    def workspace_status(self):
-        status = super().workspace_status()
+    def read_params(
+        self, flatten: bool = True, **kwargs: typing.Any
+    ) -> Dict[str, typing.Any]:
+        try:
+            config = self.read_file()
+        except MissingParamsFile:
+            config = {}
 
-        if status.get(str(self)) == "deleted":
-            return status
+        if not self.params:
+            return config
+
+        ret = {}
+        if flatten:
+            for param in self.params:
+                try:
+                    ret[param] = dpath.util.get(config, param, separator=".")
+                except KeyError:
+                    continue
+            return ret
+
+        from dpath.util import merge
+
+        for param in self.params:
+            merge(
+                ret,
+                dpath.util.search(config, param, separator="."),
+                separator=".",
+            )
+        return ret
+
+    def workspace_status(self):
+        if not self.exists:
+            return {str(self): "deleted"}
+        if self.hash_info.value is None:
+            return {str(self): "new"}
+
+        from funcy import ldistinct
 
         status = defaultdict(dict)
         info = self.hash_info.value if self.hash_info else {}
         actual = self.read_params()
-        for param in self.params:
-            if param not in actual.keys():
+
+        # NOTE: we want to preserve the order of params as specified in the
+        # status. In case of tracking the whole file, the order is top-level
+        # keys in the file and then the keys in the `info` from `dvc.lock`
+        # (which are alphabetically sorted).
+        params = self.params or ldistinct([*actual.keys(), *info.keys()])
+        for param in params:
+            if param not in actual:
                 st = "deleted"
             elif param not in info:
                 st = "new"
@@ -88,12 +130,19 @@ class ParamsDependency(Dependency):
     def status(self):
         return self.workspace_status()
 
-    def _read(self):
+    def validate_filepath(self):
         if not self.exists:
-            return {}
+            raise MissingParamsFile(f"Parameters file '{self}' does not exist")
+        if self.isdir():
+            raise ParamsIsADirectoryError(
+                f"'{self}' is a directory, expected a parameters file"
+            )
 
-        suffix = self.repo.fs.path.suffix(self.fs_path).lower()
-        loader = LOADERS[suffix]
+    def read_file(self):
+        _, ext = os.path.splitext(self.fs_path)
+        loader = LOADERS[ext]
+
+        self.validate_filepath()
         try:
             return loader(self.fs_path, fs=self.repo.fs)
         except ParseError as exc:
@@ -101,29 +150,6 @@ class ParamsDependency(Dependency):
                 f"Unable to read parameters from '{self}'"
             ) from exc
 
-    def read_params_d(self, **kwargs):
-        config = self._read()
-
-        ret = {}
-        for param in self.params:
-            dpath.util.merge(
-                ret,
-                dpath.util.search(config, param, separator="."),
-                separator=".",
-            )
-        return ret
-
-    def read_params(self):
-        config = self._read()
-
-        ret = {}
-        for param in self.params:
-            try:
-                ret[param] = dpath.util.get(config, param, separator=".")
-            except KeyError:
-                pass
-        return ret
-
     def get_hash(self):
         info = self.read_params()
 
@@ -144,12 +170,5 @@ class ParamsDependency(Dependency):
         if not self.isfile and not self.isdir:
             raise self.IsNotFileOrDirError(self)
 
-        if self.is_empty:
-            logger.warning(f"'{self}' is empty.")
-
         self.ignore()
-
-        if self.metric or self.plot:
-            self.verify_metric()
-
         self.hash_info = self.get_hash()

dvc/dependency/repo.py

@@ -9,7 +9,7 @@ from .base import Dependency
 
 if TYPE_CHECKING:
     from dvc.hash_info import HashInfo
-    from dvc.objects.db.base import ObjectDB
+    from dvc.objects.db import ObjectDB
     from dvc.objects.file import HashFile
 
 
@@ -61,7 +61,7 @@ class RepoDependency(Dependency):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
     def download(self, to, jobs=None):
-        from dvc.objects.checkout import checkout
+        from dvc.data.checkout import checkout
 
         for odb, objs in self.get_used_objs().items():
             self.repo.cloud.pull(objs, jobs=jobs, odb=odb)
@@ -98,9 +98,10 @@ class RepoDependency(Dependency):
         self, obj_only=False, **kwargs
     ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "HashFile"]:
         from dvc.config import NoRemoteError
+        from dvc.data.stage import stage
+        from dvc.data.tree import Tree, TreeError
         from dvc.exceptions import NoOutputOrStageError, PathMissingError
-        from dvc.objects.stage import stage
-        from dvc.objects.tree import Tree
+        from dvc.utils import as_posix
 
         local_odb = self.repo.odb.local
         locked = kwargs.pop("locked", True)
@@ -112,11 +113,10 @@ class RepoDependency(Dependency):
             if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
                 self.def_repo[self.PARAM_REV_LOCK] = rev
 
-            path = os.path.abspath(os.path.join(repo.root_dir, self.def_path))
             if not obj_only:
                 try:
                     for odb, obj_ids in repo.used_objs(
-                        [path],
+                        [os.path.join(repo.root_dir, self.def_path)],
                         force=True,
                         jobs=kwargs.get("jobs"),
                         recursive=True,
@@ -132,11 +132,11 @@ class RepoDependency(Dependency):
             try:
                 staging, _, staged_obj = stage(
                     local_odb,
-                    path,
+                    as_posix(self.def_path),
                     repo.repo_fs,
                     local_odb.fs.PARAM_CHECKSUM,
                 )
-            except FileNotFoundError as exc:
+            except (FileNotFoundError, TreeError) as exc:
                 raise PathMissingError(
                     self.def_path, self.def_repo[self.PARAM_URL]
                 ) from exc
@@ -150,10 +150,10 @@ class RepoDependency(Dependency):
             return used_obj_ids, staged_obj
 
     def _check_circular_import(self, odb, obj_ids):
+        from dvc.data.db.reference import ReferenceObjectDB
+        from dvc.data.tree import Tree
         from dvc.exceptions import CircularImportError
         from dvc.fs.repo import RepoFileSystem
-        from dvc.objects.db.reference import ReferenceObjectDB
-        from dvc.objects.tree import Tree
 
         if not isinstance(odb, ReferenceObjectDB):
             return
@@ -172,17 +172,17 @@ class RepoDependency(Dependency):
                 continue
             if (
                 obj.fs.repo_url in checked_urls
-                or obj.fs.root_dir in checked_urls
+                or obj.fs.repo.root_dir in checked_urls
             ):
                 continue
             self_url = self.repo.url or self.repo.root_dir
             if (
                 obj.fs.repo_url is not None
                 and obj.fs.repo_url == self_url
-                or obj.fs.root_dir == self.repo.root_dir
+                or obj.fs.repo.root_dir == self.repo.root_dir
             ):
                 raise CircularImportError(self, obj.fs.repo_url, self_url)
-            checked_urls.update([obj.fs.repo_url, obj.fs.root_dir])
+            checked_urls.update([obj.fs.repo_url, obj.fs.repo.root_dir])
 
     def get_obj(self, filter_info=None, **kwargs):
         locked = kwargs.get("locked", True)

dvc/exceptions.py

@@ -1,4 +1,5 @@
 """Exceptions raised by the dvc."""
+from dvc.utils import format_link
 
 
 class DvcException(Exception):
@@ -128,13 +129,6 @@ class NotDvcRepoError(DvcException):
     """Thrown if a directory is not a DVC repo"""
 
 
-class DvcParserError(DvcException):
-    """Base class for CLI parser errors."""
-
-    def __init__(self):
-        super().__init__("parser error")
-
-
 class CyclicGraphError(DvcException):
     def __init__(self, stages):
         assert isinstance(stages, list)
@@ -157,9 +151,9 @@ class InitError(DvcException):
 
 
 class ReproductionError(DvcException):
-    def __init__(self, dvc_file_name):
-        self.path = dvc_file_name
-        super().__init__(f"failed to reproduce '{dvc_file_name}'")
+    def __init__(self, name):
+        self.name = name
+        super().__init__(f"failed to reproduce '{name}'")
 
 
 class BadMetricError(DvcException):
@@ -294,7 +288,6 @@ class PathMissingError(DvcException):
 
 class RemoteCacheRequiredError(DvcException):
     def __init__(self, scheme, fs_path):
-        from dvc.utils import format_link
 
         super().__init__(
             (
@@ -330,7 +323,6 @@ class MergeError(DvcException):
 
 
 class CacheLinkError(DvcException):
-    from dvc.utils import format_link
 
     SUPPORT_LINK = "See {} for more information.".format(
         format_link(

dvc/external_repo.py

@@ -3,7 +3,7 @@ import os
 import tempfile
 import threading
 from contextlib import contextmanager
-from typing import Dict
+from typing import TYPE_CHECKING, Dict
 
 from funcy import retry, wrap_with
 
@@ -19,6 +19,9 @@ from dvc.repo import Repo
 from dvc.scm import CloneError, map_scm_exception
 from dvc.utils import relpath
 
+if TYPE_CHECKING:
+    from scmrepo import Git
+
 logger = logging.getLogger(__name__)
 
 
@@ -30,6 +33,7 @@ def external_repo(
     from scmrepo.git import Git
 
     from dvc.config import NoRemoteError
+    from dvc.fs.git import GitFileSystem
 
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
@@ -45,12 +49,18 @@ def external_repo(
     config = _get_remote_config(url) if os.path.isdir(url) else {}
     config.update(cache_config)
 
-    root_dir = path if for_write else os.path.realpath(path)
+    if for_write:
+        root_dir = path
+        fs = None
+    else:
+        root_dir = os.path.realpath(path)
+        scm = Git(root_dir)
+        fs = GitFileSystem(scm=scm, rev=rev)
+
     repo_kwargs = dict(
         root_dir=root_dir,
         url=url,
-        scm=None if for_write else Git(root_dir),
-        rev=None if for_write else rev,
+        fs=fs,
         config=config,
         repo_factory=erepo_factory(url, cache_config),
         **kwargs,
@@ -86,7 +96,7 @@ def erepo_factory(url, cache_config, *args, **kwargs):
     def make_repo(path, **_kwargs):
         _config = cache_config.copy()
         if os.path.isdir(url):
-            rel = os.path.relpath(path, _kwargs["scm"].root_dir)
+            rel = os.path.relpath(path, _kwargs["fs"].fs_args["scm"].root_dir)
             repo_path = os.path.join(url, rel)
             _config.update(_get_remote_config(repo_path))
         return Repo(path, config=_config, **_kwargs)
@@ -194,12 +204,12 @@ def _clone_default_branch(url, rev, for_write=False):
                     # given repo URL it is easier/safer for us to work with
                     # full clones in this case.
                     logger.debug("erepo: unshallowing clone for '%s'", url)
-                    _unshallow(git)
+                    _pull(git, unshallow=True)
                     shallow = False
                     CLONES[url] = (clone_path, shallow)
                 else:
                     logger.debug("erepo: git pull '%s'", url)
-                    git.pull()
+                    _pull(git)
         else:
             from dvc.scm import clone
 
@@ -210,12 +220,17 @@ def _clone_default_branch(url, rev, for_write=False):
 
                 try:
                     git = clone(url, clone_path, shallow_branch=rev)
-                    shallow = True
-                    logger.debug(
-                        "erepo: using shallow clone for branch '%s'", rev
+                    shallow = os.path.exists(
+                        os.path.join(clone_path, Git.GIT_DIR, "shallow")
                     )
+                    if shallow:
+                        logger.debug(
+                            "erepo: using shallow clone for branch '%s'", rev
+                        )
                 except CloneError:
-                    pass
+                    git_dir = os.path.join(clone_path, ".git")
+                    if os.path.exists(git_dir):
+                        _remove(git_dir)
             if not git:
                 git = clone(url, clone_path)
                 shallow = False
@@ -227,17 +242,24 @@ def _clone_default_branch(url, rev, for_write=False):
     return clone_path, shallow
 
 
-def _unshallow(git):
-    if git.gitpython.repo.head.is_detached:
-        # If this is a detached head (i.e. we shallow cloned a tag) switch to
-        # the default branch
-        origin_refs = git.gitpython.repo.remotes["origin"].refs
-        ref = origin_refs["HEAD"].reference
-        branch_name = ref.name.split("/")[-1]
-        branch = git.gitpython.repo.create_head(branch_name, ref)
-        branch.set_tracking_branch(ref)
-        branch.checkout()
-    git.pull(unshallow=True)
+def _pull(git: "Git", unshallow: bool = False):
+    from dvc.repo.experiments.utils import fetch_all_exps
+
+    git.fetch(unshallow=unshallow)
+    _merge_upstream(git)
+    fetch_all_exps(git, "origin")
+
+
+def _merge_upstream(git: "Git"):
+    from scmrepo.exceptions import SCMError
+
+    try:
+        branch = git.active_branch()
+        upstream = f"refs/remotes/origin/{branch}"
+        if git.get_ref(upstream):
+            git.merge(upstream)
+    except SCMError:
+        pass
 
 
 def _git_checkout(repo_path, rev):

dvc/fs/__init__.py

@@ -114,7 +114,12 @@ def get_cloud_fs(repo, **kwargs):
         remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir
 
     url = remote_conf.pop("url")
-    fs_path = cls._strip_protocol(url)  # pylint:disable=protected-access
+    if issubclass(cls, WebDAVFileSystem):
+        # For WebDAVFileSystem, provided url is the base path itself, so it
+        # should be treated as being a root path.
+        fs_path = cls.root_marker
+    else:
+        fs_path = cls._strip_protocol(url)  # pylint:disable=protected-access
 
     extras = cls._get_kwargs_from_urls(url)  # pylint:disable=protected-access
     conf = {**extras, **remote_conf}  # remote config takes priority

dvc/fs/_callback.py

@@ -0,0 +1,190 @@
+from functools import wraps
+from typing import IO, TYPE_CHECKING, Any, Optional, TypeVar, cast
+
+import fsspec
+
+if TYPE_CHECKING:
+    from typing import Callable
+
+    from typing_extensions import ParamSpec
+
+    from dvc.fs.base import FileSystem
+    from dvc.progress import Tqdm
+    from dvc.ui._rich_progress import RichTransferProgress
+
+    _P = ParamSpec("_P")
+    _R = TypeVar("_R")
+
+
+class FsspecCallback(fsspec.Callback):
+    """FsspecCallback usable as a context manager, and a few helper methods."""
+
+    def wrap_attr(self, fobj: IO, method: str = "read") -> IO:
+        from tqdm.utils import CallbackIOWrapper
+
+        wrapped = CallbackIOWrapper(self.relative_update, fobj, method)
+        return cast(IO, wrapped)
+
+    def wrap_fn(self, fn: "Callable[_P, _R]") -> "Callable[_P, _R]":
+        @wraps(fn)
+        def wrapped(*args: "_P.args", **kwargs: "_P.kwargs") -> "_R":
+            res = fn(*args, **kwargs)
+            self.relative_update()
+            return res
+
+        return wrapped
+
+    def wrap_and_branch(
+        self, fn: "Callable", fs: "FileSystem" = None
+    ) -> "Callable":
+        """
+        Wraps a function, and pass a new child callback to it.
+        When the function completes, we increment the parent callback by 1.
+        """
+        from .local import localfs
+
+        fs = fs or localfs
+        wrapped = self.wrap_fn(fn)
+
+        def make_callback(path1, path2):
+            # pylint: disable=assignment-from-none
+            child = self.branch(fs.path.name(path1), path2, {})
+            return self.as_callback(child)
+
+        @wraps(fn)
+        def func(path1, path2, **kwargs):
+            with make_callback(path1, path2) as callback:
+                return wrapped(path1, path2, callback=callback, **kwargs)
+
+        return func
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_args):
+        self.close()
+
+    def close(self):
+        """Handle here on exit."""
+
+    @classmethod
+    def as_callback(
+        cls, maybe_callback: Optional["FsspecCallback"] = None
+    ) -> "FsspecCallback":
+        if maybe_callback is None:
+            return DEFAULT_CALLBACK
+        return maybe_callback
+
+    @classmethod
+    def as_tqdm_callback(
+        cls,
+        callback: Optional["FsspecCallback"] = None,
+        **tqdm_kwargs: Any,
+    ) -> "FsspecCallback":
+        return callback or TqdmCallback(**tqdm_kwargs)
+
+    @classmethod
+    def as_rich_callback(
+        cls, callback: Optional["FsspecCallback"] = None, **rich_kwargs
+    ):
+        return callback or RichCallback(**rich_kwargs)
+
+
+class NoOpCallback(FsspecCallback, fsspec.callbacks.NoOpCallback):
+    pass
+
+
+class TqdmCallback(FsspecCallback):
+    def __init__(self, progress_bar: "Tqdm" = None, **tqdm_kwargs):
+        from dvc.progress import Tqdm
+
+        self.progress_bar = progress_bar or Tqdm(**tqdm_kwargs)
+        super().__init__()
+
+    def __enter__(self):
+        self.progress_bar.__enter__()
+        return self
+
+    def close(self):
+        self.progress_bar.close()
+
+    def set_size(self, size):
+        if size is not None:
+            self.progress_bar.total = size
+            self.progress_bar.refresh()
+            super().set_size(size)
+
+    def relative_update(self, inc=1):
+        self.progress_bar.update(inc)
+        super().relative_update(inc)
+
+    def absolute_update(self, value):
+        self.progress_bar.update_to(value)
+        super().absolute_update(value)
+
+    def branch(self, path_1: str, path_2: str, kwargs):
+        return TqdmCallback(bytes=True, total=-1, desc=path_1, **kwargs)
+
+
+class RichCallback(FsspecCallback):
+    def __init__(
+        self,
+        progress: "RichTransferProgress" = None,
+        desc: str = None,
+        total: int = None,
+        bytes: bool = False,  # pylint: disable=redefined-builtin
+        unit: str = None,
+        disable: bool = False,
+    ) -> None:
+        from dvc.ui import ui
+        from dvc.ui._rich_progress import RichTransferProgress
+
+        self.progress = progress or RichTransferProgress(
+            transient=True,
+            disable=disable,
+            console=ui.error_console,
+        )
+        self.visible = not disable
+        self._newly_created = progress is None
+        total = 0 if total is None or total < 0 else total
+        self.task = self.progress.add_task(
+            description=desc or "",
+            total=total,
+            bytes=bytes,
+            visible=False,
+            unit=unit,
+            progress_type=None if bytes else "summary",
+        )
+        super().__init__()
+
+    def __enter__(self):
+        if self._newly_created:
+            self.progress.__enter__()
+        return self
+
+    def close(self):
+        if self._newly_created:
+            self.progress.stop()
+        try:
+            self.progress.remove_task(self.task)
+        except KeyError:
+            pass
+
+    def set_size(self, size: int = None) -> None:
+        if size is not None:
+            self.progress.update(self.task, total=size, visible=self.visible)
+            super().set_size(size)
+
+    def relative_update(self, inc: int = 1) -> None:
+        self.progress.update(self.task, advance=inc)
+        super().relative_update(inc)
+
+    def absolute_update(self, value: int) -> None:
+        self.progress.update(self.task, completed=value)
+        super().absolute_update(value)
+
+    def branch(self, path_1, path_2, kwargs):
+        return RichCallback(self.progress, desc=path_1, bytes=True, total=-1)
+
+
+DEFAULT_CALLBACK = NoOpCallback()

dvc/fs/_metadata.py

@@ -1,73 +0,0 @@
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, List
-
-from dvc.output import Output
-
-if TYPE_CHECKING:
-    from dvc.repo import Repo
-
-
-@dataclass
-class Metadata:
-    """
-    Container for storing metadata for a given path, similar in spirit to
-    `os.stat_result`.
-    """
-
-    # required field
-    fs_path: str
-    repo: "Repo"
-
-    # computed fields
-    is_output: bool = field(init=False, default=False)  # is it an output?
-    # is the path part of an output?
-    part_of_output: bool = field(init=False, default=False)
-    # does the path contain outputs?
-    contains_outputs: bool = field(init=False, default=False)
-    # is the path tracked by dvc? equivalent to (is_output or part_of_output)
-    is_dvc: bool = field(init=False, default=False)
-    # does dvc has any output in that path or inside it? equiv. to bool(outs)
-    output_exists: bool = field(init=False)
-
-    # optional fields
-    isdir: bool = False  # is it a directory?
-    is_exec: bool = False  # is it an executable?
-    outs: List[Output] = field(default_factory=list)  # list of outputs
-
-    def __post_init__(self):
-        self.output_exists = bool(self.outs)
-
-        if not self.output_exists:
-            return
-
-        # it can contain multiple outputs
-        if len(self.outs) > 1:
-            self.contains_outputs = True
-        else:
-            out = self.outs[0]
-            # or, the path itself could be an output
-            self.is_output = self.fs_path == out.fs_path
-
-            # or, a directory must have an output somewhere deep inside it
-            if not self.is_output:
-                self.contains_outputs = self.repo.fs.path.isin(
-                    out.fs_path, self.fs_path
-                )
-
-        # or, the path could be a part of an output, i.e. inside of an output
-        self.part_of_output = not self.is_output and not self.contains_outputs
-
-        self.is_dvc = self.is_output or self.part_of_output
-        # if it contains outputs, it must be a dir
-        self.isdir = self.contains_outputs
-
-    @property
-    def isfile(self):
-        # self.isdir might require additional computation, especially if it's
-        # a dir that's inside a dvc-tracked dir
-        # since, `Metadata()` does not support being used as null (i.e. when
-        # path does not exist), `isfile` becomes `isdir`'s complement.
-        return not self.isdir
-
-    def __str__(self):
-        return f"Metadata: {self.fs_path}"

dvc/fs/azure.py

@@ -10,7 +10,7 @@ from dvc.exceptions import DvcException
 from dvc.scheme import Schemes
 from dvc.utils import format_link
 
-from .fsspec_wrapper import CallbackMixin, ObjectFSWrapper
+from .base import ObjectFileSystem
 
 logger = logging.getLogger(__name__)
 _DEFAULT_CREDS_STEPS = (
@@ -38,7 +38,7 @@ def _az_config():
 
 
 # pylint:disable=abstract-method
-class AzureFileSystem(CallbackMixin, ObjectFSWrapper):
+class AzureFileSystem(ObjectFileSystem):
     scheme = Schemes.AZURE
     PARAM_CHECKSUM = "etag"
     REQUIRES = {

dvc/fs/base.py

@@ -1,29 +1,43 @@
-import contextlib
 import logging
 import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partialmethod
+import shutil
 from multiprocessing import cpu_count
-from typing import ClassVar, Dict, Optional
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    ClassVar,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    Union,
+    cast,
+    overload,
+)
 
 from funcy import cached_property
-from tqdm.utils import CallbackIOWrapper
 
 from dvc.exceptions import DvcException
-from dvc.progress import DEFAULT_CALLBACK, FsspecCallback
-from dvc.ui import ui
-from dvc.utils import tmp_fname
-from dvc.utils.fs import makedirs, move
+from dvc.utils.fs import as_atomic, makedirs
+from dvc.utils.threadpool import ThreadPoolExecutor
+
+from ._callback import DEFAULT_CALLBACK, FsspecCallback
+
+if TYPE_CHECKING:
+    from fsspec.spec import AbstractFileSystem
+    from typing_extensions import Literal
+
 
 logger = logging.getLogger(__name__)
 
 
-class RemoteCmdError(DvcException):
-    def __init__(self, remote, cmd, ret, err):
-        super().__init__(
-            "{remote} command '{cmd}' finished with non-zero return code"
-            " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
-        )
+FSPath = str
+AnyFSPath = str
+
+# An info() entry, might evolve to a TypedDict
+# in the future (e.g for properly type 'size' etc).
+Entry = Dict[str, Any]
 
 
 class RemoteActionNotImplemented(DvcException):
@@ -46,13 +60,10 @@ class FileSystem:
     HASH_JOBS = max(1, min(4, cpu_count() // 2))
     LIST_OBJECT_PAGE_SIZE = 1000
     TRAVERSE_WEIGHT_MULTIPLIER = 5
-    TRAVERSE_PREFIX_LEN = 3
+    TRAVERSE_PREFIX_LEN = 2
     TRAVERSE_THRESHOLD_SIZE = 500000
     CAN_TRAVERSE = True
 
-    # Needed for some providers, and http open()
-    CHUNK_SIZE = 64 * 1024 * 1024  # 64 MiB
-
     PARAM_CHECKSUM: ClassVar[Optional[str]] = None
 
     def __init__(self, **kwargs):
@@ -61,9 +72,11 @@ class FileSystem:
         self.jobs = kwargs.get("jobs") or self._JOBS
         self.hash_jobs = kwargs.get("checksum_jobs") or self.HASH_JOBS
         self._config = kwargs
+        self.fs_args = {"skip_instance_cache": True}
+        self.fs_args.update(self._prepare_credentials(**kwargs))
 
     @property
-    def config(self):
+    def config(self) -> Dict[str, Any]:
         return self._config
 
     @cached_property
@@ -73,21 +86,37 @@ class FileSystem:
         return Path(self.sep)
 
     @classmethod
-    def _strip_protocol(cls, path: str):
+    def _strip_protocol(cls, path: str) -> str:
         return path
 
-    def unstrip_protocol(self, path):
+    def unstrip_protocol(self, path: str) -> str:
         return path
 
+    @cached_property
+    def fs(self) -> "AbstractFileSystem":
+        raise NotImplementedError
+
     @staticmethod
-    def _get_kwargs_from_urls(urlpath):  # pylint:disable=unused-argument
+    def _get_kwargs_from_urls(urlpath: str) -> "Dict[str, Any]":
+        from fsspec.utils import infer_storage_options
+
+        options = infer_storage_options(urlpath)
+        options.pop("path", None)
+        options.pop("protocol", None)
+        return options
+
+    def _prepare_credentials(
+        self, **config: Dict[str, Any]  # pylint: disable=unused-argument
+    ) -> Dict[str, Any]:
+        """Prepare the arguments for authentication to the
+        host filesystem"""
         return {}
 
     @classmethod
-    def get_missing_deps(cls):
+    def get_missing_deps(cls) -> List[str]:
         import importlib
 
-        missing = []
+        missing: List[str] = []
         for package, module in cls.REQUIRES.items():
             try:
                 importlib.import_module(module)
@@ -135,242 +164,341 @@ class FileSystem:
             f"dependencies: {missing}. {hint}"
         )
 
-    def checksum(self, path) -> str:
-        raise NotImplementedError
+    def isdir(self, path: AnyFSPath) -> bool:
+        return self.fs.isdir(path)
 
-    def open(self, path, mode: str = "r", encoding: str = None, **kwargs):
-        raise RemoteActionNotImplemented("open", self.scheme)
+    def isfile(self, path: AnyFSPath) -> bool:
+        return self.fs.isfile(path)
 
-    def exists(self, path) -> bool:
-        raise NotImplementedError
+    def is_empty(self, path: AnyFSPath) -> bool:
+        entry = self.info(path)
+        if entry["type"] == "directory":
+            return not self.fs.ls(path)
+        return entry["size"] == 0
 
-    # pylint: disable=unused-argument
+    def open(
+        self,
+        path: AnyFSPath,
+        mode: str = "r",
+        encoding: Optional[str] = None,
+        **kwargs,
+    ) -> "IO":  # pylint: disable=arguments-differ
+        return self.fs.open(path, mode=mode, encoding=encoding, **kwargs)
 
-    def isdir(self, path):
-        """Optional: Overwrite only if the remote has a way to distinguish
-        between a directory and a file.
-        """
-        return False
+    def cat(
+        self,
+        path: Union[AnyFSPath, List[AnyFSPath]],
+        recursive: bool = False,
+        on_error: "Literal['raise', 'omit', 'return']" = "raise",
+        **kwargs: Any,
+    ) -> Union[bytes, Dict[AnyFSPath, bytes]]:
+        return self.fs.cat(
+            path, recursive=recursive, on_error=on_error, **kwargs
+        )
 
-    def isfile(self, path):
-        """Optional: Overwrite only if the remote has a way to distinguish
-        between a directory and a file.
-        """
-        return True
+    def cat_ranges(
+        self,
+        paths: List[AnyFSPath],
+        starts: List[int],
+        ends: List[int],
+        max_gap: int = None,
+        **kwargs,
+    ) -> List[bytes]:
+        return self.fs.cat_ranges(
+            paths, starts, ends, max_gap=max_gap, **kwargs
+        )
 
-    def isexec(self, path):
-        """Optional: Overwrite only if the remote has a way to distinguish
-        between executable and non-executable file.
-        """
-        return False
+    def cat_file(
+        self,
+        path: AnyFSPath,
+        start: int = None,
+        end: int = None,
+        **kwargs: Any,
+    ) -> bytes:
+        return self.fs.cat_file(path, start=start, end=end, **kwargs)
 
-    def iscopy(self, path):
-        """Check if this file is an independent copy."""
-        return False  # We can't be sure by default
+    def pipe_file(self, path: AnyFSPath, value: bytes, **kwargs: Any) -> None:
+        return self.fs.pipe_file(path, value, **kwargs)
 
-    def walk(self, top, topdown=True, onerror=None, **kwargs):
-        """Return a generator with (root, dirs, files)."""
-        raise NotImplementedError
+    def pipe(
+        self,
+        path: Union[AnyFSPath, Dict[AnyFSPath, bytes]],
+        value: Optional[bytes] = None,
+        **kwargs: Any,
+    ) -> None:
+        return self.fs.pipe(path, value=value, **kwargs)
 
-    def find(self, path, prefix=None):
-        """Return a generator with `str`s to all the files.
+    def touch(
+        self, path: AnyFSPath, truncate: bool = True, **kwargs: Any
+    ) -> None:
+        return self.fs.touch(path, truncate=truncate, **kwargs)
 
-        Optional kwargs:
-            prefix (bool): If true `path` will be treated as a prefix
-                rather than directory path.
-        """
-        raise NotImplementedError
+    def checksum(self, path: AnyFSPath) -> str:
+        return self.fs.checksum(path)
 
-    def is_empty(self, path):
-        return False
+    def copy(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
+        self.makedirs(self.path.parent(to_info))
+        self.fs.copy(from_info, to_info)
 
-    def info(self, path):
-        raise NotImplementedError
+    def exists(self, path: AnyFSPath) -> bool:
+        return self.fs.exists(path)
 
-    def getsize(self, path):
-        return self.info(path).get("size")
+    def lexists(self, path: AnyFSPath) -> bool:
+        return self.fs.lexists(path)
 
-    def remove(self, path):
-        raise RemoteActionNotImplemented("remove", self.scheme)
+    def symlink(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
+        try:
+            return self.fs.symlink(from_info, to_info)
+        except AttributeError:
+            raise RemoteActionNotImplemented("symlink", self.scheme)
 
-    def makedirs(self, path, **kwargs):
-        """Optional: Implement only if the remote needs to create
-        directories before copying/linking/moving data
-        """
+    def hardlink(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
+        try:
+            return self.fs.hardlink(from_info, to_info)
+        except AttributeError:
+            raise RemoteActionNotImplemented("hardlink", self.scheme)
 
-    def move(self, from_info, to_info):
-        self.copy(from_info, to_info)
-        self.remove(from_info)
+    def reflink(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
+        try:
+            return self.fs.reflink(from_info, to_info)
+        except AttributeError:
+            raise RemoteActionNotImplemented("reflink", self.scheme)
 
-    def copy(self, from_info, to_info):
-        raise RemoteActionNotImplemented("copy", self.scheme)
+    def is_symlink(self, path: AnyFSPath) -> bool:
+        try:
+            return self.fs.is_symlink(path)
+        except AttributeError:
+            return False
 
-    def symlink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("symlink", self.scheme)
+    def is_hardlink(self, path: AnyFSPath) -> bool:
+        try:
+            return self.fs.is_hardlink(path)
+        except AttributeError:
+            return False
 
-    def hardlink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("hardlink", self.scheme)
+    def iscopy(self, path: AnyFSPath) -> bool:
+        return self.is_symlink(path) or self.is_hardlink(path)
 
-    def reflink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("reflink", self.scheme)
+    @overload
+    def ls(
+        self, path: AnyFSPath, detail: "Literal[True]"
+    ) -> "Iterator[Entry]":
+        ...
+
+    @overload
+    def ls(self, path: AnyFSPath, detail: "Literal[False]") -> Iterator[str]:
+        ...
+
+    def ls(self, path, detail=False, **kwargs):
+        return self.fs.ls(path, detail=detail)
+
+    def find(
+        self,
+        path: AnyFSPath,
+        prefix: bool = False,  # pylint: disable=unused-argument
+    ) -> Iterator[str]:
+        yield from self.fs.find(path)
+
+    def move(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
+        self.fs.move(from_info, to_info)
+
+    def remove(self, path: AnyFSPath) -> None:
+        self.fs.rm(path, recursive=True)
+
+    def info(self, path: AnyFSPath) -> "Entry":
+        return self.fs.info(path)
+
+    def makedirs(self, path: AnyFSPath, **kwargs: Any) -> None:
+        self.fs.makedirs(path, exist_ok=kwargs.pop("exist_ok", True))
+
+    def put_file(
+        self,
+        from_file: Union[AnyFSPath, IO],
+        to_info: AnyFSPath,
+        callback: FsspecCallback = DEFAULT_CALLBACK,
+        size: int = None,
+        **kwargs,
+    ) -> None:
+        if size:
+            callback.set_size(size)
+        if hasattr(from_file, "read"):
+            stream = callback.wrap_attr(cast("IO", from_file))
+            self.upload_fobj(stream, to_info, size=size)
+        else:
+            assert isinstance(from_file, str)
+            self.fs.put_file(
+                os.fspath(from_file), to_info, callback=callback, **kwargs
+            )
+        self.fs.invalidate_cache(self.path.parent(to_info))
+
+    def get_file(
+        self,
+        from_info: AnyFSPath,
+        to_info: AnyFSPath,
+        callback: FsspecCallback = DEFAULT_CALLBACK,
+        **kwargs,
+    ) -> None:
+        self.fs.get_file(from_info, to_info, callback=callback, **kwargs)
+
+    def upload_fobj(self, fobj: IO, to_info: AnyFSPath, **kwargs) -> None:
+        self.makedirs(self.path.parent(to_info))
+        with self.open(to_info, "wb") as fdest:
+            shutil.copyfileobj(
+                fobj,
+                fdest,
+                length=getattr(fdest, "blocksize", None),  # type: ignore
+            )
+
+    def walk(
+        self,
+        path: AnyFSPath,
+        topdown: bool = True,
+        **kwargs: Any,
+    ):
+        return self.fs.walk(path, topdown=topdown, **kwargs)
+
+    def size(self, path: AnyFSPath) -> Optional[int]:
+        return self.fs.size(path)
 
     # pylint: enable=unused-argument
 
     def upload(
         self,
-        from_info,
-        to_info,
-        total=None,
-        desc=None,
-        callback=None,
-        no_progress_bar=False,
-        **pbar_args,
+        from_info: Union[AnyFSPath, IO],
+        to_info: AnyFSPath,
+        size: int = None,
+        callback: FsspecCallback = None,
     ):
-        is_file_obj = hasattr(from_info, "read")
-        method = "upload_fobj" if is_file_obj else "put_file"
-        if not hasattr(self, method):
-            raise RemoteActionNotImplemented(method, self.scheme)
-
-        if not is_file_obj:
-            from .local import localfs
-
-            desc = desc or localfs.path.name(from_info)
-
-        stack = contextlib.ExitStack()
-        if not callback:
-            pbar = ui.progress(
-                desc=desc,
-                disable=no_progress_bar,
-                bytes=True,
-                total=total or -1,
-                **pbar_args,
-            )
-            stack.enter_context(pbar)
-            callback = pbar.as_callback()
-            if total:
-                callback.set_size(total)
-
-        with stack:
-            if is_file_obj:
-                wrapped = CallbackIOWrapper(
-                    callback.relative_update, from_info
-                )
-                # `size` is used to provide hints to the WebdavFileSystem
-                # for legacy servers.
-                # pylint: disable=no-member
-                return self.upload_fobj(wrapped, to_info, size=total)
+        from .local import localfs
 
+        if not hasattr(from_info, "read"):
             logger.debug("Uploading '%s' to '%s'", from_info, to_info)
-            # pylint: disable=no-member
-            return self.put_file(
-                os.fspath(from_info), to_info, callback=callback
-            )
+            desc = localfs.path.name(from_info)
+        else:
+            desc = self.path.name(to_info)
+
+        with FsspecCallback.as_tqdm_callback(
+            callback,
+            desc=desc,
+            bytes=True,
+            total=size or -1,
+        ) as cb:
+            return self.put_file(from_info, to_info, callback=cb, size=size)
 
     def download(
         self,
-        from_info,
-        to_info,
-        name=None,
-        callback=None,
-        no_progress_bar=False,
-        jobs=None,
-        _only_file=False,
-        **kwargs,
+        from_info: AnyFSPath,
+        to_info: AnyFSPath,
+        callback: "FsspecCallback" = None,
+        jobs: int = None,
     ):
+        if not self.isdir(from_info):
+            return self.download_file(from_info, to_info, callback=callback)
+
         from .local import localfs
 
-        if not hasattr(self, "get_file"):
-            raise RemoteActionNotImplemented("get_file", self.scheme)
-
-        download_dir = not _only_file and self.isdir(from_info)
-
-        desc = name or localfs.path.name(to_info)
-        stack = contextlib.ExitStack()
-        if not callback:
-            pbar_kwargs = {"unit": "Files"} if download_dir else {}
-            pbar = ui.progress(
-                total=-1,
-                desc="Downloading directory" if download_dir else desc,
-                bytes=not download_dir,
-                disable=no_progress_bar,
-                **pbar_kwargs,
+        pairs = {
+            info: localfs.path.join(
+                to_info, *self.path.relparts(info, from_info)
             )
-            stack.enter_context(pbar)
-            callback = pbar.as_callback()
-
-        with stack:
-            if download_dir:
-                return self._download_dir(
-                    from_info, to_info, callback=callback, jobs=jobs, **kwargs
-                )
-            return self._download_file(from_info, to_info, callback=callback)
-
-    download_file = partialmethod(download, _only_file=True)
-
-    def _download_dir(
-        self,
-        from_info,
-        to_info,
-        callback=DEFAULT_CALLBACK,
-        jobs=None,
-        **kwargs,
-    ):
-        from .local import localfs
-
-        from_infos = list(self.find(from_info, **kwargs))
-        if not from_infos:
+            for info in self.find(from_info)
+        }
+        if not pairs:
             makedirs(to_info, exist_ok=True)
-            return None
-
-        to_infos = (
-            localfs.path.join(to_info, *self.path.relparts(info, from_info))
-            for info in from_infos
-        )
-        callback.set_size(len(from_infos))
-
-        download_files = FsspecCallback.wrap_fn(callback, self._download_file)
-        max_workers = jobs or self.jobs
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            futures = [
-                executor.submit(download_files, from_info, to_info)
-                for from_info, to_info in zip(from_infos, to_infos)
-            ]
+            return
 
+        with FsspecCallback.as_tqdm_callback(
+            callback,
+            total=-1,
+            desc=f"Downloading directory {self.path.name(from_info)}",
+            unit="files",
+        ) as cb:
+            cb.set_size(len(pairs))
+            download_files = cb.wrap_and_branch(self.download_file, fs=self)
+            max_workers = jobs or self.jobs
             # NOTE: unlike pulling/fetching cache, where we need to
             # download everything we can, not raising an error here might
             # turn very ugly, as the user might think that he has
             # downloaded a complete directory, while having a partial one,
             # which might cause unexpected results in his pipeline.
-            for future in as_completed(futures):
-                # NOTE: executor won't let us raise until all futures that
-                # it has are finished, so we need to cancel them ourselves
-                # before re-raising.
-                exc = future.exception()
-                if exc:
-                    for entry in futures:
-                        entry.cancel()
-                    raise exc
+            with ThreadPoolExecutor(
+                max_workers=max_workers, cancel_on_error=True
+            ) as executor:
+                list(
+                    executor.imap_unordered(
+                        lambda args: download_files(*args), pairs.items()
+                    )
+                )
 
-    def _download_file(
+    def download_file(
         self,
-        from_info,
-        to_info,
-        callback=DEFAULT_CALLBACK,
+        from_info: AnyFSPath,
+        to_info: AnyFSPath,
+        callback: FsspecCallback = None,
     ):
         from .local import localfs
 
-        makedirs(localfs.path.parent(to_info), exist_ok=True)
-        tmp_file = tmp_fname(to_info)
+        with FsspecCallback.as_tqdm_callback(
+            callback,
+            total=-1,
+            desc=self.path.name(from_info),
+            bytes=True,
+        ) as cb:
+            with as_atomic(localfs, to_info, create_parents=True) as tmp_file:
+                self.get_file(from_info, tmp_file, callback=cb)
 
-        logger.debug("Downloading '%s' to '%s'", from_info, to_info)
+
+class ObjectFileSystem(FileSystem):  # pylint: disable=abstract-method
+    TRAVERSE_PREFIX_LEN = 3
+
+    def makedirs(self, path: AnyFSPath, **kwargs: Any) -> None:
+        # For object storages make this method a no-op. The original
+        # fs.makedirs() method will only check if the bucket exists
+        # and create if it doesn't though we don't want to support
+        # that behavior, and the check will cost some time so we'll
+        # simply ignore all mkdir()/makedirs() calls.
+        return None
+
+    def _isdir(self, path: AnyFSPath) -> bool:
+        # Directory in object storages are interpreted differently
+        # among different fsspec providers, so this logic is a temporary
+        # measure for us to adapt as of now. It checks whether it is a
+        # directory (as in a prefix with contents) or whether it is an empty
+        # file where it's name ends with a forward slash
+
+        entry = self.info(path)
+        return entry["type"] == "directory" or (
+            entry["size"] == 0
+            and entry["type"] == "file"
+            and entry["name"].endswith("/")
+        )
+
+    def isdir(self, path: AnyFSPath) -> bool:
         try:
-            # noqa, pylint: disable=no-member
-            self.get_file(from_info, tmp_file, callback=callback)
-        except Exception:  # pylint: disable=broad-except
-            # do we need to rollback makedirs for previously not-existing
-            # directories?
-            with contextlib.suppress(FileNotFoundError):
-                os.unlink(tmp_file)
-            raise
+            return self._isdir(path)
+        except FileNotFoundError:
+            return False
 
-        move(tmp_file, to_info)
+    def isfile(self, path: AnyFSPath) -> bool:
+        try:
+            return not self._isdir(path)
+        except FileNotFoundError:
+            return False
+
+    def find(self, path: AnyFSPath, prefix: bool = False) -> Iterator[str]:
+        if prefix:
+            with_prefix = self.path.parent(path)
+            files = self.fs.find(with_prefix, prefix=self.path.parts(path)[-1])
+        else:
+            with_prefix = path
+            files = self.fs.find(path)
+
+        # When calling find() on a file, it returns the same file in a list.
+        # For object-based storages, the same behavior applies to empty
+        # directories since they are represented as files. This condition
+        # checks whether we should yield an empty list (if it is an empty
+        # directory) or just yield the file itself.
+        if len(files) == 1 and files[0] == with_prefix and self.isdir(path):
+            return None
+
+        yield from files