exp pull/push: better result handling in exp sharing(#7448)

fix: #7448
Current we skip on those experiments which already existed on both sides, but
do not exclude them from the result list. And if we have diverged experiment,
pulling/pushing them will stop the current progress and raise an exception, but
didn't give any information on the condition of other experiments.

1. Change the exp pull and push to match the new API in `scmrepo`.
2. Handle sync status after `exp pull/push` finished.
3. Bump scmrepo to 0.0.19

Co-authored-by:

.dvcignore

@@ -0,0 +1,4 @@
+/tests
+/dvc
+/.github
+/bin

.git_archival.txt

@@ -0,0 +1 @@
+ref-names: $Format:%D$

.gitattributes

@@ -0,0 +1 @@
+.git_archival.txt  export-subst

.github/mergify.yml

@@ -1,7 +1,7 @@
 pull_request_rules:
   - name: backport patches to 1.11
     conditions:
-      - base=master
+      - base=main
       - label=backport-1.11
     actions:
       backport:

.github/workflows/benchmarks.yaml

@@ -14,10 +14,10 @@ jobs:
         uses: styfle/[email protected]
         with:
           access_token: ${{ github.token }}
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v3
         with:
             python-version: 3.7
-      - uses: iterative/dvc-bench@master
+      - uses: iterative/dvc-bench@main
         with:
             pytest_options: "-k 'test_init or test_help'"
 

.github/workflows/packages.yaml

@@ -2,7 +2,7 @@ name: Build packages
 on:
   push:
     branches:
-    - master
+    - main
   release:
     types: [released, prereleased]
   workflow_dispatch:
@@ -10,13 +10,13 @@ jobs:
   deb:
     runs-on: ubuntu-18.04
     steps:
-    - uses: actions/checkout@v2.3.5
+    - uses: actions/checkout@v3
       with:
           fetch-depth: 0
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2.2.2
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Set up Ruby 2.4
       uses: actions/[email protected]
       with:
@@ -41,13 +41,13 @@ jobs:
   rpm:
     runs-on: ubuntu-18.04
     steps:
-    - uses: actions/checkout@v2.3.5
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2.2.2
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Set up Ruby 2.4
       uses: actions/[email protected]
       with:
@@ -72,13 +72,13 @@ jobs:
   osxpkg:
     runs-on: macos-10.15
     steps:
-    - uses: actions/checkout@v2.3.5
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2.2.2
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Set up Ruby 2.4
       uses: actions/[email protected]
       with:
@@ -106,13 +106,13 @@ jobs:
   exe:
     runs-on: windows-2019
     steps:
-    - uses: actions/checkout@v2.3.5
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2.2.2
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Install deps
       run: |
         pip install .[all]
@@ -136,11 +136,11 @@ jobs:
   pip:
     runs-on: ubuntu-18.04
     steps:
-    - uses: actions/checkout@v2.3.5
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v3
       with:
         python-version: 3.7
     - name: Install

.github/workflows/tests.yaml

@@ -20,21 +20,18 @@ jobs:
       uses: styfle/[email protected]
       with:
         access_token: ${{ github.token }}
-    - uses: actions/checkout@v2.3.5
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v3
       with:
         python-version: 3.8
     - name: Install requirements
       run: |
-        pip install wheel
-        pip install ".[all,tests,terraform]" pre-commit
-    - name: install temporary dependencies
-      run: |
-        pip install git+https://github.com/isidentical/gdrivefs@service-account
-        pip install git+https://github.com/isidentical/pydata-google-auth@service-account
+        pip install --upgrade pip setuptools wheel
+        pip install collective.checkdocs==0.2
+        pip install ".[dev]" pre-commit
     - name: Check README
       run: python setup.py checkdocs
     - uses: pre-commit/[email protected]
@@ -46,16 +43,12 @@ jobs:
       matrix:
         os: [ubuntu-20.04, windows-latest, macos-latest]
         pyv: ["3.7", "3.8", "3.9", "3.10"]
-        exclude:
-          # no wheels for pygit2 yet
-          - os: windows-latest
-            pyv: "3.10"
     steps:
-    - uses: actions/checkout@v2.3.5
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - name: Set up Python
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.pyv }}
     - name: get pip cache dir
@@ -64,7 +57,7 @@ jobs:
         echo "::set-output name=dir::$(pip cache dir)"
     - name: set pip cache
       id: pip-cache
-      uses: actions/cache@v2.1.6
+      uses: actions/cache@v3
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.cfg') }}
@@ -74,49 +67,20 @@ jobs:
       if: steps.cache.pip-cache-dir.cache-hit != 'true'
       run: |
         pip install --upgrade pip setuptools wheel
-        pip install -e ".[all,tests,terraform]"
-    - name: install temporary dependencies
-      run: |
-        pip install git+https://github.com/isidentical/gdrivefs@service-account
-        pip install git+https://github.com/isidentical/pydata-google-auth@service-account
-    - name: setup git
-      run: |
-        git config --global user.email "[email protected]"
-        git config --global user.name "DVC Tester"
-    - name: setup gs creds
-      shell: bash
-      env:
-        GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
-      if: env.GS_CREDS_JSON != ''
-      run: |
-        mkdir -p scripts/ci
-        echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
-    - name: disable gdrive tests
-      if: >-
-        github.event_name != 'schedule' &&
-        (matrix.pyv != '3.8' || matrix.os != 'ubuntu-18.04')
-      run: |
-        echo "extra_test_args=--disable-gdrive" >> $GITHUB_ENV
+        pip install -e ".[dev]"
     - name: run tests
       timeout-minutes: 40
-      env:
-        GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID}}
-        OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET}}
-        OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT}}
       run: >-
-        python -m tests -n=4
+        python -m tests -n=auto
         --cov-report=xml --cov-report=term
-        --enable-ssh ${{ env.extra_test_args }}
+        ${{ env.extra_test_args }}
     - name: upload coverage report
-      uses: codecov/codecov-action@v2.1.0
+      uses: codecov/codecov-action@v3
       with:
         file: ./coverage.xml
         fail_ci_if_error: false
   notify:
-    if: github.ref == 'refs/heads/master' && failure()
+    if: github.ref == 'refs/heads/main' && failure()
     needs:
       - lint
       - tests
@@ -127,6 +91,6 @@ jobs:
       env:
         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         SLACK_COLOR: ${{ job.status }}
-        SLACK_MESSAGE: 'CI Failed on master :boom:'
+        SLACK_MESSAGE: 'CI Failed on main :boom:'
         SLACK_TITLE: CI Status
         SLACK_USERNAME: DVC-CI

.gitignore

@@ -1,10 +1,13 @@
+__pycache__/
 neatlynx.conf
-neatlynx/__pycache__
 .idea
 /cache
 *.pyc
-.env/
-.env2.7/
+
+.env*/
+.venv
+env/
+venv/
 .python-version
 
 .dvc.conf.lock
@@ -26,6 +29,7 @@ innosetup/config.ini
 
 .coverage
 .coverage.*
+coverage.xml
 
 *.sw?
 
@@ -40,3 +44,9 @@ scripts/ci/gcp-creds.json
 
 *~
 /dvc/_dvc_version.py
+
+.mypy_cache/
+.pytest_cache/
+.nox/
+.tox/
+htmlcov/

.pre-commit-config.yaml

@@ -1,14 +1,14 @@
 ci:
-  skip: [pylint, dvc-pre-commit]
+  skip: [mypy, pylint, dvc-pre-commit]
 
 repos:
   - hooks:
       - id: black
         language_version: python3
     repo: https://github.com/ambv/black
-    rev: 21.9b0
+    rev: 22.3.0
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.2.0
     hooks:
       - id: check-toml
       - id: check-yaml
@@ -16,14 +16,14 @@ repos:
       - id: codespell
         args: 
           - --ignore-words-list
-          - ba,datas,fo,uptodate
+          - ba,datas,fo,uptodate,cachable
     repo: https://github.com/codespell-project/codespell
     rev: v2.1.0
   - hooks:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 5.9.3
+    rev: 5.10.1
   - hooks:
       - id: flake8
         language_version: python3
@@ -34,19 +34,14 @@ repos:
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
     rev: 3.9.2
-  - hooks:
-      -  id: mypy
-         language_version: python3
-         files: ^dvc/
-         additional_dependencies:
-           - types-requests
-           - types-paramiko
-           - types-tabulate
-           - types-toml
-    repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.910-1'
   - repo: local
     hooks:
+    - id: mypy
+      name: mypy
+      entry: mypy
+      files: ^dvc/
+      language: system
+      types: [python]
     - id: pylint
       name: pylint
       entry: pylint

README.rst

@@ -103,7 +103,7 @@ Snap (Snapcraft/Linux)
 
 This corresponds to the latest tagged release.
 Add ``--beta`` for the latest tagged release candidate,
-or ``--edge`` for the latest ``master`` version.
+or ``--edge`` for the latest ``main`` version.
 
 Choco (Chocolatey/Windows)
 --------------------------
@@ -183,8 +183,8 @@ Fedora / CentOS (rpm)
 Comparison to related technologies
 ==================================
 
-#. Data Engineering tools such as `AirFlow <https://airflow.apache.org/>`,
-   `Luigi <https://github.com/spotify/luigi>`, and others - in DVC data,
+#. Data Engineering tools such as `AirFlow <https://airflow.apache.org/>`_,
+   `Luigi <https://github.com/spotify/luigi>`_, and others - in DVC data,
    model and ML pipelines represent a single ML project focused on data
    scientists' experience.  Data engineering tools orchestrate multiple data
    projects and focus on efficient execution. A DVC project can be used from
@@ -212,7 +212,7 @@ Comparison to related technologies
 #. `DVC Studio <https://studio.iterative.ai/>`_ - official online
    platform for DVC projects.  It can be used to manage data and models, run
    and track experiments, and visualize and share results.  Also, it
-   integrates with `CML (CI/CD for ML) <https://cml.dev/>` for training
+   integrates with `CML (CI/CD for ML) <https://cml.dev/>`__ for training
    models in the cloud or Kubernetes.
 
 
@@ -254,7 +254,7 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://dvc.org
    :alt: DVC logo
 
-.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=master
+.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=main
    :target: https://github.com/iterative/dvc/actions
    :alt: GHA Tests
 
@@ -262,7 +262,7 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://codeclimate.com/github/iterative/dvc
    :alt: Code Climate
 
-.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
+.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/main/graph/badge.svg
    :target: https://codecov.io/gh/iterative/dvc
    :alt: Codecov
 

dvc/__main__.py

@@ -1,7 +1,7 @@
 """Main entry point for DVC command line tool."""
 import sys
 
-from dvc.main import main
+from dvc.cli import main
 
 if __name__ == "__main__":
     sys.exit(main(sys.argv[1:]))

dvc/_debug.py

@@ -1,10 +1,32 @@
 from contextlib import ExitStack, contextmanager
+from datetime import datetime
 from typing import TYPE_CHECKING, Callable, Union
 
 if TYPE_CHECKING:
     from argparse import Namespace
 
 
+@contextmanager
+def viztracer_profile(
+    path: Union[Callable[[], str], str],
+    max_stack_depth: int = -1,
+):
+    try:
+        import viztracer  # pylint: disable=import-error
+    except ImportError:
+        print("Failed to run profiler, viztracer is not installed")
+        yield
+        return
+
+    tracer = viztracer.VizTracer(max_stack_depth=max_stack_depth)
+
+    tracer.start()
+    yield
+    tracer.stop()
+
+    tracer.save(path() if callable(path) else path)
+
+
 @contextmanager
 def yappi_profile(
     path: Union[Callable[[], str], str] = None, wall_clock: bool = True
@@ -100,12 +122,18 @@ def debugtools(args: "Namespace" = None, **kwargs):
         if kw.get("instrument") or kw.get("instrument_open"):
             stack.enter_context(instrument(kw.get("instrument_open", False)))
         if kw.get("yappi"):
-            from datetime import datetime
-
             output = "callgrind.dvc-{0:%Y%m%d}_{0:%H%M%S}.out"
             stack.enter_context(
                 yappi_profile(path=lambda: output.format(datetime.now()))
             )
+        if kw.get("viztracer") or kw.get("viztracer_depth"):
+            output = "viztracer.dvc-{0:%Y%m%d}_{0:%H%M%S}.json"
+            stack.enter_context(
+                viztracer_profile(
+                    path=lambda: output.format(datetime.now()),
+                    max_stack_depth=kw.get("viztracer_depth") or -1,
+                )
+            )
         yield
 
 
@@ -118,6 +146,10 @@ def add_debugging_flags(parser):
     parser.add_argument(
         "--yappi", action="store_true", default=False, help=SUPPRESS
     )
+    parser.add_argument(
+        "--viztracer", action="store_true", default=False, help=SUPPRESS
+    )
+    parser.add_argument("--viztracer-depth", type=int, help=SUPPRESS)
     parser.add_argument("--cprofile-dump", help=SUPPRESS)
     parser.add_argument(
         "--pdb", action="store_true", default=False, help=SUPPRESS

dvc/analytics.py

@@ -82,10 +82,12 @@ def send(path):
 
 
 def _scm_in_use():
+    from scmrepo.noscm import NoSCM
+
     from dvc.exceptions import NotDvcRepoError
     from dvc.repo import Repo
-    from dvc.scm import SCM, NoSCM
-    from dvc.scm.base import SCMError
+
+    from .scm import SCM, SCMError
 
     try:
         scm = SCM(root_dir=Repo.find_root())

dvc/api.py

@@ -4,7 +4,6 @@ from contextlib import _GeneratorContextManager as GCM
 from funcy import reraise
 
 from dvc.exceptions import OutputNotFoundError, PathMissingError
-from dvc.path_info import PathInfo
 from dvc.repo import Repo
 
 
@@ -20,16 +19,17 @@ def get_url(path, repo=None, rev=None, remote=None):
     directory in the remote storage.
     """
     with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
-        path_info = PathInfo(_repo.root_dir) / path
         with reraise(FileNotFoundError, PathMissingError(path, repo)):
-            metadata = _repo.repo_fs.metadata(path_info)
+            info = _repo.repo_fs.info(path)
 
-        if not metadata.is_dvc:
+        dvc_info = info.get("dvc_info")
+        if not dvc_info:
             raise OutputNotFoundError(path, repo)
 
-        cloud = metadata.repo.cloud
-        md5 = metadata.repo.dvcfs.info(path_info)["md5"]
-        return cloud.get_url_for(remote, checksum=md5)
+        dvc_repo = info["repo"]
+        md5 = dvc_info["md5"]
+
+        return dvc_repo.cloud.get_url_for(remote, checksum=md5)
 
 
 def open(  # noqa, pylint: disable=redefined-builtin

dvc/cli/__init__.py

@@ -0,0 +1,143 @@
+"""This module provides an entrypoint to the dvc cli and parsing utils."""
+
+import logging
+import sys
+
+# Workaround for CPython bug. See [1] and [2] for more info.
+# [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
+# [2] https://bugs.python.org/issue29288
+
+"".encode("idna")
+
+logger = logging.getLogger("dvc")
+
+
+class DvcParserError(Exception):
+    """Base class for CLI parser errors."""
+
+    def __init__(self):
+        super().__init__("parser error")
+
+
+def parse_args(argv=None):
+    """Parses CLI arguments.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Raises:
+        DvcParserError: raised for argument parsing errors.
+    """
+    from .parser import get_main_parser
+
+    parser = get_main_parser()
+    args = parser.parse_args(argv)
+    args.parser = parser
+    return args
+
+
+def main(argv=None):  # noqa: C901
+    """Main entry point for dvc CLI.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Returns:
+        int: command's return code.
+    """
+    from dvc._debug import debugtools
+    from dvc.config import ConfigError
+    from dvc.exceptions import DvcException, NotDvcRepoError
+    from dvc.logger import FOOTER, disable_other_loggers
+
+    # NOTE: stderr/stdout may be closed if we are running from dvc.daemon.
+    # On Linux we directly call cli.main after double forking and closing
+    # the copied parent's standard file descriptors. If we make any logging
+    # calls in this state it will cause an exception due to writing to a closed
+    # file descriptor.
+    if sys.stderr.closed:  # pylint: disable=using-constant-test
+        logging.disable()
+    elif sys.stdout.closed:  # pylint: disable=using-constant-test
+        logging.disable(logging.INFO)
+
+    args = None
+    disable_other_loggers()
+
+    outerLogLevel = logger.level
+    try:
+        args = parse_args(argv)
+
+        level = None
+        if args.quiet:
+            level = logging.CRITICAL
+        elif args.verbose == 1:
+            level = logging.DEBUG
+        elif args.verbose > 1:
+            level = logging.TRACE
+
+        if level is not None:
+            logger.setLevel(level)
+
+        logger.trace(args)
+
+        if not sys.stdout.closed and not args.quiet:
+            from dvc.ui import ui
+
+            ui.enable()
+
+        with debugtools(args):
+            cmd = args.func(args)
+            ret = cmd.do_run()
+    except ConfigError:
+        logger.exception("configuration error")
+        ret = 251
+    except KeyboardInterrupt:
+        logger.exception("interrupted by the user")
+        ret = 252
+    except NotDvcRepoError:
+        logger.exception("")
+        ret = 253
+    except DvcException:
+        ret = 255
+        logger.exception("")
+    except DvcParserError:
+        ret = 254
+    except Exception as exc:  # noqa, pylint: disable=broad-except
+        # pylint: disable=no-member
+        import errno
+
+        if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+            from dvc.utils import error_link
+
+            logger.exception(
+                "too many open files, please visit "
+                "{} to see how to handle this "
+                "problem".format(error_link("many-files")),
+                extra={"tb_only": True},
+            )
+        else:
+            from dvc.info import get_dvc_info
+
+            logger.exception("unexpected error")
+
+            dvc_info = get_dvc_info()
+            logger.debug("Version info for developers:\n%s", dvc_info)
+
+            logger.info(FOOTER)
+        ret = 255
+
+    try:
+        from dvc import analytics
+
+        if analytics.is_enabled():
+            analytics.collect_and_send_report(args, ret)
+
+        return ret
+    finally:
+        logger.setLevel(outerLogLevel)
+
+        from dvc.external_repo import clean_repos
+
+        # Remove cached repos in the end of the call, these are anonymous
+        # so won't be reused by any other subsequent run anyway.
+        clean_repos()

dvc/cli/command.py

@@ -5,29 +5,6 @@ from abc import ABC, abstractmethod
 logger = logging.getLogger(__name__)
 
 
-def fix_subparsers(subparsers):
-    """Workaround for bug in Python 3. See more info at:
-    https://bugs.python.org/issue16308
-    https://github.com/iterative/dvc/issues/769
-
-    Args:
-        subparsers: subparsers to fix.
-    """
-    subparsers.required = True
-    subparsers.dest = "cmd"
-
-
-def append_doc_link(help_message, path):
-    from dvc.utils import format_link
-
-    if not path:
-        return help_message
-    doc_base = "https://man.dvc.org/"
-    return "{message}\nDocumentation: {link}".format(
-        message=help_message, link=format_link(doc_base + path)
-    )
-
-
 class CmdBase(ABC):
     UNINITIALIZED = False
 

dvc/cli/completion.py

@@ -1,17 +1,6 @@
-import argparse
-import logging
-
 import shtab
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.ui import ui
-
-logger = logging.getLogger(__name__)
-FILE = shtab.FILE
-DIR = shtab.DIRECTORY
-
-PREAMBLE = {
-    "bash": """
+BASH_PREAMBLE = """
 # $1=COMP_WORDS[1]
 _dvc_compgen_DVCFiles() {
   compgen -d -S '/' -- $1  # recurse into subdirs
@@ -30,11 +19,12 @@ _dvc_compgen_stages_and_files() {
 }
 
 _dvc_compgen_exps() {
-    local _dvc_exps=($(dvc exp list -q --all --names-only))
+    local _dvc_exps=($(dvc exp list -q --all-commits --names-only))
     compgen -W "${_dvc_exps[*]}" -- $1
 }
-    """,
-    "zsh": """
+"""
+
+ZSH_PREAMBLE = """
 _dvc_compadd_DVCFiles() {
     _files -g '(*?.dvc|Dvcfile|dvc.yaml)'
 }
@@ -63,49 +53,21 @@ _dvc_compadd_stages_and_files() {
 }
 
 _dvc_compadd_exps() {
-    _describe 'experiments' "($(dvc exp list -q -a --names-only))"
+    _describe 'experiments' "($(dvc exp list -q --all-commits --names-only))"
 }
-    """,
+"""
+
+PREAMBLE = {
+    "bash": BASH_PREAMBLE,
+    "zsh": ZSH_PREAMBLE,
 }
 
+FILE = shtab.FILE
+DIR = shtab.DIRECTORY
 DVC_FILE = {"bash": "_dvc_compgen_DVCFiles", "zsh": "_dvc_compadd_DVCFiles"}
-
 STAGE = {"bash": "_dvc_compgen_stages", "zsh": "_dvc_compadd_stages"}
-
 DVCFILES_AND_STAGE = {
     "bash": "_dvc_compgen_stages_and_files",
     "zsh": "_dvc_compadd_stages_and_files",
 }
-
 EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
-
-
-class CmdCompletion(CmdBaseNoRepo):
-    def run(self):
-        from dvc.cli import get_main_parser
-
-        parser = get_main_parser()
-        shell = self.args.shell
-        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
-        ui.write(script, force=True)
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    COMPLETION_HELP = "Generate shell tab completion."
-    COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
-    completion_parser = subparsers.add_parser(
-        "completion",
-        parents=[parent_parser],
-        description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
-        help=COMPLETION_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    completion_parser.add_argument(
-        "-s",
-        "--shell",
-        help="Shell syntax for completions.",
-        default="bash",
-        choices=["bash", "zsh"],
-    )
-    completion_parser.set_defaults(func=CmdCompletion)

dvc/cli/parser.py

@@ -1,11 +1,10 @@
-"""DVC command line interface"""
+"""Main parser for the dvc cli."""
 import argparse
 import logging
 import os
 import sys
 
-from ._debug import add_debugging_flags
-from .command import (
+from dvc.commands import (
     add,
     cache,
     check_ignore,
@@ -28,7 +27,6 @@ from .command import (
     imp_url,
     init,
     install,
-    live,
     ls,
     machine,
     metrics,
@@ -45,8 +43,8 @@ from .command import (
     update,
     version,
 )
-from .command.base import fix_subparsers
-from .exceptions import DvcParserError
+
+from . import DvcParserError
 
 logger = logging.getLogger(__name__)
 
@@ -87,7 +85,6 @@ COMMANDS = [
     stage,
     experiments,
     check_ignore,
-    live,
     machine,
 ]
 
@@ -96,7 +93,7 @@ def _find_parser(parser, cmd_cls):
     defaults = parser._defaults  # pylint: disable=protected-access
     if not cmd_cls or cmd_cls == defaults.get("func"):
         parser.print_help()
-        raise DvcParserError()
+        raise DvcParserError
 
     actions = parser._actions  # pylint: disable=protected-access
     for action in actions:
@@ -143,6 +140,8 @@ def get_parent_parser():
     When overwriting `-q` or `-v`, you need to instantiate a new object
     in order to prevent some weird behavior.
     """
+    from dvc._debug import add_debugging_flags
+
     parent_parser = argparse.ArgumentParser(add_help=False)
     add_debugging_flags(parent_parser)
     log_level_group = parent_parser.add_mutually_exclusive_group()
@@ -207,23 +206,11 @@ def get_main_parser():
         help="Use `dvc COMMAND --help` for command-specific help.",
     )
 
+    from .utils import fix_subparsers
+
     fix_subparsers(subparsers)
 
     for cmd in COMMANDS:
         cmd.add_parser(subparsers, parent_parser)
 
     return parser
-
-
-def parse_args(argv=None):
-    """Parses CLI arguments.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Raises:
-        dvc.exceptions.DvcParserError: raised for argument parsing errors.
-    """
-    parser = get_main_parser()
-    args = parser.parse_args(argv)
-    return args

dvc/cli/utils.py

@@ -0,0 +1,31 @@
+def fix_subparsers(subparsers):
+    """Workaround for bug in Python 3. See more info at:
+    https://bugs.python.org/issue16308
+    https://github.com/iterative/dvc/issues/769
+
+    Args:
+        subparsers: subparsers to fix.
+    """
+    subparsers.required = True
+    subparsers.dest = "cmd"
+
+
+def append_doc_link(help_message, path):
+    from dvc.utils import format_link
+
+    if not path:
+        return help_message
+    doc_base = "https://man.dvc.org/"
+    return "{message}\nDocumentation: {link}".format(
+        message=help_message, link=format_link(doc_base + path)
+    )
+
+
+def fix_plumbing_subparsers(subparsers):
+    # metavar needs to be explicitly set in order to hide plumbing subcommands
+    # from the 'positional arguments' choices list
+    # see: https://bugs.python.org/issue22848
+    cmds = [
+        cmd for cmd, parser in subparsers.choices.items() if parser.add_help
+    ]
+    subparsers.metavar = "{{{}}}".format(",".join(cmds))

dvc/command/live.py

@@ -1,93 +0,0 @@
-import argparse
-from pathlib import Path
-
-from dvc.command import completion
-from dvc.command.base import CmdBase, fix_subparsers
-from dvc.ui import ui
-
-
-class CmdLive(CmdBase):
-    UNINITIALIZED = True
-
-    def _run(self, target, revs=None):
-        metrics, plots = self.repo.live.show(target=target, revs=revs)
-
-        if plots:
-            html_path = Path.cwd() / (self.args.target + "_html")
-            from dvc.render.utils import render
-
-            index_path = render(self.repo, plots, metrics, html_path)
-            ui.write(index_path.as_uri())
-            return 0
-        return 1
-
-
-class CmdLiveShow(CmdLive):
-    def run(self):
-        return self._run(self.args.target)
-
-
-class CmdLiveDiff(CmdLive):
-    def run(self):
-        return self._run(self.args.target, self.args.revs)
-
-
-def shared_parent_parser():
-    parent_parser = argparse.ArgumentParser(add_help=False)
-    parent_parser.add_argument(
-        "target", help="Logs dir to produce summary from"
-    ).complete = completion.DIR
-    parent_parser.add_argument(
-        "-o",
-        "--out",
-        default=None,
-        help="Destination path to save plots to",
-        metavar="<path>",
-    ).complete = completion.DIR
-    return parent_parser
-
-
-def add_parser(subparsers, parent_parser):
-    LIVE_DESCRIPTION = (
-        "Commands to visualize and compare dvclive-produced logs."
-    )
-    live_parser = subparsers.add_parser(
-        "live",
-        parents=[parent_parser],
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        description=LIVE_DESCRIPTION,
-    )
-    live_subparsers = live_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc live CMD --help` to display command-specific help.",
-    )
-
-    fix_subparsers(live_subparsers)
-
-    SHOW_HELP = "Visualize dvclive directory content."
-    live_show_parser = live_subparsers.add_parser(
-        "show",
-        parents=[parent_parser, shared_parent_parser()],
-        help=SHOW_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    live_show_parser.set_defaults(func=CmdLiveShow)
-
-    DIFF_HELP = (
-        "Show multiple versions of dvclive data, "
-        "by plotting it in single view."
-    )
-    live_diff_parser = live_subparsers.add_parser(
-        "diff",
-        parents=[parent_parser, shared_parent_parser()],
-        help=DIFF_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    live_diff_parser.add_argument(
-        "--revs",
-        nargs="*",
-        default=None,
-        help="Git revision (e.g. SHA, branch, tag)",
-        metavar="<commit>",
-    )
-    live_diff_parser.set_defaults(func=CmdLiveDiff)

dvc/commands/add.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -101,9 +102,10 @@ def add_parser(subparsers, parent_parser):
         "--jobs",
         type=int,
         help=(
-            "Number of jobs to run simultaneously. "
+            "Only used along with '--to-remote'. "
+            "Number of jobs to run simultaneously "
+            "when pushing data to remote."
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/cache.py

@@ -1,8 +1,8 @@
 import argparse
 
-from dvc.command import completion
-from dvc.command.base import append_doc_link, fix_subparsers
-from dvc.command.config import CmdConfig
+from dvc.cli import completion
+from dvc.cli.utils import append_doc_link, fix_subparsers
+from dvc.commands.config import CmdConfig
 from dvc.ui import ui
 
 
@@ -28,7 +28,7 @@ class CmdCacheDir(CmdConfig):
 
 
 def add_parser(subparsers, parent_parser):
-    from dvc.command.config import parent_config_parser
+    from dvc.commands.config import parent_config_parser
 
     CACHE_HELP = "Manage cache settings."
 

dvc/commands/check_ignore.py

@@ -1,7 +1,8 @@
 import argparse
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
 
 

dvc/commands/checkout.py

@@ -1,8 +1,9 @@
 import argparse
 import operator
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import CheckoutError
 from dvc.ui import ui
 

dvc/commands/commit.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 

dvc/commands/completion.py

@@ -0,0 +1,40 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.completion import PREAMBLE
+from dvc.cli.utils import append_doc_link
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdCompletion(CmdBaseNoRepo):
+    def run(self):
+        import shtab
+
+        shell = self.args.shell
+        parser = self.args.parser
+        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
+        ui.write(script, force=True)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    COMPLETION_HELP = "Generate shell tab completion."
+    COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
+    completion_parser = subparsers.add_parser(
+        "completion",
+        parents=[parent_parser],
+        description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
+        help=COMPLETION_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    completion_parser.add_argument(
+        "-s",
+        "--shell",
+        help="Shell syntax for completions.",
+        default="bash",
+        choices=["bash", "zsh"],
+    )
+    completion_parser.set_defaults(func=CmdCompletion)

dvc/commands/config.py

@@ -2,7 +2,8 @@ import argparse
 import logging
 import os
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)

dvc/commands/daemon.py

@@ -1,5 +1,6 @@
-from dvc.command import completion
-from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.cli import completion
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import fix_subparsers
 
 
 class CmdDaemonBase(CmdBaseNoRepo):

dvc/commands/dag.py

@@ -1,6 +1,7 @@
 import argparse
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
 
 
@@ -27,6 +28,32 @@ def _show_dot(G):
     return dot_file.getvalue()
 
 
+def _show_mermaid(G, markdown: bool = False):
+    from dvc.repo.graph import get_pipelines
+
+    pipelines = get_pipelines(G)
+
+    graph = "flowchart TD"
+
+    total_nodes = 0
+    for pipeline in pipelines:
+        node_ids = {}
+        nodes = sorted(str(x) for x in pipeline.nodes)
+        for node in nodes:
+            total_nodes += 1
+            node_id = f"node{total_nodes}"
+            graph += f"\n\t{node_id}[{node}]"
+            node_ids[node] = node_id
+        edges = sorted((str(a), str(b)) for b, a in pipeline.edges)
+        for a, b in edges:
+            graph += f"\n\t{node_ids[str(a)]}-->{node_ids[str(b)]}"
+
+    if markdown:
+        return f"```mermaid\n{graph}\n```"
+
+    return graph
+
+
 def _collect_targets(repo, target, outs):
     if not target:
         return []
@@ -38,12 +65,14 @@ def _collect_targets(repo, target, outs):
     targets = []
 
     outs_trie = repo.index.outs_trie
-    for stage, info in pairs:
-        if not info:
+    for stage, path in pairs:
+        if not path:
             targets.extend([str(out) for out in stage.outs])
             continue
 
-        for out in outs_trie.itervalues(prefix=info.parts):  # noqa: B301
+        for out in outs_trie.itervalues(
+            prefix=repo.fs.path.parts(path)
+        ):  # noqa: B301
             targets.extend(str(out))
 
     return targets
@@ -102,6 +131,8 @@ class CmdDAG(CmdBase):
 
         if self.args.dot:
             ui.write(_show_dot(G))
+        elif self.args.mermaid or self.args.markdown:
+            ui.write(_show_mermaid(G, self.args.markdown))
         else:
             with ui.pager():
                 ui.write(_show_ascii(G))
@@ -124,6 +155,20 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Print DAG with .dot format.",
     )
+    dag_parser.add_argument(
+        "--mermaid",
+        action="store_true",
+        default=False,
+        help="Print DAG with mermaid format.",
+    )
+    dag_parser.add_argument(
+        "--md",
+        "--show-md",
+        action="store_true",
+        default=False,
+        dest="markdown",
+        help="Print DAG with mermaid format wrapped in Markdown block.",
+    )
     dag_parser.add_argument(
         "--full",
         action="store_true",

dvc/commands/data_sync.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -18,7 +19,7 @@ class CmdDataBase(CmdBase):
 
 class CmdDataPull(CmdDataBase):
     def log_summary(self, stats):
-        from dvc.command.checkout import log_changes
+        from dvc.commands.checkout import log_changes
 
         log_changes(stats)
         super().log_summary(stats)
@@ -97,7 +98,7 @@ class CmdDataFetch(CmdDataBase):
 
 
 def shared_parent_parser():
-    from dvc.cli import get_parent_parser
+    from dvc.cli.parser import get_parent_parser
 
     # Parent parser used in pull/push/status
     parent_parser = argparse.ArgumentParser(
@@ -110,7 +111,6 @@ def shared_parent_parser():
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )
@@ -127,7 +127,7 @@ def shared_parent_parser():
 
 
 def add_parser(subparsers, _parent_parser):
-    from dvc.command.status import CmdDataStatus
+    from dvc.commands.status import CmdDataStatus
 
     # Pull
     PULL_HELP = "Download tracked files or directories from remote storage."

dvc/commands/destroy.py

@@ -1,7 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 

dvc/commands/diff.py

@@ -2,8 +2,9 @@ import argparse
 import logging
 import os
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
@@ -127,7 +128,7 @@ class CmdDiff(CmdBase):
             show_hash = self.args.show_hash
             hide_missing = self.args.b_rev or self.args.hide_missing
             if hide_missing:
-                del diff["not in cache"]
+                diff.pop("not in cache", None)
 
             for key, entries in diff.items():
                 entries = sorted(

dvc/commands/experiments/__init__.py

@@ -0,0 +1,101 @@
+import argparse
+
+from dvc.cli.utils import (
+    append_doc_link,
+    fix_plumbing_subparsers,
+    fix_subparsers,
+)
+from dvc.commands.experiments import (
+    apply,
+    branch,
+    diff,
+    exec_run,
+    gc,
+    init,
+    ls,
+    pull,
+    push,
+    remove,
+    run,
+    show,
+)
+
+SUB_COMMANDS = [
+    apply,
+    branch,
+    diff,
+    exec_run,
+    gc,
+    init,
+    ls,
+    pull,
+    push,
+    remove,
+    run,
+    show,
+]
+
+
+def add_parser(subparsers, parent_parser):
+    EXPERIMENTS_HELP = "Commands to run and compare experiments."
+
+    experiments_parser = subparsers.add_parser(
+        "experiments",
+        parents=[parent_parser],
+        aliases=["exp"],
+        description=append_doc_link(EXPERIMENTS_HELP, "exp"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        help=EXPERIMENTS_HELP,
+    )
+
+    experiments_subparsers = experiments_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc experiments CMD --help` to display "
+        "command-specific help.",
+    )
+
+    fix_subparsers(experiments_subparsers)
+    for cmd in SUB_COMMANDS:
+        cmd.add_parser(experiments_subparsers, parent_parser)
+    fix_plumbing_subparsers(experiments_subparsers)
+
+
+def add_rev_selection_flags(
+    experiments_subcmd_parser, command: str, default: bool = True
+):
+    experiments_subcmd_parser.add_argument(
+        "-A",
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help=(
+            f"{command} all experiments in the repository "
+            "(overrides `--rev` and `--num`)."
+        ),
+    )
+    default_msg = " (HEAD by default)" if default else ""
+    msg = (
+        f"{command} experiments derived from the specified `<commit>` as "
+        f"baseline{default_msg}."
+    )
+    experiments_subcmd_parser.add_argument(
+        "--rev",
+        type=str,
+        default=None,
+        help=msg,
+        metavar="<commit>",
+    )
+    experiments_subcmd_parser.add_argument(
+        "-n",
+        "--num",
+        type=int,
+        default=1,
+        dest="num",
+        metavar="<num>",
+        help=(
+            f"{command} experiments from the last `num` commits "
+            "(first parents) starting from the `--rev` baseline. "
+            "Give a negative value to include all first-parent commits "
+            "(similar to `git log -n`)."
+        ),
+    )

dvc/commands/experiments/apply.py

@@ -0,0 +1,42 @@
+import argparse
+import logging
+
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsApply(CmdBase):
+    def run(self):
+
+        self.repo.experiments.apply(
+            self.args.experiment, force=self.args.force
+        )
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_APPLY_HELP = (
+        "Apply the changes from an experiment to your workspace."
+    )
+    experiments_apply_parser = experiments_subparsers.add_parser(
+        "apply",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_APPLY_HELP, "exp/apply"),
+        help=EXPERIMENTS_APPLY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_apply_parser.add_argument(
+        "--no-force",
+        action="store_false",
+        dest="force",
+        help="Fail if this command would overwrite conflicting changes.",
+    )
+    experiments_apply_parser.add_argument(
+        "experiment", help="Experiment to be applied."
+    ).complete = completion.EXPERIMENT
+    experiments_apply_parser.set_defaults(func=CmdExperimentsApply)

dvc/commands/experiments/branch.py

@@ -0,0 +1,34 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsBranch(CmdBase):
+    def run(self):
+
+        self.repo.experiments.branch(self.args.experiment, self.args.branch)
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_BRANCH_HELP = "Promote an experiment to a Git branch."
+    experiments_branch_parser = experiments_subparsers.add_parser(
+        "branch",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_BRANCH_HELP, "exp/branch"),
+        help=EXPERIMENTS_BRANCH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_branch_parser.add_argument(
+        "experiment", help="Experiment to be promoted."
+    )
+    experiments_branch_parser.add_argument(
+        "branch", help="Git branch name to use."
+    )
+    experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)

dvc/commands/experiments/diff.py

@@ -0,0 +1,118 @@
+import argparse
+import logging
+
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.commands.metrics import DEFAULT_PRECISION
+from dvc.exceptions import DvcException
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsDiff(CmdBase):
+    def run(self):
+
+        try:
+            diff = self.repo.experiments.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                all=self.args.all,
+                param_deps=self.args.param_deps,
+            )
+        except DvcException:
+            logger.exception("failed to show experiments diff")
+            return 1
+
+        if self.args.json:
+            ui.write_json(diff)
+        else:
+            from dvc.compare import show_diff
+
+            precision = self.args.precision or DEFAULT_PRECISION
+            diffs = [("metrics", "Metric"), ("params", "Param")]
+            for idx, (key, title) in enumerate(diffs):
+                if idx:
+                    # we are printing tables even in `--quiet` mode
+                    # so we should also be printing the "table" separator
+                    ui.write(force=True)
+
+                show_diff(
+                    diff[key],
+                    title=title,
+                    markdown=self.args.markdown,
+                    no_path=self.args.no_path,
+                    on_empty_diff="diff not supported",
+                    precision=precision if key == "metrics" else None,
+                    a_rev=self.args.a_rev,
+                    b_rev=self.args.b_rev,
+                )
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_DIFF_HELP = (
+        "Show changes between experiments in the DVC repository."
+    )
+
+    experiments_diff_parser = experiments_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_DIFF_HELP, "exp/diff"),
+        help=EXPERIMENTS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_diff_parser.add_argument(
+        "a_rev", nargs="?", help="Old experiment to compare (defaults to HEAD)"
+    ).complete = completion.EXPERIMENT
+    experiments_diff_parser.add_argument(
+        "b_rev",
+        nargs="?",
+        help="New experiment to compare (defaults to the current workspace)",
+    ).complete = completion.EXPERIMENT
+    experiments_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged metrics/params as well.",
+    )
+    experiments_diff_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
+    experiments_diff_parser.add_argument(
+        "--json",
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    experiments_diff_parser.add_argument(
+        "--md",
+        "--show-md",
+        action="store_true",
+        default=False,
+        dest="markdown",
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    experiments_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show metric/param path.",
+    )
+    experiments_diff_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics/params to `n` digits precision after the decimal "
+            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
+    experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)

dvc/commands/experiments/exec_run.py

@@ -0,0 +1,42 @@
+import logging
+
+from dvc.cli.command import CmdBaseNoRepo
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExecutorRun(CmdBaseNoRepo):
+    """Run an experiment executor."""
+
+    def run(self):
+        from dvc.repo.experiments.executor.base import (
+            BaseExecutor,
+            ExecutorInfo,
+        )
+        from dvc.utils.serialize import load_json
+
+        info = ExecutorInfo.from_dict(load_json(self.args.infofile))
+        BaseExecutor.reproduce(
+            info=info,
+            rev="",
+            queue=None,
+            log_level=logger.getEffectiveLevel(),
+            infofile=self.args.infofile,
+        )
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    EXEC_RUN_HELP = "Run an experiment executor."
+    exec_run_parser = experiments_subparsers.add_parser(
+        "exec-run",
+        parents=[parent_parser],
+        description=EXEC_RUN_HELP,
+        add_help=False,
+    )
+    exec_run_parser.add_argument(
+        "--infofile",
+        help="Path to executor info file",
+        default=None,
+    )
+    exec_run_parser.set_defaults(func=CmdExecutorRun)

dvc/commands/experiments/gc.py

@@ -0,0 +1,129 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+def _raise_error_if_all_disabled(**kwargs):
+    if not any(kwargs.values()):
+        raise InvalidArgumentError(
+            "Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags` "
+            "or `--all-commits` needs to be set."
+        )
+
+
+class CmdExperimentsGC(CmdBase):
+    def run(self):
+        _raise_error_if_all_disabled(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+        )
+
+        msg = "This will remove all experiments except those derived from "
+
+        msg += "the workspace"
+        if self.args.all_commits:
+            msg += " and all git commits"
+        elif self.args.all_branches and self.args.all_tags:
+            msg += " and all git branches and tags"
+        elif self.args.all_branches:
+            msg += " and all git branches"
+        elif self.args.all_tags:
+            msg += " and all git tags"
+        msg += " of the current repo."
+        if self.args.queued:
+            msg += " Run queued experiments will be preserved."
+        else:
+            msg += " Run queued experiments will be removed."
+
+        logger.warning(msg)
+
+        msg = "Are you sure you want to proceed?"
+        if not self.args.force and not ui.confirm(msg):
+            return 1
+
+        removed = self.repo.experiments.gc(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+            queued=self.args.queued,
+        )
+
+        if removed:
+            ui.write(
+                f"Removed {removed} experiments.",
+                "To remove unused cache files",
+                "use 'dvc gc'.",
+            )
+        else:
+            ui.write("No experiments to remove.")
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments."
+    EXPERIMENTS_GC_DESCRIPTION = (
+        "Removes all experiments which are not derived from the specified"
+        "Git revisions."
+    )
+    experiments_gc_parser = experiments_subparsers.add_parser(
+        "gc",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_GC_DESCRIPTION, "exp/gc"),
+        help=EXPERIMENTS_GC_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_gc_parser.add_argument(
+        "-w",
+        "--workspace",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from the current workspace.",
+    )
+    experiments_gc_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from the tips of all Git branches.",
+    )
+    experiments_gc_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from all Git tags.",
+    )
+    experiments_gc_parser.add_argument(
+        "-A",
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from all Git commits.",
+    )
+    experiments_gc_parser.add_argument(
+        "--queued",
+        action="store_true",
+        default=False,
+        help=(
+            "Keep queued experiments (experiments run queue will be cleared "
+            "by default)."
+        ),
+    )
+    experiments_gc_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force garbage collection - automatically agree to all prompts.",
+    )
+    experiments_gc_parser.set_defaults(func=CmdExperimentsGC)

dvc/commands/experiments/init.py

@@ -0,0 +1,202 @@
+import argparse
+import logging
+from typing import TYPE_CHECKING, List
+
+from funcy import compact
+
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
+
+if TYPE_CHECKING:
+    from dvc.dependency import Dependency
+    from dvc.stage import PipelineStage
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsInit(CmdBase):
+    DEFAULT_NAME = "train"
+    CODE = "src"
+    DATA = "data"
+    MODELS = "models"
+    DEFAULT_METRICS = "metrics.json"
+    DEFAULT_PARAMS = "params.yaml"
+    PLOTS = "plots"
+    DVCLIVE = "dvclive"
+    DEFAULTS = {
+        "code": CODE,
+        "data": DATA,
+        "models": MODELS,
+        "metrics": DEFAULT_METRICS,
+        "params": DEFAULT_PARAMS,
+        "plots": PLOTS,
+        "live": DVCLIVE,
+    }
+
+    def run(self):
+        from dvc.commands.stage import parse_cmd
+
+        cmd = parse_cmd(self.args.command)
+        if not self.args.interactive and not cmd:
+            raise InvalidArgumentError("command is not specified")
+
+        from dvc.repo.experiments.init import init
+
+        defaults = {}
+        if not self.args.explicit:
+            config = self.repo.config["exp"]
+            defaults.update({**self.DEFAULTS, **config})
+
+        cli_args = compact(
+            {
+                "cmd": cmd,
+                "code": self.args.code,
+                "data": self.args.data,
+                "models": self.args.models,
+                "metrics": self.args.metrics,
+                "params": self.args.params,
+                "plots": self.args.plots,
+                "live": self.args.live,
+            }
+        )
+
+        initialized_stage, initialized_deps = init(
+            self.repo,
+            name=self.args.name,
+            type=self.args.type,
+            defaults=defaults,
+            overrides=cli_args,
+            interactive=self.args.interactive,
+            force=self.args.force,
+        )
+        self._post_init_display(initialized_stage, initialized_deps)
+        if self.args.run:
+            self.repo.experiments.run(targets=[initialized_stage.addressing])
+        return 0
+
+    def _post_init_display(
+        self, stage: "PipelineStage", new_deps: List["Dependency"]
+    ) -> None:
+        from dvc.utils import humanize
+
+        path_fmt = "[green]{0}[/green]".format
+        if new_deps:
+            deps_paths = humanize.join(map(path_fmt, new_deps))
+            ui.write(f"Creating dependencies: {deps_paths}", styled=True)
+
+        ui.write(
+            f"Creating [b]{self.args.name}[/b] stage in [green]dvc.yaml[/]",
+            styled=True,
+        )
+        if stage.outs or not self.args.run:
+            # separate the above status-like messages with help/tips section
+            ui.write(styled=True)
+
+        if stage.outs:
+            outs_paths = humanize.join(map(path_fmt, stage.outs))
+            tips = f"Ensure your experiment command creates {outs_paths}."
+            ui.write(tips, styled=True)
+
+        if not self.args.run:
+            ui.write(
+                'You can now run your experiment using [b]"dvc exp run"[/].',
+                styled=True,
+            )
+        else:
+            # separate between `exp.run` output and `dvc exp init` output
+            ui.write(styled=True)
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_INIT_HELP = "Quickly setup any project to use experiments."
+    experiments_init_parser = experiments_subparsers.add_parser(
+        "init",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_INIT_HELP, "exp/init"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        help=EXPERIMENTS_INIT_HELP,
+    )
+    experiments_init_parser.add_argument(
+        "command",
+        nargs=argparse.REMAINDER,
+        help="Command to execute.",
+        metavar="command",
+    )
+    experiments_init_parser.add_argument(
+        "--run",
+        action="store_true",
+        help="Run the experiment after initializing it",
+    )
+    experiments_init_parser.add_argument(
+        "--interactive",
+        "-i",
+        action="store_true",
+        help="Prompt for values that are not provided",
+    )
+    experiments_init_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Overwrite existing stage",
+    )
+    experiments_init_parser.add_argument(
+        "--explicit",
+        action="store_true",
+        default=False,
+        help="Only use the path values explicitly provided",
+    )
+    experiments_init_parser.add_argument(
+        "--name",
+        "-n",
+        help="Name of the stage to create (default: %(default)s)",
+        default=CmdExperimentsInit.DEFAULT_NAME,
+    )
+    experiments_init_parser.add_argument(
+        "--code",
+        help="Path to the source file or directory "
+        "which your experiments depend"
+        f" (default: {CmdExperimentsInit.CODE})",
+    )
+    experiments_init_parser.add_argument(
+        "--data",
+        help="Path to the data file or directory "
+        "which your experiments depend"
+        f" (default: {CmdExperimentsInit.DATA})",
+    )
+    experiments_init_parser.add_argument(
+        "--models",
+        help="Path to the model file or directory for your experiments"
+        f" (default: {CmdExperimentsInit.MODELS})",
+    )
+    experiments_init_parser.add_argument(
+        "--params",
+        help="Path to the parameters file for your experiments"
+        f" (default: {CmdExperimentsInit.DEFAULT_PARAMS})",
+    )
+    experiments_init_parser.add_argument(
+        "--metrics",
+        help="Path to the metrics file for your experiments"
+        f" (default: {CmdExperimentsInit.DEFAULT_METRICS})",
+    )
+    experiments_init_parser.add_argument(
+        "--plots",
+        help="Path to the plots file or directory for your experiments"
+        f" (default: {CmdExperimentsInit.PLOTS})",
+    )
+    experiments_init_parser.add_argument(
+        "--live",
+        help="Path to log dvclive outputs for your experiments"
+        f" (default: {CmdExperimentsInit.DVCLIVE})",
+    )
+    experiments_init_parser.add_argument(
+        "--type",
+        choices=["default", "dl"],
+        default="default",
+        help="Select type of stage to create (default: %(default)s)",
+    )
+    experiments_init_parser.set_defaults(func=CmdExperimentsInit)

dvc/commands/experiments/ls.py

@@ -0,0 +1,64 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsList(CmdBase):
+    def run(self):
+        name_only = self.args.name_only
+        exps = self.repo.experiments.ls(
+            all_commits=self.args.all_commits,
+            rev=self.args.rev,
+            num=self.args.num,
+            git_remote=self.args.git_remote,
+        )
+        for baseline in exps:
+            tag = self.repo.scm.describe(baseline)
+            if not tag:
+                branch = self.repo.scm.describe(baseline, base="refs/heads")
+                if branch:
+                    tag = branch.split("/")[-1]
+            name = tag if tag else baseline[:7]
+            if not name_only:
+                print(f"{name}:")
+            for exp_name in exps[baseline]:
+                indent = "" if name_only else "\t"
+                print(f"{indent}{exp_name}")
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    from . import add_rev_selection_flags
+
+    EXPERIMENTS_LIST_HELP = "List local and remote experiments."
+    experiments_list_parser = experiments_subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_LIST_HELP, "exp/list"),
+        help=EXPERIMENTS_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    add_rev_selection_flags(experiments_list_parser, "List")
+    experiments_list_parser.add_argument(
+        "--name-only",
+        "--names-only",
+        action="store_true",
+        help="Only output experiment names (without parent commits).",
+    )
+    experiments_list_parser.add_argument(
+        "git_remote",
+        nargs="?",
+        default=None,
+        help=(
+            "Optional Git remote name or Git URL. "
+            "If provided, experiments from the specified Git repository "
+            " will be listed instead of local ones."
+        ),
+        metavar="[<git_remote>]",
+    )
+    experiments_list_parser.set_defaults(func=CmdExperimentsList)

dvc/commands/experiments/pull.py

@@ -0,0 +1,117 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsPull(CmdBase):
+    def raise_error_if_all_disabled(self):
+        if not any(
+            [self.args.experiment, self.args.all_commits, self.args.rev]
+        ):
+            raise InvalidArgumentError(
+                "Either provide an `experiment` argument, or use the "
+                "`--rev` or `--all-commits` flag."
+            )
+
+    def run(self):
+        self.raise_error_if_all_disabled()
+
+        pulled_exps = self.repo.experiments.pull(
+            self.args.git_remote,
+            self.args.experiment,
+            all_commits=self.args.all_commits,
+            rev=self.args.rev,
+            num=self.args.num,
+            force=self.args.force,
+            pull_cache=self.args.pull_cache,
+            dvc_remote=self.args.dvc_remote,
+            jobs=self.args.jobs,
+            run_cache=self.args.run_cache,
+        )
+
+        if pulled_exps:
+            ui.write(
+                f"Pulled experiment '{pulled_exps}'",
+                f"from Git remote '{self.args.git_remote}'.",
+            )
+        else:
+            ui.write("No experiments to pull.")
+        if not self.args.pull_cache:
+            ui.write(
+                "To pull cached outputs for this experiment"
+                "from DVC remote storage,"
+                "re-run this command without '--no-cache'."
+            )
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    from . import add_rev_selection_flags
+
+    EXPERIMENTS_PULL_HELP = "Pull an experiment from a Git remote."
+    experiments_pull_parser = experiments_subparsers.add_parser(
+        "pull",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_PULL_HELP, "exp/pull"),
+        help=EXPERIMENTS_PULL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    add_rev_selection_flags(experiments_pull_parser, "Pull", False)
+    experiments_pull_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="Replace local experiment already exists.",
+    )
+    experiments_pull_parser.add_argument(
+        "--no-cache",
+        action="store_false",
+        dest="pull_cache",
+        help=(
+            "Do not pull cached outputs for this experiment from DVC remote "
+            "storage."
+        ),
+    )
+    experiments_pull_parser.add_argument(
+        "-r",
+        "--remote",
+        dest="dvc_remote",
+        metavar="<name>",
+        help="Name of the DVC remote to use when pulling cached outputs.",
+    )
+    experiments_pull_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        metavar="<number>",
+        help=(
+            "Number of jobs to run simultaneously when pulling from DVC "
+            "remote storage."
+        ),
+    )
+    experiments_pull_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help="Pull run history for all stages.",
+    )
+    experiments_pull_parser.add_argument(
+        "git_remote",
+        help="Git remote name or Git URL.",
+        metavar="<git_remote>",
+    )
+    experiments_pull_parser.add_argument(
+        "experiment",
+        nargs="*",
+        default=None,
+        help="Experiments to pull.",
+        metavar="<experiment>",
+    )
+    experiments_pull_parser.set_defaults(func=CmdExperimentsPull)

dvc/commands/experiments/push.py

@@ -0,0 +1,119 @@
+import argparse
+import logging
+
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsPush(CmdBase):
+    def raise_error_if_all_disabled(self):
+        if not any(
+            [self.args.experiment, self.args.all_commits, self.args.rev]
+        ):
+            raise InvalidArgumentError(
+                "Either provide an `experiment` argument, or use the "
+                "`--rev` or `--all-commits` flag."
+            )
+
+    def run(self):
+
+        self.raise_error_if_all_disabled()
+
+        pushed_exps = self.repo.experiments.push(
+            self.args.git_remote,
+            self.args.experiment,
+            all_commits=self.args.all_commits,
+            rev=self.args.rev,
+            num=self.args.num,
+            force=self.args.force,
+            push_cache=self.args.push_cache,
+            dvc_remote=self.args.dvc_remote,
+            jobs=self.args.jobs,
+            run_cache=self.args.run_cache,
+        )
+
+        if pushed_exps:
+            ui.write(
+                f"Pushed experiment '{pushed_exps}'"
+                f"to Git remote '{self.args.git_remote}'."
+            )
+        else:
+            ui.write("No experiments to push.")
+        if not self.args.push_cache:
+            ui.write(
+                "To push cached outputs",
+                "for this experiment to DVC remote storage,"
+                "re-run this command without '--no-cache'.",
+            )
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    from . import add_rev_selection_flags
+
+    EXPERIMENTS_PUSH_HELP = "Push a local experiment to a Git remote."
+    experiments_push_parser = experiments_subparsers.add_parser(
+        "push",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_PUSH_HELP, "exp/push"),
+        help=EXPERIMENTS_PUSH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    add_rev_selection_flags(experiments_push_parser, "Push", False)
+    experiments_push_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="Replace experiment in the Git remote if it already exists.",
+    )
+    experiments_push_parser.add_argument(
+        "--no-cache",
+        action="store_false",
+        dest="push_cache",
+        help=(
+            "Do not push cached outputs for this experiment to DVC remote "
+            "storage."
+        ),
+    )
+    experiments_push_parser.add_argument(
+        "-r",
+        "--remote",
+        dest="dvc_remote",
+        metavar="<name>",
+        help="Name of the DVC remote to use when pushing cached outputs.",
+    )
+    experiments_push_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        metavar="<number>",
+        help=(
+            "Number of jobs to run simultaneously when pushing to DVC remote "
+            "storage."
+        ),
+    )
+    experiments_push_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help="Push run history for all stages.",
+    )
+    experiments_push_parser.add_argument(
+        "git_remote",
+        help="Git remote name or Git URL.",
+        metavar="<git_remote>",
+    )
+    experiments_push_parser.add_argument(
+        "experiment",
+        nargs="*",
+        default=None,
+        help="Experiments to push.",
+        metavar="<experiment>",
+    ).complete = completion.EXPERIMENT
+    experiments_push_parser.set_defaults(func=CmdExperimentsPush)

dvc/commands/experiments/remove.py

@@ -0,0 +1,73 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsRemove(CmdBase):
+    def raise_error_if_all_disabled(self):
+        if not any(
+            [
+                self.args.experiment,
+                self.args.all_commits,
+                self.args.rev,
+                self.args.queue,
+            ]
+        ):
+            raise InvalidArgumentError(
+                "Either provide an `experiment` argument, or use the "
+                "`--rev` or `--all-commits` flag."
+            )
+
+    def run(self):
+
+        self.raise_error_if_all_disabled()
+
+        removed_list = self.repo.experiments.remove(
+            exp_names=self.args.experiment,
+            all_commits=self.args.all_commits,
+            rev=self.args.rev,
+            num=self.args.num,
+            queue=self.args.queue,
+            git_remote=self.args.git_remote,
+        )
+        removed = ",".join(removed_list)
+        ui.write(f"Removed experiments: {removed}")
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    from . import add_rev_selection_flags
+
+    EXPERIMENTS_REMOVE_HELP = "Remove experiments."
+    experiments_remove_parser = experiments_subparsers.add_parser(
+        "remove",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_REMOVE_HELP, "exp/remove"),
+        help=EXPERIMENTS_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remove_group = experiments_remove_parser.add_mutually_exclusive_group()
+    add_rev_selection_flags(experiments_remove_parser, "Remove", False)
+    remove_group.add_argument(
+        "--queue", action="store_true", help="Remove all queued experiments."
+    )
+    remove_group.add_argument(
+        "-g",
+        "--git-remote",
+        metavar="<git_remote>",
+        help="Name or URL of the Git remote to remove the experiment from",
+    )
+    experiments_remove_parser.add_argument(
+        "experiment",
+        nargs="*",
+        help="Experiments to remove.",
+        metavar="<experiment>",
+    )
+    experiments_remove_parser.set_defaults(func=CmdExperimentsRemove)

dvc/commands/experiments/run.py

@@ -0,0 +1,142 @@
+import argparse
+import logging
+
+from dvc.cli import completion
+from dvc.cli.utils import append_doc_link
+from dvc.commands.repro import CmdRepro
+from dvc.commands.repro import add_arguments as add_repro_arguments
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsRun(CmdRepro):
+    def run(self):
+        from dvc.compare import show_metrics
+
+        if self.args.checkpoint_resume:
+            if self.args.reset:
+                raise InvalidArgumentError(
+                    "--reset and --rev are mutually exclusive."
+                )
+            if not (self.args.queue or self.args.tmp_dir):
+                raise InvalidArgumentError(
+                    "--rev can only be used in conjunction with "
+                    "--queue or --temp."
+                )
+
+        if self.args.reset:
+            ui.write("Any existing checkpoints will be reset and re-run.")
+
+        results = self.repo.experiments.run(
+            name=self.args.name,
+            queue=self.args.queue,
+            run_all=self.args.run_all,
+            jobs=self.args.jobs,
+            params=self.args.set_param,
+            checkpoint_resume=self.args.checkpoint_resume,
+            reset=self.args.reset,
+            tmp_dir=self.args.tmp_dir,
+            machine=self.args.machine,
+            **self._common_kwargs,
+        )
+
+        if self.args.metrics and results:
+            metrics = self.repo.metrics.show(revs=list(results))
+            metrics.pop("workspace", None)
+            show_metrics(metrics)
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+
+    EXPERIMENTS_RUN_HELP = "Run or resume an experiment."
+    experiments_run_parser = experiments_subparsers.add_parser(
+        "run",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_RUN_HELP, "exp/run"),
+        help=EXPERIMENTS_RUN_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    _add_run_common(experiments_run_parser)
+    experiments_run_parser.add_argument(
+        "-r",
+        "--rev",
+        type=str,
+        dest="checkpoint_resume",
+        help=(
+            "Continue the specified checkpoint experiment. Can only be used "
+            "in conjunction with --queue or --temp."
+        ),
+        metavar="<experiment_rev>",
+    ).complete = completion.EXPERIMENT
+    experiments_run_parser.add_argument(
+        "--reset",
+        action="store_true",
+        help="Reset existing checkpoints and restart the experiment.",
+    )
+    experiments_run_parser.set_defaults(func=CmdExperimentsRun)
+
+
+def _add_run_common(parser):
+    """Add common args for 'exp run'."""
+    # inherit arguments from `dvc repro`
+    add_repro_arguments(parser)
+    parser.add_argument(
+        "-n",
+        "--name",
+        default=None,
+        help=(
+            "Human-readable experiment name. If not specified, a name will "
+            "be auto-generated."
+        ),
+        metavar="<name>",
+    )
+    parser.add_argument(
+        "-S",
+        "--set-param",
+        action="append",
+        default=[],
+        help="Use the specified param value when reproducing pipelines.",
+        metavar="[<filename>:]<param_name>=<param_value>",
+    )
+    parser.add_argument(
+        "--queue",
+        action="store_true",
+        default=False,
+        help="Stage this experiment in the run queue for future execution.",
+    )
+    parser.add_argument(
+        "--run-all",
+        action="store_true",
+        default=False,
+        help="Execute all experiments in the run queue. Implies --temp.",
+    )
+    parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        default=1,
+        help="Run the specified number of experiments at a time in parallel.",
+        metavar="<number>",
+    )
+    parser.add_argument(
+        "--temp",
+        action="store_true",
+        dest="tmp_dir",
+        help=(
+            "Run this experiment in a separate temporary directory instead of "
+            "your workspace."
+        ),
+    )
+    parser.add_argument(
+        "--machine",
+        default=None,
+        help=argparse.SUPPRESS,
+        # help=(
+        #     "Run this experiment on the specified 'dvc machine' instance."
+        # )
+        # metavar="<name>",
+    )

dvc/commands/experiments/show.py

@@ -0,0 +1,655 @@
+import argparse
+import logging
+import os
+import re
+from collections import Counter, OrderedDict, defaultdict
+from datetime import date, datetime
+from typing import TYPE_CHECKING
+
+from funcy import lmap
+
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.commands.metrics import DEFAULT_PRECISION
+from dvc.exceptions import DvcException, InvalidArgumentError
+from dvc.ui import ui
+from dvc.utils.flatten import flatten
+from dvc.utils.serialize import encode_exception
+
+if TYPE_CHECKING:
+    from dvc.compare import TabularData
+    from dvc.ui import RichText
+
+FILL_VALUE = "-"
+FILL_VALUE_ERRORED = "!"
+
+
+logger = logging.getLogger(__name__)
+
+
+def _update_names(names, items):
+    for name, item in items:
+        item = item.get("data", {})
+        if isinstance(item, dict):
+            item = flatten(item)
+            names[name].update({key: None for key in item})
+
+
+def _collect_names(all_experiments, **kwargs):
+    metric_names = defaultdict(dict)
+    param_names = defaultdict(dict)
+    deps_names = set()
+
+    for _, experiments in all_experiments.items():
+        for exp_data in experiments.values():
+            exp = exp_data.get("data", {})
+            _update_names(metric_names, exp.get("metrics", {}).items())
+            _update_names(param_names, exp.get("params", {}).items())
+            for dep_name in exp.get("deps", {}):
+                deps_names.add(dep_name)
+
+    return metric_names, param_names, sorted(deps_names)
+
+
+experiment_types = {
+    "checkpoint_tip": "│ ╓",
+    "checkpoint_commit": "│ ╟",
+    "checkpoint_base": "├─╨",
+    "branch_commit": "├──",
+    "branch_base": "└──",
+    "baseline": "",
+}
+
+
+def _collect_rows(
+    base_rev,
+    experiments,
+    all_headers,
+    metric_headers,
+    param_headers,
+    metric_names,
+    param_names,
+    deps_names,
+    precision=DEFAULT_PRECISION,
+    sort_by=None,
+    sort_order=None,
+    fill_value=FILL_VALUE,
+    iso=False,
+):
+    from scmrepo.git import Git
+
+    if sort_by:
+        sort_path, sort_name, sort_type = _sort_column(
+            sort_by, metric_names, param_names
+        )
+        reverse = sort_order == "desc"
+        experiments = _sort_exp(
+            experiments, sort_path, sort_name, sort_type, reverse
+        )
+
+    new_checkpoint = True
+    for i, (rev, results) in enumerate(experiments.items()):
+        fill_value = FILL_VALUE_ERRORED if results.get("error") else fill_value
+        row_dict = {k: fill_value for k in all_headers}
+
+        exp = results.get("data", {})
+
+        if exp.get("running"):
+            state = "Running"
+        elif exp.get("queued"):
+            state = "Queued"
+        else:
+            state = fill_value
+
+        is_baseline = rev == "baseline"
+
+        if is_baseline:
+            name_rev = base_rev[:7] if Git.is_sha(base_rev) else base_rev
+        else:
+            name_rev = rev[:7]
+
+        tip = exp.get("checkpoint_tip")
+        parent_rev = exp.get("checkpoint_parent", "")
+        parent_exp = experiments.get(parent_rev, {}).get("data", {})
+        parent_tip = parent_exp.get("checkpoint_tip")
+
+        parent = ""
+        if is_baseline:
+            typ = "baseline"
+        elif tip:
+            if tip == parent_tip:
+                typ = (
+                    "checkpoint_tip" if new_checkpoint else "checkpoint_commit"
+                )
+            elif parent_rev == base_rev:
+                typ = "checkpoint_base"
+            else:
+                typ = "checkpoint_commit"
+                parent = parent_rev[:7]
+        elif i < len(experiments) - 1:
+            typ = "branch_commit"
+        else:
+            typ = "branch_base"
+
+        if not is_baseline:
+            new_checkpoint = not (tip and tip == parent_tip)
+
+        row_dict["Experiment"] = exp.get("name", "")
+        row_dict["rev"] = name_rev
+        row_dict["typ"] = typ
+        row_dict["Created"] = _format_time(
+            exp.get("timestamp"), fill_value, iso
+        )
+        row_dict["parent"] = parent
+        row_dict["State"] = state
+        row_dict["Executor"] = exp.get("executor", fill_value)
+
+        _extend_row(
+            row_dict,
+            metric_names,
+            metric_headers,
+            exp.get("metrics", {}).items(),
+            precision,
+            fill_value=fill_value,
+        )
+        _extend_row(
+            row_dict,
+            param_names,
+            param_headers,
+            exp.get("params", {}).items(),
+            precision,
+            fill_value=fill_value,
+        )
+        for dep in deps_names:
+            hash_info = exp.get("deps", {}).get(dep, {}).get("hash")
+            if hash_info is not None:
+                hash_info = hash_info[:7]
+            row_dict[dep] = hash_info
+        yield list(row_dict.values())
+
+
+def _sort_column(sort_by, metric_names, param_names):
+    path, _, sort_name = sort_by.rpartition(":")
+    matches = set()
+
+    if path:
+        if path in metric_names and sort_name in metric_names[path]:
+            matches.add((path, sort_name, "metrics"))
+        if path in param_names and sort_name in param_names[path]:
+            matches.add((path, sort_name, "params"))
+    else:
+        for path in metric_names:
+            if sort_name in metric_names[path]:
+                matches.add((path, sort_name, "metrics"))
+        for path in param_names:
+            if sort_name in param_names[path]:
+                matches.add((path, sort_name, "params"))
+
+    if len(matches) == 1:
+        return matches.pop()
+    if len(matches) > 1:
+        raise InvalidArgumentError(
+            "Ambiguous sort column '{}' matched '{}'".format(
+                sort_by,
+                ", ".join([f"{path}:{name}" for path, name, _ in matches]),
+            )
+        )
+    raise InvalidArgumentError(f"Unknown sort column '{sort_by}'")
+
+
+def _sort_exp(experiments, sort_path, sort_name, typ, reverse):
+    def _sort(item):
+        rev, exp = item
+        exp_data = exp.get("data", {})
+        tip = exp_data.get("checkpoint_tip")
+        if tip and tip != rev:
+            # Sort checkpoint experiments by tip commit
+            return _sort((tip, experiments[tip]))
+        data = exp_data.get(typ, {}).get(sort_path, {}).get("data", {})
+        val = flatten(data).get(sort_name)
+        return val is None, val
+
+    ret = OrderedDict()
+    if "baseline" in experiments:
+        ret["baseline"] = experiments.pop("baseline")
+
+    ret.update(sorted(experiments.items(), key=_sort, reverse=reverse))
+    return ret
+
+
+def _format_time(datetime_obj, fill_value=FILL_VALUE, iso=False):
+    if datetime_obj is None:
+        return fill_value
+
+    if iso:
+        return datetime_obj.isoformat()
+
+    if datetime_obj.date() == date.today():
+        fmt = "%I:%M %p"
+    else:
+        fmt = "%b %d, %Y"
+    return datetime_obj.strftime(fmt)
+
+
+def _extend_row(row, names, headers, items, precision, fill_value=FILL_VALUE):
+    from dvc.compare import _format_field, with_value
+
+    for fname, data in items:
+        item = data.get("data", {})
+        item = flatten(item) if isinstance(item, dict) else {fname: item}
+        for name in names[fname]:
+            value = with_value(
+                item.get(name),
+                FILL_VALUE_ERRORED if data.get("error", None) else fill_value,
+            )
+            # wrap field data in ui.rich_text, otherwise rich may
+            # interpret unescaped braces from list/dict types as rich
+            # markup tags
+            value = ui.rich_text(str(_format_field(value, precision)))
+            if name in headers:
+                row[name] = value
+            else:
+                row[f"{fname}:{name}"] = value
+
+
+def experiments_table(
+    all_experiments,
+    headers,
+    metric_headers,
+    metric_names,
+    param_headers,
+    param_names,
+    deps_names,
+    sort_by=None,
+    sort_order=None,
+    precision=DEFAULT_PRECISION,
+    fill_value=FILL_VALUE,
+    iso=False,
+) -> "TabularData":
+    from funcy import lconcat
+
+    from dvc.compare import TabularData
+
+    all_headers = lconcat(headers, metric_headers, param_headers, deps_names)
+    td = TabularData(all_headers, fill_value=fill_value)
+    for base_rev, experiments in all_experiments.items():
+        rows = _collect_rows(
+            base_rev,
+            experiments,
+            all_headers,
+            metric_headers,
+            param_headers,
+            metric_names,
+            param_names,
+            deps_names,
+            sort_by=sort_by,
+            sort_order=sort_order,
+            precision=precision,
+            fill_value=fill_value,
+            iso=iso,
+        )
+        td.extend(rows)
+
+    return td
+
+
+def prepare_exp_id(kwargs) -> "RichText":
+    exp_name = kwargs["Experiment"]
+    rev = kwargs["rev"]
+    typ = kwargs.get("typ", "baseline")
+
+    if typ == "baseline" or not exp_name:
+        text = ui.rich_text(exp_name or rev)
+    else:
+        text = ui.rich_text.assemble(rev, " [", (exp_name, "bold"), "]")
+
+    parent = kwargs.get("parent")
+    suff = f" ({parent})" if parent else ""
+    text.append(suff)
+
+    tree = experiment_types[typ]
+    pref = f"{tree} " if tree else ""
+    return ui.rich_text(pref) + text
+
+
+def baseline_styler(typ):
+    return {"style": "bold"} if typ == "baseline" else {}
+
+
+def show_experiments(
+    all_experiments,
+    keep=None,
+    drop=None,
+    pager=True,
+    csv=False,
+    markdown=False,
+    pcp=False,
+    **kwargs,
+):
+    from funcy.seqs import flatten as flatten_list
+
+    metric_names, param_names, deps_names = _collect_names(all_experiments)
+
+    headers = [
+        "Experiment",
+        "rev",
+        "typ",
+        "Created",
+        "parent",
+        "State",
+        "Executor",
+    ]
+
+    names = {**metric_names, **param_names}
+    counter = Counter(flatten_list([list(a.keys()) for a in names.values()]))
+    counter.update(headers)
+    metric_headers = _normalize_headers(metric_names, counter)
+    param_headers = _normalize_headers(param_names, counter)
+
+    td = experiments_table(
+        all_experiments,
+        headers,
+        metric_headers,
+        metric_names,
+        param_headers,
+        param_names,
+        deps_names,
+        kwargs.get("sort_by"),
+        kwargs.get("sort_order"),
+        kwargs.get("precision"),
+        kwargs.get("fill_value"),
+        kwargs.get("iso"),
+    )
+    if keep:
+        for col in td.keys():
+            if re.match(keep, col):
+                td.protect(col)
+
+    for col in ("State", "Executor"):
+        if td.is_empty(col):
+            td.drop(col)
+
+    row_styles = lmap(baseline_styler, td.column("typ"))
+
+    if not csv:
+        merge_headers = ["Experiment", "rev", "typ", "parent"]
+        td.column("Experiment")[:] = map(
+            prepare_exp_id, td.as_dict(merge_headers)
+        )
+        td.drop(*merge_headers[1:])
+
+    headers = {
+        "metrics": metric_headers,
+        "params": param_headers,
+        "deps": deps_names,
+    }
+    styles = {
+        "Experiment": {"no_wrap": True, "header_style": "black on grey93"},
+        "Created": {"header_style": "black on grey93"},
+        "State": {"header_style": "black on grey93"},
+        "Executor": {"header_style": "black on grey93"},
+    }
+    header_bg_colors = {
+        "metrics": "cornsilk1",
+        "params": "light_cyan1",
+        "deps": "plum2",
+    }
+    styles.update(
+        {
+            header: {
+                "justify": "right" if typ == "metrics" else "left",
+                "header_style": f"black on {header_bg_colors[typ]}",
+                "collapse": idx != 0,
+                "no_wrap": typ == "metrics",
+            }
+            for typ, hs in headers.items()
+            for idx, header in enumerate(hs)
+        }
+    )
+
+    if kwargs.get("only_changed", False) or pcp:
+        td.drop_duplicates("cols", ignore_empty=False)
+
+    cols_to_drop = set()
+    if drop is not None:
+        cols_to_drop = {col for col in td.keys() if re.match(drop, col)}
+    if pcp:
+        cols_to_drop.add("Created")
+    td.drop(*cols_to_drop)
+
+    if pcp:
+        subset = {x for x in td.keys() if x != "Experiment"}
+        td.dropna(
+            "rows",
+            how="all",
+            subset=subset,
+        )
+        td.drop_duplicates("rows", subset=subset)
+        if "Experiment" in td:
+            td.column("Experiment")[:] = [
+                # remove tree characters
+                str(x).encode("ascii", "ignore").strip().decode()
+                for x in td.column("Experiment")
+            ]
+        out = kwargs.get("out") or "dvc_plots"
+        output_file = os.path.join(out, "index.html")
+        ui.write(
+            td.to_parallel_coordinates(
+                output_path=os.path.abspath(output_file),
+                color_by=kwargs.get("sort_by") or "Experiment",
+            )
+        )
+        if kwargs.get("open"):
+            return ui.open_browser(output_file)
+
+    else:
+        td.render(
+            pager=pager,
+            borders="horizontals",
+            rich_table=True,
+            header_styles=styles,
+            row_styles=row_styles,
+            csv=csv,
+            markdown=markdown,
+        )
+
+
+def _normalize_headers(names, count):
+    return [
+        name if count[name] == 1 else f"{path}:{name}"
+        for path in names
+        for name in names[path]
+    ]
+
+
+def _format_json(item):
+    if isinstance(item, (date, datetime)):
+        return item.isoformat()
+    return encode_exception(item)
+
+
+class CmdExperimentsShow(CmdBase):
+    def run(self):
+        try:
+            all_experiments = self.repo.experiments.show(
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+                revs=self.args.rev,
+                num=self.args.num,
+                sha_only=self.args.sha,
+                param_deps=self.args.param_deps,
+                fetch_running=self.args.fetch_running,
+            )
+        except DvcException:
+            logger.exception("failed to show experiments")
+            return 1
+
+        if self.args.json:
+            ui.write_json(all_experiments, default=_format_json)
+        else:
+            precision = (
+                self.args.precision or None
+                if self.args.csv
+                else DEFAULT_PRECISION
+            )
+            fill_value = "" if self.args.csv else FILL_VALUE
+            iso = True if self.args.csv else False
+
+            show_experiments(
+                all_experiments,
+                keep=self.args.keep,
+                drop=self.args.drop,
+                sort_by=self.args.sort_by,
+                sort_order=self.args.sort_order,
+                precision=precision,
+                fill_value=fill_value,
+                iso=iso,
+                pager=not self.args.no_pager,
+                csv=self.args.csv,
+                markdown=self.args.markdown,
+                only_changed=self.args.only_changed,
+                pcp=self.args.pcp,
+                out=self.args.out,
+                open=self.args.open,
+            )
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    from . import add_rev_selection_flags
+
+    EXPERIMENTS_SHOW_HELP = "Print experiments."
+    experiments_show_parser = experiments_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_SHOW_HELP, "exp/show"),
+        help=EXPERIMENTS_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    add_rev_selection_flags(experiments_show_parser, "Show")
+    experiments_show_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Show experiments derived from the tip of all Git branches.",
+    )
+    experiments_show_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Show experiments derived from all Git tags.",
+    )
+    experiments_show_parser.add_argument(
+        "--no-pager",
+        action="store_true",
+        default=False,
+        help="Do not pipe output into a pager.",
+    )
+    experiments_show_parser.add_argument(
+        "--only-changed",
+        action="store_true",
+        default=False,
+        help=(
+            "Only show metrics/params with values varying "
+            "across the selected experiments."
+        ),
+    )
+    experiments_show_parser.add_argument(
+        "--drop",
+        help="Remove the columns matching the specified regex pattern.",
+        metavar="<regex_pattern>",
+    )
+    experiments_show_parser.add_argument(
+        "--keep",
+        help="Preserve the columns matching the specified regex pattern.",
+        metavar="<regex_pattern>",
+    )
+    experiments_show_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
+    experiments_show_parser.add_argument(
+        "--sort-by",
+        help="Sort related experiments by the specified metric or param.",
+        metavar="<metric/param>",
+    )
+    experiments_show_parser.add_argument(
+        "--sort-order",
+        help=(
+            "Sort order to use with --sort-by."
+            " Defaults to ascending ('asc')."
+        ),
+        choices=("asc", "desc"),
+        default="asc",
+    )
+    experiments_show_parser.add_argument(
+        "--sha",
+        action="store_true",
+        default=False,
+        help="Always show git commit SHAs instead of branch/tag names.",
+    )
+    experiments_show_parser.add_argument(
+        "--json",
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Print output in JSON format instead of a human-readable table.",
+    )
+    experiments_show_parser.add_argument(
+        "--csv",
+        "--show-csv",
+        action="store_true",
+        default=False,
+        help="Print output in csv format instead of a human-readable table.",
+    )
+    experiments_show_parser.add_argument(
+        "--md",
+        "--show-md",
+        action="store_true",
+        default=False,
+        dest="markdown",
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    experiments_show_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics/params to `n` digits precision after the decimal "
+            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
+    experiments_show_parser.add_argument(
+        "--parallel-coordinates-plot",
+        "--pcp",
+        dest="pcp",
+        action="store_true",
+        default=False,
+        help="Generate a Parallel Coordinates Plot from the tabulated output.",
+    )
+    experiments_show_parser.add_argument(
+        "-o",
+        "--out",
+        default=None,
+        help="Destination folder to save the Parallel Coordinates Plot to",
+        metavar="<path>",
+    ).complete = completion.DIR
+    experiments_show_parser.add_argument(
+        "--open",
+        action="store_true",
+        default=False,
+        help="Open the Parallel Coordinates Plot directly in the browser.",
+    )
+    experiments_show_parser.add_argument(
+        "--no-fetch",
+        dest="fetch_running",
+        action="store_false",
+        help=argparse.SUPPRESS,
+    )
+    experiments_show_parser.set_defaults(func=CmdExperimentsShow)

dvc/commands/freeze.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)

dvc/commands/gc.py

@@ -2,7 +2,8 @@ import argparse
 import logging
 import os
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
@@ -136,7 +137,6 @@ def add_parser(subparsers, parent_parser):
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/get.py

@@ -1,10 +1,11 @@
 import argparse
 import logging
 
+from dvc.cli import completion
+from dvc.cli.command import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
-from . import completion
-from .base import CmdBaseNoRepo, append_doc_link
+from ..cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -93,7 +94,6 @@ def add_parser(subparsers, parent_parser):
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/get_url.py

@@ -1,10 +1,11 @@
 import argparse
 import logging
 
+from dvc.cli import completion
+from dvc.cli.command import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
-from . import completion
-from .base import CmdBaseNoRepo, append_doc_link
+from ..cli.utils import append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -43,7 +44,6 @@ def add_parser(subparsers, parent_parser):
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/git_hook.py

@@ -1,7 +1,8 @@
 import logging
 import os
 
-from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import fix_subparsers
 from dvc.exceptions import NotDvcRepoError
 
 logger = logging.getLogger(__name__)
@@ -25,7 +26,7 @@ class CmdHookBase(CmdBaseNoRepo):
 
 class CmdPreCommit(CmdHookBase):
     def _run(self):
-        from dvc.main import main
+        from dvc.cli import main
 
         return main(["status"])
 
@@ -49,14 +50,14 @@ class CmdPostCheckout(CmdHookBase):
         if os.path.isdir(os.path.join(".git", "rebase-merge")):
             return 0
 
-        from dvc.main import main
+        from dvc.cli import main
 
         return main(["checkout"])
 
 
 class CmdPrePush(CmdHookBase):
     def _run(self):
-        from dvc.main import main
+        from dvc.cli import main
 
         return main(["push"])
 

dvc/commands/imp.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -90,7 +91,6 @@ def add_parser(subparsers, parent_parser):
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/imp_url.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -90,7 +91,6 @@ def add_parser(subparsers, parent_parser):
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/init.py

@@ -4,7 +4,8 @@ import logging
 import colorama
 
 from dvc import analytics
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
 from dvc.utils import boxify
 from dvc.utils import format_link as fmt_link
 

dvc/commands/install.py

@@ -1,7 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)

dvc/commands/ls/__init__.py

@@ -1,9 +1,10 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.command.ls.ls_colors import LsColors
+from dvc.cli import completion
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
+from dvc.commands.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
 from dvc.ui import ui
 

dvc/commands/machine.py

@@ -1,9 +1,12 @@
 import argparse
 
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
-from dvc.command.config import CmdConfig
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link, fix_subparsers
+from dvc.commands.config import CmdConfig
+from dvc.compare import TabularData
 from dvc.config import ConfigError
 from dvc.exceptions import DvcException
+from dvc.types import Dict, List
 from dvc.ui import ui
 from dvc.utils import format_link
 
@@ -71,18 +74,56 @@ class CmdMachineRemove(CmdMachineConfig):
 
 
 class CmdMachineList(CmdMachineConfig):
-    def run(self):
+    TABLE_COLUMNS = [
+        "name",
+        "cloud",
+        "region",
+        "image",
+        "spot",
+        "spot_price",
+        "instance_hdd_size",
+        "instance_type",
+        "ssh_private",
+        "startup_script",
+    ]
+
+    PRIVATE_COLUMNS = ["ssh_private", "startup_script"]
+
+    def _hide_private(self, conf):
+        for machine in conf:
+            for column in self.PRIVATE_COLUMNS:
+                if column in conf[machine]:
+                    conf[machine][column] = "***"
+
+    def _show_origin(self):
         levels = [self.args.level] if self.args.level else self.config.LEVELS
         for level in levels:
             conf = self.config.read(level)["machine"]
             if self.args.name:
                 conf = conf.get(self.args.name, {})
-            prefix = self._config_file_prefix(
-                self.args.show_origin, self.config, level
-            )
+            self._hide_private(conf)
+            prefix = self._config_file_prefix(True, self.config, level)
             configs = list(self._format_config(conf, prefix))
             if configs:
                 ui.write("\n".join(configs))
+
+    def _show_table(self):
+        td = TabularData(self.TABLE_COLUMNS, fill_value="-")
+        conf = self.config.read()["machine"]
+        if self.args.name:
+            conf = {self.args.name: conf.get(self.args.name, {})}
+        self._hide_private(conf)
+        for machine, machine_config in conf.items():
+            machine_config["name"] = machine
+            td.row_from_dict(machine_config)
+        td.dropna("cols", "all")
+        td.render()
+
+    def run(self):
+        if self.args.show_origin:
+            self._show_origin()
+        else:
+            self._show_table()
         return 0
 
 
@@ -193,18 +234,60 @@ class CmdMachineCreate(CmdBase):
 
 
 class CmdMachineStatus(CmdBase):
-    SHOWN_FIELD = ["name", "cloud", "instance_hdd_size", "instance_ip"]
+    INSTANCE_FIELD = ["name", "instance", "status"]
+    SHOWN_FIELD = [
+        "cloud",
+        "instance_ip",
+        "instance_type",
+        "instance_hdd_size",
+        "instance_gpu",
+    ]
+    FILL_VALUE = "-"
+
+    def _add_row(
+        self,
+        name: str,
+        all_status: List[Dict],
+        td: TabularData,
+    ):
+
+        if not all_status:
+            row = [
+                name,
+                self.FILL_VALUE,
+                "offline",
+            ]  # back to `None` after #7167
+            td.append(row)
+        for i, status in enumerate(all_status, start=1):
+            row = [name, f"num_{i}", "running" if status else "offline"]
+            for field in self.SHOWN_FIELD:
+                value = str(status.get(field, ""))
+                row.append(value)
+            td.append(row)
 
     def run(self):
         if self.repo.machine is None:
             raise MachineDisabledError
 
-        all_status = self.repo.machine.status(self.args.name)
-        for i, status in enumerate(all_status, start=1):
-            ui.write(f"instance_num_{i}:")
-            for field in self.SHOWN_FIELD:
-                value = status.get(field, None)
-                ui.write(f"\t{field:20}: {value}")
+        td = TabularData(
+            self.INSTANCE_FIELD + self.SHOWN_FIELD, fill_value=self.FILL_VALUE
+        )
+
+        if self.args.name:
+            all_status = list(self.repo.machine.status(self.args.name))
+            self._add_row(self.args.name, all_status, td)
+        else:
+            name_set = set()
+            for level in self.repo.config.LEVELS:
+                conf = self.repo.config.read(level)["machine"]
+                name_set.update(conf.keys())
+            name_list = list(name_set)
+            for name in sorted(name_list):
+                all_status = list(self.repo.machine.status(name))
+                self._add_row(name, all_status, td)
+
+        td.dropna("cols", "all")
+        td.render()
         return 0
 
 
@@ -227,7 +310,7 @@ class CmdMachineSsh(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    from dvc.command.config import parent_config_parser
+    from dvc.commands.config import parent_config_parser
 
     machine_HELP = "Set up and manage cloud machines."
     machine_parser = subparsers.add_parser(
@@ -373,7 +456,7 @@ def add_parser(subparsers, parent_parser):
     machine_CREATE_HELP = "Create and start a machine instance."
     machine_create_parser = machine_subparsers.add_parser(
         "create",
-        parents=[parent_config_parser, parent_parser],
+        parents=[parent_parser],
         description=append_doc_link(machine_CREATE_HELP, "machine/create"),
         help=machine_CREATE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -383,23 +466,25 @@ def add_parser(subparsers, parent_parser):
     )
     machine_create_parser.set_defaults(func=CmdMachineCreate)
 
-    machine_STATUS_HELP = "List the status of a running machine."
+    machine_STATUS_HELP = (
+        "List the status of running instances for one/all machines."
+    )
     machine_status_parser = machine_subparsers.add_parser(
         "status",
-        parents=[parent_config_parser, parent_parser],
+        parents=[parent_parser],
         description=append_doc_link(machine_STATUS_HELP, "machine/status"),
         help=machine_STATUS_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     machine_status_parser.add_argument(
-        "name", help="Name of the running machine."
+        "name", nargs="?", help="(optional) Name of the machine."
     )
     machine_status_parser.set_defaults(func=CmdMachineStatus)
 
     machine_DESTROY_HELP = "Destroy an machine instance."
     machine_destroy_parser = machine_subparsers.add_parser(
         "destroy",
-        parents=[parent_config_parser, parent_parser],
+        parents=[parent_parser],
         description=append_doc_link(machine_DESTROY_HELP, "machine/destroy"),
         help=machine_DESTROY_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -412,7 +497,7 @@ def add_parser(subparsers, parent_parser):
     machine_SSH_HELP = "Connect to a machine via SSH."
     machine_ssh_parser = machine_subparsers.add_parser(
         "ssh",
-        parents=[parent_config_parser, parent_parser],
+        parents=[parent_parser],
         description=append_doc_link(machine_SSH_HELP, "machine/ssh"),
         help=machine_SSH_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,

dvc/commands/metrics.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
 from dvc.ui import ui
 from dvc.utils.serialize import encode_exception

dvc/commands/move.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)

dvc/commands/params.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
 from dvc.ui import ui
 

dvc/commands/plots.py

@@ -1,16 +1,29 @@
 import argparse
+import json
 import logging
+import os
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from funcy import first
+
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.render.utils import find_vega, render
 from dvc.ui import ui
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
 
+def _show_json(renderers, split=False):
+    from dvc.render.convert import to_json
+
+    result = {
+        renderer.name: to_json(renderer, split) for renderer in renderers
+    }
+    ui.write_json(result)
+
+
 class CmdPlots(CmdBase):
     def _func(self, *args, **kwargs):
         raise NotImplementedError
@@ -25,6 +38,10 @@ class CmdPlots(CmdBase):
     def run(self):
         from pathlib import Path
 
+        from dvc_render import render_html
+
+        from dvc.render.match import match_renderers
+
         if self.args.show_vega:
             if not self.args.targets:
                 logger.error("please specify a target for `--show-vega`")
@@ -34,8 +51,15 @@ class CmdPlots(CmdBase):
                     "you can only specify one target for `--show-vega`"
                 )
                 return 1
+            if self.args.json:
+                logger.error(
+                    "'--show-vega' and '--json' are mutually exclusive "
+                    "options."
+                )
+                return 1
 
         try:
+
             plots_data = self._func(
                 targets=self.args.targets, props=self._props()
             )
@@ -46,35 +70,55 @@ class CmdPlots(CmdBase):
                     "visualization file will not be created."
                 )
 
-            if self.args.show_vega:
-                target = self.args.targets[0]
-                plot_json = find_vega(self.repo, plots_data, target)
-                if plot_json:
-                    ui.write(plot_json)
-                return 0
+            out: str = self.args.out or "dvc_plots"
 
-            rel: str = self.args.out or "dvc_plots"
-            path: Path = (Path.cwd() / rel).resolve()
-            index_path = render(
-                self.repo,
-                plots_data,
-                path=path,
-                html_template_path=self.args.html_template,
+            renderers_out = (
+                out if self.args.json else os.path.join(out, "static")
+            )
+            renderers = match_renderers(
+                plots_data=plots_data, out=renderers_out
             )
 
-            assert index_path.is_absolute()
-            url = index_path.as_uri()
-            ui.write(url)
+            if self.args.show_vega:
+                renderer = first(filter(lambda r: r.TYPE == "vega", renderers))
+                if renderer:
+                    ui.write_json(json.loads(renderer.get_filled_template()))
+                return 0
+            if self.args.json:
+                _show_json(renderers, self.args.split)
+                return 0
 
-            if self.args.open:
-                import webbrowser
-
-                opened = webbrowser.open(index_path)
-                if not opened:
-                    ui.error_write(
-                        "Failed to open. Please try opening it manually."
+            html_template_path = self.args.html_template
+            if not html_template_path:
+                html_template_path = self.repo.config.get("plots", {}).get(
+                    "html_template", None
+                )
+                if html_template_path and not os.path.isabs(
+                    html_template_path
+                ):
+                    html_template_path = os.path.join(
+                        self.repo.dvc_dir, html_template_path
                     )
-                    return 1
+
+            output_file: Path = (Path.cwd() / out).resolve() / "index.html"
+
+            render_html(
+                renderers=renderers,
+                output_file=output_file,
+                template_path=html_template_path,
+            )
+
+            ui.write(output_file.as_uri())
+            auto_open = self.repo.config["plots"].get("auto_open", False)
+            if self.args.open or auto_open:
+                if not auto_open:
+                    ui.write(
+                        "To enable auto opening, you can run:\n"
+                        "\n"
+                        "\tdvc config plots.auto_open true"
+                    )
+                return ui.open_browser(output_file)
+
             return 0
 
         except DvcException:
@@ -109,6 +153,37 @@ class CmdPlotsModify(CmdPlots):
         return 0
 
 
+class CmdPlotsTemplates(CmdBase):
+    TEMPLATES_CHOICES = [
+        "simple",
+        "linear",
+        "confusion",
+        "confusion_normalized",
+        "scatter",
+        "smooth",
+    ]
+
+    def run(self):
+        from dvc_render.vega_templates import dump_templates
+
+        try:
+            out = (
+                os.path.join(os.getcwd(), self.args.out)
+                if self.args.out
+                else self.repo.plots.templates_dir
+            )
+
+            targets = [self.args.target] if self.args.target else None
+            dump_templates(output=out, targets=targets)
+            templates_path = os.path.relpath(out, os.getcwd())
+            ui.write(f"Templates have been written into '{templates_path}'.")
+
+            return 0
+        except DvcException:
+            logger.exception("")
+            return 1
+
+
 def add_parser(subparsers, parent_parser):
     PLOTS_HELP = (
         "Commands to visualize and compare plot metrics in structured files "
@@ -145,7 +220,8 @@ def add_parser(subparsers, parent_parser):
         "Shows all plots by default.",
     ).complete = completion.FILE
     _add_props_arguments(plots_show_parser)
-    _add_output_arguments(plots_show_parser)
+    _add_output_argument(plots_show_parser)
+    _add_ui_arguments(plots_show_parser)
     plots_show_parser.set_defaults(func=CmdPlotsShow)
 
     PLOTS_DIFF_HELP = (
@@ -180,7 +256,8 @@ def add_parser(subparsers, parent_parser):
         "revisions", nargs="*", default=None, help="Git commits to plot from"
     )
     _add_props_arguments(plots_diff_parser)
-    _add_output_arguments(plots_diff_parser)
+    _add_output_argument(plots_diff_parser)
+    _add_ui_arguments(plots_diff_parser)
     plots_diff_parser.set_defaults(func=CmdPlotsDiff)
 
     PLOTS_MODIFY_HELP = (
@@ -206,6 +283,27 @@ def add_parser(subparsers, parent_parser):
     )
     plots_modify_parser.set_defaults(func=CmdPlotsModify)
 
+    TEMPLATES_HELP = (
+        "Write built-in plots templates to a directory "
+        "(.dvc/plots by default)."
+    )
+    plots_templates_parser = plots_subparsers.add_parser(
+        "templates",
+        parents=[parent_parser],
+        description=append_doc_link(TEMPLATES_HELP, "plots/templates"),
+        help=TEMPLATES_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_templates_parser.add_argument(
+        "target",
+        default=None,
+        nargs="?",
+        choices=CmdPlotsTemplates.TEMPLATES_CHOICES,
+        help="Template to write. Writes all templates by default.",
+    )
+    _add_output_argument(plots_templates_parser, typ="templates")
+    plots_templates_parser.set_defaults(func=CmdPlotsTemplates)
+
 
 def _add_props_arguments(parser):
     parser.add_argument(
@@ -245,20 +343,33 @@ def _add_props_arguments(parser):
     )
 
 
-def _add_output_arguments(parser):
+def _add_output_argument(parser, typ="plots"):
     parser.add_argument(
         "-o",
         "--out",
         default=None,
-        help="Destination path to save plots to",
+        help=f"Directory to save {typ} to.",
         metavar="<path>",
     ).complete = completion.DIR
+
+
+def _add_ui_arguments(parser):
     parser.add_argument(
         "--show-vega",
         action="store_true",
         default=False,
         help="Show output in Vega format.",
     )
+    parser.add_argument(
+        "--json",
+        "--show-json",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--split", action="store_true", default=False, help=argparse.SUPPRESS
+    )
     parser.add_argument(
         "--open",
         action="store_true",

dvc/commands/remote.py

@@ -1,7 +1,7 @@
 import argparse
 
-from dvc.command.base import append_doc_link, fix_subparsers
-from dvc.command.config import CmdConfig
+from dvc.cli.utils import append_doc_link, fix_subparsers
+from dvc.commands.config import CmdConfig
 from dvc.ui import ui
 from dvc.utils import format_link
 
@@ -113,8 +113,8 @@ class CmdRemoteDefault(CmdRemote):
 class CmdRemoteList(CmdRemote):
     def run(self):
         conf = self.config.read(self.args.level)
-        for name, conf in conf["remote"].items():
-            ui.write(name, conf["url"], sep="\t")
+        for name, remote_conf in conf["remote"].items():
+            ui.write(name, remote_conf["url"], sep="\t")
         return 0
 
 
@@ -151,7 +151,7 @@ class CmdRemoteRename(CmdRemote):
 
 
 def add_parser(subparsers, parent_parser):
-    from dvc.command.config import parent_config_parser
+    from dvc.commands.config import parent_config_parser
 
     REMOTE_HELP = "Set up and manage data remotes."
     remote_parser = subparsers.add_parser(

dvc/commands/remove.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)

dvc/commands/repro.py

@@ -1,15 +1,18 @@
 import argparse
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.status import CmdDataStatus
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.commands.status import CmdDataStatus
 
 
 class CmdRepro(CmdBase):
     def run(self):
         from dvc.ui import ui
 
-        stages = self.repo.reproduce(**self._repro_kwargs)
+        stages = self.repo.reproduce(
+            **self._common_kwargs, **self._repro_kwargs
+        )
         if len(stages) == 0:
             ui.write(CmdDataStatus.UP_TO_DATE_MSG)
         else:
@@ -26,7 +29,7 @@ class CmdRepro(CmdBase):
         return 0
 
     @property
-    def _repro_kwargs(self):
+    def _common_kwargs(self):
         return {
             "targets": self.args.targets,
             "single_item": self.args.single_item,
@@ -35,12 +38,17 @@ class CmdRepro(CmdBase):
             "interactive": self.args.interactive,
             "pipeline": self.args.pipeline,
             "all_pipelines": self.args.all_pipelines,
-            "run_cache": not self.args.no_run_cache,
-            "no_commit": self.args.no_commit,
             "downstream": self.args.downstream,
             "recursive": self.args.recursive,
             "force_downstream": self.args.force_downstream,
             "pull": self.args.pull,
+        }
+
+    @property
+    def _repro_kwargs(self):
+        return {
+            "run_cache": not self.args.no_run_cache,
+            "no_commit": self.args.no_commit,
             "glob": self.args.glob,
         }
 
@@ -109,15 +117,6 @@ def add_arguments(repro_parser):
         default=False,
         help="Reproduce all stages in the specified directory.",
     )
-    repro_parser.add_argument(
-        "--no-run-cache",
-        action="store_true",
-        default=False,
-        help=(
-            "Execute stage commands even if they have already been run with "
-            "the same command/dependencies/outputs/etc before."
-        ),
-    )
     repro_parser.add_argument(
         "--force-downstream",
         action="store_true",
@@ -125,12 +124,6 @@ def add_arguments(repro_parser):
         help="Reproduce all descendants of a changed stage even if their "
         "direct dependencies didn't change.",
     )
-    repro_parser.add_argument(
-        "--no-commit",
-        action="store_true",
-        default=False,
-        help="Don't put files/directories into cache.",
-    )
     repro_parser.add_argument(
         "--downstream",
         action="store_true",
@@ -146,12 +139,6 @@ def add_arguments(repro_parser):
             "from the run-cache."
         ),
     )
-    repro_parser.add_argument(
-        "--glob",
-        action="store_true",
-        default=False,
-        help="Allows targets containing shell-style wildcards.",
-    )
 
 
 def add_parser(subparsers, parent_parser):
@@ -165,5 +152,28 @@ def add_parser(subparsers, parent_parser):
         help=REPRO_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    # repro/exp run shared args
     add_arguments(repro_parser)
+    # repro only args
+    repro_parser.add_argument(
+        "--no-run-cache",
+        action="store_true",
+        default=False,
+        help=(
+            "Execute stage commands even if they have already been run with "
+            "the same command/dependencies/outputs/etc before."
+        ),
+    )
+    repro_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    repro_parser.add_argument(
+        "--glob",
+        action="store_true",
+        default=False,
+        help="Allows targets containing shell-style wildcards.",
+    )
     repro_parser.set_defaults(func=CmdRepro)

dvc/commands/root.py

@@ -1,7 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)

dvc/commands/run.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.stage import parse_cmd
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.commands.stage import parse_cmd
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -23,7 +24,7 @@ class CmdRun(CmdBase):
                 self.args.outs_persist_no_cache,
                 self.args.checkpoints,
                 self.args.params,
-                self.args.cmd,
+                self.args.command,
             ]
         ):  # pragma: no cover
             logger.error(
@@ -36,7 +37,7 @@ class CmdRun(CmdBase):
         kwargs = vars(self.args)
         kwargs.update(
             {
-                "cmd": parse_cmd(self.args.cmd),
+                "cmd": parse_cmd(self.args.command),
                 "fname": kwargs.pop("file"),
                 "no_exec": (self.args.no_exec or bool(self.args.checkpoints)),
                 "run_cache": not kwargs.pop("no_run_cache"),
@@ -52,7 +53,7 @@ class CmdRun(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    from dvc.command.stage import _add_common_args
+    from dvc.commands.stage import _add_common_args
 
     RUN_HELP = (
         "Generate a dvc.yaml file from a command and execute the command."

dvc/commands/stage.py

@@ -3,8 +3,9 @@ import logging
 from itertools import chain, filterfalse
 from typing import TYPE_CHECKING, Dict, Iterable, List
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link, fix_subparsers
 from dvc.utils.cli_parse import parse_params
 from dvc.utils.humanize import truncate_text
 
@@ -93,9 +94,7 @@ class CmdStageList(CmdBase):
         self.repo.stage_collection_error_handler = log_error
 
         stages = self._get_stages()
-        names_only = self.args.names_only
-
-        data = prepare_stages_data(stages, description=not names_only)
+        data = prepare_stages_data(stages, description=not self.args.name_only)
         ui.table(data.items())
 
         return 0
@@ -129,7 +128,7 @@ class CmdStageAdd(CmdBase):
         kwargs = vars(self.args)
         kwargs.update(
             {
-                "cmd": parse_cmd(kwargs.pop("cmd")),
+                "cmd": parse_cmd(kwargs.pop("command")),
                 "params": parse_params(self.args.params),
             }
         )
@@ -213,7 +212,7 @@ def _add_common_args(parser):
         "--live-no-summary",
         action="store_true",
         default=False,
-        help="Signal dvclive logger to not dump latest metrics file.",
+        help=argparse.SUPPRESS,
     )
     parser.add_argument(
         "--live-no-html",
@@ -281,7 +280,7 @@ def _add_common_args(parser):
         ),
     )
     parser.add_argument(
-        "cmd",
+        "command",
         nargs=argparse.REMAINDER,
         help="Command to execute.",
         metavar="command",
@@ -357,6 +356,7 @@ def add_parser(subparsers, parent_parser):
         help="List all stages inside the specified directory.",
     )
     stage_list_parser.add_argument(
+        "--name-only",
         "--names-only",
         action="store_true",
         default=False,

dvc/commands/status.py

@@ -1,6 +1,6 @@
 import logging
 
-from dvc.command.data_sync import CmdDataBase
+from dvc.commands.data_sync import CmdDataBase
 from dvc.exceptions import DvcException
 from dvc.ui import ui
 from dvc.utils import format_link

dvc/commands/unprotect.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)

dvc/commands/update.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -70,7 +71,6 @@ def add_parser(subparsers, parent_parser):
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/commands/version.py

@@ -1,7 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)

dvc/compare.py

@@ -11,7 +11,9 @@ from typing import (
     List,
     Mapping,
     MutableSequence,
+    Optional,
     Sequence,
+    Set,
     Tuple,
     Union,
     overload,
@@ -20,6 +22,7 @@ from typing import (
 from funcy import reraise
 
 if TYPE_CHECKING:
+    from dvc.types import StrPath
     from dvc.ui.table import CellT
 
 
@@ -36,11 +39,21 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         self._columns: Dict[str, Column] = {name: Column() for name in columns}
         self._keys: List[str] = list(columns)
         self._fill_value = fill_value
+        self._protected: Set[str] = set()
 
     @property
     def columns(self) -> List[Column]:
         return list(map(self.column, self.keys()))
 
+    def is_protected(self, col_name) -> bool:
+        return col_name in self._protected
+
+    def protect(self, *col_names: str):
+        self._protected.update(col_names)
+
+    def unprotect(self, *col_names: str):
+        self._protected = self._protected.difference(col_names)
+
     def column(self, name: str) -> Column:
         return self._columns[name]
 
@@ -115,6 +128,8 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             del col[item]
 
     def __len__(self) -> int:
+        if not self._columns:
+            return 0
         return len(self.columns[0])
 
     @property
@@ -122,9 +137,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         return len(self.columns), len(self)
 
     def drop(self, *col_names: str) -> None:
-        for col_name in col_names:
-            self._keys.remove(col_name)
-            self._columns.pop(col_name)
+        for col in col_names:
+            if not self.is_protected(col):
+                self._keys.remove(col)
+                self._columns.pop(col)
 
     def rename(self, from_col_name: str, to_col_name: str) -> None:
         self._columns[to_col_name] = self._columns.pop(from_col_name)
@@ -150,6 +166,24 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             writer.writerow(row)
         return buff.getvalue()
 
+    def to_parallel_coordinates(
+        self, output_path: "StrPath", color_by: str = None
+    ) -> "StrPath":
+        from dvc_render.html import render_html
+        from dvc_render.plotly import ParallelCoordinatesRenderer
+
+        render_html(
+            renderers=[
+                ParallelCoordinatesRenderer(
+                    self.as_dict(),
+                    color_by=color_by,
+                    fill_value=self._fill_value,
+                )
+            ],
+            output_file=output_path,
+        )
+        return output_path
+
     def add_column(self, name: str) -> None:
         self._columns[name] = Column([self._fill_value] * len(self))
         self._keys.append(name)
@@ -181,6 +215,109 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             {k: self._columns[k][i] for k in keys} for i in range(len(self))
         ]
 
+    def dropna(
+        self,
+        axis: str = "rows",
+        how="any",
+        subset: Optional[Iterable[str]] = None,
+    ):
+        if axis not in ["rows", "cols"]:
+            raise ValueError(
+                f"Invalid 'axis' value {axis}."
+                "Choose one of ['rows', 'cols']"
+            )
+        if how not in ["any", "all"]:
+            raise ValueError(
+                f"Invalid 'how' value {how}." "Choose one of ['any', 'all']"
+            )
+
+        match_line: Set = set()
+        match_any = True
+        if how == "all":
+            match_any = False
+
+        for n_row, row in enumerate(self):
+            for n_col, col in enumerate(row):
+                if subset and self.keys()[n_col] not in subset:
+                    continue
+                if (col == self._fill_value) is match_any:
+                    if axis == "rows":
+                        match_line.add(n_row)
+                        break
+                    else:
+                        match_line.add(self.keys()[n_col])
+
+        to_drop = match_line
+        if how == "all":
+            if axis == "rows":
+                to_drop = set(range(len(self)))
+            else:
+                to_drop = set(self.keys())
+            to_drop -= match_line
+
+        if axis == "rows":
+            for name in self.keys():
+                self._columns[name] = Column(
+                    [
+                        x
+                        for n, x in enumerate(self._columns[name])
+                        if n not in to_drop
+                    ]
+                )
+        else:
+            self.drop(*to_drop)
+
+    def drop_duplicates(
+        self,
+        axis: str = "rows",
+        subset: Optional[Iterable[str]] = None,
+        ignore_empty: bool = True,
+    ):
+        if axis not in ["rows", "cols"]:
+            raise ValueError(
+                f"Invalid 'axis' value {axis}."
+                "Choose one of ['rows', 'cols']"
+            )
+
+        if axis == "cols":
+            cols_to_drop: List[str] = []
+            for n_col, col in enumerate(self.columns):
+                if subset and self.keys()[n_col] not in subset:
+                    continue
+                # Cast to str because Text is not hashable error
+                unique_vals = {str(x) for x in col}
+                if ignore_empty and self._fill_value in unique_vals:
+                    unique_vals -= {self._fill_value}
+                if len(unique_vals) == 1:
+                    cols_to_drop.append(self.keys()[n_col])
+            self.drop(*cols_to_drop)
+
+        elif axis == "rows":
+            unique_rows = []
+            rows_to_drop: List[int] = []
+            for n_row, row in enumerate(self):
+                if subset:
+                    row = [
+                        col
+                        for n_col, col in enumerate(row)
+                        if self.keys()[n_col] in subset
+                    ]
+
+                tuple_row = tuple(row)
+                if tuple_row in unique_rows:
+                    rows_to_drop.append(n_row)
+                else:
+                    unique_rows.append(tuple_row)
+
+            for name in self.keys():
+                self._columns[name] = Column(
+                    [
+                        x
+                        for n, x in enumerate(self._columns[name])
+                        if n not in rows_to_drop
+                    ]
+                )
+
 
 def _normalize_float(val: float, precision: int):
     return f"{val:.{precision}g}"

dvc/config.py

@@ -8,7 +8,6 @@ from functools import partial
 from funcy import cached_property, compact, memoize, re_find
 
 from dvc.exceptions import DvcException, NotDvcRepoError
-from dvc.path_info import PathInfo
 
 logger = logging.getLogger(__name__)
 
@@ -95,7 +94,7 @@ class Config(dict):
             try:
                 from dvc.repo import Repo
 
-                self.dvc_dir = os.path.join(Repo.find_dvc_dir())
+                self.dvc_dir = Repo.find_dvc_dir()
             except NotDvcRepoError:
                 self.dvc_dir = None
         else:
@@ -229,6 +228,7 @@ class Config(dict):
 
     @staticmethod
     def _to_relpath(conf_dir, path):
+        from dvc.fs.local import localfs
         from dvc.utils import relpath
 
         from .config_schema import RelPath
@@ -238,8 +238,9 @@ class Config(dict):
 
         if isinstance(path, RelPath) or not os.path.isabs(path):
             path = relpath(path, conf_dir)
+            return localfs.path.as_posix(path)
 
-        return PathInfo(path).as_posix()
+        return path
 
     @staticmethod
     def _save_paths(conf, filename):
@@ -265,7 +266,12 @@ class Config(dict):
                     "key_path": func,
                 }
             },
-            "machine": {str: {"startup_script": func}},
+            "machine": {
+                str: {
+                    "startup_script": func,
+                    "setup_script": func,
+                }
+            },
         }
         return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
 

dvc/config_schema.py

@@ -79,13 +79,13 @@ REMOTE_COMMON = {
     "checksum_jobs": All(Coerce(int), Range(1)),
     "jobs": All(Coerce(int), Range(1)),
     Optional("no_traverse"): Bool,  # obsoleted
-    "verify": Bool,
 }
 LOCAL_COMMON = {
     "type": supported_cache_type,
     Optional("protected", default=False): Bool,  # obsoleted
     "shared": All(Lower, Choices("group")),
     Optional("slow_link_warning", default=True): Bool,
+    Optional("verify", default=False): Bool,
 }
 HTTP_COMMON = {
     "auth": All(Lower, Choices("basic", "digest", "custom")),
@@ -95,6 +95,7 @@ HTTP_COMMON = {
     "ask_password": Bool,
     "ssl_verify": Any(Bool, str),
     "method": str,
+    Optional("verify", default=False): Bool,
 }
 WEBDAV_COMMON = {
     "user": str,
@@ -105,6 +106,7 @@ WEBDAV_COMMON = {
     "key_path": str,
     "timeout": Coerce(int),
     "ssl_verify": Any(Bool, str),
+    Optional("verify", default=False): Bool,
 }
 
 SCHEMA = {
@@ -158,11 +160,13 @@ SCHEMA = {
                     "cache_regions": bool,
                     "read_timeout": Coerce(int),
                     "connect_timeout": Coerce(int),
+                    Optional("verify", default=False): Bool,
                     **REMOTE_COMMON,
                 },
                 "gs": {
                     "projectname": str,
                     "credentialpath": str,
+                    Optional("verify", default=False): Bool,
                     **REMOTE_COMMON,
                 },
                 "ssh": {
@@ -175,14 +179,18 @@ SCHEMA = {
                     "timeout": Coerce(int),
                     "gss_auth": Bool,
                     "allow_agent": Bool,
+                    Optional("verify", default=False): Bool,
                     **REMOTE_COMMON,
                 },
                 "hdfs": {"user": str, "kerb_ticket": str, **REMOTE_COMMON},
                 "webhdfs": {
-                    "hdfscli_config": str,
-                    "webhdfs_token": str,
-                    "user": str,
-                    "webhdfs_alias": str,
+                    "kerberos": Bool,
+                    "kerberos_principal": str,
+                    "proxy_to": str,
+                    "ssl_verify": Any(Bool, str),
+                    "token": str,
+                    "use_https": Bool,
+                    Optional("verify", default=False): Bool,
                     **REMOTE_COMMON,
                 },
                 "azure": {
@@ -198,12 +206,14 @@ SCHEMA = {
                     "exclude_visual_studio_code_credential": Bool,
                     "exclude_shared_token_cache_credential": Bool,
                     "exclude_managed_identity_credential": Bool,
+                    Optional("verify", default=False): Bool,
                     **REMOTE_COMMON,
                 },
                 "oss": {
                     "oss_key_id": str,
                     "oss_key_secret": str,
                     "oss_endpoint": str,
+                    Optional("verify", default=True): Bool,
                     **REMOTE_COMMON,
                 },
                 "gdrive": {
@@ -214,6 +224,7 @@ SCHEMA = {
                     "gdrive_service_account_user_email": str,
                     "gdrive_service_account_json_file_path": str,
                     Optional("gdrive_trash_only", default=False): Bool,
+                    Optional("verify", default=True): Bool,
                     **REMOTE_COMMON,
                 },
                 "http": {**HTTP_COMMON, **REMOTE_COMMON},
@@ -239,7 +250,6 @@ SCHEMA = {
                 Lower, Choices("us-west", "us-east", "eu-west", "eu-north")
             ),
             "image": str,
-            "name": str,
             "spot": Bool,
             "spot_price": Coerce(float),
             "instance_hdd_size": Coerce(int),
@@ -247,6 +257,7 @@ SCHEMA = {
             "instance_gpu": Lower,
             "ssh_private": str,
             "startup_script": str,
+            "setup_script": str,
         },
     },
     # section for experimental features
@@ -255,7 +266,10 @@ SCHEMA = {
         # enabled by default. It's of no use, kept for backward compatibility.
         Optional("parametrization", default=True): Bool,
     },
-    "plots": {"html_template": str},
+    "plots": {
+        "html_template": str,
+        Optional("auto_open", default=False): Bool,
+    },
     "exp": {
         "code": str,
         "data": str,

dvc/daemon.py

@@ -42,7 +42,7 @@ def _spawn_windows(cmd, env):
 
 
 def _spawn_posix(cmd, env):
-    from dvc.main import main
+    from dvc.cli import main
 
     # NOTE: using os._exit instead of sys.exit, because dvc built
     # with PyInstaller has trouble with SystemExit exception and throws
@@ -106,8 +106,9 @@ def daemon(args):
     cmd = ["daemon", "-q"] + args
 
     env = fix_env()
-    file_path = os.path.abspath(inspect.stack()[0][1])
-    env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
+    if not is_binary():
+        file_path = os.path.abspath(inspect.stack()[0][1])
+        env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
     env[DVC_DAEMON] = "1"
 
     _spawn(cmd, env)

dvc/data/__init__.py

@@ -0,0 +1,33 @@
+import logging
+from typing import TYPE_CHECKING, Iterator, Union
+
+from .tree import Tree
+
+if TYPE_CHECKING:
+    from dvc.hash_info import HashInfo
+    from dvc.objects.db import ObjectDB
+    from dvc.objects.file import HashFile
+
+logger = logging.getLogger(__name__)
+
+
+def check(odb: "ObjectDB", obj: "HashFile", **kwargs):
+    if isinstance(obj, Tree):
+        for _, _, oid in obj:
+            odb.check(oid, **kwargs)
+
+    odb.check(obj.hash_info, **kwargs)
+
+
+def load(odb: "ObjectDB", hash_info: "HashInfo") -> "HashFile":
+    if hash_info.isdir:
+        return Tree.load(odb, hash_info)
+    return odb.get(hash_info)
+
+
+def iterobjs(
+    obj: Union["Tree", "HashFile"]
+) -> Iterator[Union["Tree", "HashFile"]]:
+    if isinstance(obj, Tree):
+        yield from (entry_obj for _, entry_obj in obj)
+    yield obj

dvc/data/_progress.py

@@ -0,0 +1,33 @@
+from dvc.progress import Tqdm
+
+
+class QueryingProgress(Tqdm):
+    def __init__(self, iterable=None, total=None, name=None, phase="Querying"):
+        msg_part = "cache in " + f"'{name}'" if name else "remote cache"
+        msg_fmt = "{phase} " + msg_part
+
+        self._estimating_msg = msg_fmt.format(phase="Estimating size of")
+        self._listing_msg = msg_fmt.format(phase="Querying")
+        self.desc = desc = msg_fmt.format(phase=phase)
+        super().__init__(
+            iterable=iterable,
+            desc=desc,
+            total=total,
+            unit="files",
+            unit_scale=False,
+            bar_format=self.BAR_FMT_DEFAULT,
+        )
+
+    def callback(self, phase, *args):
+        total = args[0] if args else self.total
+        completed = args[1] if len(args) > 1 else self.n
+        desc = self.desc
+        if phase == "estimating":
+            desc = self._estimating_msg
+        elif phase == "querying":
+            desc = self._listing_msg
+
+        if desc != self.desc:
+            # let Tqdm take care of when to update
+            self.set_description_str(desc, refresh=False)
+        self.update_to(completed, total)

dvc/data/checkout.py

@@ -0,0 +1,265 @@
+import logging
+from itertools import chain
+
+from dvc import prompt
+from dvc.exceptions import CacheLinkError, CheckoutError, ConfirmRemoveError
+from dvc.fs.utils import test_links, transfer
+from dvc.ignore import DvcIgnoreFilter
+from dvc.types import Optional
+
+from .diff import ROOT
+from .diff import diff as odiff
+from .slow_link_detection import slow_link_guard  # type: ignore[attr-defined]
+from .stage import stage
+
+logger = logging.getLogger(__name__)
+
+
+def _remove(fs_path, fs, in_cache, force=False):
+    if not fs.exists(fs_path):
+        return
+
+    if force:
+        fs.remove(fs_path)
+        return
+
+    if not in_cache:
+        msg = (
+            f"file/directory '{fs_path}' is going to be removed. "
+            "Are you sure you want to proceed?"
+        )
+
+        if not prompt.confirm(msg):
+            raise ConfirmRemoveError(fs_path)
+
+    fs.remove(fs_path)
+
+
+def _relink(link, cache, cache_info, fs, path, in_cache, force):
+    _remove(path, fs, in_cache, force=force)
+    link(cache, cache_info, fs, path)
+    # NOTE: Depending on a file system (e.g. on NTFS), `_remove` might reset
+    # read-only permissions in order to delete a hardlink to protected object,
+    # which will also reset it for the object itself, making it unprotected,
+    # so we need to protect it back.
+    cache.protect(cache_info)
+
+
+def _checkout_file(
+    link,
+    fs_path,
+    fs,
+    change,
+    cache,
+    force,
+    progress_callback=None,
+    relink=False,
+    state=None,
+):
+    """The file is changed we need to checkout a new copy"""
+    modified = False
+
+    cache_fs_path = cache.hash_to_path(change.new.oid.value)
+    if change.old.oid:
+        if relink:
+            if fs.iscopy(fs_path) and cache.cache_types[0] == "copy":
+                cache.unprotect(fs_path)
+            else:
+                _relink(
+                    link,
+                    cache,
+                    cache_fs_path,
+                    fs,
+                    fs_path,
+                    change.old.in_cache,
+                    force=force,
+                )
+        else:
+            modified = True
+            _relink(
+                link,
+                cache,
+                cache_fs_path,
+                fs,
+                fs_path,
+                change.old.in_cache,
+                force=force,
+            )
+    else:
+        link(cache, cache_fs_path, fs, fs_path)
+        modified = True
+
+    if state:
+        state.save(fs_path, fs, change.new.oid)
+
+    if progress_callback:
+        progress_callback(fs_path)
+
+    return modified
+
+
+def _diff(
+    fs_path,
+    fs,
+    obj,
+    cache,
+    relink=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+):
+    old = None
+    try:
+        _, _, old = stage(
+            cache,
+            fs_path,
+            fs,
+            obj.hash_info.name if obj else cache.fs.PARAM_CHECKSUM,
+            dry_run=True,
+            dvcignore=dvcignore,
+        )
+    except FileNotFoundError:
+        pass
+
+    diff = odiff(old, obj, cache)
+
+    if relink:
+        diff.modified.extend(diff.unchanged)
+
+    return diff
+
+
+class Link:
+    def __init__(self, links):
+        self._links = links
+
+    @slow_link_guard
+    def __call__(self, cache, from_path, to_fs, to_path):
+        if to_fs.exists(to_path):
+            to_fs.remove(to_path)  # broken symlink
+
+        cache.makedirs(cache.fs.path.parent(to_path))
+        try:
+            transfer(cache.fs, from_path, to_fs, to_path, links=self._links)
+        except FileNotFoundError as exc:
+            raise CheckoutError([to_path]) from exc
+        except OSError as exc:
+            raise CacheLinkError([to_path]) from exc
+
+
+def _checkout(
+    diff,
+    fs_path,
+    fs,
+    cache,
+    force=False,
+    progress_callback=None,
+    relink=False,
+    state=None,
+):
+    if not diff:
+        return
+
+    links = test_links(cache.cache_types, cache.fs, cache.fs_path, fs, fs_path)
+    if not links:
+        raise CacheLinkError([fs_path])
+    link = Link(links)
+
+    for change in diff.deleted:
+        entry_path = (
+            fs.path.join(fs_path, *change.old.key)
+            if change.old.key != ROOT
+            else fs_path
+        )
+        _remove(entry_path, fs, change.old.in_cache, force=force)
+
+    failed = []
+    for change in chain(diff.added, diff.modified):
+        entry_path = (
+            fs.path.join(fs_path, *change.new.key)
+            if change.new.key != ROOT
+            else fs_path
+        )
+        if change.new.oid.isdir:
+            fs.makedirs(entry_path)
+            continue
+
+        try:
+            _checkout_file(
+                link,
+                entry_path,
+                fs,
+                change,
+                cache,
+                force,
+                progress_callback,
+                relink,
+                state=state,
+            )
+        except CheckoutError as exc:
+            failed.extend(exc.target_infos)
+
+    if failed:
+        raise CheckoutError(failed)
+
+
+def checkout(
+    fs_path,
+    fs,
+    obj,
+    cache,
+    force=False,
+    progress_callback=None,
+    relink=False,
+    quiet=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+    state=None,
+):
+    # if scheme(fs_path) not in ["local", cache.fs.scheme]:
+    #    raise NotImplementedError
+
+    diff = _diff(
+        fs_path,
+        fs,
+        obj,
+        cache,
+        relink=relink,
+        dvcignore=dvcignore,
+    )
+
+    failed = []
+    if not obj:
+        if not quiet:
+            logger.warning(
+                "No file hash info found for '%s'. It won't be created.",
+                fs_path,
+            )
+        failed.append(fs_path)
+    elif not diff:
+        logger.trace("Data '%s' didn't change.", fs_path)  # type: ignore
+
+    try:
+        _checkout(
+            diff,
+            fs_path,
+            fs,
+            cache,
+            force=force,
+            progress_callback=progress_callback,
+            relink=relink,
+            state=state,
+        )
+    except CheckoutError as exc:
+        failed.extend(exc.target_infos)
+
+    if diff and state:
+        state.save_link(fs_path, fs)
+        if not failed:
+            state.save(fs_path, fs, obj.hash_info)
+
+    if failed or not diff:
+        if progress_callback and obj:
+            progress_callback(fs_path, len(obj))
+        if failed:
+            raise CheckoutError(failed)
+        return
+
+    return bool(diff) and not relink

dvc/data/db/__init__.py

@@ -6,22 +6,15 @@ if TYPE_CHECKING:
     from .index import ObjectDBIndexBase
 
 
-def get_odb(fs, path_info, **config):
-    from .base import ObjectDB
-    from .gdrive import GDriveObjectDB
+def get_odb(fs, fs_path, **config):
+    from dvc.objects.db import ObjectDB
+
     from .local import LocalObjectDB
-    from .oss import OSSObjectDB
 
     if fs.scheme == Schemes.LOCAL:
-        return LocalObjectDB(fs, path_info, **config)
+        return LocalObjectDB(fs, fs_path, **config)
 
-    if fs.scheme == Schemes.GDRIVE:
-        return GDriveObjectDB(fs, path_info, **config)
-
-    if fs.scheme == Schemes.OSS:
-        return OSSObjectDB(fs, path_info, **config)
-
-    return ObjectDB(fs, path_info, **config)
+    return ObjectDB(fs, fs_path, **config)
 
 
 def _get_odb(repo, settings):
@@ -30,9 +23,9 @@ def _get_odb(repo, settings):
     if not settings:
         return None
 
-    cls, config, path_info = get_cloud_fs(repo, **settings)
+    cls, config, fs_path = get_cloud_fs(repo, **settings)
     config["tmp_dir"] = repo.tmp_dir
-    return get_odb(cls(**config), path_info, state=repo.state, **config)
+    return get_odb(cls(**config), fs_path, state=repo.state, **config)
 
 
 def get_index(odb) -> "ObjectDBIndexBase":
@@ -43,8 +36,9 @@ def get_index(odb) -> "ObjectDBIndexBase":
     cls = ObjectDBIndex if odb.tmp_dir else ObjectDBIndexNoop
     return cls(
         odb.tmp_dir,
-        hashlib.sha256(odb.path_info.url.encode("utf-8")).hexdigest(),
-        odb.fs.CHECKSUM_DIR_SUFFIX,
+        hashlib.sha256(
+            odb.fs.unstrip_protocol(odb.fs_path).encode("utf-8")
+        ).hexdigest(),
     )
 
 

dvc/data/db/index.py

@@ -1,9 +1,10 @@
 import logging
 import os
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Iterable, Optional, Set
+from typing import TYPE_CHECKING, Iterable, Set
 
-from ..errors import ObjectDBError
+from dvc.objects.errors import ObjectDBError
+from dvc.utils.decorators import with_diskcache
 
 if TYPE_CHECKING:
     from dvc.types import StrPath
@@ -14,7 +15,9 @@ logger = logging.getLogger(__name__)
 class ObjectDBIndexBase(ABC):
     @abstractmethod
     def __init__(
-        self, tmp_dir: "StrPath", name: str, dir_suffix: Optional[str] = None
+        self,
+        tmp_dir: "StrPath",
+        name: str,
     ):
         pass
 
@@ -50,7 +53,9 @@ class ObjectDBIndexNoop(ObjectDBIndexBase):
     """No-op class for ODBs which are not indexed."""
 
     def __init__(
-        self, tmp_dir: "StrPath", name: str, dir_suffix: Optional[str] = None
+        self,
+        tmp_dir: "StrPath",
+        name: str,
     ):  # pylint: disable=super-init-not-called
         pass
 
@@ -80,9 +85,11 @@ class ObjectDBIndex(ObjectDBIndexBase):
     INDEX_DIR = "index"
 
     def __init__(
-        self, tmp_dir: "StrPath", name: str, dir_suffix: Optional[str] = None
+        self,
+        tmp_dir: "StrPath",
+        name: str,
     ):  # pylint: disable=super-init-not-called
-        from diskcache import Index
+        from diskcache import Cache, Index
 
         from dvc.fs.local import LocalFileSystem
         from dvc.utils.fs import makedirs
@@ -90,25 +97,28 @@ class ObjectDBIndex(ObjectDBIndexBase):
         self.index_dir = os.path.join(tmp_dir, self.INDEX_DIR, name)
         makedirs(self.index_dir, exist_ok=True)
         self.fs = LocalFileSystem()
-        self.index = Index(self.index_dir)
-
-        if not dir_suffix:
-            dir_suffix = self.fs.CHECKSUM_DIR_SUFFIX
-        self.dir_suffix = dir_suffix
+        self.index = Index.fromcache(
+            Cache(
+                self.index_dir,
+                disk_pickle_protocol=4,
+                eviction_policy="none",
+            )
+        )
 
+    @with_diskcache(name="index")
     def __iter__(self):
         return iter(self.index)
 
+    @with_diskcache(name="index")
     def __contains__(self, hash_):
         return hash_ in self.index
 
+    @with_diskcache(name="index")
     def dir_hashes(self):
         """Iterate over .dir hashes stored in the index."""
         yield from (hash_ for hash_, is_dir in self.index.items() if is_dir)
 
-    def is_dir_hash(self, hash_: str):
-        return hash_.endswith(self.dir_suffix)
-
+    @with_diskcache(name="index")
     def clear(self):
         """Clear this index (to force re-indexing later)."""
         from diskcache import Timeout
@@ -118,6 +128,7 @@ class ObjectDBIndex(ObjectDBIndexBase):
         except Timeout as exc:
             raise ObjectDBError("Failed to clear ODB index") from exc
 
+    @with_diskcache(name="index")
     def update(self, dir_hashes: Iterable[str], file_hashes: Iterable[str]):
         """Update this index, adding the specified hashes."""
         from diskcache import Timeout
@@ -132,6 +143,7 @@ class ObjectDBIndex(ObjectDBIndexBase):
         except Timeout as exc:
             raise ObjectDBError("Failed to update ODB index") from exc
 
+    @with_diskcache(name="index")
     def intersection(self, hashes: Set[str]):
         """Iterate over values from `hashes` which exist in the index."""
         yield from hashes.intersection(self.index.keys())

dvc/data/db/local.py

@@ -1,19 +1,17 @@
 import logging
 import os
 import stat
+from functools import partial
 
 from funcy import cached_property
 from shortuuid import uuid
 
 from dvc.hash_info import HashInfo
+from dvc.objects.db import ObjectDB, noop, wrap_iter
 from dvc.objects.errors import ObjectFormatError
-from dvc.path_info import PathInfo
-from dvc.progress import Tqdm
 from dvc.utils import relpath
 from dvc.utils.fs import copyfile, remove, umask, walk_files
 
-from .base import ObjectDB
-
 logger = logging.getLogger(__name__)
 
 
@@ -22,9 +20,9 @@ class LocalObjectDB(ObjectDB):
     CACHE_MODE = 0o444
     UNPACKED_DIR_SUFFIX = ".unpacked"
 
-    def __init__(self, fs, path_info, **config):
-        super().__init__(fs, path_info, **config)
-        self.cache_dir = path_info.fspath
+    def __init__(self, fs, fs_path, **config):
+        super().__init__(fs, fs_path, **config)
+        self.cache_dir = fs_path
 
         shared = config.get("shared")
         if shared:
@@ -36,11 +34,11 @@ class LocalObjectDB(ObjectDB):
 
     @property
     def cache_dir(self):
-        return self.path_info.fspath if self.path_info else None
+        return self.fs_path if self.fs_path else None
 
     @cache_dir.setter
     def cache_dir(self, value):
-        self.path_info = PathInfo(value) if value else None
+        self.fs_path = value
 
     @cached_property
     def cache_path(self):
@@ -50,10 +48,10 @@ class LocalObjectDB(ObjectDB):
         super().move(from_info, to_info)
         os.chmod(to_info, self._file_mode)
 
-    def makedirs(self, path_info):
+    def makedirs(self, fs_path):
         from dvc.utils.fs import makedirs
 
-        makedirs(path_info, exist_ok=True, mode=self._dir_mode)
+        makedirs(fs_path, exist_ok=True, mode=self._dir_mode)
 
     def hash_to_path(self, hash_):
         # NOTE: `self.cache_path` is already normalized so we can simply use
@@ -62,15 +60,12 @@ class LocalObjectDB(ObjectDB):
         return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
 
     def hashes_exist(
-        self, hashes, jobs=None, name=None
+        self, hashes, jobs=None, progress=noop
     ):  # pylint: disable=unused-argument
         ret = []
+        progress = partial(progress, "querying", len(hashes))
 
-        for hash_ in Tqdm(
-            hashes,
-            unit="file",
-            desc="Querying " + ("cache in " + name if name else "local cache"),
-        ):
+        for hash_ in wrap_iter(hashes, progress):
             hash_info = HashInfo(self.fs.PARAM_CHECKSUM, hash_)
             try:
                 self.check(hash_info)
@@ -80,27 +75,26 @@ class LocalObjectDB(ObjectDB):
 
         return ret
 
-    def _list_paths(self, prefix=None, progress_callback=None):
-        assert self.path_info is not None
+    def _list_paths(self, prefix=None):
+        assert self.fs_path is not None
         if prefix:
-            path_info = self.path_info / prefix[:2]
-            if not self.fs.exists(path_info):
+            fs_path = self.fs.path.join(self.fs_path, prefix[:2])
+            if not self.fs.exists(fs_path):
                 return
         else:
-            path_info = self.path_info
+            fs_path = self.fs_path
+
         # NOTE: use utils.fs walk_files since fs.walk_files will not follow
         # symlinks
-        if progress_callback:
-            for path in walk_files(path_info):
-                progress_callback()
-                yield path
-        else:
-            yield from walk_files(path_info)
+        yield from walk_files(fs_path)
 
     def _remove_unpacked_dir(self, hash_):
-        info = self.hash_to_path_info(hash_)
-        path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
-        self.fs.remove(path_info)
+        hash_fs_path = self.hash_to_path(hash_)
+        fs_path = self.fs.path.with_name(
+            hash_fs_path,
+            self.fs.path.name(hash_fs_path) + self.UNPACKED_DIR_SUFFIX,
+        )
+        self.fs.remove(fs_path)
 
     def _unprotect_file(self, path):
         if self.fs.is_symlink(path) or self.fs.is_hardlink(path):
@@ -126,47 +120,47 @@ class LocalObjectDB(ObjectDB):
         os.chmod(path, self._file_mode)
 
     def _unprotect_dir(self, path):
-        for fname in self.fs.walk_files(path):
+        for fname in self.fs.find(path):
             self._unprotect_file(fname)
 
-    def unprotect(self, path_info):
-        if not os.path.exists(path_info):
+    def unprotect(self, fs_path):
+        if not os.path.exists(fs_path):
             from dvc.exceptions import DvcException
 
             raise DvcException(
-                f"can't unprotect non-existing data '{path_info}'"
+                f"can't unprotect non-existing data '{fs_path}'"
             )
 
-        if os.path.isdir(path_info):
-            self._unprotect_dir(path_info)
+        if os.path.isdir(fs_path):
+            self._unprotect_dir(fs_path)
         else:
-            self._unprotect_file(path_info)
+            self._unprotect_file(fs_path)
 
-    def protect(self, path_info):
+    def protect(self, fs_path):
         try:
-            os.chmod(path_info, self.CACHE_MODE)
+            os.chmod(fs_path, self.CACHE_MODE)
         except OSError:
             # NOTE: not being able to protect cache file is not fatal, it
             # might happen on funky filesystems (e.g. Samba, see #5255),
             # read-only filesystems or in a shared cache scenario.
-            logger.trace("failed to protect '%s'", path_info, exc_info=True)
+            logger.trace("failed to protect '%s'", fs_path, exc_info=True)
 
-    def is_protected(self, path_info):
+    def is_protected(self, fs_path):
         try:
-            mode = os.stat(path_info).st_mode
+            mode = os.stat(fs_path).st_mode
         except FileNotFoundError:
             return False
 
         return stat.S_IMODE(mode) == self.CACHE_MODE
 
-    def set_exec(self, path_info):
-        mode = os.stat(path_info).st_mode | stat.S_IEXEC
+    def set_exec(self, fs_path):
+        mode = os.stat(fs_path).st_mode | stat.S_IEXEC
         try:
-            os.chmod(path_info, mode)
+            os.chmod(fs_path, mode)
         except OSError:
             logger.trace(
                 "failed to chmod '%s' '%s'",
                 oct(mode),
-                path_info,
+                fs_path,
                 exc_info=True,
             )

dvc/data/db/reference.py

@@ -1,18 +1,16 @@
-import io
 import logging
 import os
 from typing import TYPE_CHECKING, Dict
 
+from dvc.objects.db import ObjectDB
+from dvc.objects.errors import ObjectFormatError
 from dvc.scheme import Schemes
 
-from ..errors import ObjectFormatError
 from ..reference import ReferenceHashFile
-from .base import ObjectDB
 
 if TYPE_CHECKING:
-    from dvc.fs.base import BaseFileSystem
+    from dvc.fs.base import AnyFSPath, FileSystem
     from dvc.hash_info import HashInfo
-    from dvc.types import AnyPath, DvcPath
 
 logger = logging.getLogger(__name__)
 
@@ -26,7 +24,7 @@ class ReferenceObjectDB(ObjectDB):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self._fs_cache: Dict[tuple, "BaseFileSystem"] = {}
+        self._fs_cache: Dict[tuple, "FileSystem"] = {}
         self._obj_cache: Dict["HashInfo", "ReferenceHashFile"] = {}
 
     def get(self, hash_info: "HashInfo"):
@@ -36,41 +34,43 @@ class ReferenceObjectDB(ObjectDB):
             return self._obj_cache[hash_info]
         except KeyError:
             pass
-        path_info = self.hash_to_path(hash_info.value)
+        fs_path = self.hash_to_path(hash_info.value)
         try:
-            with self.fs.open(path_info, "rb") as fobj:
-                ref_file = ReferenceHashFile.from_bytes(
-                    fobj.read(), fs_cache=self._fs_cache
-                )
+            ref_file = ReferenceHashFile.from_bytes(
+                self.fs.cat_file(fs_path),
+                fs_cache=self._fs_cache,
+            )
         except OSError:
             raise FileNotFoundError
         try:
             ref_file.check(self, check_hash=False)
         except ObjectFormatError:
-            self.fs.remove(path_info)
+            self.fs.remove(fs_path)
             raise
         self._obj_cache[hash_info] = ref_file
         return ref_file
 
     def _add_file(
         self,
-        from_fs: "BaseFileSystem",
-        from_info: "AnyPath",
-        to_info: "DvcPath",
+        from_fs: "FileSystem",
+        from_info: "AnyFSPath",
+        to_info: "AnyFSPath",
         hash_info: "HashInfo",
-        move: bool = False,
+        hardlink: bool = False,
     ):
-        self.makedirs(to_info.parent)
+        self.makedirs(self.fs.path.parent(to_info))
         if hash_info.isdir:
             return super()._add_file(
-                from_fs, from_info, to_info, hash_info, move
+                from_fs,
+                from_info,
+                to_info,
+                hash_info,
+                hardlink=hardlink,
             )
         ref_file = ReferenceHashFile(from_info, from_fs, hash_info)
         self._obj_cache[hash_info] = ref_file
-        ref_fobj = io.BytesIO(ref_file.to_bytes())
-        ref_fobj.seek(0)
         try:
-            self.fs.upload(ref_fobj, to_info)
+            self.fs.pipe_file(to_info, ref_file.to_bytes())
         except OSError as exc:
             if isinstance(exc, FileExistsError) or (
                 os.name == "nt"

dvc/data/diff.py

@@ -3,8 +3,7 @@ from typing import TYPE_CHECKING, List, Optional, Tuple
 
 if TYPE_CHECKING:
     from dvc.hash_info import HashInfo
-
-    from .file import HashFile
+    from dvc.objects.file import HashFile
 
 ADD = "add"
 MODIFY = "modify"
@@ -96,7 +95,7 @@ def diff(
         return entry_obj.hash_info if entry_obj else None
 
     def _in_cache(oid, cache):
-        from .errors import ObjectFormatError
+        from dvc.objects.errors import ObjectFormatError
 
         if not oid:
             return False

dvc/data/gc.py

@@ -0,0 +1,40 @@
+def gc(odb, used, jobs=None, cache_odb=None, shallow=True):
+    from dvc.data.tree import Tree
+    from dvc.objects.errors import ObjectDBPermissionError
+
+    from ._progress import QueryingProgress
+
+    if odb.read_only:
+        raise ObjectDBPermissionError("Cannot gc read-only ODB")
+    if not cache_odb:
+        cache_odb = odb
+    used_hashes = set()
+    for hash_info in used:
+        used_hashes.add(hash_info.value)
+        if hash_info.isdir and not shallow:
+            tree = Tree.load(cache_odb, hash_info)
+            used_hashes.update(
+                entry_obj.hash_info.value for _, entry_obj in tree
+            )
+
+    def _is_dir_hash(_hash):
+        from dvc.hash_info import HASH_DIR_SUFFIX
+
+        return _hash.endswith(HASH_DIR_SUFFIX)
+
+    removed = False
+    # hashes must be sorted to ensure we always remove .dir files first
+
+    hashes = QueryingProgress(odb.all(jobs), name=odb.fs_path)
+    for hash_ in sorted(hashes, key=_is_dir_hash, reverse=True):
+        if hash_ in used_hashes:
+            continue
+        fs_path = odb.hash_to_path(hash_)
+        if _is_dir_hash(hash_):
+            # backward compatibility
+            # pylint: disable=protected-access
+            odb._remove_unpacked_dir(hash_)
+        odb.fs.remove(fs_path)
+        removed = True
+
+    return removed

dvc/data/meta.py

@@ -1,15 +1,25 @@
 from collections import OrderedDict
 from dataclasses import dataclass, field
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from dvc.objects.db import ObjectDB
+    from dvc.objects.file import HashFile
 
 
 @dataclass
 class Meta:
     PARAM_SIZE = "size"
     PARAM_NFILES = "nfiles"
+    PARAM_ISEXEC = "isexec"
 
     size: Optional[int] = field(default=None)
     nfiles: Optional[int] = field(default=None)
+    isexec: Optional[bool] = field(default=False)
+
+    obj: Optional["HashFile"] = field(default=None)
+    odb: Optional["ObjectDB"] = field(default=None)
+    remote: Optional[str] = field(default=None)
 
     @classmethod
     def from_dict(cls, d):
@@ -18,7 +28,9 @@ class Meta:
 
         size = d.pop(cls.PARAM_SIZE, None)
         nfiles = d.pop(cls.PARAM_NFILES, None)
-        return cls(size=size, nfiles=nfiles)
+        isexec = d.pop(cls.PARAM_ISEXEC, False)
+
+        return cls(size=size, nfiles=nfiles, isexec=isexec)
 
     def to_dict(self):
         ret = OrderedDict()
@@ -29,4 +41,7 @@ class Meta:
         if self.nfiles is not None:
             ret[self.PARAM_NFILES] = self.nfiles
 
+        if self.isexec:
+            ret[self.PARAM_ISEXEC] = self.isexec
+
         return ret

dvc/data/reference.py

@@ -4,15 +4,13 @@ import os
 import pickle
 from typing import TYPE_CHECKING, Optional
 
-from .errors import ObjectFormatError
-from .file import HashFile
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.file import HashFile
 
 if TYPE_CHECKING:
-    from dvc.fs.base import BaseFileSystem
+    from dvc.fs.base import AnyFSPath, FileSystem
     from dvc.hash_info import HashInfo
-    from dvc.types import AnyPath
-
-    from .db.base import ObjectDB
+    from dvc.objects.db import ObjectDB
 
 logger = logging.getLogger(__name__)
 
@@ -25,23 +23,23 @@ class ReferenceHashFile(HashFile):
 
     def __init__(
         self,
-        path_info: "AnyPath",
-        fs: "BaseFileSystem",
+        fs_path: "AnyFSPath",
+        fs: "FileSystem",
         hash_info: "HashInfo",
         checksum: Optional[str] = None,
         **kwargs,
     ):
-        super().__init__(path_info, fs, hash_info, **kwargs)
-        self.checksum = checksum or fs.checksum(path_info)
+        super().__init__(fs_path, fs, hash_info, **kwargs)
+        self.checksum = checksum or fs.checksum(fs_path)
 
     def __str__(self):
-        return f"ref object {self.hash_info} -> {self.path_info}"
+        return f"ref object {self.hash_info} -> {self.fs_path}"
 
     def check(self, odb: "ObjectDB", check_hash: bool = True):
         assert self.fs
-        if not self.fs.exists(self.path_info):
+        if not self.fs.exists(self.fs_path):
             raise FileNotFoundError(
-                errno.ENOENT, os.strerror(errno.ENOENT), self.path_info
+                errno.ENOENT, os.strerror(errno.ENOENT), self.fs_path
             )
         if self.checksum != self._get_checksum():
             raise ObjectFormatError(f"{self} is changed")
@@ -50,23 +48,17 @@ class ReferenceHashFile(HashFile):
 
     def _get_checksum(self) -> str:
         assert self.fs
-        return self.fs.checksum(self.path_info)
+        return self.fs.checksum(self.fs_path)
 
     def to_bytes(self):
-        from dvc.fs.repo import RepoFileSystem
-        from dvc.path_info import PathInfo
-
         # NOTE: dumping reference FS's this way is insecure, as the
         # fully parsed remote FS config will include credentials
         #
         # ReferenceHashFiles should currently only be serialized in
         # memory and not to disk
-        path_info = self.path_info
-        if isinstance(self.fs, RepoFileSystem):
-            path_info = PathInfo(path_info).relative_to(self.fs.root_dir)
-
+        fs_path = self.fs_path
         dict_ = {
-            self.PARAM_PATH: path_info,
+            self.PARAM_PATH: fs_path,
             self.PARAM_HASH: self.hash_info,
             self.PARAM_CHECKSUM: self.checksum,
             self.PARAM_FS_CONFIG: self.config_tuple(self.fs),
@@ -79,7 +71,7 @@ class ReferenceHashFile(HashFile):
     @classmethod
     def from_bytes(cls, data: bytes, fs_cache: Optional[dict] = None):
         from dvc.fs import get_fs_cls
-        from dvc.fs.repo import RepoFileSystem
+        from dvc.fs.repo import RepoFileSystem, _RepoFileSystem
 
         try:
             dict_ = pickle.loads(data)
@@ -87,7 +79,7 @@ class ReferenceHashFile(HashFile):
             raise ObjectFormatError("ReferenceHashFile is corrupted") from exc
 
         try:
-            path_info = dict_[cls.PARAM_PATH]
+            fs_path = dict_[cls.PARAM_PATH]
             hash_info = dict_[cls.PARAM_HASH]
         except KeyError as exc:
             raise ObjectFormatError("ReferenceHashFile is corrupted") from exc
@@ -96,21 +88,20 @@ class ReferenceHashFile(HashFile):
         fs = fs_cache.get((scheme, config_pairs)) if fs_cache else None
         if not fs:
             config = dict(config_pairs)
-            if RepoFileSystem.PARAM_REPO_URL in config:
-                fs = RepoFileSystem(**config)
-                path_info = fs.root_dir / path_info
+            if _RepoFileSystem.PARAM_REPO_URL in config:
+                fs_cls = RepoFileSystem
             else:
-                fs_cls = get_fs_cls(config, scheme=path_info.scheme)
-                fs = fs_cls(**config)
+                fs_cls = get_fs_cls(config, scheme=scheme)
+            fs = fs_cls(**config)
         return ReferenceHashFile(
-            path_info,
+            fs_path,
             fs,
             hash_info,
             checksum=dict_.get(cls.PARAM_CHECKSUM),
         )
 
     @staticmethod
-    def config_tuple(fs: "BaseFileSystem"):
+    def config_tuple(fs: "FileSystem"):
         return (
             fs.scheme,
             tuple(

dvc/data/slow_link_detection.py

@@ -25,15 +25,15 @@ this.message = (
 
 def slow_link_guard(f):
     @wraps(f)
-    def wrapper(odb, *args, **kwargs):
+    def wrapper(self, odb, *args, **kwargs):
         if this.already_displayed:
-            return f(odb, *args, **kwargs)
+            return f(self, odb, *args, **kwargs)
 
         if not odb.slow_link_warning or odb.cache_types:
-            return f(odb, *args, **kwargs)
+            return f(self, odb, *args, **kwargs)
 
         start = time.time()
-        result = f(odb, *args, **kwargs)
+        result = f(self, odb, *args, **kwargs)
         delta = time.time() - start
 
         if delta >= this.timeout_seconds:

dvc/data/stage.py

@@ -1,26 +1,23 @@
 import hashlib
 import logging
 import os
-import pathlib
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
-from typing import TYPE_CHECKING, Dict, Tuple
+from typing import TYPE_CHECKING, Dict, Optional, Tuple
 
 from dvc.exceptions import DvcIgnoreInCollectedDirError
 from dvc.hash_info import HashInfo
 from dvc.ignore import DvcIgnore
+from dvc.objects.file import HashFile
 from dvc.progress import Tqdm
-from dvc.utils import file_md5
+from dvc.utils import file_md5, is_exec
 
 from .db.reference import ReferenceObjectDB
-from .file import HashFile
 from .meta import Meta
 
 if TYPE_CHECKING:
-    from dvc.fs.base import BaseFileSystem
-    from dvc.types import AnyPath, DvcPath
-
-    from .db.base import ObjectDB
+    from dvc.fs.base import AnyFSPath, FileSystem
+    from dvc.objects.db import ObjectDB
 
 logger = logging.getLogger(__name__)
 
@@ -28,74 +25,97 @@ logger = logging.getLogger(__name__)
 _STAGING_MEMFS_PATH = "dvc-staging"
 
 
-def _upload_file(path_info, fs, odb, upload_odb):
+def _upload_file(from_fs_path, fs, odb, upload_odb, callback=None):
+    from dvc.fs._callback import FsspecCallback
     from dvc.utils import tmp_fname
     from dvc.utils.stream import HashedStreamReader
 
-    tmp_info = upload_odb.path_info / tmp_fname()
-    with fs.open(path_info, mode="rb", chunk_size=fs.CHUNK_SIZE) as stream:
+    fs_path = upload_odb.fs.path
+    tmp_info = fs_path.join(upload_odb.fs_path, tmp_fname())
+    with fs.open(from_fs_path, mode="rb") as stream:
         stream = HashedStreamReader(stream)
-        size = fs.getsize(path_info)
-        upload_odb.fs.upload(stream, tmp_info, desc=path_info.name, total=size)
+        size = fs.size(from_fs_path)
+        with FsspecCallback.as_tqdm_callback(
+            callback,
+            desc=fs_path.name(from_fs_path),
+            bytes=True,
+            total=size,
+        ) as cb:
+            upload_odb.fs.upload(stream, tmp_info, size=size, callback=cb)
 
     odb.add(tmp_info, upload_odb.fs, stream.hash_info)
     meta = Meta(size=size)
-    return path_info, meta, odb.get(stream.hash_info)
+    return from_fs_path, meta, odb.get(stream.hash_info)
 
 
-def _get_file_hash(path_info, fs, name):
-    info = fs.info(path_info)
+def _adapt_info(info, scheme):
+    if scheme == "s3" and "ETag" in info:
+        info["etag"] = info["ETag"].strip('"')
+    elif scheme == "gs" and "etag" in info:
+        import base64
+
+        info["etag"] = base64.b64decode(info["etag"]).hex()
+    elif scheme.startswith("http") and (
+        "ETag" in info or "Content-MD5" in info
+    ):
+        info["checksum"] = info.get("ETag") or info.get("Content-MD5")
+    return info
+
+
+def _get_file_hash(fs_path, fs, name):
+    info = _adapt_info(fs.info(fs_path), fs.scheme)
+
     if name in info:
         assert not info[name].endswith(".dir")
         hash_value = info[name]
     elif hasattr(fs, name):
         func = getattr(fs, name)
-        hash_value = func(path_info)
+        hash_value = func(fs_path)
     elif name == "md5":
-        hash_value = file_md5(path_info, fs)
+        hash_value = file_md5(fs_path, fs)
     else:
         raise NotImplementedError
 
-    meta = Meta(size=info["size"])
+    meta = Meta(size=info["size"], isexec=is_exec(info.get("mode", 0)))
     hash_info = HashInfo(name, hash_value)
     return meta, hash_info
 
 
-def get_file_hash(path_info, fs, name, state=None):
+def get_file_hash(fs_path, fs, name, state=None):
     if state:
         meta, hash_info = state.get(  # pylint: disable=assignment-from-none
-            path_info, fs
+            fs_path, fs
         )
         if hash_info:
             return meta, hash_info
 
-    meta, hash_info = _get_file_hash(path_info, fs, name)
+    meta, hash_info = _get_file_hash(fs_path, fs, name)
 
     if state:
         assert ".dir" not in hash_info.value
-        state.save(path_info, fs, hash_info)
+        state.save(fs_path, fs, hash_info)
 
     return meta, hash_info
 
 
-def _stage_file(path_info, fs, name, odb=None, upload_odb=None, dry_run=False):
+def _stage_file(fs_path, fs, name, odb=None, upload_odb=None, dry_run=False):
     state = odb.state if odb else None
-    meta, hash_info = get_file_hash(path_info, fs, name, state=state)
+    meta, hash_info = get_file_hash(fs_path, fs, name, state=state)
     if upload_odb and not dry_run:
         assert odb and name == "md5"
-        return _upload_file(path_info, fs, odb, upload_odb)
+        return _upload_file(fs_path, fs, odb, upload_odb)
 
     if dry_run:
-        obj = HashFile(path_info, fs, hash_info)
+        obj = HashFile(fs_path, fs, hash_info)
     else:
-        odb.add(path_info, fs, hash_info, move=False)
+        odb.add(fs_path, fs, hash_info, hardlink=False)
         obj = odb.get(hash_info)
 
-    return path_info, meta, obj
+    return fs_path, meta, obj
 
 
 def _build_objects(
-    path_info,
+    fs_path,
     fs,
     name,
     dvcignore=None,
@@ -104,9 +124,9 @@ def _build_objects(
     **kwargs,
 ):
     if dvcignore:
-        walk_iterator = dvcignore.walk_files(fs, path_info)
+        walk_iterator = dvcignore.find(fs, fs_path)
     else:
-        walk_iterator = fs.walk_files(path_info)
+        walk_iterator = fs.find(fs_path)
     with Tqdm(
         unit="md5",
         desc="Computing file/dir hashes (only done once)",
@@ -126,18 +146,18 @@ def _build_objects(
             yield from executor.map(worker, walk_iterator)
 
 
-def _iter_objects(path_info, fs, name, **kwargs):
-    yield from _build_objects(path_info, fs, name, **kwargs)
+def _iter_objects(fs_path, fs, name, **kwargs):
+    yield from _build_objects(fs_path, fs, name, **kwargs)
 
 
-def _build_tree(path_info, fs, name, **kwargs):
+def _build_tree(fs_path, fs, name, **kwargs):
     from .tree import Tree
 
     tree_meta = Meta(size=0, nfiles=0)
     tree = Tree(None, None, None)
-    for file_info, meta, obj in _iter_objects(path_info, fs, name, **kwargs):
-        if DvcIgnore.DVCIGNORE_FILE == file_info.name:
-            raise DvcIgnoreInCollectedDirError(file_info.parent)
+    for file_fs_path, meta, obj in _iter_objects(fs_path, fs, name, **kwargs):
+        if DvcIgnore.DVCIGNORE_FILE == fs.path.name(file_fs_path):
+            raise DvcIgnoreInCollectedDirError(fs.path.parent(file_fs_path))
 
         # NOTE: this is lossy transformation:
         #   "hey\there" -> "hey/there"
@@ -147,15 +167,18 @@ def _build_tree(path_info, fs, name, **kwargs):
         #
         # Yes, this is a BUG, as long as we permit "/" in
         # filenames on Windows and "\" on Unix
-        tree.add(file_info.relative_to(path_info).parts, meta, obj.hash_info)
 
-        tree_meta.size += meta.size
+        key = fs.path.relparts(file_fs_path, fs_path)
+        assert key
+        tree.add(key, meta, obj.hash_info)
+
+        tree_meta.size += meta.size or 0
         tree_meta.nfiles += 1
 
     return tree_meta, tree
 
 
-def _stage_tree(path_info, fs, fs_info, name, odb=None, **kwargs):
+def _stage_tree(fs_path, fs, fs_info, name, odb=None, **kwargs):
     from .tree import Tree
 
     value = fs_info.get(name)
@@ -167,41 +190,46 @@ def _stage_tree(path_info, fs, fs_info, name, odb=None, **kwargs):
         except FileNotFoundError:
             pass
 
-    meta, tree = _build_tree(path_info, fs, name, odb=odb, **kwargs)
+    meta, tree = _build_tree(fs_path, fs, name, odb=odb, **kwargs)
     state = odb.state if odb and odb.state else None
     hash_info = None
     if state:
         _, hash_info = state.get(  # pylint: disable=assignment-from-none
-            path_info, fs
+            fs_path, fs
         )
     tree.digest(hash_info=hash_info)
-    odb.add(tree.path_info, tree.fs, tree.hash_info, move=True)
+    odb.add(tree.fs_path, tree.fs, tree.hash_info, hardlink=False)
     raw = odb.get(tree.hash_info)
     # cleanup unneeded memfs tmpfile and return tree based on the
     # ODB fs/path
     if odb.fs != tree.fs:
-        tree.fs.remove(tree.path_info)
+        tree.fs.remove(tree.fs_path)
     tree.fs = raw.fs
-    tree.path_info = raw.path_info
+    tree.fs_path = raw.fs_path
     return meta, tree
 
 
 _url_cache: Dict[str, str] = {}
 
 
-def _make_staging_url(path_info: "AnyPath"):
-    from dvc.path_info import CloudURLInfo
+def _make_staging_url(
+    fs: "FileSystem", odb: "ObjectDB", fs_path: Optional[str]
+):
     from dvc.scheme import Schemes
 
-    url = CloudURLInfo(f"{Schemes.MEMORY}://{_STAGING_MEMFS_PATH}")
-    if path_info:
-        if isinstance(path_info, (str, pathlib.PurePath)):
-            path = os.path.abspath(path_info)
-        else:
-            path = str(path_info)
-        if path not in _url_cache:
-            _url_cache[path] = hashlib.sha256(path.encode("utf-8")).hexdigest()
-        url /= _url_cache[path]
+    url = f"{Schemes.MEMORY}://{_STAGING_MEMFS_PATH}"
+
+    if fs_path is not None:
+        if odb.fs.scheme == Schemes.LOCAL:
+            fs_path = os.path.abspath(fs_path)
+
+        if fs_path not in _url_cache:
+            _url_cache[fs_path] = hashlib.sha256(
+                fs_path.encode("utf-8")
+            ).hexdigest()
+
+        url = fs.path.join(url, _url_cache[fs_path])
+
     return url
 
 
@@ -214,18 +242,19 @@ def _get_staging(odb: "ObjectDB") -> "ObjectDB":
     from dvc.fs.memory import MemoryFileSystem
 
     fs = MemoryFileSystem()
-    path_info = _make_staging_url(odb.path_info)
+    fs_path = _make_staging_url(fs, odb, odb.fs_path)
     state = odb.state
-    return ReferenceObjectDB(fs, path_info, state=state)
+    return ReferenceObjectDB(fs, fs_path, state=state)
 
 
-def _load_from_state(odb, staging, path_info, fs, name):
+def _load_from_state(odb, staging, fs_path, fs, name):
+    from dvc.objects.errors import ObjectFormatError
+
     from . import check, load
-    from .errors import ObjectFormatError
     from .tree import Tree
 
     state = odb.state
-    meta, hash_info = state.get(path_info, fs)
+    meta, hash_info = state.get(fs_path, fs)
     if hash_info:
         for odb_ in (odb, staging):
             if odb_.exists(hash_info):
@@ -248,10 +277,10 @@ def _stage_external_tree_info(odb, tree, name):
     # that would be universal for all clouds.
     assert odb and name != "md5"
 
-    odb.add(tree.path_info, tree.fs, tree.hash_info)
+    odb.add(tree.fs_path, tree.fs, tree.hash_info)
     raw = odb.get(tree.hash_info)
-    _, hash_info = get_file_hash(raw.path_info, raw.fs, name, state=odb.state)
-    tree.path_info = raw.path_info
+    _, hash_info = get_file_hash(raw.fs_path, raw.fs, name, state=odb.state)
+    tree.fs_path = raw.fs_path
     tree.fs = raw.fs
     tree.hash_info.name = hash_info.name
     tree.hash_info.value = hash_info.value
@@ -262,8 +291,8 @@ def _stage_external_tree_info(odb, tree, name):
 
 def stage(
     odb: "ObjectDB",
-    path_info: "DvcPath",
-    fs: "BaseFileSystem",
+    fs_path: "AnyFSPath",
+    fs: "FileSystem",
     name: str,
     upload: bool = False,
     dry_run: bool = False,
@@ -282,19 +311,20 @@ def stage(
     ODB filesystem, and staged objects will reference the uploaded path rather
     than the original source path.
     """
-    assert path_info and path_info.scheme == fs.scheme
+    assert fs_path
+    # assert scheme(fs_path) == fs.scheme
 
-    details = fs.info(path_info)
+    details = fs.info(fs_path)
     staging = _get_staging(odb)
     if odb:
         try:
-            return _load_from_state(odb, staging, path_info, fs, name)
+            return _load_from_state(odb, staging, fs_path, fs, name)
         except FileNotFoundError:
             pass
 
     if details["type"] == "directory":
         meta, obj = _stage_tree(
-            path_info,
+            fs_path,
             fs,
             details,
             name,
@@ -308,7 +338,7 @@ def stage(
             obj = _stage_external_tree_info(odb, obj, name)
     else:
         _, meta, obj = _stage_file(
-            path_info,
+            fs_path,
             fs,
             name,
             odb=staging,
@@ -317,6 +347,6 @@ def stage(
         )
 
     if odb and odb.state and obj.hash_info:
-        odb.state.save(path_info, fs, obj.hash_info)
+        odb.state.save(fs_path, fs, obj.hash_info)
 
     return staging, meta, obj

dvc/data/status.py

@@ -7,9 +7,10 @@ from dvc.scheme import Schemes
 from .tree import Tree
 
 if TYPE_CHECKING:
-    from .db.base import ObjectDB
+    from dvc.objects.db import ObjectDB
+    from dvc.objects.file import HashFile
+
     from .db.index import ObjectDBIndexBase
-    from .file import HashFile
 
 logger = logging.getLogger(__name__)
 
@@ -26,14 +27,20 @@ class CompareStatusResult(NamedTuple):
     deleted: Set["HashInfo"]
 
 
-def _indexed_dir_hashes(odb, index, dir_objs, name, cache_odb):
+def _indexed_dir_hashes(odb, index, dir_objs, name, cache_odb, jobs=None):
     # Validate our index by verifying all indexed .dir hashes
     # still exist on the remote
+    from ._progress import QueryingProgress
+
     dir_hashes = set(dir_objs.keys())
     indexed_dirs = set(index.dir_hashes())
     indexed_dir_exists = set()
     if indexed_dirs:
-        indexed_dir_exists.update(odb.list_hashes_exists(indexed_dirs))
+        hashes = QueryingProgress(
+            odb.list_hashes_exists(indexed_dirs, jobs=jobs),
+            total=len(indexed_dirs),
+        )
+        indexed_dir_exists.update(hashes)
         missing_dirs = indexed_dirs.difference(indexed_dir_exists)
         if missing_dirs:
             logger.debug(
@@ -45,7 +52,13 @@ def _indexed_dir_hashes(odb, index, dir_objs, name, cache_odb):
 
     # Check if non-indexed (new) dir hashes exist on remote
     dir_exists = dir_hashes.intersection(indexed_dir_exists)
-    dir_exists.update(odb.list_hashes_exists(dir_hashes - dir_exists))
+    dir_missing = dir_hashes - dir_exists
+    dir_exists.update(
+        QueryingProgress(
+            odb.list_hashes_exists(dir_missing, jobs=jobs),
+            total=len(dir_missing),
+        )
+    )
 
     # If .dir hash exists in the ODB, assume directory contents
     # also exists
@@ -75,6 +88,7 @@ def status(
     index: Optional["ObjectDBIndexBase"] = None,
     cache_odb: Optional["ObjectDB"] = None,
     shallow: bool = True,
+    jobs: Optional[int] = None,
     **kwargs,
 ) -> "StatusResult":
     """Return status of whether or not the specified objects exist odb.
@@ -86,7 +100,7 @@ def status(
         exists: objs that exist in odb
         missing: objs that do not exist in ODB
     """
-    logger.debug("Preparing to collect status from '%s'", odb.path_info)
+    logger.debug("Preparing to collect status from '%s'", odb.fs_path)
     if not name:
         name = odb.fs.PARAM_CHECKSUM
 
@@ -116,11 +130,13 @@ def status(
     hashes: Set[str] = set(hash_infos.keys())
     exists: Set[str] = set()
 
-    logger.debug("Collecting status from '%s'", odb.path_info)
+    logger.debug("Collecting status from '%s'", odb.fs_path)
     if index and hashes:
         if dir_objs:
             exists = hashes.intersection(
-                _indexed_dir_hashes(odb, index, dir_objs, name, cache_odb)
+                _indexed_dir_hashes(
+                    odb, index, dir_objs, name, cache_odb, jobs=jobs
+                )
             )
             hashes.difference_update(exists)
         if hashes:
@@ -128,9 +144,12 @@ def status(
             hashes.difference_update(exists)
 
     if hashes:
-        exists.update(
-            odb.hashes_exist(hashes, name=str(odb.path_info), **kwargs)
-        )
+        from ._progress import QueryingProgress
+
+        with QueryingProgress(phase="Checking", name=odb.fs_path) as pbar:
+            exists.update(
+                odb.hashes_exist(hashes, jobs=jobs, progress=pbar.callback)
+            )
     return StatusResult(
         {hash_infos[hash_] for hash_ in exists},
         {hash_infos[hash_] for hash_ in (hashes - exists)},
@@ -145,6 +164,7 @@ def compare_status(
     check_deleted: bool = True,
     src_index: Optional["ObjectDBIndexBase"] = None,
     dest_index: Optional["ObjectDBIndexBase"] = None,
+    jobs: Optional[int] = None,
     **kwargs,
 ) -> "CompareStatusResult":
     """Compare status for the specified objects between two ODBs.
@@ -158,13 +178,13 @@ def compare_status(
     if "cache_odb" not in kwargs:
         kwargs["cache_odb"] = src
     dest_exists, dest_missing = status(
-        dest, obj_ids, index=dest_index, **kwargs
+        dest, obj_ids, index=dest_index, jobs=jobs, **kwargs
     )
     # for transfer operations we can skip src status check when all objects
     # already exist in dest
     if dest_missing or check_deleted:
         src_exists, src_missing = status(
-            src, obj_ids, index=src_index, **kwargs
+            src, obj_ids, index=src_index, jobs=jobs, **kwargs
         )
     else:
         src_exists = dest_exists

dvc/data/transfer.py

@@ -10,8 +10,8 @@ from dvc.utils.threadpool import ThreadPoolExecutor
 
 if TYPE_CHECKING:
     from dvc.hash_info import HashInfo
+    from dvc.objects.db import ObjectDB
 
-    from .db.base import ObjectDB
     from .db.index import ObjectDBIndexBase
     from .tree import Tree
 
@@ -20,9 +20,9 @@ logger = logging.getLogger(__name__)
 
 def _log_exceptions(func):
     @wraps(func)
-    def wrapper(path_info, *args, **kwargs):
+    def wrapper(fs_path, *args, **kwargs):
         try:
-            func(path_info, *args, **kwargs)
+            func(fs_path, *args, **kwargs)
             return 0
         except Exception as exc:  # pylint: disable=broad-except
             # NOTE: this means we ran out of file descriptors and there is no
@@ -31,7 +31,7 @@ def _log_exceptions(func):
             if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
                 raise
 
-            logger.exception("failed to transfer '%s'", path_info)
+            logger.exception("failed to transfer '%s'", fs_path)
             return 1
 
     return wrapper
@@ -40,7 +40,8 @@ def _log_exceptions(func):
 def find_tree_by_obj_id(
     odbs: Iterable[Optional["ObjectDB"]], obj_id: "HashInfo"
 ) -> Optional["Tree"]:
-    from .errors import ObjectFormatError
+    from dvc.objects.errors import ObjectFormatError
+
     from .tree import Tree
 
     for odb in odbs:
@@ -87,8 +88,8 @@ def _do_transfer(
             )
             logger.error(
                 "failed to upload '%s' to '%s'",
-                src.get(dir_obj.hash_info).path_info,
-                dest.get(dir_obj.hash_info).path_info,
+                src.get(dir_obj.hash_info).fs_path,
+                dest.get(dir_obj.hash_info).fs_path,
             )
             total_fails += dir_fails + 1
         elif entry_ids.intersection(missing_ids):
@@ -133,7 +134,7 @@ def transfer(
     obj_ids: Iterable["HashInfo"],
     jobs: Optional[int] = None,
     verify: bool = False,
-    move: bool = False,
+    hardlink: bool = False,
     **kwargs,
 ) -> int:
     """Transfer (copy) the specified objects from one ODB to another.
@@ -144,20 +145,26 @@ def transfer(
 
     logger.debug(
         "Preparing to transfer data from '%s' to '%s'",
-        src.path_info,
-        dest.path_info,
+        src.fs_path,
+        dest.fs_path,
     )
     if src == dest:
         return 0
 
-    status = compare_status(src, dest, obj_ids, check_deleted=False, **kwargs)
+    status = compare_status(
+        src, dest, obj_ids, check_deleted=False, jobs=jobs, **kwargs
+    )
     if not status.new:
         return 0
 
     def func(hash_info: "HashInfo") -> None:
         obj = src.get(hash_info)
         return dest.add(
-            obj.path_info, obj.fs, obj.hash_info, verify=verify, move=move
+            obj.fs_path,
+            obj.fs,
+            obj.hash_info,
+            verify=verify,
+            hardlink=hardlink,
         )
 
     total = len(status.new)

dvc/data/tree.py

@@ -1,22 +1,44 @@
 import json
 import logging
 import posixpath
-from typing import TYPE_CHECKING, Dict, Optional, Tuple
+from typing import TYPE_CHECKING, Dict, Iterable, Optional, Tuple
 
 from funcy import cached_property
 
-from .errors import ObjectFormatError
-from .file import HashFile
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.file import HashFile
+
 from .stage import get_file_hash
 
 if TYPE_CHECKING:
     from dvc.hash_info import HashInfo
+    from dvc.objects.db import ObjectDB
 
     from .meta import Meta
 
 logger = logging.getLogger(__name__)
 
 
+class TreeError(Exception):
+    pass
+
+
+def _try_load(
+    odbs: Iterable["ObjectDB"],
+    hash_info: "HashInfo",
+) -> Optional["HashFile"]:
+    for odb in odbs:
+        if not odb:
+            continue
+
+        try:
+            return Tree.load(odb, hash_info)
+        except (FileNotFoundError, ObjectFormatError):
+            pass
+
+    return None
+
+
 class Tree(HashFile):
     PARAM_RELPATH = "relpath"
 
@@ -25,7 +47,7 @@ class Tree(HashFile):
         self._dict: Dict[Tuple[str], Tuple["Meta", "HashInfo"]] = {}
 
     @cached_property
-    def trie(self):
+    def _trie(self):
         from pygtrie import Trie
 
         return Trie(self._dict)
@@ -36,22 +58,44 @@ class Tree(HashFile):
 
     def digest(self, hash_info: Optional["HashInfo"] = None):
         from dvc.fs.memory import MemoryFileSystem
-        from dvc.path_info import CloudURLInfo
         from dvc.utils import tmp_fname
 
         memfs = MemoryFileSystem()
-        path_info = CloudURLInfo("memory://{}".format(tmp_fname("")))
-        with memfs.open(path_info, "wb") as fobj:
-            fobj.write(self.as_bytes())
+        fs_path = "memory://{}".format(tmp_fname(""))
+        memfs.pipe_file(fs_path, self.as_bytes())
         self.fs = memfs
-        self.path_info = path_info
+        self.fs_path = fs_path
         if hash_info:
             self.hash_info = hash_info
         else:
-            _, self.hash_info = get_file_hash(path_info, memfs, "md5")
+            _, self.hash_info = get_file_hash(fs_path, memfs, "md5")
             assert self.hash_info.value
             self.hash_info.value += ".dir"
 
+    def _load(self, key, meta, hash_info):
+        if hash_info and hash_info.isdir and not meta.obj:
+            meta.obj = _try_load([meta.odb, meta.remote], hash_info)
+            if meta.obj:
+                for ikey, value in meta.obj.iteritems():
+                    self._trie[key + ikey] = value
+                    self._dict[key + ikey] = value
+
+    def iteritems(self, prefix=None):
+        kwargs = {}
+        if prefix:
+            kwargs = {"prefix": prefix}
+            item = self._trie.longest_prefix(prefix)
+            if item:
+                key, (meta, hash_info) = item
+                self._load(key, meta, hash_info)
+
+        for key, (meta, hash_info) in self._trie.iteritems(**kwargs):
+            self._load(key, meta, hash_info)
+            yield key, (meta, hash_info)
+
+    def shortest_prefix(self, *args, **kwargs):
+        return self._trie.shortest_prefix(*args, **kwargs)
+
     def __len__(self):
         return len(self._dict)
 
@@ -101,7 +145,7 @@ class Tree(HashFile):
         obj = odb.get(hash_info)
 
         try:
-            with obj.fs.open(obj.path_info, "r") as fobj:
+            with obj.fs.open(obj.fs_path, "r") as fobj:
                 raw = json.load(fobj)
         except ValueError as exc:
             raise ObjectFormatError(f"{obj} is corrupted") from exc
@@ -109,12 +153,12 @@ class Tree(HashFile):
         if not isinstance(raw, list):
             logger.error(
                 "dir cache file format error '%s' [skipping the file]",
-                obj.path_info,
+                obj.fs_path,
             )
             raise ObjectFormatError(f"{obj} is corrupted")
 
         tree = cls.from_list(raw)
-        tree.path_info = obj.path_info
+        tree.fs_path = obj.fs_path
         tree.fs = obj.fs
         tree.hash_info = hash_info
 
@@ -125,13 +169,13 @@ class Tree(HashFile):
         inside prefix.
 
         The returned tree will contain the original tree's hash_info and
-        path_info.
+        fs_path.
 
         Returns an empty tree if no object exists at the specified prefix.
         """
-        tree = Tree(self.path_info, self.fs, self.hash_info)
+        tree = Tree(self.fs_path, self.fs, self.hash_info)
         try:
-            for key, (meta, oid) in self.trie.items(prefix):
+            for key, (meta, oid) in self._trie.items(prefix):
                 tree.add(key, meta, oid)
         except KeyError:
             pass
@@ -149,19 +193,39 @@ class Tree(HashFile):
         tree = Tree(None, None, None)
         depth = len(prefix)
         try:
-            for key, (meta, entry_oid) in self.trie.items(prefix):
+            for key, (meta, entry_oid) in self._trie.items(prefix):
                 tree.add(key[depth:], meta, entry_oid)
         except KeyError:
             return None
         tree.digest()
         return tree
 
+    def ls(self, prefix=None):
+        kwargs = {}
+        if prefix:
+            kwargs["prefix"] = prefix
+
+        meta, hash_info = self._trie.get(prefix, (None, None))
+        if hash_info and hash_info.isdir and meta and not meta.obj:
+            raise TreeError
+
+        ret = []
+
+        def node_factory(_, key, children, *args):
+            if key == prefix:
+                list(children)
+            else:
+                ret.append(key[-1])
+
+        self._trie.traverse(node_factory, **kwargs)
+
+        return ret
+
 
 def du(odb, tree):
     try:
         return sum(
-            odb.fs.getsize(odb.hash_to_path_info(oid.value))
-            for _, _, oid in tree
+            odb.fs.size(odb.hash_to_path(oid.value)) for _, _, oid in tree
         )
     except FileNotFoundError:
         return None

dvc/data_cloud.py

@@ -3,11 +3,11 @@
 import logging
 from typing import TYPE_CHECKING, Iterable, Optional
 
-from dvc.objects.db import get_index
+from dvc.data.db import get_index
 
 if TYPE_CHECKING:
     from dvc.hash_info import HashInfo
-    from dvc.objects.db.base import ObjectDB
+    from dvc.objects.db import ObjectDB
 
 logger = logging.getLogger(__name__)
 
@@ -55,12 +55,12 @@ class DataCloud:
         raise NoRemoteError(error_msg)
 
     def _init_odb(self, name):
+        from dvc.data.db import get_odb
         from dvc.fs import get_cloud_fs
-        from dvc.objects.db import get_odb
 
-        cls, config, path_info = get_cloud_fs(self.repo, name=name)
+        cls, config, fs_path = get_cloud_fs(self.repo, name=name)
         config["tmp_dir"] = self.repo.index_db_dir
-        return get_odb(cls(**config), path_info, **config)
+        return get_odb(cls(**config), fs_path, **config)
 
     def push(
         self,
@@ -78,7 +78,7 @@ class DataCloud:
                 By default remote from core.remote config option is used.
             odb: optional ODB to push to. Overrides remote.
         """
-        from dvc.objects.transfer import transfer
+        from dvc.data.transfer import transfer
 
         if not odb:
             odb = self.get_remote_odb(remote, "push")
@@ -107,7 +107,7 @@ class DataCloud:
                 By default remote from core.remote config option is used.
             odb: optional ODB to pull from. Overrides remote.
         """
-        from dvc.objects.transfer import transfer
+        from dvc.data.transfer import transfer
 
         if not odb:
             odb = self.get_remote_odb(remote, "pull")
@@ -141,7 +141,7 @@ class DataCloud:
             log_missing: log warning messages if file doesn't exist
                 neither in cache, neither in cloud.
         """
-        from dvc.objects.status import compare_status
+        from dvc.data.status import compare_status
 
         if not odb:
             odb = self.get_remote_odb(remote, "status")
@@ -156,5 +156,6 @@ class DataCloud:
         )
 
     def get_url_for(self, remote, checksum):
-        remote_odb = self.get_remote_odb(remote)
-        return str(remote_odb.hash_to_path_info(checksum))
+        odb = self.get_remote_odb(remote)
+        path = odb.hash_to_path(checksum)
+        return odb.fs.unstrip_protocol(path)

dvc/dependency/__init__.py

@@ -47,16 +47,29 @@ def loads_from(stage, s_list, erepo=None):
 def _merge_params(s_list):
     d = defaultdict(list)
     default_file = ParamsDependency.DEFAULT_PARAMS_FILE
+
+    # figure out completely tracked params file, and ignore specific keys
+    wholly_tracked = set()
+    for key in s_list:
+        if not isinstance(key, dict):
+            continue
+        wholly_tracked.update(k for k, params in key.items() if not params)
+
     for key in s_list:
         if isinstance(key, str):
-            d[default_file].append(key)
+            if default_file not in wholly_tracked:
+                d[default_file].append(key)
             continue
+
         if not isinstance(key, dict):
             msg = "Only list of str/dict is supported. Got: "
             msg += f"'{type(key).__name__}'."
             raise ValueError(msg)
 
         for k, params in key.items():
+            if k in wholly_tracked:
+                d[k] = []
+                continue
             if not isinstance(params, list):
                 msg = "Expected list of params for custom params file "
                 msg += f"'{k}', got '{type(params).__name__}'."

dvc/dependency/param.py

@@ -1,6 +1,8 @@
 import logging
 import os
+import typing
 from collections import defaultdict
+from typing import Dict
 
 import dpath.util
 from voluptuous import Any
@@ -18,6 +20,14 @@ class MissingParamsError(DvcException):
     pass
 
 
+class MissingParamsFile(DvcException):
+    pass
+
+
+class ParamsIsADirectoryError(DvcException):
+    pass
+
+
 class BadParamFileError(DvcException):
     pass
 
@@ -27,51 +37,83 @@ class ParamsDependency(Dependency):
     PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
     DEFAULT_PARAMS_FILE = "params.yaml"
 
-    def __init__(self, stage, path, params):
-        info = {}
-        self.params = []
-        if params:
-            if isinstance(params, list):
-                self.params = params
-            else:
-                assert isinstance(params, dict)
-                self.params = list(params.keys())
-                info = {self.PARAM_PARAMS: params}
-
-        super().__init__(
-            stage,
-            path
-            or os.path.join(stage.repo.root_dir, self.DEFAULT_PARAMS_FILE),
-            info=info,
+    def __init__(self, stage, path, params=None, repo=None):
+        self.params = list(params) if params else []
+        info = (
+            {self.PARAM_PARAMS: params} if isinstance(params, dict) else None
         )
+        repo = repo or stage.repo
+        path = path or os.path.join(repo.root_dir, self.DEFAULT_PARAMS_FILE)
+        super().__init__(stage, path, info=info, repo=repo)
 
     def dumpd(self):
         ret = super().dumpd()
         if not self.hash_info:
-            ret[self.PARAM_PARAMS] = self.params
+            ret[self.PARAM_PARAMS] = self.params or {}
         return ret
 
     def fill_values(self, values=None):
         """Load params values dynamically."""
-        if not values:
+        if values is None:
             return
+
         info = {}
+        if not self.params:
+            info.update(values)
         for param in self.params:
             if param in values:
                 info[param] = values[param]
         self.hash_info = HashInfo(self.PARAM_PARAMS, info)
 
-    def workspace_status(self):
-        status = super().workspace_status()
+    def read_params(
+        self, flatten: bool = True, **kwargs: typing.Any
+    ) -> Dict[str, typing.Any]:
+        try:
+            config = self.read_file()
+        except MissingParamsFile:
+            config = {}
 
-        if status.get(str(self)) == "deleted":
-            return status
+        if not self.params:
+            return config
+
+        ret = {}
+        if flatten:
+            for param in self.params:
+                try:
+                    ret[param] = dpath.util.get(config, param, separator=".")
+                except KeyError:
+                    continue
+            return ret
+
+        from dpath.util import merge
+
+        for param in self.params:
+            merge(
+                ret,
+                dpath.util.search(config, param, separator="."),
+                separator=".",
+            )
+        return ret
+
+    def workspace_status(self):
+        if not self.exists:
+            return {str(self): "deleted"}
+        if self.hash_info.value is None:
+            return {str(self): "new"}
+
+        from funcy import ldistinct
 
         status = defaultdict(dict)
         info = self.hash_info.value if self.hash_info else {}
         actual = self.read_params()
-        for param in self.params:
-            if param not in actual.keys():
+
+        # NOTE: we want to preserve the order of params as specified in the
+        # status. In case of tracking the whole file, the order is top-level
+        # keys in the file and then the keys in the `info` from `dvc.lock`
+        # (which are alphabetically sorted).
+        params = self.params or ldistinct([*actual.keys(), *info.keys()])
+        for param in params:
+            if param not in actual:
                 st = "deleted"
             elif param not in info:
                 st = "new"
@@ -88,42 +130,26 @@ class ParamsDependency(Dependency):
     def status(self):
         return self.workspace_status()
 
-    def _read(self):
+    def validate_filepath(self):
         if not self.exists:
-            return {}
+            raise MissingParamsFile(f"Parameters file '{self}' does not exist")
+        if self.isdir():
+            raise ParamsIsADirectoryError(
+                f"'{self}' is a directory, expected a parameters file"
+            )
 
-        suffix = self.path_info.suffix.lower()
-        loader = LOADERS[suffix]
+    def read_file(self):
+        _, ext = os.path.splitext(self.fs_path)
+        loader = LOADERS[ext]
+
+        self.validate_filepath()
         try:
-            return loader(self.path_info, fs=self.repo.fs)
+            return loader(self.fs_path, fs=self.repo.fs)
         except ParseError as exc:
             raise BadParamFileError(
                 f"Unable to read parameters from '{self}'"
             ) from exc
 
-    def read_params_d(self, **kwargs):
-        config = self._read()
-
-        ret = {}
-        for param in self.params:
-            dpath.util.merge(
-                ret,
-                dpath.util.search(config, param, separator="."),
-                separator=".",
-            )
-        return ret
-
-    def read_params(self):
-        config = self._read()
-
-        ret = {}
-        for param in self.params:
-            try:
-                ret[param] = dpath.util.get(config, param, separator=".")
-            except KeyError:
-                pass
-        return ret
-
     def get_hash(self):
         info = self.read_params()
 
@@ -144,12 +170,5 @@ class ParamsDependency(Dependency):
         if not self.isfile and not self.isdir:
             raise self.IsNotFileOrDirError(self)
 
-        if self.is_empty:
-            logger.warning(f"'{self}' is empty.")
-
         self.ignore()
-
-        if self.metric or self.plot:
-            self.verify_metric()
-
         self.hash_info = self.get_hash()

dvc/dependency/repo.py

@@ -5,13 +5,11 @@ from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple
 
 from voluptuous import Required
 
-from dvc.path_info import PathInfo
-
 from .base import Dependency
 
 if TYPE_CHECKING:
     from dvc.hash_info import HashInfo
-    from dvc.objects.db.base import ObjectDB
+    from dvc.objects.db import ObjectDB
     from dvc.objects.file import HashFile
 
 
@@ -34,7 +32,7 @@ class RepoDependency(Dependency):
         self._staged_objs: Dict[str, "HashFile"] = {}
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, fs, path_info):
+    def _parse_path(self, fs, fs_path):
         return None
 
     @property
@@ -63,14 +61,14 @@ class RepoDependency(Dependency):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
     def download(self, to, jobs=None):
-        from dvc.objects.checkout import checkout
+        from dvc.data.checkout import checkout
 
         for odb, objs in self.get_used_objs().items():
             self.repo.cloud.pull(objs, jobs=jobs, odb=odb)
 
         obj = self.get_obj()
         checkout(
-            to.path_info,
+            to.fs_path,
             to.fs,
             obj,
             self.repo.odb.local,
@@ -100,9 +98,10 @@ class RepoDependency(Dependency):
         self, obj_only=False, **kwargs
     ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "HashFile"]:
         from dvc.config import NoRemoteError
+        from dvc.data.stage import stage
+        from dvc.data.tree import Tree, TreeError
         from dvc.exceptions import NoOutputOrStageError, PathMissingError
-        from dvc.objects.stage import stage
-        from dvc.objects.tree import Tree
+        from dvc.utils import as_posix
 
         local_odb = self.repo.odb.local
         locked = kwargs.pop("locked", True)
@@ -114,11 +113,10 @@ class RepoDependency(Dependency):
             if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
                 self.def_repo[self.PARAM_REV_LOCK] = rev
 
-            path_info = PathInfo(repo.root_dir) / str(self.def_path)
             if not obj_only:
                 try:
                     for odb, obj_ids in repo.used_objs(
-                        [os.fspath(path_info)],
+                        [os.path.join(repo.root_dir, self.def_path)],
                         force=True,
                         jobs=kwargs.get("jobs"),
                         recursive=True,
@@ -134,11 +132,11 @@ class RepoDependency(Dependency):
             try:
                 staging, _, staged_obj = stage(
                     local_odb,
-                    path_info,
+                    as_posix(self.def_path),
                     repo.repo_fs,
                     local_odb.fs.PARAM_CHECKSUM,
                 )
-            except FileNotFoundError as exc:
+            except (FileNotFoundError, TreeError) as exc:
                 raise PathMissingError(
                     self.def_path, self.def_repo[self.PARAM_URL]
                 ) from exc
@@ -152,10 +150,10 @@ class RepoDependency(Dependency):
             return used_obj_ids, staged_obj
 
     def _check_circular_import(self, odb, obj_ids):
+        from dvc.data.db.reference import ReferenceObjectDB
+        from dvc.data.tree import Tree
         from dvc.exceptions import CircularImportError
         from dvc.fs.repo import RepoFileSystem
-        from dvc.objects.db.reference import ReferenceObjectDB
-        from dvc.objects.tree import Tree
 
         if not isinstance(odb, ReferenceObjectDB):
             return
@@ -174,17 +172,17 @@ class RepoDependency(Dependency):
                 continue
             if (
                 obj.fs.repo_url in checked_urls
-                or obj.fs.root_dir in checked_urls
+                or obj.fs.repo.root_dir in checked_urls
             ):
                 continue
             self_url = self.repo.url or self.repo.root_dir
             if (
                 obj.fs.repo_url is not None
                 and obj.fs.repo_url == self_url
-                or obj.fs.root_dir == self.repo.root_dir
+                or obj.fs.repo.root_dir == self.repo.root_dir
             ):
                 raise CircularImportError(self, obj.fs.repo_url, self_url)
-            checked_urls.update([obj.fs.repo_url, obj.fs.root_dir])
+            checked_urls.update([obj.fs.repo_url, obj.fs.repo.root_dir])
 
     def get_obj(self, filter_info=None, **kwargs):
         locked = kwargs.get("locked", True)

dvc/dvcfile.py

@@ -60,7 +60,7 @@ def is_lock_file(path):
 
 def is_git_ignored(repo, path):
     from dvc.fs.local import LocalFileSystem
-    from dvc.scm.base import NoSCMError
+    from dvc.scm import NoSCMError
 
     try:
         return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored(
@@ -199,7 +199,7 @@ class SingleStageFile(FileMixin):
             check_dvcfile_path(self.repo, self.path)
         logger.debug(f"Saving information to '{relpath(self.path)}'.")
         dump_yaml(self.path, serialize.to_single_stage_file(stage))
-        self.repo.scm.track_file(self.relpath)
+        self.repo.scm_context.track_file(self.relpath)
 
     def remove_stage(self, stage):  # pylint: disable=unused-argument
         self.remove()
@@ -268,7 +268,7 @@ class PipelineFile(FileMixin):
             else:
                 data["stages"].update(stage_data)
 
-        self.repo.scm.track_file(self.relpath)
+        self.repo.scm_context.track_file(self.relpath)
 
     @property
     def stage(self):
@@ -388,7 +388,7 @@ class Lockfile(FileMixin):
             data["stages"].update(stage_data)
 
         if modified:
-            self.repo.scm.track_file(self.relpath)
+            self.repo.scm_context.track_file(self.relpath)
 
     def remove_stage(self, stage):
         if not self.exists():

dvc/exceptions.py

@@ -1,4 +1,5 @@
 """Exceptions raised by the dvc."""
+from dvc.utils import format_link
 
 
 class DvcException(Exception):
@@ -128,13 +129,6 @@ class NotDvcRepoError(DvcException):
     """Thrown if a directory is not a DVC repo"""
 
 
-class DvcParserError(DvcException):
-    """Base class for CLI parser errors."""
-
-    def __init__(self):
-        super().__init__("parser error")
-
-
 class CyclicGraphError(DvcException):
     def __init__(self, stages):
         assert isinstance(stages, list)
@@ -157,9 +151,9 @@ class InitError(DvcException):
 
 
 class ReproductionError(DvcException):
-    def __init__(self, dvc_file_name):
-        self.path = dvc_file_name
-        super().__init__(f"failed to reproduce '{dvc_file_name}'")
+    def __init__(self, name):
+        self.name = name
+        super().__init__(f"failed to reproduce '{name}'")
 
 
 class BadMetricError(DvcException):
@@ -216,18 +210,6 @@ class DvcIgnoreInCollectedDirError(DvcException):
         )
 
 
-class GitHookAlreadyExistsError(DvcException):
-    def __init__(self, hook_name):
-        from dvc.utils import format_link
-
-        super().__init__(
-            "Hook '{}' already exists. Please refer to {} for more "
-            "info.".format(
-                hook_name, format_link("https://man.dvc.org/install")
-            )
-        )
-
-
 class FileTransferError(DvcException):
     _METHOD = "transfer"
 
@@ -305,8 +287,7 @@ class PathMissingError(DvcException):
 
 
 class RemoteCacheRequiredError(DvcException):
-    def __init__(self, path_info):
-        from dvc.utils import format_link
+    def __init__(self, scheme, fs_path):
 
         super().__init__(
             (
@@ -314,8 +295,8 @@ class RemoteCacheRequiredError(DvcException):
                 "existing cache on '{}' remote. See {} for information on how "
                 "to set up remote cache."
             ).format(
-                path_info,
-                path_info.scheme,
+                fs_path,
+                scheme,
                 format_link("https://man.dvc.org/config#cache"),
             )
         )
@@ -342,7 +323,6 @@ class MergeError(DvcException):
 
 
 class CacheLinkError(DvcException):
-    from dvc.utils import format_link
 
     SUPPORT_LINK = "See {} for more information.".format(
         format_link(
@@ -350,12 +330,12 @@ class CacheLinkError(DvcException):
         )
     )
 
-    def __init__(self, path_infos):
+    def __init__(self, fs_paths):
         msg = "No possible cache link types for '{}'. {}".format(
-            ", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK
+            ", ".join(fs_paths), self.SUPPORT_LINK
         )
         super().__init__(msg)
-        self.path_infos = path_infos
+        self.fs_paths = fs_paths
 
 
 class CircularImportError(DvcException):

dvc/external_repo.py

@@ -3,7 +3,7 @@ import os
 import tempfile
 import threading
 from contextlib import contextmanager
-from typing import Dict
+from typing import TYPE_CHECKING, Dict
 
 from funcy import retry, wrap_with
 
@@ -16,17 +16,24 @@ from dvc.exceptions import (
     PathMissingError,
 )
 from dvc.repo import Repo
+from dvc.scm import CloneError, map_scm_exception
 from dvc.utils import relpath
 
+if TYPE_CHECKING:
+    from scmrepo import Git
+
 logger = logging.getLogger(__name__)
 
 
 @contextmanager
+@map_scm_exception()
 def external_repo(
     url, rev=None, for_write=False, cache_dir=None, cache_types=None, **kwargs
 ):
+    from scmrepo.git import Git
+
     from dvc.config import NoRemoteError
-    from dvc.scm.git import Git
+    from dvc.fs.git import GitFileSystem
 
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
@@ -42,12 +49,18 @@ def external_repo(
     config = _get_remote_config(url) if os.path.isdir(url) else {}
     config.update(cache_config)
 
-    root_dir = path if for_write else os.path.realpath(path)
+    if for_write:
+        root_dir = path
+        fs = None
+    else:
+        root_dir = os.path.realpath(path)
+        scm = Git(root_dir)
+        fs = GitFileSystem(scm=scm, rev=rev)
+
     repo_kwargs = dict(
         root_dir=root_dir,
         url=url,
-        scm=None if for_write else Git(root_dir),
-        rev=None if for_write else rev,
+        fs=fs,
         config=config,
         repo_factory=erepo_factory(url, cache_config),
         **kwargs,
@@ -83,7 +96,7 @@ def erepo_factory(url, cache_config, *args, **kwargs):
     def make_repo(path, **_kwargs):
         _config = cache_config.copy()
         if os.path.isdir(url):
-            rel = os.path.relpath(path, _kwargs["scm"].root_dir)
+            rel = os.path.relpath(path, _kwargs["fs"].fs_args["scm"].root_dir)
             repo_path = os.path.join(url, rel)
             _config.update(_get_remote_config(repo_path))
         return Repo(path, config=_config, **_kwargs)
@@ -173,7 +186,7 @@ def _clone_default_branch(url, rev, for_write=False):
 
     The cloned is reactualized with git pull unless rev is a known sha.
     """
-    from dvc.scm.git import Git
+    from scmrepo.git import Git
 
     clone_path, shallow = CLONES.get(url, (None, False))
 
@@ -191,29 +204,35 @@ def _clone_default_branch(url, rev, for_write=False):
                     # given repo URL it is easier/safer for us to work with
                     # full clones in this case.
                     logger.debug("erepo: unshallowing clone for '%s'", url)
-                    _unshallow(git)
+                    _pull(git, unshallow=True)
                     shallow = False
                     CLONES[url] = (clone_path, shallow)
                 else:
                     logger.debug("erepo: git pull '%s'", url)
-                    git.pull()
+                    _pull(git)
         else:
+            from dvc.scm import clone
+
             logger.debug("erepo: git clone '%s' to a temporary dir", url)
             clone_path = tempfile.mkdtemp("dvc-clone")
             if not for_write and rev and not Git.is_sha(rev):
                 # If rev is a tag or branch name try shallow clone first
-                from dvc.scm.base import CloneError
 
                 try:
-                    git = Git.clone(url, clone_path, shallow_branch=rev)
-                    shallow = True
-                    logger.debug(
-                        "erepo: using shallow clone for branch '%s'", rev
+                    git = clone(url, clone_path, shallow_branch=rev)
+                    shallow = os.path.exists(
+                        os.path.join(clone_path, Git.GIT_DIR, "shallow")
                     )
+                    if shallow:
+                        logger.debug(
+                            "erepo: using shallow clone for branch '%s'", rev
+                        )
                 except CloneError:
-                    pass
+                    git_dir = os.path.join(clone_path, ".git")
+                    if os.path.exists(git_dir):
+                        _remove(git_dir)
             if not git:
-                git = Git.clone(url, clone_path)
+                git = clone(url, clone_path)
                 shallow = False
             CLONES[url] = (clone_path, shallow)
     finally:
@@ -223,21 +242,28 @@ def _clone_default_branch(url, rev, for_write=False):
     return clone_path, shallow
 
 
-def _unshallow(git):
-    if git.gitpython.repo.head.is_detached:
-        # If this is a detached head (i.e. we shallow cloned a tag) switch to
-        # the default branch
-        origin_refs = git.gitpython.repo.remotes["origin"].refs
-        ref = origin_refs["HEAD"].reference
-        branch_name = ref.name.split("/")[-1]
-        branch = git.gitpython.repo.create_head(branch_name, ref)
-        branch.set_tracking_branch(ref)
-        branch.checkout()
-    git.pull(unshallow=True)
+def _pull(git: "Git", unshallow: bool = False):
+    from dvc.repo.experiments.utils import fetch_all_exps
+
+    git.fetch(unshallow=unshallow)
+    _merge_upstream(git)
+    fetch_all_exps(git, "origin")
+
+
+def _merge_upstream(git: "Git"):
+    from scmrepo.exceptions import SCMError
+
+    try:
+        branch = git.active_branch()
+        upstream = f"refs/remotes/origin/{branch}"
+        if git.get_ref(upstream):
+            git.merge(upstream)
+    except SCMError:
+        pass
 
 
 def _git_checkout(repo_path, rev):
-    from dvc.scm.git import Git
+    from scmrepo.git import Git
 
     logger.debug("erepo: git checkout %s@%s", repo_path, rev)
     git = Git(repo_path)