run: update -h message (#6197)

* update dvc exp run -h message

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.coveragerc

@@ -13,3 +13,4 @@ exclude_lines =
     raise NotImplementedError
     # pragma: no cover
     #  pragma: no cover
+    \.\.\.$

.github/CODEOWNERS

@@ -0,0 +1 @@
+*       @iterative/dvc

.github/ISSUE_TEMPLATE/bug_report.md

@@ -48,14 +48,17 @@ A clear and concise description of what you expect to happen.
 This is required to ensure that we can reproduce the bug.
 -->
 
-**Output of `dvc version`:**
+**Output of `dvc doctor`:**
 
 ```console
-$ dvc version
+$ dvc doctor
 ```
 
 **Additional Information (if any):**
 
 <!--
+Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
+
 If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
--->
+If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
+-->

.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+
+updates:
+  - directory: "/"
+    package-ecosystem: "pip"
+    schedule:
+      interval: "daily"
+    labels:
+      - "maintenance"
+
+  - directory: "/"
+    package-ecosystem: "github-actions"
+    schedule:
+      interval: "daily"
+    labels:
+      - "maintenance"

.github/workflows/packages.yaml

@@ -6,91 +6,130 @@ on:
   release:
     types: [released, prereleased]
 jobs:
-  linux:
-    runs-on: ubuntu-16.04
+  deb:
+    runs-on: ubuntu-18.04
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Set up Ruby 2.4
-      uses: actions/setup-ruby@v1
+      uses: actions/setup-ruby@v1.1.3
       with:
         ruby-version: 2.4
-    - name: Build
-      run: ./scripts/build_posix.sh
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build deb
+      run: python scripts/build.py deb
     - name: Publish deb
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_path: scripts/fpm/dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_content_type: binary/octet-stream
-    - name: Publish rpm
-      if: github.event_name == 'release'
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
-        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
-        asset_content_type: binary/octet-stream
-  mac:
-    runs-on: macos-10.15
+  rpm:
+    runs-on: ubuntu-18.04
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Set up Ruby 2.4
-      uses: actions/setup-ruby@v1
+      uses: actions/setup-ruby@v1.1.3
       with:
         ruby-version: 2.4
-    - name: Build
-      run: ./scripts/build_posix.sh
-    - name: Publish
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build rpm
+      run: python scripts/build.py rpm
+    - name: Publish rpm
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_content_type: binary/octet-stream
+  osxpkg:
+    runs-on: macos-10.15
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Set up Ruby 2.4
+      uses: actions/[email protected]
+      with:
+        ruby-version: 2.4
+    - name: Install deps
+      run: |
+        # available lxml (webdav dependency) wheels are built with an old
+        # SDK, which breaks osxpkg notarization. Building from scratch
+        # instead.
+        pip install .[all] --no-binary lxml
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build
+      run: python scripts/build.py osxpkg
+    - name: Publish
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}.pkg
         asset_name: dvc-${{ github.event.release.tag_name }}.pkg
         asset_content_type: binary/octet-stream
-  windows:
+  exe:
     runs-on: windows-2019
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+    - name: Pull images
+      run: |
+        dvc pull
     - name: Build
-      run: scripts\build_windows.cmd
+      run: python scripts\build.py exe
       shell: cmd
     - name: Publish
       if: github.event_name == 'release'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.exe
+        asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe
         asset_name: dvc-${{ github.event.release.tag_name }}.exe
         asset_content_type: binary/octet-stream
-  python:
-    runs-on: ubuntu-16.04
+  pip:
+    runs-on: ubuntu-18.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.7
     - name: Install
@@ -103,29 +142,3 @@ jobs:
       with:
         user: __token__
         password: ${{ secrets.PYPI_TOKEN }}
-  snap:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/[email protected]
-    - name: Build the package
-      uses: snapcore/action-build@v1
-      id: snapcraft
-#    - name: Test the package
-#      run: |
-#        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
-#        dvc version
-#        sudo snap remove dvc
-    - name: Publish to the edge channel
-      uses: snapcore/action-publish@v1
-      if: github.event_name == 'push'
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: edge,v2/edge
-    - name: Publish to the stable channel
-      uses: snapcore/action-publish@v1
-      if: ${{ github.event_name == 'release' && github.event.release.released }}
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: stable,v2/stable

.github/workflows/tests.yaml

@@ -7,34 +7,34 @@ on:
   push:
     branches:
       - master
+  schedule:
+    - cron: '5 1 * * *'  # every day at 01:05
 
 env:
   DVC_TEST: "true"
   HOMEBREW_NO_AUTO_UPDATE: 1
   SHELL: /bin/bash
-  GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.8
     - name: Install requirements
-      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+      run: |
+        pip install wheel
+        pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
     - name: install temporary dependencies
       run: |
         pip install git+https://github.com/isidentical/gdrivefs@service-account
         pip install git+https://github.com/isidentical/pydata-google-auth@service-account
     - name: Check README
       run: python setup.py checkdocs
-    - uses: pre-commit/[email protected].0
+    - uses: pre-commit/[email protected].3
   tests:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -51,9 +51,9 @@ jobs:
             pyv: "3.9"
 
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: ${{ matrix.pyv }}
     - name: get pip cache dir
@@ -62,7 +62,7 @@ jobs:
         echo "::set-output name=dir::$(pip cache dir)"
     - name: set pip cache
       id: pip-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v2.1.6
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
@@ -83,25 +83,44 @@ jobs:
         git config --global user.name "DVC Tester"
     - name: setup gs creds
       shell: bash
+      env:
+        GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
       if: env.GS_CREDS_JSON != ''
       run: |
         mkdir -p scripts/ci
         echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
+    - name: disable gdrive tests
+      if: >-
+        github.event_name != 'schedule' &&
+        (matrix.pyv != '3.8' || matrix.os != 'ubuntu-18.04')
+      run: |
+        echo "extra_test_args=--disable-gdrive" >> $GITHUB_ENV
     - name: run tests
-      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined -n=4
+      env:
+        GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      run: >-
+        python -m tests -n=4
+        --cov-report=xml --cov-report=term
+        --all ${{ env.extra_test_args }}
     - name: upload coverage report
-      uses: codecov/codecov-action@v1.0.13
+      uses: codecov/codecov-action@v1.5.2
       with:
         file: ./coverage.xml
         fail_ci_if_error: false
-    - name: Post TAP
-      # Running custom fork with an option, that does not fail CI on error
-      uses: skshetry/flaky-service/packages/action@master
-      if: always() && github.ref == 'refs/heads/master'
-      with:
-        file-path: ${{github.workspace}}/testresults.tap
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        os: ${{runner.os}}
-        matrix: ${{toJson(matrix)}}
-        repo-description: ${{github.event.repository.description}}
-        fail_ci_if_error: false
+  notify:
+    if: github.ref == 'refs/heads/master' && failure()
+    needs:
+      - lint
+      - tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Slack Notification
+      uses: rtCamp/[email protected]
+      env:
+        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        SLACK_COLOR: ${{ job.status }}
+        SLACK_MESSAGE: 'CI Failed on master :boom:'
+        SLACK_TITLE: CI Status
+        SLACK_USERNAME: DVC-CI

.pre-commit-config.yaml

@@ -1,14 +1,17 @@
+ci:
+  skip: [pylint, dvc-pre-commit]
+
 repos:
   - hooks:
       - id: black
         language_version: python3
     repo: https://github.com/ambv/black
-    rev: 19.10b0
+    rev: 21.6b0
   - hooks:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 5.6.4
+    rev: 5.9.1
   - hooks:
       - id: flake8
         language_version: python3
@@ -18,13 +21,18 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.2
   - hooks:
       -  id: mypy
          language_version: python3
          files: ^dvc/
+         additional_dependencies:
+           - types-requests
+           - types-paramiko
+           - types-tabulate
+           - types-toml
     repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.790'
+    rev: 'v0.910'
   - repo: local
     hooks:
     - id: pylint
@@ -39,7 +47,7 @@ repos:
         id: beautysh
         language_version: python3
     repo: https://github.com/lovesegfault/beautysh
-    rev: 6.0.1
+    rev: v6.1.0
   - hooks:
       - args:
           - git-hook

.pylintrc

@@ -15,10 +15,13 @@ ignore-patterns=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-init-hook="import sys; \
-           from pathlib import Path; \
-           from pylint.config import find_pylintrc; \
-           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
+init-hook="import sys;
+           from pathlib import Path;
+           from pylint.config import find_pylintrc;
+
+           tests_folder = Path(find_pylintrc()).parent / 'tests'
+           sys.path.append(str(tests_folder));
+           "
 
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
@@ -178,7 +181,10 @@ disable=format,
         # we use it to speedup/optimize
         import-outside-toplevel,
         fixme,
-
+        # we use cause of the exception for error message output
+        # so some of the times we don't use `raise from` to not show the cause
+        # of the error
+        raise-missing-from,
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -307,6 +313,7 @@ missing-member-hint-distance=1
 # showing a hint for a missing member.
 missing-member-max-choices=1
 
+signature-mutators=funcy.decorators.decorator,
 
 [VARIABLES]
 

MANIFEST.in

@@ -1,2 +1,3 @@
 include fastentrypoints.py
 include LICENSE
+include test_requirements.txt

README.rst

@@ -129,7 +129,11 @@ Conda (Anaconda)
 
 .. code-block:: bash
 
-   conda install -c conda-forge dvc
+   conda install -c conda-forge mamba # installs much faster than conda
+   mamba install -c conda-forge dvc
+
+Depending on the remote storage type you plan to use to keep and share your data, you might need to
+install optional dependencies: `dvc-s3`, `dvc-azure`, `dvc-gdrive`, `dvc-gs`, `dvc-oss`, `dvc-ssh`.
 
 pip (PyPI)
 ----------
@@ -178,6 +182,12 @@ Fedora / CentOS (rpm)
 Comparison to related technologies
 ==================================
 
+#. *Data Engineering* tools such as `AirFlow <https://airflow.apache.org/>`_, `Luigi <https://github.com/spotify/luigi>`_ and
+   others - in DVC data, model and ML pipelines represent a single ML project and focuses on data scientist' experience while
+   data engineering pipeline orchestrates multiple data projects and focuses on efficent execution. DVC project can be used
+   from data pipeline as a single execution step. `AirFlow DVC <https://github.com/covid-genomics/airflow-dvc>`_ is an example
+   of such integration.
+
 #. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (which should
    not be in a Git repository) in a local key-value store, and uses file hardlinks/symlinks instead of
    copying/duplicating files.
@@ -191,7 +201,12 @@ Comparison to related technologies
 #. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_ - DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-#. `DAGsHub <https://dagshub.com/>`_ - This is a Github equivalent for DVC. Pushing Git+DVC based repositories to DAGsHub will produce in a high level project dashboard; including DVC pipelines and metrics visualizations, as well as links to any DVC-managed files present in cloud storage.
+#. `DAGsHub <https://dagshub.com/>`_ - online service of Git+DVC repositories with pipeline and metrics visualization and
+   DVC-specific cloud storage.
+
+#. `DVC Studio <https://studio.iterative.ai/>`_ - online service of DVC repositories visualisation from DVC team. Also,
+   integrated with `CML (CI/CD for ML) <https://cml.dev/>`_ for training models in clouds and Kubernetes.
+
 
 Contributing
 ============
@@ -222,6 +237,9 @@ Citation
 Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
 `DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
 
+Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and Source Code - Empirical Study of DVC Projects <https://mcis.cs.queensu.ca/publications/2021/saner.pdf>`_ , in Proceedings of the 28th IEEE International Conference on Software Analysis, Evolution, and Reengineering, SANER 2021. Hawaii, USA.
+
+
 .. |Banner| image:: https://dvc.org/img/logo-github-readme.png
    :target: https://dvc.org
    :alt: DVC logo

debian/changelog

@@ -1,5 +0,0 @@
-dvc (0.75.b1) experimental; urgency=low
-
-  * First native Debian package version.
-
- -- Dmitry Petrov <[email protected]>  Sat, 07 Dec 2019 09:49:28 +0300

debian/compat

@@ -1 +0,0 @@
-10

debian/control

@@ -1,33 +0,0 @@
-Source: dvc
-Maintainer: Dmitry Petrov <[email protected]>
-Section: python
-Priority: optional
-Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
-Standards-Version: 4.0.0
-Homepage: http://dvc.org
-Vcs-Git: https://github.com/iterative/dvc.git
-Vcs-browser: https://github.com/iterative/dvc
-
-Package: dvc
-Architecture: all
-Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
-Recommends: python3-arrow (=0.14.0),
-            python3-paramiko (>=2.5.0),
-            python3-gssapi (>= 2.5.0),
-            python3-boto3 (>=1.9.86),
-            python3-pydrive (=1.3.1),
-            python3-azure-storage-blob (=12.1.0),
-            python3-oss2 (=2.6.1),
-            python3-google-cloud-storage (=1.19.0)
-Description: Git for data scientists - manage your code and data together
- Data Version Control or DVC is an open-source tool for data science and machine learning projects.
- .
- Key features:
- .
- 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
- .
- 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
- .
- 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
- .
- 4. Helps manage experiments with Git tags/branches and metrics tracking.

debian/copyright

@@ -1,11 +0,0 @@
-Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: DVC
-Upstream-Contact: Dmitry Petrov <[email protected]>
-Source: https://github.com/iterative/dvc
-
-Files: *
-Copyright: Dmitry Petrov <[email protected]>
-License: Apache 2.0
- On Debian systems, the full text of the Apache 2.0
- License can be found in the file
- `/usr/share/common-licenses/Apache-2.0`.

debian/rules

@@ -1,6 +0,0 @@
-#!/usr/bin/make -f
-
-export PYBUILD_NAME=dvc
-%:
-	dh $@ --with python3 --buildsystem=pybuild
-

debian/source/format

@@ -1 +0,0 @@
-3.0 (native)

debian/watch

@@ -1,4 +0,0 @@
-# please also check http://pypi.debian.net/dvc/watch
-version=3
-opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
-http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

dvc/_debug.py

@@ -77,15 +77,15 @@ def add_debugging_flags(parser):
     from argparse import SUPPRESS
 
     parser.add_argument(
-        "--cprofile", action="store_true", default=False, help=SUPPRESS,
+        "--cprofile", action="store_true", default=False, help=SUPPRESS
     )
     parser.add_argument("--cprofile-dump", help=SUPPRESS)
     parser.add_argument(
-        "--pdb", action="store_true", default=False, help=SUPPRESS,
+        "--pdb", action="store_true", default=False, help=SUPPRESS
     )
     parser.add_argument(
-        "--instrument", action="store_true", default=False, help=SUPPRESS,
+        "--instrument", action="store_true", default=False, help=SUPPRESS
     )
     parser.add_argument(
-        "--instrument-open", action="store_true", default=False, help=SUPPRESS,
+        "--instrument-open", action="store_true", default=False, help=SUPPRESS
     )

dvc/api.py

@@ -4,7 +4,6 @@ from contextlib import _GeneratorContextManager as GCM
 from funcy import reraise
 
 from dvc.exceptions import OutputNotFoundError, PathMissingError
-from dvc.objects.stage import get_hash
 from dvc.path_info import PathInfo
 from dvc.repo import Repo
 
@@ -29,8 +28,8 @@ def get_url(path, repo=None, rev=None, remote=None):
             raise OutputNotFoundError(path, repo)
 
         cloud = metadata.repo.cloud
-        hash_info = get_hash(path_info, _repo.repo_fs, "md5", _repo.odb)
-        return cloud.get_url_for(remote, checksum=hash_info.value)
+        md5 = metadata.repo.dvcfs.info(path_info)["md5"]
+        return cloud.get_url_for(remote, checksum=md5)
 
 
 def open(  # noqa, pylint: disable=redefined-builtin

dvc/checkout.py

@@ -2,18 +2,21 @@ import logging
 
 from shortuuid import uuid
 
-import dvc.prompt as prompt
+from dvc import prompt
 from dvc.exceptions import (
     CacheLinkError,
     CheckoutError,
     ConfirmRemoveError,
     DvcException,
 )
-from dvc.objects import ObjectFormatError, check, load
-from dvc.objects.stage import get_hash
+from dvc.ignore import DvcIgnoreFilter
+from dvc.objects import check, load
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.stage import stage
 from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
     slow_link_guard,
 )
+from dvc.types import Optional
 
 logger = logging.getLogger(__name__)
 
@@ -30,7 +33,7 @@ def _changed(path_info, fs, obj, cache):
         return True
 
     try:
-        actual = get_hash(path_info, fs, obj.hash_info.name, cache)
+        actual = stage(cache, path_info, fs, obj.hash_info.name).hash_info
     except FileNotFoundError:
         logger.debug("'%s' doesn't exist.", path_info)
         return True
@@ -56,7 +59,7 @@ def _remove(path_info, fs, cache, force=False):
         fs.remove(path_info)
         return
 
-    current = get_hash(path_info, fs, fs.PARAM_CHECKSUM, cache)
+    current = stage(cache, path_info, fs, fs.PARAM_CHECKSUM).hash_info
     try:
         obj = load(cache, current)
         check(cache, obj)
@@ -94,7 +97,7 @@ def _do_link(cache, from_info, to_info, link_method):
     link_method(from_info, to_info)
 
     logger.debug(
-        "Created '%s': %s -> %s", cache.cache_types[0], from_info, to_info,
+        "Created '%s': %s -> %s", cache.cache_types[0], from_info, to_info
     )
 
 
@@ -131,7 +134,7 @@ def _cache_is_copy(cache, path_info):
         return True
 
     workspace_file = path_info.with_name("." + uuid())
-    test_cache_file = cache.fs.path_info / ".cache_type_test_file"
+    test_cache_file = cache.path_info / ".cache_type_test_file"
     if not cache.fs.exists(test_cache_file):
         cache.makedirs(test_cache_file.parent)
         with cache.fs.open(test_cache_file, "wb") as fobj:
@@ -147,7 +150,14 @@ def _cache_is_copy(cache, path_info):
 
 
 def _checkout_file(
-    path_info, fs, obj, cache, force, progress_callback=None, relink=False,
+    path_info,
+    fs,
+    obj,
+    cache,
+    force,
+    progress_callback=None,
+    relink=False,
+    state=None,
 ):
     """The file is changed we need to checkout a new copy"""
     modified = False
@@ -167,17 +177,24 @@ def _checkout_file(
         _link(cache, cache_info, path_info)
         modified = True
 
-    fs.repo.state.save(path_info, fs, obj.hash_info)
+    if state:
+        state.save(path_info, fs, obj.hash_info)
+
     if progress_callback:
         progress_callback(str(path_info))
 
     return modified
 
 
-def _remove_redundant_files(path_info, fs, dir_info, cache, force):
-    existing_files = set(fs.walk_files(path_info))
+def _remove_redundant_files(
+    path_info, fs, obj, cache, force, dvcignore: Optional[DvcIgnoreFilter]
+):
+    if dvcignore:
+        existing_files = set(dvcignore.walk_files(fs, path_info))
+    else:
+        existing_files = set(fs.walk_files(path_info))
 
-    needed_files = {path_info.joinpath(*key) for key, _ in dir_info.items()}
+    needed_files = {path_info.joinpath(*key) for key, _ in obj}
     redundant_files = existing_files - needed_files
     for path in redundant_files:
         _remove(path, fs, cache, force)
@@ -186,9 +203,17 @@ def _remove_redundant_files(path_info, fs, dir_info, cache, force):
 
 
 def _checkout_dir(
-    path_info, fs, obj, cache, force, progress_callback=None, relink=False,
+    path_info,
+    fs,
+    obj,
+    cache,
+    force,
+    progress_callback=None,
+    relink=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+    state=None,
 ):
-    modified = False, False
+    modified = False
     # Create dir separately so that dir is created
     # even if there are no files in it
     if not fs.exists(path_info):
@@ -197,27 +222,29 @@ def _checkout_dir(
 
     logger.debug("Linking directory '%s'.", path_info)
 
-    for entry_key, entry_hash_info in obj.hash_info.dir_info.items():
+    for entry_key, entry_obj in obj:
         entry_modified = _checkout_file(
             path_info.joinpath(*entry_key),
             fs,
-            cache.get(entry_hash_info),
+            entry_obj,
             cache,
             force,
             progress_callback,
             relink,
+            state=None,
         )
         if entry_modified:
             modified = True
 
     modified = (
         _remove_redundant_files(
-            path_info, fs, obj.hash_info.dir_info, cache, force
+            path_info, fs, obj, cache, force, dvcignore=dvcignore
         )
         or modified
     )
 
-    fs.repo.state.save(path_info, fs, obj.hash_info)
+    if state:
+        state.save(path_info, fs, obj.hash_info)
 
     # relink is not modified, assume it as nochange
     return modified and not relink
@@ -231,17 +258,28 @@ def _checkout(
     force=False,
     progress_callback=None,
     relink=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+    state=None,
 ):
     if not obj.hash_info.isdir:
         ret = _checkout_file(
-            path_info, fs, obj, cache, force, progress_callback, relink
+            path_info, fs, obj, cache, force, progress_callback, relink, state
         )
     else:
         ret = _checkout_dir(
-            path_info, fs, obj, cache, force, progress_callback, relink,
+            path_info,
+            fs,
+            obj,
+            cache,
+            force,
+            progress_callback,
+            relink,
+            dvcignore=dvcignore,
+            state=state,
         )
 
-    fs.repo.state.save_link(path_info, fs)
+    if state:
+        state.save_link(path_info, fs)
 
     return ret
 
@@ -255,6 +293,8 @@ def checkout(
     progress_callback=None,
     relink=False,
     quiet=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+    state=None,
 ):
     if path_info.scheme not in ["local", cache.fs.scheme]:
         raise NotImplementedError
@@ -271,7 +311,7 @@ def checkout(
         failed = path_info
 
     elif not relink and not _changed(path_info, fs, obj, cache):
-        logger.trace("Data '%s' didn't change.", path_info)
+        logger.trace("Data '%s' didn't change.", path_info)  # type: ignore
         skip = True
     else:
         try:
@@ -288,9 +328,7 @@ def checkout(
 
     if failed or skip:
         if progress_callback and obj:
-            progress_callback(
-                str(path_info), len(obj),
-            )
+            progress_callback(str(path_info), len(obj))
         if failed:
             raise CheckoutError([failed])
         return
@@ -298,5 +336,13 @@ def checkout(
     logger.debug("Checking out '%s' with cache '%s'.", path_info, obj)
 
     return _checkout(
-        path_info, fs, obj, cache, force, progress_callback, relink,
+        path_info,
+        fs,
+        obj,
+        cache,
+        force,
+        progress_callback,
+        relink,
+        dvcignore=dvcignore,
+        state=state,
     )

dvc/command/add.py

@@ -117,6 +117,6 @@ def add_parser(subparsers, parent_parser):
         ),
     )
     parser.add_argument(
-        "targets", nargs="+", help="Input files/directories to add.",
+        "targets", nargs="+", help="Input files/directories to add."
     ).complete = completion.FILE
     parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -7,11 +7,11 @@ logger = logging.getLogger(__name__)
 
 def fix_subparsers(subparsers):
     """Workaround for bug in Python 3. See more info at:
-        https://bugs.python.org/issue16308
-        https://github.com/iterative/dvc/issues/769
+    https://bugs.python.org/issue16308
+    https://github.com/iterative/dvc/issues/769
 
-        Args:
-            subparsers: subparsers to fix.
+    Args:
+        subparsers: subparsers to fix.
     """
     subparsers.required = True
     subparsers.dest = "cmd"
@@ -45,6 +45,10 @@ class CmdBase(ABC):
             updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
             updater.check()
 
+    def do_run(self):
+        with self.repo:
+            return self.run()
+
     @abstractmethod
     def run(self):
         pass
@@ -55,3 +59,6 @@ class CmdBaseNoRepo(CmdBase):
         self.args = args
 
         os.chdir(args.cd)
+
+    def do_run(self):
+        return self.run()

dvc/command/cache.py

@@ -1,11 +1,9 @@
 import argparse
-import logging
 
 from dvc.command import completion
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 class CmdCacheDir(CmdConfig):
@@ -17,7 +15,7 @@ class CmdCacheDir(CmdConfig):
                 # Use merged config with default values
                 conf = self.config
             self._check(conf, False, "cache", "dir")
-            logger.info(conf["cache"]["dir"])
+            ui.write(conf["cache"]["dir"])
             return 0
         with self.config.edit(level=self.args.level) as conf:
             if self.args.unset:

dvc/command/check_ignore.py

@@ -1,16 +1,14 @@
 import argparse
-import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 class CmdCheckIgnore(CmdBase):
     def __init__(self, args):
         super().__init__(args)
-        self.ignore_filter = self.repo.fs.dvcignore
+        self.ignore_filter = self.repo.dvcignore
 
     def _show_results(self, result):
         if not result.match and not self.args.non_matching:
@@ -22,9 +20,9 @@ class CmdCheckIgnore(CmdBase):
                 patterns = patterns[-1:]
 
             for pattern in patterns:
-                logger.info("{}\t{}".format(pattern, result.file))
+                ui.write(pattern, result.file, sep="\t")
         else:
-            logger.info(result.file)
+            ui.write(result.file)
 
     def _check_one_file(self, target):
         result = self.ignore_filter.check_ignore(target)
@@ -122,6 +120,6 @@ def add_parser(subparsers, parent_parser):
         help="Read paths from standard input instead of providing `targets`.",
     )
     parser.add_argument(
-        "targets", nargs="*", help="File or directory paths to check",
+        "targets", nargs="*", help="File or directory paths to check"
     ).complete = completion.FILE
     parser.set_defaults(func=CmdCheckIgnore)

dvc/command/checkout.py

@@ -1,38 +1,28 @@
 import argparse
-import logging
 import operator
 
-import colorama
-
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import CheckoutError
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 def log_changes(stats):
-    colors = [
-        ("modified", colorama.Fore.YELLOW,),
-        ("added", colorama.Fore.GREEN),
-        ("deleted", colorama.Fore.RED,),
-    ]
+    colors = {
+        "modified": "yellow",
+        "added": "green",
+        "deleted": "red",
+    }
 
-    for state, color in colors:
+    for state, color in colors.items():
         entries = stats.get(state)
 
         if not entries:
             continue
 
         for entry in entries:
-            logger.info(
-                "{color}{state}{nc}{spacing}{entry}".format(
-                    color=color,
-                    state=state[0].upper(),
-                    nc=colorama.Fore.RESET,
-                    spacing="\t",
-                    entry=entry,
-                )
+            ui.write(
+                f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t"
             )
 
 
@@ -58,7 +48,7 @@ class CmdCheckout(CmdBase):
             msg = get_summary(
                 sorted(stats.items(), key=operator.itemgetter(0))
             )
-            logger.info(msg or default_message)
+            ui.write(msg or default_message)
         else:
             log_changes(stats)
 

dvc/command/commit.py

@@ -71,5 +71,5 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="stages or .dvc files to commit. Optional. "
         "(Finds all DVC files in the workspace by default.)",
-    ).complete = completion.DVC_FILE
+    ).complete = completion.DVCFILES_AND_STAGE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -4,6 +4,7 @@ import logging
 import shtab
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 FILE = shtab.FILE
@@ -67,25 +68,16 @@ _dvc_compadd_exps() {
     """,
 }
 
-DVC_FILE = {
-    "bash": "_dvc_compgen_DVCFiles",
-    "zsh": "_dvc_compadd_DVCFiles",
-}
+DVC_FILE = {"bash": "_dvc_compgen_DVCFiles", "zsh": "_dvc_compadd_DVCFiles"}
 
-STAGE = {
-    "bash": "_dvc_compgen_stages",
-    "zsh": "_dvc_compadd_stages",
-}
+STAGE = {"bash": "_dvc_compgen_stages", "zsh": "_dvc_compadd_stages"}
 
 DVCFILES_AND_STAGE = {
     "bash": "_dvc_compgen_stages_and_files",
     "zsh": "_dvc_compadd_stages_and_files",
 }
 
-EXPERIMENT = {
-    "bash": "_dvc_compgen_exps",
-    "zsh": "_dvc_compadd_exps",
-}
+EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
 
 
 class CmdCompletion(CmdBaseNoRepo):
@@ -95,7 +87,7 @@ class CmdCompletion(CmdBaseNoRepo):
         parser = get_main_parser()
         shell = self.args.shell
         script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
-        print(script)
+        ui.write(script, force=True)
         return 0
 
 

dvc/command/config.py

@@ -3,6 +3,7 @@ import logging
 import os
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -71,7 +72,9 @@ class CmdConfig(CmdBaseNoRepo):
             prefix = self._config_file_prefix(
                 self.args.show_origin, self.config, level
             )
-            logger.info("\n".join(self._format_config(conf, prefix)))
+            configs = list(self._format_config(conf, prefix))
+            if configs:
+                ui.write("\n".join(configs))
 
         return 0
 
@@ -96,7 +99,7 @@ class CmdConfig(CmdBaseNoRepo):
                 prefix = self._config_file_prefix(
                     self.args.show_origin, self.config, level
                 )
-                logger.info("{}{}".format(prefix, conf[section][opt]))
+                ui.write(prefix, conf[section][opt], sep="")
                 break
 
         return 0

dvc/command/daemon.py

@@ -67,6 +67,6 @@ def add_parser(subparsers, parent_parser):
         help=DAEMON_ANALYTICS_HELP,
     )
     daemon_analytics_parser.add_argument(
-        "target", help="Analytics file.",
+        "target", help="Analytics file."
     ).complete = completion.FILE
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/dag.py

@@ -1,9 +1,7 @@
 import argparse
-import logging
 
 from dvc.command.base import CmdBase, append_doc_link
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 def _show_ascii(G):
@@ -101,7 +99,7 @@ class CmdDAG(CmdBase):
         )
 
         if self.args.dot:
-            logger.info(_show_dot(G))
+            ui.write(_show_dot(G))
         else:
             from dvc.utils.pager import pager
 

dvc/command/data_sync.py

@@ -9,10 +9,11 @@ logger = logging.getLogger(__name__)
 
 class CmdDataBase(CmdBase):
     def log_summary(self, stats):
+        from dvc.ui import ui
         from dvc.utils.humanize import get_summary
 
         default_msg = "Everything is up to date."
-        logger.info(get_summary(stats.items()) or default_msg)
+        ui.write(get_summary(stats.items()) or default_msg)
 
 
 class CmdDataPull(CmdDataBase):
@@ -139,7 +140,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pull_parser.add_argument(
-        "-r", "--remote", help="Remote storage to pull from", metavar="<name>",
+        "-r", "--remote", help="Remote storage to pull from", metavar="<name>"
     )
     pull_parser.add_argument(
         "-a",
@@ -156,6 +157,7 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     pull_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -207,7 +209,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     push_parser.add_argument(
-        "-r", "--remote", help="Remote storage to push to", metavar="<name>",
+        "-r", "--remote", help="Remote storage to push to", metavar="<name>"
     )
     push_parser.add_argument(
         "-a",
@@ -224,6 +226,7 @@ def add_parser(subparsers, _parent_parser):
         help="Push cache for all tags.",
     )
     push_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -270,10 +273,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     fetch_parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to fetch from",
-        metavar="<name>",
+        "-r", "--remote", help="Remote storage to fetch from", metavar="<name>"
     )
     fetch_parser.add_argument(
         "-a",
@@ -290,6 +290,7 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     fetch_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -370,6 +371,7 @@ def add_parser(subparsers, _parent_parser):
         "for all tags.",
     )
     status_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,

dvc/command/destroy.py

@@ -1,7 +1,6 @@
 import argparse
 import logging
 
-import dvc.prompt as prompt
 from dvc.command.base import CmdBase, append_doc_link
 
 logger = logging.getLogger(__name__)
@@ -10,6 +9,7 @@ logger = logging.getLogger(__name__)
 class CmdDestroy(CmdBase):
     def run(self):
         from dvc.exceptions import DvcException
+        from dvc.ui import ui
 
         try:
             statement = (
@@ -19,7 +19,7 @@ class CmdDestroy(CmdBase):
                 "Are you sure you want to continue?"
             )
 
-            if not self.args.force and not prompt.confirm(statement):
+            if not self.args.force and not ui.confirm(statement):
                 raise DvcException(
                     "cannot destroy without a confirmation from the user."
                     " Use `-f` to force."

dvc/command/diff.py

@@ -3,10 +3,9 @@ import json
 import logging
 import os
 
-import colorama
-
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -18,13 +17,12 @@ def _digest(checksum):
 
 
 def _show_md(diff, show_hash=False, hide_missing=False):
-    from dvc.utils.diff import table
-
-    header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
+    headers = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
     rows = []
     statuses = ["added", "deleted", "renamed", "modified"]
     if not hide_missing:
         statuses.append("not in cache")
+
     for status in statuses:
         entries = diff.get(status, [])
         if not entries:
@@ -39,12 +37,12 @@ def _show_md(diff, show_hash=False, hide_missing=False):
             else:
                 rows.append([status, path])
 
-    return table(header, rows, True)
+    ui.table(rows, headers=headers, markdown=True)
 
 
 class CmdDiff(CmdBase):
     @staticmethod
-    def _format(diff, hide_missing=False):
+    def _show_diff(diff, hide_missing=False):
         """
         Given a diff structure, generate a string of paths separated
         by new lines and grouped together by their state.
@@ -71,15 +69,14 @@ class CmdDiff(CmdBase):
         """
 
         colors = {
-            "added": colorama.Fore.GREEN,
-            "modified": colorama.Fore.YELLOW,
-            "deleted": colorama.Fore.RED,
-            "renamed": colorama.Fore.GREEN,
-            "not in cache": colorama.Fore.YELLOW,
+            "added": "green",
+            "modified": "yellow",
+            "deleted": "red",
+            "renamed": "green",
+            "not in cache": "yellow",
         }
 
         summary = {}
-        groups = []
 
         states = ["added", "deleted", "renamed", "modified"]
         if not hide_missing:
@@ -91,7 +88,8 @@ class CmdDiff(CmdBase):
             if not entries:
                 continue
 
-            content = []
+            header = state.capitalize()
+            ui.write(f"[{colors[state]}]{header}[/]:", styled=True)
 
             for entry in entries:
                 path = entry["path"]
@@ -99,7 +97,7 @@ class CmdDiff(CmdBase):
                     path = f"{path['old']} -> {path['new']}"
                 checksum = entry.get("hash")
                 summary[state] += 1 if not path.endswith(os.sep) else 0
-                content.append(
+                ui.write(
                     "{space}{checksum}{separator}{path}".format(
                         space="    ",
                         checksum=_digest(checksum) if checksum else "",
@@ -108,14 +106,7 @@ class CmdDiff(CmdBase):
                     )
                 )
 
-            groups.append(
-                "{color}{header}{nc}:\n{content}".format(
-                    color=colors[state],
-                    header=state.capitalize(),
-                    nc=colorama.Fore.RESET,
-                    content="\n".join(content),
-                )
-            )
+            ui.write()
 
         if not sum(summary.values()):
             return None
@@ -126,9 +117,8 @@ class CmdDiff(CmdBase):
         )
         if not hide_missing:
             fmt += ", {not in cache} not in cache"
-        groups.append(fmt.format_map(summary))
 
-        return "\n\n".join(groups)
+        ui.write(fmt.format_map(summary))
 
     def run(self):
         from dvc.exceptions import DvcException
@@ -155,13 +145,11 @@ class CmdDiff(CmdBase):
                 diff[key] = entries
 
             if self.args.show_json:
-                logger.info(json.dumps(diff))
+                ui.write(json.dumps(diff))
             elif self.args.show_md:
-                logger.info(_show_md(diff, show_hash, hide_missing))
+                _show_md(diff, show_hash, hide_missing)
             elif diff:
-                output = self._format(diff, hide_missing)
-                if output:
-                    logger.info(output)
+                self._show_diff(diff, hide_missing)
 
         except DvcException:
             logger.exception("failed to get diff")

dvc/command/experiments.py

@@ -1,24 +1,33 @@
 import argparse
 import logging
 from collections import Counter, OrderedDict, defaultdict
-from collections.abc import Mapping
 from datetime import date, datetime
-from itertools import groupby
-from typing import Dict, Iterable, Optional
+from fnmatch import fnmatch
+from typing import TYPE_CHECKING, Dict, Iterable, Optional
 
-import dvc.prompt as prompt
+from funcy import lmap
+
+from dvc import prompt
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.command.metrics import DEFAULT_PRECISION
 from dvc.command.repro import CmdRepro
 from dvc.command.repro import add_arguments as add_repro_arguments
 from dvc.exceptions import DvcException, InvalidArgumentError
+from dvc.ui import ui
 from dvc.utils.flatten import flatten
 
+if TYPE_CHECKING:
+    from rich.text import Text
+
+    from dvc.compare import TabularData
+
+
 logger = logging.getLogger(__name__)
 
 
 SHOW_MAX_WIDTH = 1024
+FILL_VALUE = "-"
 
 
 def _filter_name(names, label, filter_strs):
@@ -27,32 +36,23 @@ def _filter_name(names, label, filter_strs):
 
     for filter_s in filter_strs:
         path, _, name = filter_s.rpartition(":")
-        path_filters[path].append(tuple(name.split(".")))
+        path_filters[path].append(name)
 
     for path, filters in path_filters.items():
         if path:
             match_paths = [path]
         else:
             match_paths = names.keys()
-        for length, groups in groupby(filters, len):
-            for group in groups:
-                for match_path in match_paths:
-                    possible_names = [
-                        tuple(name.split(".")) for name in names[match_path]
-                    ]
-                    matches = [
-                        name
-                        for name in possible_names
-                        if name[:length] == group
-                    ]
-                    if not matches:
-                        name = ".".join(group)
-                        raise InvalidArgumentError(
-                            f"'{name}' does not match any known {label}"
-                        )
-                    ret[match_path].update(
-                        {".".join(match): None for match in matches}
+        for match_path in match_paths:
+            for f in filters:
+                matches = [
+                    name for name in names[match_path] if fnmatch(name, f)
+                ]
+                if not matches:
+                    raise InvalidArgumentError(
+                        f"'{f}' does not match any known {label}"
                     )
+                ret[match_path].update({match: None for match in matches})
 
     return ret
 
@@ -103,7 +103,6 @@ def _collect_names(all_experiments, **kwargs):
         for exp in experiments.values():
             _update_names(metric_names, exp.get("metrics", {}).items())
             _update_names(param_names, exp.get("params", {}).items())
-
     metric_names = _filter_names(
         metric_names,
         "metrics",
@@ -111,7 +110,7 @@ def _collect_names(all_experiments, **kwargs):
         kwargs.get("exclude_metrics"),
     )
     param_names = _filter_names(
-        (param_names),
+        param_names,
         "params",
         kwargs.get("include_params"),
         kwargs.get("exclude_params"),
@@ -120,13 +119,22 @@ def _collect_names(all_experiments, **kwargs):
     return metric_names, param_names
 
 
+experiment_types = {
+    "checkpoint_tip": "│ ╓",
+    "checkpoint_commit": "│ ╟",
+    "checkpoint_base": "├─╨",
+    "branch_commit": "├──",
+    "branch_base": "└──",
+    "baseline": "",
+}
+
+
 def _collect_rows(
     base_rev,
     experiments,
     metric_names,
     param_names,
     precision=DEFAULT_PRECISION,
-    no_timestamp=False,
     sort_by=None,
     sort_order=None,
 ):
@@ -143,56 +151,56 @@ def _collect_rows(
 
     new_checkpoint = True
     for i, (rev, exp) in enumerate(experiments.items()):
-        row = []
-        style = None
-        queued = "*" if exp.get("queued", False) else ""
+        queued = str(exp.get("queued") or "")
+        is_baseline = rev == "baseline"
 
-        tip = exp.get("checkpoint_tip")
-        parent = ""
-        if rev == "baseline":
-            if Git.is_sha(base_rev):
-                name_rev = base_rev[:7]
-            else:
-                name_rev = base_rev
-            name = exp.get("name", name_rev)
-            row.append(f"{name}")
-            style = "bold"
+        if is_baseline:
+            name_rev = base_rev[:7] if Git.is_sha(base_rev) else base_rev
         else:
-            if tip:
-                parent_rev = exp.get("checkpoint_parent", "")
-                parent_exp = experiments.get(parent_rev, {})
-                parent_tip = parent_exp.get("checkpoint_tip")
-                if tip == parent_tip:
-                    if new_checkpoint:
-                        tree = "│ ╓"
-                    else:
-                        tree = "│ ╟"
-                    new_checkpoint = False
-                else:
-                    if parent_rev == base_rev:
-                        tree = "├─╨"
-                    else:
-                        tree = "│ ╟"
-                        parent = f" ({parent_rev[:7]})"
-                    new_checkpoint = True
+            name_rev = rev[:7]
+
+        exp_name = exp.get("name", "")
+        tip = exp.get("checkpoint_tip")
+
+        parent_rev = exp.get("checkpoint_parent", "")
+        parent_exp = experiments.get(parent_rev, {})
+        parent_tip = parent_exp.get("checkpoint_tip")
+
+        parent = ""
+        if is_baseline:
+            typ = "baseline"
+        elif tip:
+            if tip == parent_tip:
+                typ = (
+                    "checkpoint_tip" if new_checkpoint else "checkpoint_commit"
+                )
+            elif parent_rev == base_rev:
+                typ = "checkpoint_base"
             else:
-                if i < len(experiments) - 1:
-                    tree = "├──"
-                else:
-                    tree = "└──"
-                new_checkpoint = True
-            name = exp.get("name", rev[:7])
-            row.append(f"{tree} {queued}{name}{parent}")
+                typ = "checkpoint_commit"
+                parent = parent_rev[:7]
+        elif i < len(experiments) - 1:
+            typ = "branch_commit"
+        else:
+            typ = "branch_base"
 
-        if not no_timestamp:
-            row.append(_format_time(exp.get("timestamp")))
+        if not is_baseline:
+            new_checkpoint = not (tip and tip == parent_tip)
 
+        row = [
+            exp_name,
+            name_rev,
+            queued,
+            typ,
+            _format_time(exp.get("timestamp")),
+            parent,
+        ]
         _extend_row(
             row, metric_names, exp.get("metrics", {}).items(), precision
         )
         _extend_row(row, param_names, exp.get("params", {}).items(), precision)
 
-        yield row, style
+        yield row
 
 
 def _sort_column(sort_by, metric_names, param_names):
@@ -233,7 +241,7 @@ def _sort_exp(experiments, sort_path, sort_name, typ, reverse):
             return _sort((tip, experiments[tip]))
         data = exp.get(typ, {}).get(sort_path, {})
         val = flatten(data).get(sort_name)
-        return (val is None, val)
+        return val is None, val
 
     ret = OrderedDict()
     if "baseline" in experiments:
@@ -245,7 +253,7 @@ def _sort_exp(experiments, sort_path, sort_name, typ, reverse):
 
 def _format_time(timestamp):
     if timestamp is None:
-        return "-"
+        return FILL_VALUE
     if timestamp.date() == date.today():
         fmt = "%I:%M %p"
     else:
@@ -253,42 +261,23 @@ def _format_time(timestamp):
     return timestamp.strftime(fmt)
 
 
-def _format_field(val, precision=DEFAULT_PRECISION):
-    if isinstance(val, float):
-        fmt = f"{{:.{precision}g}}"
-        return fmt.format(val)
-    if isinstance(val, Mapping):
-        return {k: _format_field(v) for k, v in val.items()}
-    if isinstance(val, list):
-        return [_format_field(x) for x in val]
-    return str(val)
-
-
 def _extend_row(row, names, items, precision):
     from rich.text import Text
 
+    from dvc.compare import _format_field, with_value
+
     if not items:
-        row.extend(["-"] * len(names))
+        row.extend(FILL_VALUE for keys in names.values() for _ in keys)
         return
 
     for fname, item in items:
-        if isinstance(item, dict):
-            item = flatten(item)
-        else:
-            item = {fname: item}
+        item = flatten(item) if isinstance(item, dict) else {fname: item}
         for name in names[fname]:
-            if name in item:
-                value = item[name]
-                if value is None:
-                    text = "-"
-                else:
-                    # wrap field data in rich.Text, otherwise rich may
-                    # interpret unescaped braces from list/dict types as rich
-                    # markup tags
-                    text = Text(str(_format_field(value, precision)))
-                row.append(text)
-            else:
-                row.append("-")
+            value = with_value(item.get(name), FILL_VALUE)
+            # wrap field data in rich.Text, otherwise rich may
+            # interpret unescaped braces from list/dict types as rich
+            # markup tags
+            row.append(Text(str(_format_field(value, precision))))
 
 
 def _parse_filter_list(param_list):
@@ -302,9 +291,68 @@ def _parse_filter_list(param_list):
     return ret
 
 
-def _experiments_table(all_experiments, **kwargs):
-    from dvc.utils.table import Table
+def experiments_table(
+    all_experiments,
+    metric_headers,
+    metric_names,
+    param_headers,
+    param_names,
+    sort_by=None,
+    sort_order=None,
+    precision=DEFAULT_PRECISION,
+) -> "TabularData":
+    from funcy import lconcat
 
+    from dvc.compare import TabularData
+
+    headers = ["Experiment", "rev", "queued", "typ", "Created", "parent"]
+    td = TabularData(
+        lconcat(headers, metric_headers, param_headers), fill_value=FILL_VALUE
+    )
+    for base_rev, experiments in all_experiments.items():
+        rows = _collect_rows(
+            base_rev,
+            experiments,
+            metric_names,
+            param_names,
+            sort_by=sort_by,
+            sort_order=sort_order,
+            precision=precision,
+        )
+        td.extend(rows)
+
+    return td
+
+
+def prepare_exp_id(kwargs) -> "Text":
+    from rich.text import Text
+
+    exp_name = kwargs["Experiment"]
+    rev = kwargs["rev"]
+    typ = kwargs.get("typ", "baseline")
+
+    if typ == "baseline" or not exp_name:
+        text = Text(exp_name or rev)
+    else:
+        text = Text.assemble(rev, " [", (exp_name, "bold"), "]")
+
+    parent = kwargs.get("parent")
+    suff = f" ({parent})" if parent else ""
+    text.append(suff)
+
+    tree = experiment_types[typ]
+    queued = "*" if kwargs.get("queued") else ""
+    pref = (f"{tree} " if tree else "") + queued
+    return Text(pref) + text
+
+
+def baseline_styler(typ):
+    return {"style": "bold"} if typ == "baseline" else {}
+
+
+def show_experiments(
+    all_experiments, pager=True, no_timestamp=False, **kwargs
+):
     include_metrics = _parse_filter_list(kwargs.pop("include_metrics", []))
     exclude_metrics = _parse_filter_list(kwargs.pop("exclude_metrics", []))
     include_params = _parse_filter_list(kwargs.pop("include_params", []))
@@ -317,44 +365,66 @@ def _experiments_table(all_experiments, **kwargs):
         include_params=include_params,
         exclude_params=exclude_params,
     )
+    metric_headers = _normalize_headers(metric_names)
+    param_headers = _normalize_headers(param_names)
 
-    table = Table()
-    table.add_column(
-        "Experiment", no_wrap=True, header_style="black on grey93"
-    )
-    if not kwargs.get("no_timestamp", False):
-        table.add_column("Created", header_style="black on grey93")
-    _add_data_columns(
-        table,
+    td = experiments_table(
+        all_experiments,
+        metric_headers,
         metric_names,
-        justify="right",
-        no_wrap=True,
-        header_style="black on cornsilk1",
-    )
-    _add_data_columns(
-        table, param_names, justify="left", header_style="black on light_cyan1"
+        param_headers,
+        param_names,
+        kwargs.get("sort_by"),
+        kwargs.get("sort_order"),
+        kwargs.get("precision"),
     )
 
-    for base_rev, experiments in all_experiments.items():
-        for row, _, in _collect_rows(
-            base_rev, experiments, metric_names, param_names, **kwargs,
-        ):
-            table.add_row(*row)
+    if no_timestamp:
+        td.drop("Created")
 
-    return table
+    row_styles = lmap(baseline_styler, td.column("typ"))
+
+    merge_headers = ["Experiment", "rev", "queued", "typ", "parent"]
+    td.column("Experiment")[:] = map(prepare_exp_id, td.as_dict(merge_headers))
+    td.drop(*merge_headers[1:])
+
+    headers = {"metrics": metric_headers, "params": param_headers}
+    styles = {
+        "Experiment": {"no_wrap": True, "header_style": "black on grey93"},
+        "Created": {"header_style": "black on grey93"},
+    }
+    header_bg_colors = {"metrics": "cornsilk1", "params": "light_cyan1"}
+    styles.update(
+        {
+            header: {
+                "justify": "right" if typ == "metrics" else "left",
+                "header_style": f"black on {header_bg_colors[typ]}",
+                "collapse": idx != 0,
+                "no_wrap": typ == "metrics",
+            }
+            for typ, hs in headers.items()
+            for idx, header in enumerate(hs)
+        }
+    )
+
+    td.render(
+        pager=pager,
+        borders=True,
+        rich_table=True,
+        header_styles=styles,
+        row_styles=row_styles,
+    )
 
 
-def _add_data_columns(table, names, **kwargs):
+def _normalize_headers(names):
     count = Counter(
         name for path in names for name in names[path] for path in names
     )
-    first = True
-    for path in names:
-        for name in names[path]:
-            col_name = name if count[name] == 1 else f"{path}:{name}"
-            kwargs["collapse"] = False if first else True
-            table.add_column(col_name, **kwargs)
-            first = False
+    return [
+        name if count[name] == 1 else f"{path}:{name}"
+        for path in names
+        for name in names[path]
+    ]
 
 
 def _format_json(item):
@@ -365,8 +435,6 @@ def _format_json(item):
 
 class CmdExperimentsShow(CmdBase):
     def run(self):
-        from rich.console import Console
-
         try:
             all_experiments = self.repo.experiments.show(
                 all_branches=self.args.all_branches,
@@ -374,20 +442,18 @@ class CmdExperimentsShow(CmdBase):
                 all_commits=self.args.all_commits,
                 sha_only=self.args.sha,
                 num=self.args.num,
+                param_deps=self.args.param_deps,
             )
+        except DvcException:
+            logger.exception("failed to show experiments")
+            return 1
 
-            if self.args.show_json:
-                import json
+        if self.args.show_json:
+            import json
 
-                logger.info(json.dumps(all_experiments, default=_format_json))
-                return 0
-
-            if self.args.precision is None:
-                precision = DEFAULT_PRECISION
-            else:
-                precision = self.args.precision
-
-            table = _experiments_table(
+            ui.write(json.dumps(all_experiments, default=_format_json))
+        else:
+            show_experiments(
                 all_experiments,
                 include_metrics=self.args.include_metrics,
                 exclude_metrics=self.args.exclude_metrics,
@@ -396,29 +462,9 @@ class CmdExperimentsShow(CmdBase):
                 no_timestamp=self.args.no_timestamp,
                 sort_by=self.args.sort_by,
                 sort_order=self.args.sort_order,
-                precision=precision,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                pager=not self.args.no_pager,
             )
-
-            console = Console()
-            if self.args.no_pager:
-                console.print(table)
-            else:
-                from dvc.utils.pager import DvcPager
-
-                # NOTE: rich does not have native support for unlimited width
-                # via pager. we override rich table compression by setting
-                # console width to the full width of the table
-                measurement = table.__rich_measure__(console, SHOW_MAX_WIDTH)
-                console._width = (  # pylint: disable=protected-access
-                    measurement.maximum
-                )
-                with console.pager(pager=DvcPager(), styles=True):
-                    console.print(table)
-
-        except DvcException:
-            logger.exception("failed to show experiments")
-            return 1
-
         return 0
 
 
@@ -432,43 +478,6 @@ class CmdExperimentsApply(CmdBase):
         return 0
 
 
-def _show_diff(
-    diff,
-    title="",
-    markdown=False,
-    no_path=False,
-    old=False,
-    precision=DEFAULT_PRECISION,
-):
-    from dvc.utils.diff import table
-
-    rows = []
-    for fname, diff_ in diff.items():
-        sorted_diff = OrderedDict(sorted(diff_.items()))
-        for item, change in sorted_diff.items():
-            row = [] if no_path else [fname]
-            row.append(item)
-            if old:
-                row.append(_format_field(change.get("old"), precision))
-            row.append(_format_field(change["new"], precision))
-            row.append(
-                _format_field(
-                    change.get("diff", "diff not supported"), precision
-                )
-            )
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append(title)
-    if old:
-        header.extend(["Old", "New"])
-    else:
-        header.append("Value")
-    header.append("Change")
-
-    return table(header, rows, markdown)
-
-
 class CmdExperimentsDiff(CmdBase):
     def run(self):
 
@@ -477,45 +486,57 @@ class CmdExperimentsDiff(CmdBase):
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
                 all=self.args.all,
+                param_deps=self.args.param_deps,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                if self.args.precision is None:
-                    precision = DEFAULT_PRECISION
-                else:
-                    precision = self.args.precision
-
-                diffs = [("metrics", "Metric"), ("params", "Param")]
-                for key, title in diffs:
-                    table = _show_diff(
-                        diff[key],
-                        title=title,
-                        markdown=self.args.show_md,
-                        no_path=self.args.no_path,
-                        old=self.args.old,
-                        precision=precision,
-                    )
-                    if table:
-                        logger.info(table)
-                        logger.info("")
-
         except DvcException:
             logger.exception("failed to show experiments diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            ui.write(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            precision = self.args.precision or DEFAULT_PRECISION
+            diffs = [("metrics", "Metric"), ("params", "Param")]
+            for idx, (key, title) in enumerate(diffs):
+                if idx:
+                    # we are printing tables even in `--quiet` mode
+                    # so we should also be printing the "table" separator
+                    ui.write(force=True)
+
+                show_diff(
+                    diff[key],
+                    title=title,
+                    markdown=self.args.show_md,
+                    no_path=self.args.no_path,
+                    old=self.args.old,
+                    on_empty_diff="diff not supported",
+                    precision=precision if key == "metrics" else None,
+                )
+
         return 0
 
 
 class CmdExperimentsRun(CmdRepro):
     def run(self):
-        from dvc.command.metrics import _show_metrics
+        from dvc.compare import show_metrics
+
+        if self.args.checkpoint_resume:
+            if self.args.reset:
+                raise InvalidArgumentError(
+                    "--reset and --rev are mutually exclusive."
+                )
+            if not (self.args.queue or self.args.tmp_dir):
+                raise InvalidArgumentError(
+                    "--rev can only be used in conjunction with "
+                    "--queue or --temp."
+                )
 
         if self.args.reset:
-            logger.info("Any existing checkpoints will be reset and re-run.")
+            ui.write("Any existing checkpoints will be reset and re-run.")
 
         results = self.repo.experiments.run(
             name=self.args.name,
@@ -532,7 +553,7 @@ class CmdExperimentsRun(CmdRepro):
         if self.args.metrics and results:
             metrics = self.repo.metrics.show(revs=list(results))
             metrics.pop("workspace", None)
-            logger.info(_show_metrics(metrics))
+            show_metrics(metrics)
 
         return 0
 
@@ -586,12 +607,13 @@ class CmdExperimentsGC(CmdRepro):
         )
 
         if removed:
-            logger.info(
-                f"Removed {removed} experiments. To remove unused cache files "
-                "use 'dvc gc'."
+            ui.write(
+                f"Removed {removed} experiments.",
+                "To remove unused cache files",
+                "use 'dvc gc'.",
             )
         else:
-            logger.info("No experiments to remove.")
+            ui.write("No experiments to remove.")
         return 0
 
 
@@ -640,15 +662,15 @@ class CmdExperimentsPush(CmdBase):
             run_cache=self.args.run_cache,
         )
 
-        logger.info(
-            "Pushed experiment '%s' to Git remote '%s'.",
-            self.args.experiment,
-            self.args.git_remote,
+        ui.write(
+            f"Pushed experiment '{self.args.experiment}'"
+            f"to Git remote '{self.args.git_remote}'."
         )
         if not self.args.push_cache:
-            logger.info(
-                "To push cached outputs for this experiment to DVC remote "
-                "storage, re-run this command without '--no-cache'."
+            ui.write(
+                "To push cached outputs",
+                "for this experiment to DVC remote storage,"
+                "re-run this command without '--no-cache'.",
             )
 
         return 0
@@ -667,15 +689,15 @@ class CmdExperimentsPull(CmdBase):
             run_cache=self.args.run_cache,
         )
 
-        logger.info(
-            "Pulled experiment '%s' from Git remote '%s'. ",
-            self.args.experiment,
-            self.args.git_remote,
+        ui.write(
+            f"Pulled experiment '{self.args.experiment}'",
+            f"from Git remote '{self.args.git_remote}'.",
         )
         if not self.args.pull_cache:
-            logger.info(
-                "To pull cached outputs for this experiment from DVC remote "
-                "storage, re-run this command without '--no-cache'."
+            ui.write(
+                "To pull cached outputs for this experiment"
+                "from DVC remote storage,"
+                "re-run this command without '--no-cache'."
             )
 
         return 0
@@ -685,7 +707,7 @@ class CmdExperimentsRemove(CmdBase):
     def run(self):
 
         self.repo.experiments.remove(
-            exp_names=self.args.experiment, queue=self.args.queue,
+            exp_names=self.args.experiment, queue=self.args.queue
         )
 
         return 0
@@ -783,6 +805,12 @@ def add_parser(subparsers, parent_parser):
         help="Exclude the specified params from output table.",
         metavar="<params_list>",
     )
+    experiments_show_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     experiments_show_parser.add_argument(
         "--sort-by",
         help="Sort related experiments by the specified metric or param.",
@@ -840,7 +868,7 @@ def add_parser(subparsers, parent_parser):
         help="Fail if this command would overwrite conflicting changes.",
     )
     experiments_apply_parser.add_argument(
-        "experiment", help="Experiment to be applied.",
+        "experiment", help="Experiment to be applied."
     ).complete = completion.EXPERIMENT
     experiments_apply_parser.set_defaults(func=CmdExperimentsApply)
 
@@ -868,6 +896,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show unchanged metrics/params as well.",
     )
+    experiments_diff_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     experiments_diff_parser.add_argument(
         "--show-json",
         action="store_true",
@@ -903,9 +937,7 @@ def add_parser(subparsers, parent_parser):
     )
     experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)
 
-    EXPERIMENTS_RUN_HELP = (
-        "Reproduce complete or partial experiment pipelines."
-    )
+    EXPERIMENTS_RUN_HELP = "Run or resume an experiment."
     experiments_run_parser = experiments_subparsers.add_parser(
         "run",
         parents=[parent_parser],
@@ -920,9 +952,8 @@ def add_parser(subparsers, parent_parser):
         type=str,
         dest="checkpoint_resume",
         help=(
-            "Continue the specified checkpoint experiment. "
-            "(Only required for explicitly resuming checkpoints in queued "
-            "or temp dir runs.)"
+            "Continue the specified checkpoint experiment. Can only be used "
+            "in conjunction with --queue or --temp."
         ),
         metavar="<experiment_rev>",
     ).complete = completion.EXPERIMENT
@@ -967,6 +998,7 @@ def add_parser(subparsers, parent_parser):
         help="Keep experiments derived from all Git tags.",
     )
     experiments_gc_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -999,10 +1031,10 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     experiments_branch_parser.add_argument(
-        "experiment", help="Experiment to be promoted.",
+        "experiment", help="Experiment to be promoted."
     )
     experiments_branch_parser.add_argument(
-        "branch", help="Git branch name to use.",
+        "branch", help="Git branch name to use."
     )
     experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)
 
@@ -1025,7 +1057,7 @@ def add_parser(subparsers, parent_parser):
         metavar="<rev>",
     )
     experiments_list_parser.add_argument(
-        "--all", action="store_true", help="List all experiments.",
+        "--all", action="store_true", help="List all experiments."
     )
     experiments_list_parser.add_argument(
         "--names-only",
@@ -1097,7 +1129,7 @@ def add_parser(subparsers, parent_parser):
         metavar="<git_remote>",
     )
     experiments_push_parser.add_argument(
-        "experiment", help="Experiment to push.", metavar="<experiment>",
+        "experiment", help="Experiment to push.", metavar="<experiment>"
     ).complete = completion.EXPERIMENT
     experiments_push_parser.set_defaults(func=CmdExperimentsPush)
 
@@ -1153,7 +1185,7 @@ def add_parser(subparsers, parent_parser):
         metavar="<git_remote>",
     )
     experiments_pull_parser.add_argument(
-        "experiment", help="Experiment to pull.", metavar="<experiment>",
+        "experiment", help="Experiment to pull.", metavar="<experiment>"
     )
     experiments_pull_parser.set_defaults(func=CmdExperimentsPull)
 
@@ -1166,7 +1198,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     experiments_remove_parser.add_argument(
-        "--queue", action="store_true", help="Remove all queued experiments.",
+        "--queue", action="store_true", help="Remove all queued experiments."
     )
     experiments_remove_parser.add_argument(
         "experiment",
@@ -1215,6 +1247,7 @@ def _add_run_common(parser):
         "-j",
         "--jobs",
         type=int,
+        default=1,
         help="Run the specified number of experiments at a time in parallel.",
         metavar="<number>",
     )

dvc/command/freeze.py

@@ -40,7 +40,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     freeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to freeze",
+        "targets", nargs="+", help="Stages or .dvc files to freeze"
     ).complete = completion.DVC_FILE
     freeze_parser.set_defaults(func=CmdFreeze)
 
@@ -53,6 +53,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unfreeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to unfreeze",
+        "targets", nargs="+", help="Stages or .dvc files to unfreeze"
     ).complete = completion.DVC_FILE
     unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/gc.py

@@ -2,8 +2,8 @@ import argparse
 import logging
 import os
 
-import dvc.prompt as prompt
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -44,7 +44,7 @@ class CmdGC(CmdBase):
         logger.warning(msg)
 
         msg = "Are you sure you want to proceed?"
-        if not self.args.force and not prompt.confirm(msg):
+        if not self.args.force and not ui.confirm(msg):
             return 1
 
         self.repo.gc(
@@ -97,6 +97,7 @@ def add_parser(subparsers, parent_parser):
         help="Keep data files for all Git tags.",
     )
     gc_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,

dvc/command/get.py

@@ -12,12 +12,13 @@ logger = logging.getLogger(__name__)
 class CmdGet(CmdBaseNoRepo):
     def _show_url(self):
         from dvc.api import get_url
+        from dvc.ui import ui
 
         try:
             url = get_url(
                 self.args.path, repo=self.args.url, rev=self.args.rev
             )
-            logger.info(url)
+            ui.write(url, force=True)
         except DvcException:
             logger.exception("failed to show URL")
             return 1
@@ -64,7 +65,7 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path", help="Path to a file or directory within the repository",
+        "path", help="Path to a file or directory within the repository"
     ).complete = completion.FILE
     get_parser.add_argument(
         "-o",

dvc/command/get_url.py

@@ -34,7 +34,7 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out", nargs="?", help="Destination path to put data to.",
+        "out", nargs="?", help="Destination path to put data to."
     ).complete = completion.DIR
     get_parser.add_argument(
         "-j",

dvc/command/git_hook.py

@@ -105,7 +105,7 @@ def add_parser(subparsers, parent_parser):
         help=PRE_COMMIT_HELP,
     )
     pre_commit_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
     )
     pre_commit_parser.set_defaults(func=CmdPreCommit)
 
@@ -117,7 +117,7 @@ def add_parser(subparsers, parent_parser):
         help=POST_CHECKOUT_HELP,
     )
     post_checkout_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
     )
     post_checkout_parser.set_defaults(func=CmdPostCheckout)
 
@@ -129,7 +129,7 @@ def add_parser(subparsers, parent_parser):
         help=PRE_PUSH_HELP,
     )
     pre_push_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
     )
     pre_push_parser.set_defaults(func=CmdPrePush)
 
@@ -146,9 +146,7 @@ def add_parser(subparsers, parent_parser):
         help="Ancestor's version of the conflicting file.",
     )
     merge_driver_parser.add_argument(
-        "--our",
-        required=True,
-        help="Current version of the conflicting file.",
+        "--our", required=True, help="Current version of the conflicting file."
     )
     merge_driver_parser.add_argument(
         "--their",

dvc/command/imp.py

@@ -48,7 +48,7 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path", help="Path to a file or directory within the repository",
+        "path", help="Path to a file or directory within the repository"
     ).complete = completion.FILE
     import_parser.add_argument(
         "-o",

dvc/command/imp_url.py

@@ -58,7 +58,7 @@ def add_parser(subparsers, parent_parser):
         "remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to.",
+        "out", nargs="?", help="Destination path to put files to."
     ).complete = completion.DIR
     import_parser.add_argument(
         "--file",

dvc/command/init.py

@@ -12,8 +12,10 @@ logger = logging.getLogger(__name__)
 
 
 def _welcome_message():
+    from dvc.ui import ui
+
     if analytics.is_enabled():
-        logger.info(
+        ui.write(
             boxify(
                 "DVC has enabled anonymous aggregate usage analytics.\n"
                 "Read the analytics documentation (and how to opt-out) here:\n"
@@ -30,7 +32,7 @@ def _welcome_message():
         f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
     ).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)
 
-    logger.info(msg)
+    ui.write(msg)
 
 
 class CmdInit(CmdBaseNoRepo):
@@ -39,14 +41,14 @@ class CmdInit(CmdBaseNoRepo):
         from dvc.repo import Repo
 
         try:
-            self.repo = Repo.init(
+            with Repo.init(
                 ".",
                 no_scm=self.args.no_scm,
                 force=self.args.force,
                 subdir=self.args.subdir,
-            )
-            self.config = self.repo.config
-            _welcome_message()
+            ) as repo:
+                self.config = repo.config
+                _welcome_message()
         except InitError:
             logger.exception("failed to initiate DVC")
             return 1

dvc/command/live.py

@@ -1,12 +1,9 @@
 import argparse
-import logging
 import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, fix_subparsers
-from dvc.utils.html import write
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 class CmdLive(CmdBase):
@@ -16,9 +13,9 @@ class CmdLive(CmdBase):
         metrics, plots = self.repo.live.show(target=target, revs=revs)
 
         html_path = self.args.target + ".html"
-        write(html_path, plots, metrics)
+        self.repo.plots.write_html(html_path, plots, metrics)
 
-        logger.info(f"\nfile://{os.path.abspath(html_path)}")
+        ui.write("\nfile://", os.path.abspath(html_path), sep="")
 
         return 0
 
@@ -36,7 +33,7 @@ class CmdLiveDiff(CmdLive):
 def shared_parent_parser():
     parent_parser = argparse.ArgumentParser(add_help=False)
     parent_parser.add_argument(
-        "target", help="Logs dir to produce summary from",
+        "target", help="Logs dir to produce summary from"
     ).complete = completion.DIR
     parent_parser.add_argument(
         "-o",

dvc/command/ls/__init__.py

@@ -1,11 +1,11 @@
 import argparse
 import logging
-import sys
 
 from dvc.command import completion
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.command.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -34,9 +34,13 @@ class CmdList(CmdBaseNoRepo):
                 recursive=self.args.recursive,
                 dvc_only=self.args.dvc_only,
             )
-            if entries:
-                entries = _prettify(entries, sys.stdout.isatty())
-                logger.info("\n".join(entries))
+            if self.args.show_json:
+                import json
+
+                ui.write(json.dumps(entries))
+            elif entries:
+                entries = _prettify(entries, with_color=True)
+                ui.write("\n".join(entries))
             return 0
         except DvcException:
             logger.exception(f"failed to list '{self.args.url}'")
@@ -65,6 +69,9 @@ def add_parser(subparsers, parent_parser):
     list_parser.add_argument(
         "--dvc-only", action="store_true", help="Show only DVC outputs."
     )
+    list_parser.add_argument(
+        "--show-json", action="store_true", help="Show output in JSON format."
+    )
     list_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/metrics.py

@@ -4,6 +4,7 @@ import logging
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import BadMetricError, DvcException
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -11,64 +12,6 @@ logger = logging.getLogger(__name__)
 DEFAULT_PRECISION = 5
 
 
-def _show_metrics(
-    metrics,
-    markdown=False,
-    all_branches=False,
-    all_tags=False,
-    all_commits=False,
-    precision=None,
-):
-    from dvc.utils.diff import format_dict, table
-    from dvc.utils.flatten import flatten
-
-    # When `metrics` contains a `None` key, it means that some files
-    # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
-    missing = metrics.pop(None, None)
-    with_rev = any([all_branches, all_tags, all_commits])
-    header_set = set()
-    rows = []
-
-    if precision is None:
-        precision = DEFAULT_PRECISION
-
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
-        return val
-
-    for _branch, val in metrics.items():
-        for _fname, metric in val.items():
-            if not isinstance(metric, dict):
-                header_set.add("")
-                continue
-            for key, _val in flatten(format_dict(metric)).items():
-                header_set.add(key)
-    header = sorted(header_set)
-    for branch, val in metrics.items():
-        for fname, metric in val.items():
-            row = []
-            if with_rev:
-                row.append(branch)
-            row.append(fname)
-            if not isinstance(metric, dict):
-                row.append(str(metric))
-                rows.append(row)
-                continue
-            flattened_val = flatten(format_dict(metric))
-
-            for i in header:
-                row.append(_round(flattened_val.get(i)))
-            rows.append(row)
-    header.insert(0, "Path")
-    if with_rev:
-        header.insert(0, "Revision")
-
-    if missing:
-        raise BadMetricError(missing)
-    return table(header, rows, markdown)
-
-
 class CmdMetricsBase(CmdBase):
     UNINITIALIZED = True
 
@@ -83,61 +26,37 @@ class CmdMetricsShow(CmdMetricsBase):
                 all_commits=self.args.all_commits,
                 recursive=self.args.recursive,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(metrics))
-            else:
-                table = _show_metrics(
-                    metrics,
-                    self.args.show_md,
-                    self.args.all_branches,
-                    self.args.all_tags,
-                    self.args.all_commits,
-                )
-                if table:
-                    logger.info(table)
         except DvcException:
             logger.exception("")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            ui.write(json.dumps(metrics))
+        else:
+            from dvc.compare import show_metrics
+
+            # When `metrics` contains a `None` key, it means that some files
+            # specified as `targets` in `repo.metrics.show` didn't contain any
+            # metrics.
+            missing = metrics.pop(None, None)
+            if missing:
+                raise BadMetricError(missing)
+
+            show_metrics(
+                metrics,
+                markdown=self.args.show_md,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                round_digits=True,
+            )
+
         return 0
 
 
-def _show_diff(diff, markdown=False, no_path=False, precision=None):
-    from collections import OrderedDict
-
-    from dvc.utils.diff import table
-
-    if precision is None:
-        precision = DEFAULT_PRECISION
-
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
-
-        return val
-
-    rows = []
-    for fname, mdiff in diff.items():
-        sorted_mdiff = OrderedDict(sorted(mdiff.items()))
-        for metric, change in sorted_mdiff.items():
-            row = [] if no_path else [fname]
-            row.append(metric)
-            row.append(_round(change.get("old")))
-            row.append(_round(change["new"]))
-            row.append(_round(change.get("diff")))
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append("Metric")
-    header.extend(["Old", "New"])
-    header.append("Change")
-
-    return table(header, rows, markdown)
-
-
 class CmdMetricsDiff(CmdMetricsBase):
     def run(self):
         try:
@@ -148,25 +67,26 @@ class CmdMetricsDiff(CmdMetricsBase):
                 recursive=self.args.recursive,
                 all=self.args.all,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                table = _show_diff(
-                    diff,
-                    self.args.show_md,
-                    self.args.no_path,
-                    precision=self.args.precision,
-                )
-                if table:
-                    logger.info(table)
-
         except DvcException:
             logger.exception("failed to show metrics diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            ui.write(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            show_diff(
+                diff,
+                title="Metric",
+                markdown=self.args.show_md,
+                no_path=self.args.no_path,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                round_digits=True,
+            )
+
         return 0
 
 
@@ -219,6 +139,7 @@ def add_parser(subparsers, parent_parser):
         help="Show metrics for all tags.",
     )
     metrics_show_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,

dvc/command/move.py

@@ -37,9 +37,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src", help="Source path to a data file or directory.",
+        "src", help="Source path to a data file or directory."
     ).complete = completion.FILE
     move_parser.add_argument(
-        "dst", help="Destination path.",
+        "dst", help="Destination path."
     ).complete = completion.FILE
     move_parser.set_defaults(func=CmdMove)

dvc/command/params.py

@@ -1,35 +1,14 @@
 import argparse
 import logging
-from collections import OrderedDict
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
-def _show_diff(diff, markdown=False, no_path=False):
-    from dvc.utils.diff import table
-
-    rows = []
-    for fname, pdiff in diff.items():
-        sorted_pdiff = OrderedDict(sorted(pdiff.items()))
-        for param, change in sorted_pdiff.items():
-            row = [] if no_path else [fname]
-            row.append(param)
-            row.append(change["old"])
-            row.append(change["new"])
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append("Param")
-    header.append("Old")
-    header.append("New")
-
-    return table(header, rows, markdown)
-
-
 class CmdParamsDiff(CmdBase):
     UNINITIALIZED = True
 
@@ -40,21 +19,27 @@ class CmdParamsDiff(CmdBase):
                 b_rev=self.args.b_rev,
                 targets=self.args.targets,
                 all=self.args.all,
+                deps=self.args.deps,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                table = _show_diff(diff, self.args.show_md, self.args.no_path)
-                if table:
-                    logger.info(table)
-
         except DvcException:
             logger.exception("failed to show params diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            ui.write(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            show_diff(
+                diff,
+                title="Param",
+                markdown=self.args.show_md,
+                no_path=self.args.no_path,
+                show_changes=False,
+            )
+
         return 0
 
 
@@ -111,6 +96,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show unchanged params as well.",
     )
+    params_diff_parser.add_argument(
+        "--deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     params_diff_parser.add_argument(
         "--show-json",
         action="store_true",

dvc/command/plots.py

@@ -1,12 +1,11 @@
 import argparse
 import logging
-import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
+from dvc.ui import ui
 from dvc.utils import format_link
-from dvc.utils.html import write
 
 logger = logging.getLogger(__name__)
 
@@ -23,6 +22,8 @@ class CmdPlots(CmdBase):
         return {k: v for k, v in props.items() if v is not None}
 
     def run(self):
+        from pathlib import Path
+
         if self.args.show_vega:
             if not self.args.targets:
                 logger.error("please specify a target for `--show-vega`")
@@ -38,20 +39,32 @@ class CmdPlots(CmdBase):
 
             if self.args.show_vega:
                 target = self.args.targets[0]
-                logger.info(plots[target])
+                ui.write(plots[target])
                 return 0
 
-            path = self.args.out or "plots.html"
-            path = os.path.join(os.getcwd(), path)
-
-            write(path, plots)
-
-            logger.info(f"file://{path}")
+            rel: str = self.args.out or "plots.html"
+            path = (Path.cwd() / rel).resolve()
+            self.repo.plots.write_html(
+                path, plots=plots, html_template_path=self.args.html_template
+            )
 
         except DvcException:
             logger.exception("")
             return 1
 
+        assert path.is_absolute()  # as_uri throws ValueError if not absolute
+        url = path.as_uri()
+        ui.write(url)
+        if self.args.open:
+            import webbrowser
+
+            opened = webbrowser.open(rel)
+            if not opened:
+                ui.error_write(
+                    "Failed to open. Please try opening it manually."
+                )
+                return 1
+
         return 0
 
 
@@ -77,7 +90,7 @@ class CmdPlotsDiff(CmdPlots):
 class CmdPlotsModify(CmdPlots):
     def run(self):
         self.repo.plots.modify(
-            self.args.target, props=self._props(), unset=self.args.unset,
+            self.args.target, props=self._props(), unset=self.args.unset
         )
         return 0
 
@@ -85,7 +98,7 @@ class CmdPlotsModify(CmdPlots):
 def add_parser(subparsers, parent_parser):
     PLOTS_HELP = (
         "Commands to visualize and compare plot metrics in structured files "
-        "(JSON, YAML, CSV, TSV)"
+        "(JSON, YAML, CSV, TSV)."
     )
 
     plots_parser = subparsers.add_parser(
@@ -150,7 +163,7 @@ def add_parser(subparsers, parent_parser):
         help=argparse.SUPPRESS,
     )
     plots_diff_parser.add_argument(
-        "revisions", nargs="*", default=None, help="Git commits to plot from",
+        "revisions", nargs="*", default=None, help="Git commits to plot from"
     )
     _add_props_arguments(plots_diff_parser)
     _add_output_arguments(plots_diff_parser)
@@ -165,7 +178,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_modify_parser.add_argument(
-        "target", help="Metric file to set properties to",
+        "target", help="Metric file to set properties to"
     ).complete = completion.FILE
     _add_props_arguments(plots_modify_parser)
     plots_modify_parser.add_argument(
@@ -229,3 +242,15 @@ def _add_output_arguments(parser):
         default=False,
         help="Show output in Vega format.",
     )
+    parser.add_argument(
+        "--open",
+        action="store_true",
+        default=False,
+        help="Open plot file directly in the browser.",
+    )
+    parser.add_argument(
+        "--html-template",
+        default=None,
+        help="Custom HTML template for VEGA visualization.",
+        metavar="<path>",
+    )

dvc/command/remote.py

@@ -1,12 +1,10 @@
 import argparse
-import logging
 
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
+from dvc.ui import ui
 from dvc.utils import format_link
 
-logger = logging.getLogger(__name__)
-
 
 class CmdRemote(CmdConfig):
     def __init__(self, args):
@@ -27,7 +25,7 @@ class CmdRemoteAdd(CmdRemote):
         from dvc.config import ConfigError
 
         if self.args.default:
-            logger.info(f"Setting '{self.args.name}' as a default remote.")
+            ui.write(f"Setting '{self.args.name}' as a default remote.")
 
         with self.config.edit(self.args.level) as conf:
             if self.args.name in conf["remote"] and not self.args.force:
@@ -90,7 +88,7 @@ class CmdRemoteDefault(CmdRemote):
             try:
                 print(conf["core"]["remote"])
             except KeyError:
-                logger.info("No default remote set")
+                ui.write("No default remote set")
                 return 1
         else:
             with self.config.edit(self.args.level) as conf:
@@ -116,7 +114,7 @@ class CmdRemoteList(CmdRemote):
     def run(self):
         conf = self.config.read(self.args.level)
         for name, conf in conf["remote"].items():
-            logger.info("{}\t{}".format(name, conf["url"]))
+            ui.write(name, conf["url"], sep="\t")
         return 0
 
 

dvc/command/repro.py

@@ -1,27 +1,27 @@
 import argparse
-import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.metrics import _show_metrics
 from dvc.command.status import CmdDataStatus
 
-logger = logging.getLogger(__name__)
-
 
 class CmdRepro(CmdBase):
     def run(self):
+        from dvc.ui import ui
+
         stages = self.repo.reproduce(**self._repro_kwargs)
         if len(stages) == 0:
-            logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+            ui.write(CmdDataStatus.UP_TO_DATE_MSG)
         else:
-            logger.info(
+            ui.write(
                 "Use `dvc push` to send your updates to " "remote storage."
             )
 
         if self.args.metrics:
+            from dvc.compare import show_metrics
+
             metrics = self.repo.metrics.show()
-            logger.info(_show_metrics(metrics))
+            show_metrics(metrics)
 
         return 0
 

dvc/command/root.py

@@ -10,8 +10,9 @@ logger = logging.getLogger(__name__)
 class CmdRoot(CmdBaseNoRepo):
     def run(self):
         from dvc.repo import Repo
+        from dvc.ui import ui
 
-        logger.info(relpath(Repo.find_root()))
+        ui.write(relpath(Repo.find_root()))
         return 0
 
 

dvc/command/run.py

@@ -33,36 +33,17 @@ class CmdRun(CmdBase):
             )
             return 1
 
+        kwargs = vars(self.args)
+        kwargs.update(
+            {
+                "cmd": parse_cmd(self.args.cmd),
+                "fname": kwargs.pop("file"),
+                "no_exec": (self.args.no_exec or bool(self.args.checkpoints)),
+                "run_cache": not kwargs.pop("no_run_cache"),
+            }
+        )
         try:
-            self.repo.run(
-                cmd=parse_cmd(self.args.cmd),
-                outs=self.args.outs,
-                outs_no_cache=self.args.outs_no_cache,
-                metrics=self.args.metrics,
-                metrics_no_cache=self.args.metrics_no_cache,
-                plots=self.args.plots,
-                plots_no_cache=self.args.plots_no_cache,
-                live=self.args.live,
-                live_no_cache=self.args.live_no_cache,
-                live_no_summary=self.args.live_no_summary,
-                live_no_html=self.args.live_no_html,
-                deps=self.args.deps,
-                params=self.args.params,
-                fname=self.args.file,
-                wdir=self.args.wdir,
-                no_exec=(self.args.no_exec or bool(self.args.checkpoints)),
-                force=self.args.force,
-                run_cache=not self.args.no_run_cache,
-                no_commit=self.args.no_commit,
-                outs_persist=self.args.outs_persist,
-                outs_persist_no_cache=self.args.outs_persist_no_cache,
-                checkpoints=self.args.checkpoints,
-                always_changed=self.args.always_changed,
-                name=self.args.name,
-                single_stage=self.args.single_stage,
-                external=self.args.external,
-                desc=self.args.desc,
-            )
+            self.repo.run(**kwargs)
         except DvcException:
             logger.exception("")
             return 1
@@ -83,11 +64,9 @@ def add_parser(subparsers, parent_parser):
         help=RUN_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    run_parser.add_argument("-n", "--name", help="Stage name.")
     run_parser.add_argument(
-        "-n", "--name", help="Stage name.",
-    )
-    run_parser.add_argument(
-        "--file", metavar="<filename>", help=argparse.SUPPRESS,
+        "--file", metavar="<filename>", help=argparse.SUPPRESS
     )
     run_parser.add_argument(
         "--no-exec",

dvc/command/stage.py

@@ -9,7 +9,7 @@ from dvc.utils.cli_parse import parse_params
 from dvc.utils.humanize import truncate_text
 
 if TYPE_CHECKING:
-    from dvc.output.base import BaseOutput
+    from dvc.output import Output
     from dvc.stage import Stage
 
 logger = logging.getLogger(__name__)
@@ -18,7 +18,7 @@ MAX_TEXT_LENGTH = 80
 
 
 def generate_description(stage: "Stage") -> str:
-    def part_desc(outs: Iterable["BaseOutput"]) -> str:
+    def part_desc(outs: Iterable["Output"]) -> str:
         return ", ".join(out.def_path for out in outs)
 
     if not stage.deps and not stage.outs:
@@ -27,7 +27,7 @@ def generate_description(stage: "Stage") -> str:
     if not stage.outs and stage.deps:
         return "Depends on " + part_desc(stage.deps)
 
-    def is_plot_or_metric(out: "BaseOutput"):
+    def is_plot_or_metric(out: "Output"):
         return bool(out.plot) or bool(out.metric)
 
     desc: List[str] = []
@@ -75,16 +75,14 @@ class CmdStageList(CmdBase):
         # removing duplicates while maintaining order
         collected = chain.from_iterable(
             self.repo.stage.collect(
-                target=target,
-                recursive=self.args.recursive,
-                accept_group=True,
+                target=target, recursive=self.args.recursive, accept_group=True
             )
             for target in self.args.targets
         )
         return dict.fromkeys(collected).keys()
 
     def run(self):
-        from dvc.utils.diff import table
+        from dvc.ui import ui
 
         def log_error(relpath: str, exc: Exception):
             if self.args.fail:
@@ -98,8 +96,7 @@ class CmdStageList(CmdBase):
         names_only = self.args.names_only
 
         data = prepare_stages_data(stages, description=not names_only)
-        if data:
-            print(table(header=(), rows=data.items()))
+        ui.table(data.items())
 
         return 0
 
@@ -116,8 +113,11 @@ def parse_cmd(commands: List[str]) -> str:
     """
 
     def quote_argument(arg: str):
-        should_quote = " " in arg and '"' not in arg
-        return f'"{arg}"' if should_quote else arg
+        if not arg:
+            return '""'
+        if " " in arg and '"' not in arg:
+            return f'"{arg}"'
+        return arg
 
     if len(commands) < 2:
         return " ".join(commands)
@@ -202,7 +202,7 @@ def _add_common_args(parser):
         metavar="<path>",
     )
     parser.add_argument(
-        "--live", help="Declare output as dvclive.", metavar="<path>",
+        "--live", help="Declare output as dvclive.", metavar="<path>"
     )
     parser.add_argument(
         "--live-no-cache",
@@ -255,7 +255,8 @@ def _add_common_args(parser):
         "--checkpoints",
         action="append",
         default=[],
-        help=argparse.SUPPRESS,
+        help="Declare checkpoint output file or directory for 'dvc exp run'. "
+        "Not compatible with 'dvc repro'.",
         metavar="<filename>",
     ).complete = completion.FILE
     parser.add_argument(

dvc/command/status.py

@@ -2,6 +2,7 @@ import logging
 
 from dvc.command.data_sync import CmdDataBase
 from dvc.exceptions import DvcException
+from dvc.ui import ui
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
@@ -26,7 +27,7 @@ class CmdDataStatus(CmdDataBase):
         ind = indent * self.STATUS_INDENT
 
         if isinstance(status, str):
-            logger.info(f"{ind}{status}")
+            ui.write(f"{ind}{status}")
             return
 
         if isinstance(status, list):
@@ -38,25 +39,32 @@ class CmdDataStatus(CmdDataBase):
 
         for key, value in status.items():
             if isinstance(value, str):
-                logger.info("{}{}{}".format(ind, self._normalize(value), key))
+                ui.write("{}{}{}".format(ind, self._normalize(value), key))
             elif value:
-                logger.info(f"{ind}{key}:")
+                ui.write(f"{ind}{key}:")
                 self._show(value, indent + 1)
 
     def run(self):
+        from dvc.repo import lock_repo
+
         indent = 1 if self.args.cloud else 0
-        try:
-            st = self.repo.status(
-                targets=self.args.targets,
-                jobs=self.args.jobs,
-                cloud=self.args.cloud,
-                remote=self.args.remote,
-                all_branches=self.args.all_branches,
-                all_tags=self.args.all_tags,
-                all_commits=self.args.all_commits,
-                with_deps=self.args.with_deps,
-                recursive=self.args.recursive,
-            )
+
+        with lock_repo(self.repo):
+            try:
+                st = self.repo.status(
+                    targets=self.args.targets,
+                    jobs=self.args.jobs,
+                    cloud=self.args.cloud,
+                    remote=self.args.remote,
+                    all_branches=self.args.all_branches,
+                    all_tags=self.args.all_tags,
+                    all_commits=self.args.all_commits,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("")
+                return 1
 
             if self.args.quiet:
                 return bool(st)
@@ -64,20 +72,22 @@ class CmdDataStatus(CmdDataBase):
             if self.args.show_json:
                 import json
 
-                logger.info(json.dumps(st))
-            elif st:
+                ui.write(json.dumps(st))
+                return 0
+
+            if st:
                 self._show(st, indent)
-            elif not self.repo.stages:
-                logger.info(self.EMPTY_PROJECT_MSG)
+                return 0
+
+            # additional hints for the user
+            if not self.repo.stages:
+                ui.write(self.EMPTY_PROJECT_MSG)
             elif self.args.cloud or self.args.remote:
                 remote = self.args.remote or self.repo.config["core"].get(
                     "remote"
                 )
-                logger.info(self.IN_SYNC_MSG.format(remote=remote))
+                ui.write(self.IN_SYNC_MSG.format(remote=remote))
             else:
-                logger.info(self.UP_TO_DATE_MSG)
+                ui.write(self.UP_TO_DATE_MSG)
 
-        except DvcException:
-            logger.exception("")
-            return 1
         return 0

dvc/command/unprotect.py

@@ -23,7 +23,7 @@ class CmdUnprotect(CmdBase):
 def add_parser(subparsers, parent_parser):
     UNPROTECT_HELP = (
         "Unprotect tracked files or directories (when hardlinks or symlinks "
-        "have been enabled with `dvc config cache.type`)"
+        "have been enabled with `dvc config cache.type`)."
     )
     unprotect_parser = subparsers.add_parser(
         "unprotect",
@@ -33,6 +33,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets", nargs="+", help="Data files/directories to unprotect.",
+        "targets", nargs="+", help="Data files/directories to unprotect."
     ).complete = completion.FILE
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -36,7 +36,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets", nargs="+", help=".dvc files to update.",
+        "targets", nargs="+", help=".dvc files to update."
     ).complete = completion.DVC_FILE
     update_parser.add_argument(
         "--rev",

dvc/command/version.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -11,7 +12,7 @@ class CmdVersion(CmdBaseNoRepo):
         from dvc.info import get_dvc_info
 
         dvc_info = get_dvc_info()
-        logger.info(dvc_info)
+        ui.write(dvc_info, force=True)
         return 0
 
 

dvc/compare.py

@@ -0,0 +1,321 @@
+from collections import abc
+from itertools import chain, repeat, zip_longest
+from operator import itemgetter
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    ItemsView,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    MutableSequence,
+    Sequence,
+    Tuple,
+    Union,
+    overload,
+)
+
+from funcy import reraise
+
+if TYPE_CHECKING:
+    from dvc.ui.table import CellT
+
+
+class Column(List["CellT"]):
+    pass
+
+
+def with_value(value, default):
+    return default if value is None else value
+
+
+class TabularData(MutableSequence[Sequence["CellT"]]):
+    def __init__(self, columns: Sequence[str], fill_value: str = ""):
+        self._columns: Dict[str, Column] = {name: Column() for name in columns}
+        self._keys: List[str] = list(columns)
+        self._fill_value = fill_value
+
+    @property
+    def columns(self) -> List[Column]:
+        return list(map(self.column, self.keys()))
+
+    def column(self, name: str) -> Column:
+        return self._columns[name]
+
+    def items(self) -> ItemsView[str, Column]:
+        projection = {k: self.column(k) for k in self.keys()}
+        return projection.items()
+
+    def keys(self) -> List[str]:
+        return self._keys
+
+    def _iter_col_row(
+        self, row: Sequence["CellT"]
+    ) -> Iterator[Tuple["CellT", Column]]:
+        for val, col in zip_longest(row, self.columns):
+            if col is None:
+                break
+            yield with_value(val, self._fill_value), col
+
+    def append(self, value: Sequence["CellT"]) -> None:
+        for val, col in self._iter_col_row(value):
+            col.append(val)
+
+    def extend(self, values: Iterable[Sequence["CellT"]]) -> None:
+        for row in values:
+            self.append(row)
+
+    def insert(self, index: int, value: Sequence["CellT"]) -> None:
+        for val, col in self._iter_col_row(value):
+            col.insert(index, val)
+
+    def __iter__(self) -> Iterator[List["CellT"]]:
+        return map(list, zip(*self.columns))
+
+    def __getattr__(self, item: str) -> Column:
+        with reraise(KeyError, AttributeError):
+            return self.column(item)
+
+    def __getitem__(self, item: Union[int, slice]):
+        func = itemgetter(item)
+        it = map(func, self.columns)
+        if isinstance(item, slice):
+            it = map(list, zip(*it))
+        return list(it)
+
+    @overload
+    def __setitem__(self, item: int, value: Sequence["CellT"]) -> None:
+        ...
+
+    @overload
+    def __setitem__(
+        self, item: slice, value: Iterable[Sequence["CellT"]]
+    ) -> None:
+        ...
+
+    def __setitem__(self, item, value) -> None:
+        it = value
+        if isinstance(item, slice):
+            n = len(self.columns)
+            normalized_rows = (
+                chain(val, repeat(self._fill_value, n - len(val)))
+                for val in value
+            )
+            # we need to transpose those rows into columnar format
+            # as we work in terms of column-based arrays
+            it = zip(*normalized_rows)
+
+        for i, col in self._iter_col_row(it):
+            col[item] = i
+
+    def __delitem__(self, item: Union[int, slice]) -> None:
+        for col in self.columns:
+            del col[item]
+
+    def __len__(self) -> int:
+        return len(self.columns[0])
+
+    @property
+    def shape(self) -> Tuple[int, int]:
+        return len(self.columns), len(self)
+
+    def drop(self, *col_names: str) -> None:
+        for col_name in col_names:
+            self._keys.remove(col_name)
+            self._columns.pop(col_name)
+
+    def rename(self, from_col_name: str, to_col_name: str) -> None:
+        self._columns[to_col_name] = self._columns.pop(from_col_name)
+        self._keys[self._keys.index(from_col_name)] = to_col_name
+
+    def project(self, *col_names: str) -> None:
+        self.drop(*(set(self._keys) - set(col_names)))
+        self._keys = list(col_names)
+
+    def to_csv(self) -> str:
+        import csv
+        from io import StringIO
+
+        buff = StringIO()
+        writer = csv.writer(buff)
+        writer.writerow(self.keys())
+
+        for row in self:
+            writer.writerow(row)
+        return buff.getvalue()
+
+    def add_column(self, name: str) -> None:
+        self._columns[name] = Column([self._fill_value] * len(self))
+        self._keys.append(name)
+
+    def row_from_dict(self, d: Mapping[str, "CellT"]) -> None:
+        keys = self.keys()
+        for key in d:
+            if key not in keys:
+                self.add_column(key)
+
+        row: List["CellT"] = [
+            with_value(d.get(key), self._fill_value) for key in self.keys()
+        ]
+        self.append(row)
+
+    def render(self, **kwargs: Any):
+        from dvc.ui import ui
+
+        ui.table(self, headers=self.keys(), **kwargs)
+
+    def as_dict(
+        self, cols: Iterable[str] = None
+    ) -> Iterable[Dict[str, "CellT"]]:
+        keys = self.keys() if cols is None else set(cols)
+        return [
+            {k: self._columns[k][i] for k in keys} for i in range(len(self))
+        ]
+
+
+def _normalize_float(val: float, precision: int):
+    return f"{val:.{precision}g}"
+
+
+def _format_field(
+    val: Any, precision: int = None, round_digits: bool = False
+) -> str:
+    def _format(_val):
+        if isinstance(_val, float) and precision:
+            func = round if round_digits else _normalize_float
+            return func(_val, precision)
+        if isinstance(_val, abc.Mapping):
+            return {k: _format(v) for k, v in _val.items()}
+        if isinstance(_val, list):
+            return [_format(x) for x in _val]
+        return _val
+
+    return str(_format(val))
+
+
+def diff_table(
+    diff,
+    title: str,
+    old: bool = True,
+    no_path: bool = False,
+    show_changes: bool = True,
+    precision: int = None,
+    round_digits: bool = False,
+    on_empty_diff: str = None,
+) -> TabularData:
+    headers: List[str] = ["Path", title, "Old", "New", "Change"]
+    fill_value = "-"
+    td = TabularData(headers, fill_value=fill_value)
+
+    for fname, diff_in_file in diff.items():
+        for item, change in sorted(diff_in_file.items()):
+            old_value = with_value(change.get("old"), fill_value)
+            new_value = with_value(change.get("new"), fill_value)
+            diff_value = with_value(
+                change.get("diff", on_empty_diff), fill_value
+            )
+            td.append(
+                [
+                    fname,
+                    str(item),
+                    _format_field(old_value, precision, round_digits),
+                    _format_field(new_value, precision, round_digits),
+                    _format_field(diff_value, precision, round_digits),
+                ]
+            )
+
+    if no_path:
+        td.drop("Path")
+
+    if not show_changes:
+        td.drop("Change")
+
+    if not old:
+        td.drop("Old")
+        td.rename("New", "Value")
+
+    return td
+
+
+def show_diff(
+    diff,
+    title: str,
+    old: bool = True,
+    no_path: bool = False,
+    show_changes: bool = True,
+    precision: int = None,
+    round_digits: bool = False,
+    on_empty_diff: str = None,
+    markdown: bool = False,
+) -> None:
+    td = diff_table(
+        diff,
+        title=title,
+        old=old,
+        no_path=no_path,
+        show_changes=show_changes,
+        precision=precision,
+        round_digits=round_digits,
+        on_empty_diff=on_empty_diff,
+    )
+    td.render(markdown=markdown)
+
+
+def metrics_table(
+    metrics,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    all_commits: bool = False,
+    precision: int = None,
+    round_digits: bool = False,
+):
+    from dvc.utils.diff import format_dict
+    from dvc.utils.flatten import flatten
+
+    td = TabularData(["Revision", "Path"], fill_value="-")
+
+    for branch, val in metrics.items():
+        for fname, metric in val.items():
+            row_data: Dict[str, str] = {"Revision": branch, "Path": fname}
+            flattened = (
+                flatten(format_dict(metric))
+                if isinstance(metric, dict)
+                else {"": metric}
+            )
+            row_data.update(
+                {
+                    k: _format_field(v, precision, round_digits)
+                    for k, v in flattened.items()
+                }
+            )
+            td.row_from_dict(row_data)
+
+    rev, path, *metrics_headers = td.keys()
+    td.project(rev, path, *sorted(metrics_headers))
+
+    if not any([all_branches, all_tags, all_commits]):
+        td.drop("Revision")
+
+    return td
+
+
+def show_metrics(
+    metrics,
+    markdown: bool = False,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    all_commits: bool = False,
+    precision: int = None,
+    round_digits: bool = False,
+) -> None:
+    td = metrics_table(
+        metrics,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        all_commits=all_commits,
+        precision=precision,
+        round_digits=round_digits,
+    )
+    td.render(markdown=markdown)

dvc/config.py

@@ -63,7 +63,7 @@ class Config(dict):
     CONFIG_LOCAL = "config.local"
 
     def __init__(
-        self, dvc_dir=None, validate=True, fs=None, config=None,
+        self, dvc_dir=None, validate=True, fs=None, config=None
     ):  # pylint: disable=super-init-not-called
         from dvc.fs.local import LocalFileSystem
 
@@ -79,7 +79,7 @@ class Config(dict):
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
-        self.wfs = LocalFileSystem(None, {"url": self.dvc_dir})
+        self.wfs = LocalFileSystem(url=self.dvc_dir)
         self.fs = fs or self.wfs
 
         self.load(validate=validate, config=config)
@@ -154,7 +154,7 @@ class Config(dict):
         filename = self.files[level]
         fs = self._get_fs(level)
 
-        if fs.exists(filename, use_dvcignore=False):
+        if fs.exists(filename):
             with fs.open(filename) as fobj:
                 conf_obj = ConfigObj(fobj)
         else:

dvc/config_schema.py

@@ -93,7 +93,7 @@ HTTP_COMMON = {
     "user": str,
     "password": str,
     "ask_password": Bool,
-    "ssl_verify": Bool,
+    "ssl_verify": Any(Bool, str),
     "method": str,
 }
 WEBDAV_COMMON = {
@@ -104,6 +104,7 @@ WEBDAV_COMMON = {
     "cert_path": str,
     "key_path": str,
     "timeout": Coerce(int),
+    "ssl_verify": Any(Bool, str),
 }
 
 SCHEMA = {
@@ -145,6 +146,7 @@ SCHEMA = {
                     "session_token": str,
                     Optional("listobjects", default=False): Bool,  # obsoleted
                     Optional("use_ssl", default=True): Bool,
+                    "ssl_verify": Any(Bool, str),
                     "sse": str,
                     "sse_kms_key_id": str,
                     "acl": str,
@@ -171,7 +173,7 @@ SCHEMA = {
                     "allow_agent": Bool,
                     **REMOTE_COMMON,
                 },
-                "hdfs": {"user": str, **REMOTE_COMMON},
+                "hdfs": {"user": str, "kerb_ticket": str, **REMOTE_COMMON},
                 "webhdfs": {
                     "hdfscli_config": str,
                     "webhdfs_token": str,
@@ -187,6 +189,7 @@ SCHEMA = {
                     "tenant_id": str,
                     "client_id": str,
                     "client_secret": str,
+                    "allow_anonymous_login": bool,
                     **REMOTE_COMMON,
                 },
                 "oss": {
@@ -222,4 +225,5 @@ SCHEMA = {
         # enabled by default. It's of no use, kept for backward compatibility.
         Optional("parametrization", default=True): Bool
     },
+    "plots": {"html_template": str},
 }

dvc/daemon.py

@@ -33,7 +33,7 @@ def _spawn_windows(cmd, env):
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
     _popen(
-        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo,
+        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo
     ).communicate()
 
 

dvc/dagascii.py

@@ -3,52 +3,14 @@
 import logging
 import math
 import os
-import pydoc
-import sys
 
 from grandalf.graphs import Edge, Graph, Vertex
 from grandalf.layouts import SugiyamaLayout
 from grandalf.routing import EdgeViewer, route_with_lines
 
-from dvc.env import DVC_PAGER
-from dvc.utils import format_link
-
 logger = logging.getLogger(__name__)
 
 
-DEFAULT_PAGER = "less"
-DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
-    DEFAULT_PAGER
-)
-
-
-def make_pager(cmd):
-    def pager(text):
-        return pydoc.tempfilepager(pydoc.plain(text), cmd)
-
-    return pager
-
-
-def find_pager():
-    if not sys.stdout.isatty():
-        return pydoc.plainpager
-
-    env_pager = os.getenv(DVC_PAGER)
-    if env_pager:
-        return make_pager(env_pager)
-
-    if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
-        return make_pager(DEFAULT_PAGER_FORMATTED)
-
-    logger.warning(
-        "Unable to find `less` in the PATH. Check out "
-        "{} for more info.".format(
-            format_link("https://man.dvc.org/pipeline/show")
-        )
-    )
-    return pydoc.plainpager
-
-
 class VertexViewer:
     """Class to define vertex box boundaries that will be accounted for during
     graph building by grandalf.

dvc/data_cloud.py

@@ -1,6 +1,11 @@
 """Manages dvc remotes that user can use with push/pull/status commands."""
 
 import logging
+from typing import TYPE_CHECKING, Iterable, Optional
+
+if TYPE_CHECKING:
+    from dvc.objects.file import HashFile
+    from dvc.remote.base import Remote
 
 logger = logging.getLogger(__name__)
 
@@ -19,7 +24,11 @@ class DataCloud:
     def __init__(self, repo):
         self.repo = repo
 
-    def get_remote(self, name=None, command="<command>"):
+    def get_remote(
+        self,
+        name: Optional[str] = None,
+        command: str = "<command>",
+    ) -> "Remote":
         from dvc.config import NoRemoteError
 
         if not name:
@@ -49,97 +58,87 @@ class DataCloud:
         return get_remote(self.repo, name=name)
 
     def push(
-        self, cache, jobs=None, remote=None, show_checksums=False,
+        self,
+        objs: Iterable["HashFile"],
+        jobs: Optional[int] = None,
+        remote: Optional[str] = None,
+        show_checksums: bool = False,
     ):
         """Push data items in a cloud-agnostic way.
 
         Args:
-            cache (NamedCache): named checksums to push to the cloud.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to push to.
+            objs: objects to push to the cloud.
+            jobs: number of jobs that can be running simultaneously.
+            remote: optional remote to push to.
                 By default remote from core.remote config option is used.
-            show_checksums (bool): show checksums instead of file names in
+            show_checksums: show checksums instead of file names in
                 information messages.
         """
-        remote = self.get_remote(remote, "push")
+        remote_obj = self.get_remote(remote, "push")
 
-        return remote.push(
+        return remote_obj.push(
             self.repo.odb.local,
-            cache,
+            objs,
             jobs=jobs,
             show_checksums=show_checksums,
         )
 
     def pull(
-        self, cache, jobs=None, remote=None, show_checksums=False,
+        self,
+        objs: Iterable["HashFile"],
+        jobs: Optional[int] = None,
+        remote: Optional[str] = None,
+        show_checksums: bool = False,
     ):
         """Pull data items in a cloud-agnostic way.
 
         Args:
-            cache (NamedCache): named checksums to pull from the cloud.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to pull from.
+            objs: objects to pull from the cloud.
+            jobs: number of jobs that can be running simultaneously.
+            remote: optional remote to pull from.
                 By default remote from core.remote config option is used.
-            show_checksums (bool): show checksums instead of file names in
+            show_checksums: show checksums instead of file names in
                 information messages.
         """
-        remote = self.get_remote(remote, "pull")
+        remote_obj = self.get_remote(remote, "pull")
 
-        return remote.pull(
+        return remote_obj.pull(
             self.repo.odb.local,
-            cache,
+            objs,
             jobs=jobs,
             show_checksums=show_checksums,
         )
 
-    def transfer(self, source, jobs=None, remote=None, command=None):
-        """Transfer data items in a cloud-agnostic way.
-
-        Args:
-            source (str): url for the source location.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to compare
-                cache to. By default remote from core.remote config option
-                is used.
-            command (str): the command which is benefitting from this function
-                (to be used for reporting better error messages).
-        """
-        from dvc.fs import get_cloud_fs
-
-        from_fs = get_cloud_fs(self.repo, url=source)
-        remote = self.get_remote(remote, command)
-        return remote.transfer(from_fs, from_fs.path_info, jobs=jobs)
-
     def status(
         self,
-        cache,
-        jobs=None,
-        remote=None,
-        show_checksums=False,
-        log_missing=True,
+        objs: Iterable["HashFile"],
+        jobs: Optional[int] = None,
+        remote: Optional[str] = None,
+        show_checksums: bool = False,
+        log_missing: bool = True,
     ):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
-            cache (NamedCache): named checksums to check status for.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to compare
+            objs: objects to check status for.
+            jobs: number of jobs that can be running simultaneously.
+            remote: optional remote to compare
                 cache to. By default remote from core.remote config option
                 is used.
-            show_checksums (bool): show checksums instead of file names in
+            show_checksums: show checksums instead of file names in
                 information messages.
-            log_missing (bool): log warning messages if file doesn't exist
+            log_missing: log warning messages if file doesn't exist
                 neither in cache, neither in cloud.
         """
-        remote = self.get_remote(remote, "status")
-        return remote.status(
+        remote_obj = self.get_remote(remote, "status")
+        return remote_obj.status(
             self.repo.odb.local,
-            cache,
+            objs,
             jobs=jobs,
             show_checksums=show_checksums,
             log_missing=log_missing,
         )
 
     def get_url_for(self, remote, checksum):
-        remote = self.get_remote(remote)
-        return str(remote.odb.hash_to_path_info(checksum))
+        remote_obj = self.get_remote(remote)
+        return str(remote_obj.odb.hash_to_path_info(checksum))

dvc/dependency/__init__.py

@@ -1,58 +1,24 @@
 from collections import defaultdict
-from urllib.parse import urlparse
+from typing import Any, Mapping
 
-import dvc.output as output
-from dvc.dependency.azure import AzureDependency
-from dvc.dependency.gs import GSDependency
-from dvc.dependency.hdfs import HDFSDependency
-from dvc.dependency.http import HTTPDependency
-from dvc.dependency.https import HTTPSDependency
-from dvc.dependency.local import LocalDependency
-from dvc.dependency.param import ParamsDependency
-from dvc.dependency.s3 import S3Dependency
-from dvc.dependency.ssh import SSHDependency
-from dvc.dependency.webdav import WebDAVDependency
-from dvc.dependency.webdavs import WebDAVSDependency
-from dvc.dependency.webhdfs import WebHDFSDependency
-from dvc.output.base import BaseOutput
-from dvc.scheme import Schemes
+from dvc.output import ARTIFACT_SCHEMA, Output
 
-from ..fs import get_cloud_fs
+from .base import Dependency
+from .param import ParamsDependency
 from .repo import RepoDependency
 
-DEP_MAP = {
-    Schemes.LOCAL: LocalDependency,
-    Schemes.SSH: SSHDependency,
-    Schemes.S3: S3Dependency,
-    Schemes.AZURE: AzureDependency,
-    Schemes.GS: GSDependency,
-    Schemes.HDFS: HDFSDependency,
-    Schemes.HTTP: HTTPDependency,
-    Schemes.HTTPS: HTTPSDependency,
-    Schemes.WEBDAV: WebDAVDependency,
-    Schemes.WEBDAVS: WebDAVSDependency,
-    Schemes.WEBHDFS: WebHDFSDependency,
-}
-
-
 # NOTE: schema for dependencies is basically the same as for outputs, but
 # without output-specific entries like 'cache' (whether or not output is
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
 # output is a metrics file and how to parse it, see `-M` flag for `dvc run`).
-SCHEMA = output.SCHEMA.copy()
-del SCHEMA[BaseOutput.PARAM_CACHE]
-del SCHEMA[BaseOutput.PARAM_METRIC]
-del SCHEMA[BaseOutput.PARAM_DESC]
-SCHEMA.update(RepoDependency.REPO_SCHEMA)
-SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
+SCHEMA: Mapping[str, Any] = {
+    **ARTIFACT_SCHEMA,
+    **RepoDependency.REPO_SCHEMA,
+    **ParamsDependency.PARAM_SCHEMA,
+}
 
 
 def _get(stage, p, info):
-    parsed = urlparse(p) if p else None
-    if parsed and parsed.scheme == "remote":
-        fs = get_cloud_fs(stage.repo, name=parsed.netloc)
-        return DEP_MAP[fs.scheme](stage, p, info, fs=fs)
-
     if info and info.get(RepoDependency.PARAM_REPO):
         repo = info.pop(RepoDependency.PARAM_REPO)
         return RepoDependency(repo, stage, p, info)
@@ -61,14 +27,13 @@ def _get(stage, p, info):
         params = info.pop(ParamsDependency.PARAM_PARAMS)
         return ParamsDependency(stage, p, params)
 
-    dep_cls = DEP_MAP.get(parsed.scheme, LocalDependency)
-    return dep_cls(stage, p, info)
+    return Dependency(stage, p, info)
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(BaseOutput.PARAM_PATH, None)
+        p = d.pop(Output.PARAM_PATH, None)
         ret.append(_get(stage, p, d))
     return ret
 

dvc/dependency/azure.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.azure import AzureFileSystem
-
-
-class AzureDependency(BaseDependency, BaseOutput):
-    FS_CLS = AzureFileSystem

dvc/dependency/base.py

@@ -1,6 +1,7 @@
 from typing import Type
 
 from dvc.exceptions import DvcException
+from dvc.output import Output
 
 
 class DependencyDoesNotExistError(DvcException):
@@ -20,7 +21,7 @@ class DependencyIsStageFileError(DvcException):
         super().__init__(f"DVC file '{path}' cannot be a dependency.")
 
 
-class BaseDependency:
+class Dependency(Output):
     IS_DEPENDENCY = True
 
     DoesNotExistError = DependencyDoesNotExistError  # type: Type[DvcException]

dvc/dependency/gs.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.gs import GSFileSystem
-
-
-class GSDependency(BaseDependency, BaseOutput):
-    FS_CLS = GSFileSystem

dvc/dependency/hdfs.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.hdfs import HDFSOutput
-
-
-class HDFSDependency(BaseDependency, HDFSOutput):
-    pass

dvc/dependency/http.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.http import HTTPFileSystem
-
-
-class HTTPDependency(BaseDependency, BaseOutput):
-    FS_CLS = HTTPFileSystem

dvc/dependency/https.py

@@ -1,6 +0,0 @@
-from ..fs.https import HTTPSFileSystem
-from .http import HTTPDependency
-
-
-class HTTPSDependency(HTTPDependency):
-    FS_CLS = HTTPSFileSystem

dvc/dependency/local.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.local import LocalOutput
-
-
-class LocalDependency(BaseDependency, LocalOutput):
-    pass

dvc/dependency/param.py

@@ -4,11 +4,12 @@ from collections import defaultdict
 import dpath.util
 from voluptuous import Any
 
-from dvc.dependency.local import LocalDependency
 from dvc.exceptions import DvcException
 from dvc.hash_info import HashInfo
 from dvc.utils.serialize import LOADERS, ParseError
 
+from .base import Dependency
+
 
 class MissingParamsError(DvcException):
     pass
@@ -18,7 +19,7 @@ class BadParamFileError(DvcException):
     pass
 
 
-class ParamsDependency(LocalDependency):
+class ParamsDependency(Dependency):
     PARAM_PARAMS = "params"
     PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
     DEFAULT_PARAMS_FILE = "params.yaml"
@@ -84,19 +85,34 @@ class ParamsDependency(LocalDependency):
     def status(self):
         return self.workspace_status()
 
-    def read_params(self):
+    def _read(self):
         if not self.exists:
             return {}
 
         suffix = self.path_info.suffix.lower()
         loader = LOADERS[suffix]
         try:
-            config = loader(self.path_info, fs=self.repo.fs)
+            return loader(self.path_info, fs=self.repo.fs)
         except ParseError as exc:
             raise BadParamFileError(
                 f"Unable to read parameters from '{self}'"
             ) from exc
 
+    def read_params_d(self):
+        config = self._read()
+
+        ret = {}
+        for param in self.params:
+            dpath.util.merge(
+                ret,
+                dpath.util.search(config, param, separator="."),
+                separator=".",
+            )
+        return ret
+
+    def read_params(self):
+        config = self._read()
+
         ret = {}
         for param in self.params:
             try:
@@ -112,7 +128,7 @@ class ParamsDependency(LocalDependency):
         if missing_params:
             raise MissingParamsError(
                 "Parameters '{}' are missing from '{}'.".format(
-                    ", ".join(missing_params), self,
+                    ", ".join(missing_params), self
                 )
             )
 

dvc/dependency/repo.py

@@ -1,11 +1,19 @@
+import os
+from collections import defaultdict
+from typing import TYPE_CHECKING, Dict, Optional, Set
+
 from voluptuous import Required
 
 from dvc.path_info import PathInfo
 
-from .local import LocalDependency
+from .base import Dependency
+
+if TYPE_CHECKING:
+    from dvc.objects.db.base import ObjectDB
+    from dvc.objects.file import HashFile
 
 
-class RepoDependency(LocalDependency):
+class RepoDependency(Dependency):
     PARAM_REPO = "repo"
     PARAM_URL = "url"
     PARAM_REV = "rev"
@@ -21,46 +29,22 @@ class RepoDependency(LocalDependency):
 
     def __init__(self, def_repo, stage, *args, **kwargs):
         self.def_repo = def_repo
+        self._staged_objs: Dict[str, "HashFile"] = {}
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, fs, path):
+    def _parse_path(self, fs, path_info):
         return None
 
     @property
     def is_in_repo(self):
         return False
 
-    @property
-    def repo_pair(self):
-        d = self.def_repo
-        rev = d.get(self.PARAM_REV_LOCK) or d.get(self.PARAM_REV)
-        return d[self.PARAM_URL], rev
-
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 
-    def _make_repo(self, *, locked=True, **kwargs):
-        from dvc.external_repo import external_repo
-
-        d = self.def_repo
-        rev = (d.get("rev_lock") if locked else None) or d.get("rev")
-        return external_repo(d["url"], rev=rev, **kwargs)
-
-    def _get_hash(self, locked=True):
-        from dvc.objects.stage import stage
-
-        with self._make_repo(locked=locked) as repo:
-            path_info = PathInfo(repo.root_dir) / self.def_path
-            return stage(
-                self.repo.odb.local,
-                path_info,
-                repo.repo_fs,
-                follow_subrepos=False,
-            ).hash_info
-
     def workspace_status(self):
-        current = self._get_hash(locked=True)
-        updated = self._get_hash(locked=False)
+        current = self.get_obj(locked=True).hash_info
+        updated = self.get_obj(locked=False).hash_info
 
         if current != updated:
             return {str(self): "update available"}
@@ -78,32 +62,28 @@ class RepoDependency(LocalDependency):
 
     def download(self, to, jobs=None):
         from dvc.checkout import checkout
-        from dvc.config import NoRemoteError
-        from dvc.exceptions import NoOutputOrStageError
         from dvc.objects import save
-        from dvc.objects.stage import stage
+        from dvc.objects.db.git import GitObjectDB
+        from dvc.repo.fetch import fetch_from_odb
 
-        odb = self.repo.odb.local
+        for odb, objs in self.get_used_objs().items():
+            if not isinstance(odb, GitObjectDB):
+                fetch_from_odb(self.repo, odb, objs, jobs=jobs)
 
-        with self._make_repo(cache_dir=odb.cache_dir) as repo:
-            if self.def_repo.get(self.PARAM_REV_LOCK) is None:
-                self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
-            path_info = PathInfo(repo.root_dir) / self.def_path
-            try:
-                repo.fetch([path_info.fspath], jobs=jobs, recursive=True)
-            except (NoOutputOrStageError, NoRemoteError):
-                pass
-            obj = stage(
-                odb, path_info, repo.repo_fs, jobs=jobs, follow_subrepos=False,
-            )
-            save(odb, obj, jobs=jobs)
-
-        checkout(to.path_info, to.fs, obj, odb)
+        obj = self.get_obj()
+        save(self.repo.odb.local, obj, jobs=jobs)
+        checkout(
+            to.path_info,
+            to.fs,
+            obj,
+            self.repo.odb.local,
+            dvcignore=None,
+            state=self.repo.state,
+        )
 
     def update(self, rev=None):
         if rev:
             self.def_repo[self.PARAM_REV] = rev
-
         with self._make_repo(locked=False) as repo:
             self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
 
@@ -112,3 +92,100 @@ class RepoDependency(LocalDependency):
         # origin project url and rev_lock, and it makes RepoDependency
         # immutable, hence its impossible for checksum to change.
         return False
+
+    def get_used_objs(
+        self, **kwargs
+    ) -> Dict[Optional["ObjectDB"], Set["HashFile"]]:
+        from dvc.config import NoRemoteError
+        from dvc.exceptions import NoOutputOrStageError, PathMissingError
+        from dvc.objects.db.git import GitObjectDB
+        from dvc.objects.stage import stage
+
+        local_odb = self.repo.odb.local
+        locked = kwargs.pop("locked", True)
+        with self._make_repo(
+            locked=locked, cache_dir=local_odb.cache_dir
+        ) as repo:
+            used_objs = defaultdict(set)
+            rev = repo.get_rev()
+            if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
+                self.def_repo[self.PARAM_REV_LOCK] = rev
+
+            path_info = PathInfo(repo.root_dir) / str(self.def_path)
+            try:
+                for odb, objs in repo.used_objs(
+                    [os.fspath(path_info)],
+                    force=True,
+                    jobs=kwargs.get("jobs"),
+                    recursive=True,
+                ).items():
+                    if odb is None:
+                        odb = repo.cloud.get_remote().odb
+                    self._check_circular_import(odb)
+                    used_objs[odb].update(objs)
+            except (NoRemoteError, NoOutputOrStageError):
+                pass
+
+            try:
+                staged_obj = stage(
+                    local_odb,
+                    path_info,
+                    repo.repo_fs,
+                    local_odb.fs.PARAM_CHECKSUM,
+                )
+            except FileNotFoundError as exc:
+                raise PathMissingError(
+                    self.def_path, self.def_repo[self.PARAM_URL]
+                ) from exc
+
+            self._staged_objs[rev] = staged_obj
+            git_odb = GitObjectDB(repo.repo_fs, repo.root_dir)
+            used_objs[git_odb].add(staged_obj)
+            return used_objs
+
+    def _check_circular_import(self, odb):
+        from dvc.exceptions import CircularImportError
+        from dvc.fs.repo import RepoFileSystem
+
+        if not odb or not isinstance(odb.fs, RepoFileSystem):
+            return
+
+        self_url = self.repo.url or self.repo.root_dir
+        if odb.fs.repo_url is not None and odb.fs.repo_url == self_url:
+            raise CircularImportError(self, odb.fs.repo_url, self_url)
+
+    def get_obj(self, filter_info=None, **kwargs):
+        from dvc.objects.stage import stage
+
+        odb = self.repo.odb.local
+        locked = kwargs.pop("locked", True)
+        with self._make_repo(locked=locked, cache_dir=odb.cache_dir) as repo:
+            rev = repo.get_rev()
+            if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
+                self.def_repo[self.PARAM_REV_LOCK] = rev
+            obj = self._staged_objs.get(rev)
+            if obj is not None:
+                return obj
+
+            path_info = PathInfo(repo.root_dir) / str(self.def_path)
+            obj = stage(
+                odb,
+                path_info,
+                repo.repo_fs,
+                odb.fs.PARAM_CHECKSUM,
+            )
+            self._staged_objs[rev] = obj
+            return obj
+
+    def _make_repo(self, locked=True, **kwargs):
+        from dvc.external_repo import external_repo
+
+        d = self.def_repo
+        rev = self._get_rev(locked=locked)
+        return external_repo(d[self.PARAM_URL], rev=rev, **kwargs)
+
+    def _get_rev(self, locked=True):
+        d = self.def_repo
+        return (d.get(self.PARAM_REV_LOCK) if locked else None) or d.get(
+            self.PARAM_REV
+        )

dvc/dependency/s3.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.s3 import S3Output
-
-
-class S3Dependency(BaseDependency, S3Output):
-    pass

dvc/dependency/ssh.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.ssh import SSHOutput
-
-
-class SSHDependency(BaseDependency, SSHOutput):
-    pass

dvc/dependency/webdav.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.webdav import WebDAVFileSystem
-
-
-class WebDAVDependency(BaseDependency, BaseOutput):
-    FS_CLS = WebDAVFileSystem

dvc/dependency/webdavs.py

@@ -1,6 +0,0 @@
-from ..fs.webdavs import WebDAVSFileSystem
-from .webdav import WebDAVDependency
-
-
-class WebDAVSDependency(WebDAVDependency):
-    FS_CLS = WebDAVSFileSystem

dvc/dependency/webhdfs.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.webhdfs import WebHDFSOutput
-
-
-class WebHDFSDependency(BaseDependency, WebHDFSOutput):
-    pass

dvc/dir_info.py

@@ -1,117 +0,0 @@
-import posixpath
-from operator import itemgetter
-
-from funcy import cached_property
-from pygtrie import Trie
-
-from .hash_info import HashInfo
-
-
-def _diff(ancestor, other, allow_removed=False):
-    from dictdiffer import diff
-
-    from dvc.exceptions import MergeError
-
-    allowed = ["add"]
-    if allow_removed:
-        allowed.append("remove")
-
-    result = list(diff(ancestor, other))
-    for typ, _, _ in result:
-        if typ not in allowed:
-            raise MergeError(
-                "unable to auto-merge directories with diff that contains "
-                f"'{typ}'ed files"
-            )
-    return result
-
-
-def _merge(ancestor, our, their):
-    import copy
-
-    from dictdiffer import patch
-
-    our_diff = _diff(ancestor, our)
-    if not our_diff:
-        return copy.deepcopy(their)
-
-    their_diff = _diff(ancestor, their)
-    if not their_diff:
-        return copy.deepcopy(our)
-
-    # make sure there are no conflicting files
-    _diff(our, their, allow_removed=True)
-
-    return patch(our_diff + their_diff, ancestor)
-
-
-class DirInfo:
-    PARAM_RELPATH = "relpath"
-
-    def __init__(self):
-        self._dict = {}
-
-    def as_dict(self):
-        return self._dict.copy()
-
-    def add(self, key, hash_info):
-        self.__dict__.pop("trie", None)
-        self._dict[key] = hash_info
-
-    def get(self, key, default=None):
-        return self._dict.get(key, default)
-
-    @cached_property
-    def trie(self):
-        return Trie(self._dict)
-
-    @property
-    def size(self):
-        try:
-            return sum(hash_info.size for _, hash_info in self._dict.items())
-        except TypeError:
-            return None
-
-    @property
-    def nfiles(self):
-        return len(self._dict)
-
-    def items(self):
-        for key, hash_info in self._dict.items():
-            yield key, hash_info
-
-    @classmethod
-    def from_list(cls, lst):
-        ret = DirInfo()
-        for _entry in lst:
-            entry = _entry.copy()
-            relpath = entry.pop(cls.PARAM_RELPATH)
-            parts = tuple(relpath.split(posixpath.sep))
-            hash_info = HashInfo.from_dict(entry)
-            ret.add(parts, hash_info)
-        return ret
-
-    def to_list(self):
-        # Sorting the list by path to ensure reproducibility
-        return sorted(
-            (
-                {
-                    # NOTE: not using hash_info.to_dict() because we don't want
-                    # size/nfiles fields at this point.
-                    hash_info.name: hash_info.value,
-                    self.PARAM_RELPATH: posixpath.sep.join(parts),
-                }
-                for parts, hash_info in self._dict.items()  # noqa: B301
-            ),
-            key=itemgetter(self.PARAM_RELPATH),
-        )
-
-    def merge(self, ancestor, their):
-        merged_dict = _merge(
-            ancestor.as_dict(), self.as_dict(), their.as_dict(),
-        )
-
-        merged = DirInfo()
-        for key, hi in merged_dict.items():
-            merged.add(key, hi)
-        return merged

dvc/dvcfile.py

@@ -37,8 +37,12 @@ PIPELINE_LOCK = "dvc.lock"
 
 
 class FileIsGitIgnored(DvcException):
-    def __init__(self, path):
-        super().__init__(f"'{path}' is git-ignored.")
+    def __init__(self, path, pipeline_file=False):
+        super().__init__(
+            "{}'{}' is git-ignored.".format(
+                "bad DVC file name " if pipeline_file else "", path
+            )
+        )
 
 
 class LockfileCorruptedError(DvcException):
@@ -66,7 +70,19 @@ def is_lock_file(path):
     return os.path.basename(path) == PIPELINE_LOCK
 
 
-def check_dvc_filename(path):
+def is_git_ignored(repo, path):
+    from dvc.fs.local import LocalFileSystem
+    from dvc.scm.base import NoSCMError
+
+    try:
+        return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored(
+            path
+        )
+    except NoSCMError:
+        return False
+
+
+def check_dvcfile_path(repo, path):
     if not is_valid_filename(path):
         raise StageFileBadNameError(
             "bad DVC file name '{}'. DVC files should be named "
@@ -75,6 +91,9 @@ def check_dvc_filename(path):
             )
         )
 
+    if is_git_ignored(repo, path):
+        raise FileIsGitIgnored(relpath(path), True)
+
 
 class FileMixin:
     SCHEMA = None
@@ -105,18 +124,15 @@ class FileMixin:
         return relpath(self.path)
 
     def exists(self):
-        return self.repo.fs.exists(self.path)
+        is_ignored = self.repo.dvcignore.is_ignored_file(self.path)
+        return self.repo.fs.exists(self.path) and not is_ignored
 
     def _is_git_ignored(self):
-        from dvc.fs.local import LocalFileSystem
-
-        return isinstance(
-            self.repo.fs, LocalFileSystem
-        ) and self.repo.scm.is_ignored(self.path)
+        return is_git_ignored(self.repo, self.path)
 
     def _verify_filename(self):
         if self.verify:
-            check_dvc_filename(self.path)
+            check_dvcfile_path(self.repo, self.path)
 
     def _check_gitignored(self):
         if self._is_git_ignored():
@@ -129,8 +145,10 @@ class FileMixin:
         # 3. path doesn't represent a regular file
         # 4. when the file is git ignored
         if not self.exists():
-            is_ignored = self.repo.fs.exists(self.path, use_dvcignore=False)
-            raise StageFileDoesNotExistError(self.path, dvc_ignored=is_ignored)
+            dvc_ignored = self.repo.dvcignore.is_ignored_file(self.path)
+            raise StageFileDoesNotExistError(
+                self.path, dvc_ignored=dvc_ignored
+            )
 
         self._verify_filename()
         if not self.repo.fs.isfile(self.path):
@@ -181,7 +199,7 @@ class SingleStageFile(FileMixin):
 
         assert not isinstance(stage, PipelineStage)
         if self.verify:
-            check_dvc_filename(self.path)
+            check_dvcfile_path(self.repo, self.path)
         logger.debug(
             "Saving information to '{file}'.".format(file=relpath(self.path))
         )
@@ -218,7 +236,7 @@ class PipelineFile(FileMixin):
 
         assert isinstance(stage, PipelineStage)
         if self.verify:
-            check_dvc_filename(self.path)
+            check_dvcfile_path(self.repo, self.path)
 
         if update_pipeline and not stage.is_data_source:
             self._dump_pipeline_file(stage)

dvc/env.py

@@ -6,3 +6,4 @@ DVCLIVE_PATH = "DVCLIVE_PATH"
 DVCLIVE_SUMMARY = "DVCLIVE_SUMMARY"
 DVCLIVE_HTML = "DVCLIVE_HTML"
 DVCLIVE_RESUME = "DVCLIVE_RESUME"
+DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY"

dvc/exceptions.py

@@ -35,7 +35,7 @@ class OutputDuplicationError(DvcException):
             )
         else:
             msg = "output '{}' is already specified in stages:\n{}".format(
-                output, "\n".join(f"\t- {s.addressing}" for s in stages),
+                output, "\n".join(f"\t- {s.addressing}" for s in stages)
             )
         super().__init__(msg)
         self.stages = stages
@@ -283,7 +283,7 @@ class CheckoutError(DvcException):
         m = (
             "Checkout failed for following targets:\n{}\nIs your "
             "cache up to date?\n{}".format(
-                "\n".join(targets), error_link("missing-files"),
+                "\n".join(targets), error_link("missing-files")
             )
         )
         super().__init__(m)
@@ -380,7 +380,15 @@ class CacheLinkError(DvcException):
 
     def __init__(self, path_infos):
         msg = "No possible cache link types for '{}'. {}".format(
-            ", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK,
+            ", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK
         )
         super().__init__(msg)
         self.path_infos = path_infos
+
+
+class CircularImportError(DvcException):
+    def __init__(self, dep, a, b):
+        super().__init__(
+            f"'{dep}' contains invalid circular import. "
+            f"DVC repo '{a}' already imports from '{b}'."
+        )

dvc/external_repo.py

@@ -36,10 +36,7 @@ def external_repo(
     rev = rev or "refs/remotes/origin/HEAD"
 
     cache_config = {
-        "cache": {
-            "dir": cache_dir or _get_cache_dir(url),
-            "type": cache_types,
-        }
+        "cache": {"dir": cache_dir or _get_cache_dir(url), "type": cache_types}
     }
 
     config = _get_remote_config(url) if os.path.isdir(url) else {}
@@ -144,7 +141,7 @@ def _cached_clone(url, rev, for_write=False):
     revision checked out. If for_write is set prevents reusing this dir via
     cache.
     """
-    from distutils.dir_util import copy_tree
+    from shutil import copytree
 
     # even if we have already cloned this repo, we may need to
     # fetch/fast-forward to get specified rev
@@ -156,7 +153,7 @@ def _cached_clone(url, rev, for_write=False):
     # Copy to a new dir to keep the clone clean
     repo_path = tempfile.mkdtemp("dvc-erepo")
     logger.debug("erepo: making a copy of %s clone", url)
-    copy_tree(clone_path, repo_path)
+    copytree(clone_path, repo_path)
 
     # Check out the specified revision
     if for_write:

dvc/fs/__init__.py

@@ -12,8 +12,7 @@ from .local import LocalFileSystem
 from .oss import OSSFileSystem
 from .s3 import S3FileSystem
 from .ssh import SSHFileSystem
-from .webdav import WebDAVFileSystem
-from .webdavs import WebDAVSFileSystem
+from .webdav import WebDAVFileSystem, WebDAVSFileSystem
 from .webhdfs import WebHDFSFileSystem
 
 FS_MAP = {
@@ -82,9 +81,35 @@ def get_cloud_fs(repo, **kwargs):
     from dvc.config import ConfigError
     from dvc.config_schema import SCHEMA, Invalid
 
-    remote_conf = get_fs_config(repo.config, **kwargs)
+    repo_config = repo.config if repo else {}
+    core_config = repo_config.get("core", {})
+
+    remote_conf = get_fs_config(repo_config, **kwargs)
     try:
         remote_conf = SCHEMA["remote"][str](remote_conf)
     except Invalid as exc:
         raise ConfigError(str(exc)) from None
-    return get_fs_cls(remote_conf)(repo, remote_conf)
+
+    if "jobs" not in remote_conf:
+        jobs = core_config.get("jobs")
+        if jobs:
+            remote_conf["jobs"] = jobs
+
+    if "checksum_jobs" not in remote_conf:
+        checksum_jobs = core_config.get("checksum_jobs")
+        if checksum_jobs:
+            remote_conf["checksum_jobs"] = checksum_jobs
+
+    cls = get_fs_cls(remote_conf)
+
+    if cls == GDriveFileSystem and repo:
+        remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir
+
+    url = remote_conf.pop("url")
+    path_info = cls.PATH_CLS(
+        cls._strip_protocol(url)  # pylint:disable=protected-access
+    )
+
+    extras = cls._get_kwargs_from_urls(url)  # pylint:disable=protected-access
+    conf = {**extras, **remote_conf}  # remote config takes priority
+    return cls, conf, path_info

dvc/fs/_metadata.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, List
 
-from dvc.output import BaseOutput
+from dvc.output import Output
 from dvc.path_info import PathInfo
 
 if TYPE_CHECKING:
@@ -33,7 +33,7 @@ class Metadata:
     # optional fields
     isdir: bool = False  # is it a directory?
     is_exec: bool = False  # is it an executable?
-    outs: List[BaseOutput] = field(default_factory=list)  # list of outputs
+    outs: List[Output] = field(default_factory=list)  # list of outputs
 
     def __post_init__(self):
         self.output_exists = bool(self.outs)

dvc/fs/azure.py

@@ -1,15 +1,19 @@
+import asyncio
 import logging
 import os
+import sys
 import threading
+from contextlib import contextmanager
 
-from funcy import cached_property, wrap_prop
+from fsspec.utils import infer_storage_options
+from funcy import cached_property, memoize, wrap_prop
 
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.scheme import Schemes
 from dvc.utils import format_link
 
-from .fsspec_wrapper import FSSpecWrapper
+from .fsspec_wrapper import ObjectFSWrapper
 
 logger = logging.getLogger(__name__)
 _DEFAULT_CREDS_STEPS = (
@@ -19,11 +23,66 @@ _DEFAULT_CREDS_STEPS = (
 )
 
 
+@contextmanager
+def _temp_event_loop():
+    """When trying to initalize azure filesystem instances
+    with DefaultCredentials, the authentication process requires
+    to have an access to a separate event loop. The normal calls
+    are run in a separate loop managed by the fsspec, but the
+    DefaultCredentials assumes that the callee is managing their
+    own event loop. This function checks whether is there any
+    event loop set, and if not it creates a temporary event loop
+    and set it. After the context is finalized, it restores the
+    original event loop back (if there is any)."""
+
+    try:
+        original_loop = asyncio.get_event_loop()
+        original_policy = asyncio.get_event_loop_policy()
+    except RuntimeError:
+        original_loop = None
+        original_policy = None
+
+    # From 3.8>= and onwards, asyncio changed the default
+    # loop policy for windows to use proactor loops instead
+    # of selector based ones. Due to that, proxied connections
+    # doesn't work with the aiohttp and this is most likely an
+    # upstream bug that needs to be solved outside of DVC. Until
+    # such issue is resolved, we need to manage this;
+    # https://github.com/aio-libs/aiohttp/issues/4536
+    if sys.version_info >= (3, 8) and os.name == "nt":
+        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+    loop = original_loop or asyncio.new_event_loop()
+
+    try:
+        asyncio.set_event_loop(loop)
+        yield
+    finally:
+        if original_loop is None:
+            loop.close()
+        asyncio.set_event_loop(original_loop)
+        asyncio.set_event_loop_policy(original_policy)
+
+
 class AzureAuthError(DvcException):
     pass
 
 
-class AzureFileSystem(FSSpecWrapper):
+@memoize
+def _az_config():
+    # NOTE: ideally we would've used get_default_cli().config from
+    # azure.cli.core, but azure-cli-core has a lot of conflicts with other
+    # dependencies. So instead we are just use knack directly
+    from knack.config import CLIConfig
+
+    config_dir = os.getenv(
+        "AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure"))
+    )
+    return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE")
+
+
+# pylint:disable=abstract-method
+class AzureFileSystem(ObjectFSWrapper):
     scheme = Schemes.AZURE
     PATH_CLS = CloudURLInfo
     PARAM_CHECKSUM = "etag"
@@ -34,45 +93,65 @@ class AzureFileSystem(FSSpecWrapper):
         "azure-identity": "azure.identity",
     }
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    @classmethod
+    def _strip_protocol(cls, path: str):
+        bucket = infer_storage_options(path).get("host")
+        if bucket:
+            return path
 
-        url = config.get("url")
-        self.path_info = self.PATH_CLS(url)
-        self.bucket = self.path_info.bucket
+        bucket = _az_config().get("storage", "container_name", None)
+        return f"azure://{bucket}"
 
-        if not self.bucket:
-            container = self._az_config.get("storage", "container_name", None)
-            url = f"azure://{container}"
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):
+        ops = infer_storage_options(urlpath)
+        if "host" in ops:
+            return {"bucket": ops["host"]}
+        return {}
 
-        self.path_info = self.PATH_CLS(url)
-        self.bucket = self.path_info.bucket
-
-        self.login_method, self.login_info = self._prepare_credentials(config)
-
-    def _prepare_credentials(self, config):
+    def _prepare_credentials(self, **config):
         from azure.identity.aio import DefaultAzureCredential
 
+        # Disable spam from failed cred types for DefaultAzureCredential
+        logging.getLogger("azure.identity.aio").setLevel(logging.ERROR)
+
         login_info = {}
         login_info["connection_string"] = config.get(
             "connection_string",
-            self._az_config.get("storage", "connection_string", None),
+            _az_config().get("storage", "connection_string", None),
         )
         login_info["account_name"] = config.get(
-            "account_name", self._az_config.get("storage", "account", None)
+            "account_name", _az_config().get("storage", "account", None)
         )
         login_info["account_key"] = config.get(
-            "account_key", self._az_config.get("storage", "key", None)
+            "account_key", _az_config().get("storage", "key", None)
         )
         login_info["sas_token"] = config.get(
-            "sas_token", self._az_config.get("storage", "sas_token", None)
+            "sas_token", _az_config().get("storage", "sas_token", None)
         )
         login_info["tenant_id"] = config.get("tenant_id")
         login_info["client_id"] = config.get("client_id")
         login_info["client_secret"] = config.get("client_secret")
 
-        if not any(login_info.values()):
-            login_info["credentials"] = DefaultAzureCredential()
+        if not (login_info["account_name"] or login_info["connection_string"]):
+            raise AzureAuthError(
+                "Authentication to Azure Blob Storage requires either "
+                "account_name or connection_string.\nLearn more about "
+                "configuration settings at "
+                + format_link("https://man.dvc.org/remote/modify")
+            )
+
+        any_secondary = any(
+            value for key, value in login_info.items() if key != "account_name"
+        )
+        if (
+            login_info["account_name"]
+            and not any_secondary
+            and not config.get("allow_anonymous_login", False)
+        ):
+            login_info["credential"] = DefaultAzureCredential(
+                exclude_interactive_browser_credential=False
+            )
 
         for login_method, required_keys in [  # noqa
             ("connection string", ["connection_string"]),
@@ -82,27 +161,19 @@ class AzureFileSystem(FSSpecWrapper):
             ),
             ("account key", ["account_name", "account_key"]),
             ("SAS token", ["account_name", "sas_token"]),
+            (
+                f"default credentials ({_DEFAULT_CREDS_STEPS})",
+                ["account_name", "credential"],
+            ),
             ("anonymous login", ["account_name"]),
-            (f"default credentials ({_DEFAULT_CREDS_STEPS})", ["credentials"]),
         ]:
             if all(login_info.get(key) is not None for key in required_keys):
                 break
         else:
             login_method = None
 
-        return login_method, login_info
-
-    @cached_property
-    def _az_config(self):
-        # NOTE: ideally we would've used get_default_cli().config from
-        # azure.cli.core, but azure-cli-core has a lot of conflicts with other
-        # dependencies. So instead we are just use knack directly
-        from knack.config import CLIConfig
-
-        config_dir = os.getenv(
-            "AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure"))
-        )
-        return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE")
+        self.login_method = login_method
+        return login_info
 
     @wrap_prop(threading.Lock())
     @cached_property
@@ -111,11 +182,8 @@ class AzureFileSystem(FSSpecWrapper):
         from azure.core.exceptions import AzureError
 
         try:
-            file_system = AzureBlobFileSystem(**self.login_info)
-            if self.bucket not in [
-                container.rstrip("/") for container in file_system.ls("/")
-            ]:
-                file_system.mkdir(self.bucket)
+            with _temp_event_loop():
+                file_system = AzureBlobFileSystem(**self.fs_args)
         except (ValueError, AzureError) as e:
             raise AzureAuthError(
                 f"Authentication to Azure Blob Storage via {self.login_method}"
@@ -124,3 +192,9 @@ class AzureFileSystem(FSSpecWrapper):
             ) from e
 
         return file_system
+
+    def open(
+        self, path_info, mode="r", **kwargs
+    ):  # pylint: disable=arguments-differ
+        with _temp_event_loop():
+            return self.fs.open(self._with_bucket(path_info), mode=mode)

dvc/fs/base.py

@@ -1,11 +1,9 @@
 import logging
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial
+from functools import partial, partialmethod
 from multiprocessing import cpu_count
 from typing import Any, ClassVar, Dict, FrozenSet, Optional
 
-from funcy import cached_property
-
 from dvc.exceptions import DvcException
 from dvc.path_info import URLInfo
 from dvc.progress import Tqdm
@@ -35,10 +33,12 @@ class RemoteMissingDepsError(DvcException):
 
 
 class BaseFileSystem:
+    sep = "/"
+
     scheme = "base"
     REQUIRES: ClassVar[Dict[str, str]] = {}
     PATH_CLS = URLInfo  # type: Any
-    JOBS = 4 * cpu_count()
+    _JOBS = 4 * cpu_count()
 
     CHECKSUM_DIR_SUFFIX = ".dir"
     HASH_JOBS = max(1, min(4, cpu_count() // 2))
@@ -54,29 +54,19 @@ class BaseFileSystem:
     PARAM_CHECKSUM: ClassVar[Optional[str]] = None
     DETAIL_FIELDS: FrozenSet[str] = frozenset()
 
-    def __init__(self, repo, config):
-        self.repo = repo
-        self.config = config
+    def __init__(self, **kwargs):
+        self._check_requires(**kwargs)
 
-        self._check_requires()
+        self.jobs = kwargs.get("jobs") or self._JOBS
+        self.hash_jobs = kwargs.get("checksum_jobs") or self.HASH_JOBS
 
-        self.path_info = None
+    @classmethod
+    def _strip_protocol(cls, path: str):
+        return path
 
-    @cached_property
-    def jobs(self):
-        return (
-            self.config.get("jobs")
-            or (self.repo and self.repo.config["core"].get("jobs"))
-            or self.JOBS
-        )
-
-    @cached_property
-    def hash_jobs(self):
-        return (
-            self.config.get("checksum_jobs")
-            or (self.repo and self.repo.config["core"].get("checksum_jobs"))
-            or self.HASH_JOBS
-        )
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):  # pylint:disable=unused-argument
+        return {}
 
     @classmethod
     def get_missing_deps(cls):
@@ -91,7 +81,7 @@ class BaseFileSystem:
 
         return missing
 
-    def _check_requires(self):
+    def _check_requires(self, **kwargs):
         from ..scheme import Schemes
         from ..utils import format_link
         from ..utils.pkg import PKG
@@ -100,7 +90,7 @@ class BaseFileSystem:
         if not missing:
             return
 
-        url = self.config.get("url", f"{self.scheme}://")
+        url = kwargs.get("url", f"{self.scheme}://")
 
         scheme = self.scheme
         if scheme == Schemes.WEBDAVS:
@@ -139,7 +129,7 @@ class BaseFileSystem:
 
         raise RemoteActionNotImplemented("open", self.scheme)
 
-    def exists(self, path_info, use_dvcignore=True) -> bool:
+    def exists(self, path_info) -> bool:
         raise NotImplementedError
 
     # pylint: disable=unused-argument
@@ -166,6 +156,10 @@ class BaseFileSystem:
         """Check if this file is an independent copy."""
         return False  # We can't be sure by default
 
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
+        """Return a generator with (root, dirs, files)."""
+        raise NotImplementedError
+
     def walk_files(self, path_info, **kwargs):
         """Return a generator with `PathInfo`s to all the files.
 
@@ -175,9 +169,12 @@ class BaseFileSystem:
         """
         raise NotImplementedError
 
-    def ls(self, path_info, detail=False, **kwargs):
+    def ls(self, path_info, detail=False):
         raise RemoteActionNotImplemented("ls", self.scheme)
 
+    def find(self, path_info, detail=False, prefix=None):
+        raise RemoteActionNotImplemented("find", self.scheme)
+
     def is_empty(self, path_info):
         return False
 
@@ -219,9 +216,7 @@ class BaseFileSystem:
             return False
         return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
 
-    def upload(
-        self, from_info, to_info, name=None, no_progress_bar=False,
-    ):
+    def upload(self, from_info, to_info, name=None, no_progress_bar=False):
         if not hasattr(self, "_upload"):
             raise RemoteActionNotImplemented("upload", self.scheme)
 
@@ -242,14 +237,23 @@ class BaseFileSystem:
             no_progress_bar=no_progress_bar,
         )
 
-    def upload_fobj(self, fobj, to_info, no_progress_bar=False, **pbar_args):
+    def upload_fobj(
+        self, fobj, to_info, no_progress_bar=False, size=None, **pbar_args
+    ):
         if not hasattr(self, "_upload_fobj"):
             raise RemoteActionNotImplemented("upload_fobj", self.scheme)
 
         with Tqdm.wrapattr(
-            fobj, "read", disable=no_progress_bar, bytes=True, **pbar_args
+            fobj,
+            "read",
+            disable=no_progress_bar,
+            bytes=True,
+            total=size,
+            **pbar_args,
         ) as wrapped:
-            self._upload_fobj(wrapped, to_info)  # pylint: disable=no-member
+            self._upload_fobj(  # pylint: disable=no-member
+                wrapped, to_info, size=size
+            )
 
     def download(
         self,
@@ -258,6 +262,7 @@ class BaseFileSystem:
         name=None,
         no_progress_bar=False,
         jobs=None,
+        _only_file=False,
         **kwargs,
     ):
         if not hasattr(self, "_download"):
@@ -273,16 +278,21 @@ class BaseFileSystem:
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        if self.isdir(from_info):
+        if not _only_file and self.isdir(from_info):
             return self._download_dir(
-                from_info, to_info, name, no_progress_bar, jobs, **kwargs,
+                from_info, to_info, name, no_progress_bar, jobs, **kwargs
             )
-        return self._download_file(from_info, to_info, name, no_progress_bar,)
+        return self._download_file(from_info, to_info, name, no_progress_bar)
+
+    download_file = partialmethod(download, _only_file=True)
 
     def _download_dir(
-        self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
+        self, from_info, to_info, name, no_progress_bar, jobs, **kwargs
     ):
         from_infos = list(self.walk_files(from_info, **kwargs))
+        if not from_infos:
+            makedirs(to_info, exist_ok=True)
+            return None
         to_infos = (
             to_info / info.relative_to(from_info) for info in from_infos
         )
@@ -294,7 +304,7 @@ class BaseFileSystem:
             disable=no_progress_bar,
         ) as pbar:
             download_files = pbar.wrap_fn(
-                partial(self._download_file, name=name, no_progress_bar=True,)
+                partial(self._download_file, name=name, no_progress_bar=True)
             )
             max_workers = jobs or self.jobs
             with ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -318,9 +328,7 @@ class BaseFileSystem:
                             entry.cancel()
                         raise exc
 
-    def _download_file(
-        self, from_info, to_info, name, no_progress_bar,
-    ):
+    def _download_file(self, from_info, to_info, name, no_progress_bar):
         makedirs(to_info.parent, exist_ok=True)
 
         logger.debug("Downloading '%s' to '%s'", from_info, to_info)

dvc/fs/dvc.py

@@ -10,7 +10,7 @@ from ._metadata import Metadata
 from .base import BaseFileSystem
 
 if typing.TYPE_CHECKING:
-    from dvc.output.base import BaseOutput
+    from dvc.output import Output
 
 
 logger = logging.getLogger(__name__)
@@ -23,11 +23,14 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         repo: DVC repo.
     """
 
+    sep = os.sep
+
     scheme = "local"
     PARAM_CHECKSUM = "md5"
 
-    def __init__(self, repo):
-        super().__init__(repo, {"url": repo.root_dir})
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.repo = kwargs["repo"]
 
     def _find_outs(self, path, *args, **kwargs):
         outs = self.repo.find_outs_by_path(path, *args, **kwargs)
@@ -42,15 +45,15 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         return outs
 
     def _get_granular_hash(
-        self, path_info: PathInfo, out: "BaseOutput", remote=None
+        self, path_info: PathInfo, out: "Output", remote=None
     ):
         assert isinstance(path_info, PathInfo)
         # NOTE: use string paths here for performance reasons
         key = tuple(relpath(path_info, out.path_info).split(os.sep))
         out.get_dir_cache(remote=remote)
-        file_hash = out.hash_info.dir_info.get(key)
-        if file_hash:
-            return file_hash
+        obj = out.obj.trie.get(key)
+        if obj:
+            return obj.hash_info
         raise FileNotFoundError
 
     def open(  # type: ignore
@@ -67,13 +70,17 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             raise IsADirectoryError
 
         out = outs[0]
+
+        if not out.hash_info:
+            raise FileNotFoundError
+
         if out.changed_cache(filter_info=path):
             from dvc.config import NoRemoteError
 
             try:
                 remote_obj = self.repo.cloud.get_remote(remote)
-            except NoRemoteError:
-                raise FileNotFoundError
+            except NoRemoteError as exc:
+                raise FileNotFoundError from exc
             if out.is_dir_checksum:
                 checksum = self._get_granular_hash(path, out).value
             else:
@@ -131,21 +138,14 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         # pull dir cache if needed
         out.get_dir_cache(**kwargs)
 
-        dir_cache = out.dir_cache
-        if not dir_cache:
-            raise FileNotFoundError
-
-        from dvc.objects import Tree
-
-        hash_info = Tree.save_dir_info(out.odb, dir_cache)
-        if hash_info != out.hash_info:
+        if not out.obj:
             raise FileNotFoundError
 
     def _add_dir(self, trie, out, **kwargs):
         self._fetch_dir(out, **kwargs)
 
         base = out.path_info.parts
-        for key, _ in out.dir_cache.items():  # noqa: B301
+        for key, _ in out.obj:  # noqa: B301
             trie[base + key] = None
 
     def _walk(self, root, trie, topdown=True, **kwargs):
@@ -241,9 +241,9 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         elif meta.part_of_output:
             (out,) = meta.outs
             key = path_info.relative_to(out.path_info).parts
-            hash_info = out.hash_info.dir_info.trie.get(key)
-            if hash_info:
-                ret["size"] = hash_info.size
-                ret[hash_info.name] = hash_info.value
+            obj = out.obj.trie.get(key)
+            if obj:
+                ret["size"] = obj.size
+                ret[obj.hash_info.name] = obj.hash_info.value
 
         return ret

dvc/fs/fsspec_wrapper.py

@@ -1,5 +1,8 @@
 import os
 import shutil
+from functools import lru_cache
+
+from funcy import cached_property
 
 from dvc.progress import Tqdm
 
@@ -8,27 +11,40 @@ from .base import BaseFileSystem
 
 # pylint: disable=no-member
 class FSSpecWrapper(BaseFileSystem):
+    TRAVERSE_PREFIX_LEN = 2
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.fs_args = {"skip_instance_cache": True}
+        self.fs_args.update(self._prepare_credentials(**kwargs))
+
+    @cached_property
     def fs(self):
         raise NotImplementedError
 
+    @lru_cache(512)
     def _with_bucket(self, path):
         if isinstance(path, self.PATH_CLS):
             return f"{path.bucket}/{path.path}"
         return path
 
     def _strip_bucket(self, entry):
-        _, entry = entry.split("/", 1)
-        return entry
+        try:
+            bucket, path = entry.split("/", 1)
+        except ValueError:
+            # If there is no path attached, only returns
+            # the bucket (top-level).
+            bucket, path = entry, None
 
-    def _strip_buckets(self, entries, detail, prefix=None):
+        return path or bucket
+
+    def _strip_buckets(self, entries, detail=False):
         for entry in entries:
             if detail:
                 entry = self._entry_hook(entry.copy())
                 entry["name"] = self._strip_bucket(entry["name"])
             else:
-                entry = self._strip_bucket(
-                    f"{prefix}/{entry}" if prefix else entry
-                )
+                entry = self._strip_bucket(entry)
             yield entry
 
     def _entry_hook(self, entry):
@@ -36,11 +52,33 @@ class FSSpecWrapper(BaseFileSystem):
         entries within info() and ls(detail=True) calls"""
         return entry
 
-    def isdir(self, path_info):
+    def _prepare_credentials(
+        self, **config
+    ):  # pylint: disable=unused-argument
+        """Prepare the arguments for authentication to the
+        host filesystem"""
+        return {}
+
+    def _isdir(self, path_info):
         return self.fs.isdir(self._with_bucket(path_info))
 
+    def isdir(self, path_info):
+        try:
+            return self._isdir(path_info)
+        except FileNotFoundError:
+            return False
+
     def isfile(self, path_info):
-        return self.fs.isfile(self._with_bucket(path_info))
+        try:
+            return not self._isdir(path_info)
+        except FileNotFoundError:
+            return False
+
+    def is_empty(self, path_info):
+        entry = self.info(path_info)
+        if entry["type"] == "directory":
+            return not self.fs.ls(self._with_bucket(path_info))
+        return entry["size"] == 0
 
     def open(
         self, path_info, mode="r", **kwargs
@@ -50,24 +88,25 @@ class FSSpecWrapper(BaseFileSystem):
     def copy(self, from_info, to_info):
         self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
 
-    def exists(self, path_info, use_dvcignore=False):
+    def exists(self, path_info) -> bool:
         return self.fs.exists(self._with_bucket(path_info))
 
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
+    def ls(self, path_info, detail=False):
         path = self._with_bucket(path_info)
-        if recursive:
-            for root, _, files in self.fs.walk(path, detail=detail):
-                if detail:
-                    files = files.values()
-                yield from self._strip_buckets(files, detail, prefix=root)
-            return None
+        files = self.fs.ls(path, detail=detail)
+        yield from self._strip_buckets(files, detail=detail)
 
-        yield from self._strip_buckets(self.ls(path, detail=detail), detail)
+    # pylint: disable=unused-argument
+    def find(self, path_info, detail=False, prefix=None):
+        path = self._with_bucket(path_info)
+        files = self.fs.find(path, detail=detail)
+        if detail:
+            files = files.values()
+
+        yield from self._strip_buckets(files, detail=detail)
 
     def walk_files(self, path_info, **kwargs):
-        for file in self.ls(path_info, recursive=True):
+        for file in self.find(path_info, **kwargs):
             yield path_info.replace(path=file)
 
     def remove(self, path_info):
@@ -79,20 +118,20 @@ class FSSpecWrapper(BaseFileSystem):
         info["name"] = self._strip_bucket(info["name"])
         return info
 
-    def _upload_fobj(self, fobj, to_info):
+    def _upload_fobj(self, fobj, to_info, size=None):
         with self.open(to_info, "wb") as fdest:
             shutil.copyfileobj(fobj, fdest, length=fdest.blocksize)
 
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **kwargs
     ):
-        total = os.path.getsize(from_file)
+        size = os.path.getsize(from_file)
         with open(from_file, "rb") as fobj:
             self.upload_fobj(
                 fobj,
                 self._with_bucket(to_info),
+                size=size,
                 desc=name,
-                total=total,
                 no_progress_bar=no_progress_bar,
             )
         self.fs.invalidate_cache(self._with_bucket(to_info.parent))
@@ -113,3 +152,45 @@ class FSSpecWrapper(BaseFileSystem):
             ) as wrapped:
                 with open(to_file, "wb") as fdest:
                     shutil.copyfileobj(wrapped, fdest, length=fobj.blocksize)
+
+
+# pylint: disable=abstract-method
+class ObjectFSWrapper(FSSpecWrapper):
+    TRAVERSE_PREFIX_LEN = 3
+
+    def _isdir(self, path_info):
+        # Directory in object storages are interpreted differently
+        # among different fsspec providers, so this logic is a temporary
+        # measure for us to adapt as of now. It checks whether it is a
+        # directory (as in a prefix with contents) or whether it is an empty
+        # file where it's name ends with a forward slash
+
+        entry = self.info(path_info)
+        return entry["type"] == "directory" or (
+            entry["size"] == 0
+            and entry["type"] == "file"
+            and entry["name"].endswith("/")
+        )
+
+    def find(self, path_info, detail=False, prefix=None):
+        if prefix:
+            path = self._with_bucket(path_info.parent)
+            files = self.fs.find(
+                path, detail=detail, prefix=path_info.parts[-1]
+            )
+        else:
+            path = self._with_bucket(path_info)
+            files = self.fs.find(path, detail=detail)
+
+        if detail:
+            files = files.values()
+
+        # When calling find() on a file, it returns the same file in a list.
+        # For object-based storages, the same behavior applies to empty
+        # directories since they are represented as files. This condition
+        # checks whether we should yield an empty list (if it is an empty
+        # directory) or just yield the file itself.
+        if len(files) == 1 and files[0] == path and self.isdir(path_info):
+            return None
+
+        yield from self._strip_buckets(files, detail=detail)

dvc/fs/gdrive.py

@@ -85,7 +85,7 @@ class GDriveURLInfo(CloudURLInfo):
         self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
 
 
-class GDriveFileSystem(BaseFileSystem):
+class GDriveFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     scheme = Schemes.GDRIVE
     PATH_CLS = GDriveURLInfo
     PARAM_CHECKSUM = "checksum"
@@ -100,8 +100,8 @@ class GDriveFileSystem(BaseFileSystem):
     DEFAULT_GDRIVE_CLIENT_ID = "710796635688-iivsgbgsb6uv1fap6635dhvuei09o66c.apps.googleusercontent.com"  # noqa: E501
     DEFAULT_GDRIVE_CLIENT_SECRET = "a1Fz59uTpVNeG_VGuSKDLJXv"
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
         self.path_info = self.PATH_CLS(config["url"])
 
@@ -126,17 +126,26 @@ class GDriveFileSystem(BaseFileSystem):
         self._client_id = config.get("gdrive_client_id")
         self._client_secret = config.get("gdrive_client_secret")
         self._validate_config()
+
+        tmp_dir = config["gdrive_credentials_tmp_dir"]
+        assert tmp_dir
+
+        self._gdrive_service_credentials_path = tmp_fname(
+            os.path.join(tmp_dir, "")
+        )
         self._gdrive_user_credentials_path = (
-            tmp_fname(os.path.join(self.repo.tmp_dir, ""))
+            tmp_fname(os.path.join(tmp_dir, ""))
             if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
             else config.get(
                 "gdrive_user_credentials_file",
-                os.path.join(
-                    self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE,
-                ),
+                os.path.join(tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE),
             )
         )
 
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):
+        return {"url": urlpath}
+
     def _validate_config(self):
         # Validate Service Account configuration
         if (
@@ -208,8 +217,15 @@ class GDriveFileSystem(BaseFileSystem):
         from pydrive2.auth import GoogleAuth
         from pydrive2.drive import GoogleDrive
 
-        if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
-            with open(self._gdrive_user_credentials_path, "w") as cred_file:
+        temporary_save_path = self._gdrive_user_credentials_path
+        is_credentials_temp = os.getenv(
+            GDriveFileSystem.GDRIVE_CREDENTIALS_DATA
+        )
+        if self._use_service_account:
+            temporary_save_path = self._gdrive_service_credentials_path
+
+        if is_credentials_temp:
+            with open(temporary_save_path, "w") as cred_file:
                 cred_file.write(
                     os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
                 )
@@ -217,6 +233,7 @@ class GDriveFileSystem(BaseFileSystem):
         auth_settings = {
             "client_config_backend": "settings",
             "save_credentials": True,
+            "save_credentials_backend": "file",
             "save_credentials_file": self._gdrive_user_credentials_path,
             "get_refresh_token": True,
             "oauth_scope": [
@@ -226,13 +243,16 @@ class GDriveFileSystem(BaseFileSystem):
         }
 
         if self._use_service_account:
-            auth_settings["save_credentials_backend"] = "json"
             auth_settings["service_config"] = {
                 "client_user_email": self._service_account_user_email,
                 "client_json_file_path": self._service_account_json_file_path,
             }
+            if is_credentials_temp:
+                auth_settings["service_config"][
+                    "client_json_file_path"
+                ] = temporary_save_path
+
         else:
-            auth_settings["save_credentials_backend"] = "file"
             auth_settings["client_config"] = {
                 "client_id": self._client_id or self.DEFAULT_GDRIVE_CLIENT_ID,
                 "client_secret": self._client_secret
@@ -267,8 +287,8 @@ class GDriveFileSystem(BaseFileSystem):
         except Exception as exc:
             raise GDriveAuthError(self.credentials_location) from exc
         finally:
-            if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
-                os.remove(self._gdrive_user_credentials_path)
+            if is_credentials_temp:
+                os.remove(temporary_save_path)
 
         return GoogleDrive(gauth)
 
@@ -503,7 +523,7 @@ class GDriveFileSystem(BaseFileSystem):
         assert not create
         raise FileMissingError(path_info, hint)
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         try:
             self._get_item_id(path_info)
         except FileMissingError:
@@ -518,7 +538,7 @@ class GDriveFileSystem(BaseFileSystem):
         query = f"({query}) and trashed=false"
         return self._gdrive_list(query)
 
-    def _ls_recursive(self, path_info, detail=False):
+    def find(self, path_info, detail=False, prefix=None):
         root_path = path_info.path
         seen_paths = set()
 
@@ -555,13 +575,7 @@ class GDriveFileSystem(BaseFileSystem):
                 else:
                     yield item_path
 
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
-        if recursive:
-            yield from self._ls_recursive(path_info, detail=detail)
-            return None
-
+    def ls(self, path_info, detail=False):
         cached = path_info.path in self._ids_cache["dirs"]
         if cached:
             dir_ids = self._ids_cache["dirs"][path_info.path]
@@ -591,8 +605,8 @@ class GDriveFileSystem(BaseFileSystem):
             self._cache_path_id(root_path, *dir_ids)
 
     def walk_files(self, path_info, **kwargs):
-        for filename in self.ls(path_info, recursive=True):
-            yield path_info.replace(path=filename)
+        for file in self.find(path_info):
+            yield path_info.replace(path=file)
 
     def remove(self, path_info):
         item_id = self._get_item_id(path_info)
@@ -604,22 +618,27 @@ class GDriveFileSystem(BaseFileSystem):
         gdrive_file.FetchMetadata(fields="fileSize")
         return {"size": gdrive_file.get("fileSize")}
 
-    def _upload_fobj(self, fobj, to_info):
+    def _upload_fobj(self, fobj, to_info, **kwargs):
         dirname = to_info.parent
         assert dirname
         parent_id = self._get_item_id(dirname, create=True)
         self._gdrive_upload_fobj(fobj, parent_id, to_info.name)
 
     def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+        self,
+        from_file,
+        to_info,
+        name=None,
+        no_progress_bar=False,
+        **_kwargs,
     ):
         with open(from_file, "rb") as fobj:
             self.upload_fobj(
                 fobj,
                 to_info,
+                size=os.path.getsize(from_file),
                 no_progress_bar=no_progress_bar,
                 desc=name or to_info.name,
-                total=os.path.getsize(from_file),
             )
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):

dvc/fs/git.py

@@ -1,8 +1,6 @@
 import errno
 import os
 
-from funcy import cached_property
-
 from dvc.utils import is_exec, relpath
 
 from .base import BaseFileSystem
@@ -11,43 +9,31 @@ from .base import BaseFileSystem
 class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     """Proxies the repo file access methods to Git objects"""
 
-    def __init__(
-        self, root_dir, trie, use_dvcignore=False, dvcignore_root=None
-    ):
-        super().__init__(None, {})
-        self._fs_root = root_dir
+    sep = os.sep
+
+    scheme = "local"
+
+    def __init__(self, root_dir, trie):
+        super().__init__()
+        self._root = root_dir
         self.trie = trie
-        self.use_dvcignore = use_dvcignore
-        self.dvcignore_root = dvcignore_root
 
     @property
     def rev(self):
         return self.trie.rev
 
-    @property
-    def fs_root(self):
-        return self._fs_root
-
     def _get_key(self, path):
         if isinstance(path, str):
             if not os.path.isabs(path):
                 relparts = path.split(os.sep)
             else:
-                relparts = relpath(path, self.fs_root).split(os.sep)
+                relparts = relpath(path, self._root).split(os.sep)
         else:
-            relparts = path.relative_to(self.fs_root).parts
+            relparts = path.relative_to(self._root).parts
         if relparts == ["."]:
             return ()
         return tuple(relparts)
 
-    @cached_property
-    def dvcignore(self):
-        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
-
-        root = self.dvcignore_root or self.fs_root
-        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
-        return cls(self, root)
-
     def open(
         self, path, mode="r", encoding=None
     ):  # pylint: disable=arguments-differ
@@ -66,50 +52,27 @@ class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 errno.EISDIR, os.strerror(errno.EISDIR), path
             ) from exc
 
-    def exists(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        def _is_ignored(path):
-            return self.dvcignore.is_ignored_file(
-                path
-            ) or self.dvcignore.is_ignored_dir(path)
-
-        if use_dvcignore and _is_ignored(path):
-            return False
-
-        key = self._get_key(path)
+    def exists(self, path_info) -> bool:
+        key = self._get_key(path_info)
         return self.trie.exists(key)
 
-    def isdir(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        if use_dvcignore and self.dvcignore.is_ignored_dir(path):
-            return False
-        key = self._get_key(path)
+    def isdir(self, path_info) -> bool:
+        key = self._get_key(path_info)
         return self.trie.isdir(key)
 
-    def isfile(self, path):  # pylint: disable=arguments-differ
-        if self.dvcignore.is_ignored_file(path):
-            return False
-        key = self._get_key(path)
+    def isfile(self, path_info) -> bool:
+        key = self._get_key(path_info)
         return self.trie.isfile(key)
 
-    def walk(
-        self,
-        top,
-        topdown=True,
-        onerror=None,
-        use_dvcignore=True,
-        ignore_subrepos=True,
-    ):
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
         - no support for symlinks
         """
-        if not self.isdir(top, use_dvcignore=use_dvcignore):
+        if not self.isdir(top):
             if onerror:
-                if self.exists(top, use_dvcignore=use_dvcignore):
+                if self.exists(top):
                     exc = NotADirectoryError(
                         errno.ENOTDIR, os.strerror(errno.ENOTDIR), top
                     )
@@ -123,13 +86,9 @@ class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         key = self._get_key(top)
         for prefix, dirs, files in self.trie.walk(key, topdown=topdown):
             if prefix:
-                root = os.path.join(self.fs_root, os.sep.join(prefix))
+                root = os.path.join(self._root, os.sep.join(prefix))
             else:
-                root = self.fs_root
-            if use_dvcignore:
-                dirs[:], files[:] = self.dvcignore(
-                    root, dirs, files, ignore_subrepos=ignore_subrepos,
-                )
+                root = self._root
             yield root, dirs, files
 
     def isexec(self, path_info):
@@ -158,17 +117,8 @@ class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 errno.ENOENT, os.strerror(errno.ENOENT), path
             )
 
-    @property
-    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
-        # NOTE: gitpython is not threadsafe. See
-        # https://github.com/iterative/dvc/issues/4079
-        return 1
-
-    def walk_files(self, top):  # pylint: disable=arguments-differ
-        for root, _, files in self.walk(top):
+    def walk_files(self, path_info, **kwargs):
+        for root, _, files in self.walk(path_info, **kwargs):
             for file in files:
                 # NOTE: os.path.join is ~5.5 times slower
                 yield f"{root}{os.sep}{file}"
-
-    def _reset(self):
-        return self.__dict__.pop("dvcignore", None)

dvc/fs/gs.py

@@ -6,32 +6,27 @@ from funcy import cached_property, wrap_prop
 from dvc.path_info import CloudURLInfo
 from dvc.scheme import Schemes
 
-from .fsspec_wrapper import FSSpecWrapper
+from .fsspec_wrapper import ObjectFSWrapper
 
 
-class GSFileSystem(FSSpecWrapper):
+# pylint:disable=abstract-method
+class GSFileSystem(ObjectFSWrapper):
     scheme = Schemes.GS
     PATH_CLS = CloudURLInfo
     REQUIRES = {"gcsfs": "gcsfs"}
     PARAM_CHECKSUM = "etag"
     DETAIL_FIELDS = frozenset(("etag", "size"))
+    TRAVERSE_PREFIX_LEN = 2
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-
-        url = config.get("url", "gs://")
-        self.path_info = self.PATH_CLS(url)
-
-        self.login_info = self._prepare_credentials(config)
-
-    def _prepare_credentials(self, config):
-        login_info = {}
+    def _prepare_credentials(self, **config):
+        login_info = {"consistency": None}
         login_info["project"] = config.get("projectname")
         login_info["token"] = config.get("credentialpath")
         return login_info
 
     def _entry_hook(self, entry):
-        entry["etag"] = base64.b64decode(entry["etag"]).hex()
+        if "etag" in entry:
+            entry["etag"] = base64.b64decode(entry["etag"]).hex()
         return entry
 
     @wrap_prop(threading.Lock())
@@ -39,4 +34,4 @@ class GSFileSystem(FSSpecWrapper):
     def fs(self):
         from gcsfs import GCSFileSystem
 
-        return GCSFileSystem(**self.login_info)
+        return GCSFileSystem(**self.fs_args)

dvc/fs/hdfs.py

@@ -6,7 +6,6 @@ import shutil
 import subprocess
 from collections import deque
 from contextlib import closing, contextmanager
-from urllib.parse import urlparse
 
 from dvc.hash_info import HashInfo
 from dvc.progress import Tqdm
@@ -65,27 +64,23 @@ class HDFSFileSystem(BaseFileSystem):
     PARAM_CHECKSUM = "checksum"
     TRAVERSE_PREFIX_LEN = 2
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
-        self.path_info = None
-        url = config.get("url")
-        if not url:
-            return
+        self.host = config["host"]
+        self.user = config.get("user")
+        self.port = config.get("port")
 
-        parsed = urlparse(url)
-        user = parsed.username or config.get("user")
-
-        self.path_info = self.PATH_CLS.from_parts(
-            scheme=self.scheme,
-            host=parsed.hostname,
-            user=user,
-            port=parsed.port,
-            path=parsed.path,
-        )
+        self.kerb_ticket = config.get("kerb_ticket")
 
     @staticmethod
-    def hdfs(path_info):
+    def _get_kwargs_from_urls(urlpath):
+        from fsspec.implementations.hdfs import PyArrowHDFS
+
+        # pylint:disable=protected-access
+        return PyArrowHDFS._get_kwargs_from_urls(urlpath)
+
+    def hdfs(self, path_info):
         import pyarrow.fs
 
         # NOTE: HadoopFileSystem is not meant to be closed by us and doesn't
@@ -100,6 +95,7 @@ class HDFSFileSystem(BaseFileSystem):
             path_info.host,
             path_info.port,
             user=path_info.user,
+            kerb_ticket=self.kerb_ticket,
         )
 
     @contextmanager
@@ -121,7 +117,7 @@ class HDFSFileSystem(BaseFileSystem):
                 raise FileNotFoundError(*e.args)
             raise
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         assert not isinstance(path_info, list)
         assert path_info.scheme == "hdfs"
         with self.hdfs(path_info) as hdfs:
@@ -157,15 +153,15 @@ class HDFSFileSystem(BaseFileSystem):
         if not topdown:
             yield root, dirs, nondirs
 
-    def walk(self, path_info, **kwargs):
-        if not self.isdir(path_info):
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
+        if not self.isdir(top):
             return
 
-        with self.hdfs(path_info) as hdfs:
+        with self.hdfs(top) as hdfs:
             for root, dnames, fnames in self._walk(
-                hdfs, path_info.path, **kwargs
+                hdfs, top.path, topdown=topdown
             ):
-                yield path_info.replace(path=root), dnames, fnames
+                yield top.replace(path=root), dnames, fnames
 
     def walk_files(self, path_info, **kwargs):
         for root, _, fnames in self.walk(path_info):
@@ -234,7 +230,7 @@ class HDFSFileSystem(BaseFileSystem):
             size=self.getsize(path_info),
         )
 
-    def _upload_fobj(self, fobj, to_info):
+    def _upload_fobj(self, fobj, to_info, **kwargs):
         with self.hdfs(to_info) as hdfs:
             with hdfs.open_output_stream(to_info.path) as fdest:
                 shutil.copyfileobj(fobj, fdest)

dvc/fs/http.py

@@ -1,10 +1,12 @@
 import logging
 import os.path
 import threading
+from typing import Optional
+from urllib.parse import urlparse
 
 from funcy import cached_property, memoize, wrap_prop, wrap_with
 
-import dvc.prompt as prompt
+from dvc import prompt
 from dvc.exceptions import DvcException, HTTPError
 from dvc.path_info import HTTPURLInfo
 from dvc.progress import Tqdm
@@ -35,16 +37,10 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     REQUEST_TIMEOUT = 60
     CHUNK_SIZE = 2 ** 16
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
-        url = config.get("url")
-        if url:
-            self.path_info = self.PATH_CLS(url)
-            self.user = config.get("user", None)
-            self.host = self.path_info.host
-        else:
-            self.path_info = None
+        self.user = config.get("user", None)
 
         self.auth = config.get("auth", None)
         self.custom_auth_header = config.get("custom_auth_header", None)
@@ -54,12 +50,12 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         self.ssl_verify = config.get("ssl_verify", True)
         self.method = config.get("method", "POST")
 
-    def _auth_method(self):
+    def _auth_method(self, url):
         from requests.auth import HTTPBasicAuth, HTTPDigestAuth
 
         if self.auth:
             if self.ask_password and self.password is None:
-                self.password = ask_password(self.host, self.user)
+                self.password = ask_password(urlparse(url).hostname, self.user)
             if self.auth == "basic":
                 return HTTPBasicAuth(self.user, self.password)
             if self.auth == "digest":
@@ -102,7 +98,7 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             res = self._session.request(
                 method,
                 url,
-                auth=self._auth_method(),
+                auth=self._auth_method(url),
                 headers=self.headers,
                 **kwargs,
             )
@@ -137,7 +133,7 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
         return response
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         res = self._head(path_info.url)
         if res.status_code == 404:
             return False
@@ -146,13 +142,12 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         raise HTTPError(res.status_code, res.reason)
 
     def info(self, path_info):
-        url = path_info.url
-        headers = self._head(url).headers
-        etag = headers.get("ETag") or headers.get("Content-MD5")
-        size = int(headers.get("Content-Length"))
+        resp = self._head(path_info.url)
+        etag = resp.headers.get("ETag") or resp.headers.get("Content-MD5")
+        size = self._content_length(resp)
         return {"etag": etag, "size": size}
 
-    def _upload_fobj(self, fobj, to_info):
+    def _upload_fobj(self, fobj, to_info, **kwargs):
         def chunks(fobj):
             while True:
                 chunk = fobj.read(self.CHUNK_SIZE)
@@ -189,12 +184,12 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             self.upload_fobj(
                 fobj,
                 to_info,
+                size=None if no_progress_bar else os.path.getsize(from_file),
                 no_progress_bar=no_progress_bar,
                 desc=name or to_info.url,
-                total=None if no_progress_bar else os.path.getsize(from_file),
             )
 
     @staticmethod
-    def _content_length(response):
+    def _content_length(response) -> Optional[int]:
         res = response.headers.get("Content-Length")
         return int(res) if res else None

dvc/fs/local.py

@@ -2,8 +2,6 @@ import logging
 import os
 import stat
 
-from funcy import cached_property
-
 from dvc.path_info import PathInfo
 from dvc.scheme import Schemes
 from dvc.system import System
@@ -16,76 +14,40 @@ logger = logging.getLogger(__name__)
 
 
 class LocalFileSystem(BaseFileSystem):
+    sep = os.sep
+
     scheme = Schemes.LOCAL
     PATH_CLS = PathInfo
     PARAM_CHECKSUM = "md5"
     PARAM_PATH = "path"
     TRAVERSE_PREFIX_LEN = 2
 
-    def __init__(self, repo, config, use_dvcignore=False, dvcignore_root=None):
-        super().__init__(repo, config)
-        url = config.get("url")
-        self.path_info = self.PATH_CLS(url) if url else None
-        self.use_dvcignore = use_dvcignore
-        self.dvcignore_root = dvcignore_root
+    def __init__(self, **config):
+        from fsspec.implementations.local import LocalFileSystem as LocalFS
 
-    @property
-    def fs_root(self):
-        return self.config.get("url")
-
-    @cached_property
-    def dvcignore(self):
-        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
-
-        root = self.dvcignore_root or self.fs_root
-        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
-        return cls(self, root)
+        super().__init__(**config)
+        self.fs = LocalFS()
 
     @staticmethod
     def open(path_info, mode="r", encoding=None, **kwargs):
         return open(path_info, mode=mode, encoding=encoding)
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         assert isinstance(path_info, str) or path_info.scheme == "local"
-        if self.repo:
-            ret = os.path.lexists(path_info)
-        else:
-            ret = os.path.exists(path_info)
-        if not ret:
-            return False
-        if not use_dvcignore:
-            return True
+        return self.fs.exists(path_info)
 
-        return not self.dvcignore.is_ignored_file(
-            path_info
-        ) and not self.dvcignore.is_ignored_dir(path_info)
+    def isfile(self, path_info) -> bool:
+        return os.path.isfile(path_info)
 
-    def isfile(self, path_info):
-        if not os.path.isfile(path_info):
-            return False
-
-        return not self.dvcignore.is_ignored_file(path_info)
-
-    def isdir(
-        self, path_info, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        if not os.path.isdir(path_info):
-            return False
-        return not (use_dvcignore and self.dvcignore.is_ignored_dir(path_info))
+    def isdir(self, path_info) -> bool:
+        return os.path.isdir(path_info)
 
     def iscopy(self, path_info):
         return not (
             System.is_symlink(path_info) or System.is_hardlink(path_info)
         )
 
-    def walk(
-        self,
-        top,
-        topdown=True,
-        onerror=None,
-        use_dvcignore=True,
-        ignore_subrepos=True,
-    ):
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
         """Directory fs generator.
 
         See `os.walk` for the docs. Differences:
@@ -94,14 +56,6 @@ class LocalFileSystem(BaseFileSystem):
         for root, dirs, files in os.walk(
             top, topdown=topdown, onerror=onerror
         ):
-            if use_dvcignore:
-                dirs[:], files[:] = self.dvcignore(
-                    os.path.abspath(root),
-                    dirs,
-                    files,
-                    ignore_subrepos=ignore_subrepos,
-                )
-
             yield os.path.normpath(root), dirs, files
 
     def walk_files(self, path_info, **kwargs):
@@ -133,9 +87,6 @@ class LocalFileSystem(BaseFileSystem):
         return is_exec(mode)
 
     def stat(self, path):
-        if self.dvcignore.is_ignored(path):
-            raise FileNotFoundError
-
         return os.stat(path)
 
     def move(self, from_info, to_info):
@@ -154,7 +105,7 @@ class LocalFileSystem(BaseFileSystem):
             self.remove(tmp_info)
             raise
 
-    def _upload_fobj(self, fobj, to_info):
+    def _upload_fobj(self, fobj, to_info, **kwargs):
         self.makedirs(to_info.parent)
         tmp_info = to_info.parent / tmp_fname("")
         try:
@@ -210,7 +161,7 @@ class LocalFileSystem(BaseFileSystem):
         }
 
     def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs,
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
         makedirs(to_info.parent, exist_ok=True)
 
@@ -227,6 +178,3 @@ class LocalFileSystem(BaseFileSystem):
         copyfile(
             from_info, to_file, no_progress_bar=no_progress_bar, name=name
         )
-
-    def _reset(self):
-        return self.__dict__.pop("dvcignore", None)

dvc/fs/memory.py

@@ -5,21 +5,24 @@ class MemoryFileSystem(BaseFileSystem):
     scheme = "local"
     PARAM_CHECKSUM = "md5"
 
-    def __init__(self, repo, config):
+    def __init__(self, **kwargs):
         from fsspec.implementations.memory import MemoryFileSystem as MemFS
 
-        super().__init__(repo, config)
+        super().__init__(**kwargs)
 
         self.fs = MemFS()
 
-    def exists(self, path_info, use_dvcignore=True):
-        return self.fs.exists(path_info.path)
+    def exists(self, path_info) -> bool:
+        return self.fs.exists(path_info.fspath)
 
     def open(self, path_info, mode="r", encoding=None, **kwargs):
         return self.fs.open(
             path_info.fspath, mode=mode, encoding=encoding, **kwargs
         )
 
+    def info(self, path_info):
+        return self.fs.info(path_info.fspath)
+
     def stat(self, path_info):
         import os
 

dvc/fs/oss.py

@@ -39,13 +39,9 @@ class OSSFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     COPY_POLL_SECONDS = 5
     LIST_OBJECT_PAGE_SIZE = 100
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
-        url = config.get("url")
-        self.path_info = self.PATH_CLS(url) if url else None
-
-        self.bucket = self.path_info.bucket
         self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")
 
         self.key_id = (
@@ -68,27 +64,19 @@ class OSSFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         logger.debug(f"key id: {self.key_id}")
         logger.debug(f"key secret: {self.key_secret}")
 
-        auth = oss2.Auth(self.key_id, self.key_secret)
-        bucket = oss2.Bucket(auth, self.endpoint, self.bucket)
+        return oss2.Auth(self.key_id, self.key_secret)
 
-        # Ensure bucket exists
-        try:
-            bucket.get_bucket_info()
-        except oss2.exceptions.NoSuchBucket:
-            bucket.create_bucket(
-                oss2.BUCKET_ACL_PUBLIC_READ,
-                oss2.models.BucketCreateConfig(
-                    oss2.BUCKET_STORAGE_CLASS_STANDARD
-                ),
-            )
-        return bucket
+    def _get_bucket(self, bucket):
+        import oss2
+
+        return oss2.Bucket(self.oss_service, self.endpoint, bucket)
 
     def _generate_download_url(self, path_info, expires=3600):
-        assert path_info.bucket == self.bucket
+        return self._get_bucket(path_info.bucket).sign_url(
+            "GET", path_info.path, expires
+        )
 
-        return self.oss_service.sign_url("GET", path_info.path, expires)
-
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         paths = self._list_paths(path_info)
         return any(path_info.path == path for path in paths)
 
@@ -96,7 +84,7 @@ class OSSFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         import oss2
 
         for blob in oss2.ObjectIterator(
-            self.oss_service, prefix=path_info.path
+            self._get_bucket(path_info.bucket), prefix=path_info.path
         ):
             yield blob.key
 
@@ -114,16 +102,17 @@ class OSSFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             raise NotImplementedError
 
         logger.debug(f"Removing oss://{path_info}")
-        self.oss_service.delete_object(path_info.path)
+        self._get_bucket(path_info.bucket).delete_object(path_info.path)
 
-    def _upload_fobj(self, fobj, to_info):
-        self.oss_service.put_object(to_info.path, fobj)
+    def _upload_fobj(self, fobj, to_info, **kwargs):
+        self._get_bucket(to_info.bucket).put_object(to_info.path, fobj)
 
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
         with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
-            self.oss_service.put_object_from_file(
+            bucket = self._get_bucket(to_info.bucket)
+            bucket.put_object_from_file(
                 to_info.path, from_file, progress_callback=pbar.update_to
             )
 
@@ -133,8 +122,9 @@ class OSSFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
             import oss2
 
+            bucket = self._get_bucket(from_info.bucket)
             oss2.resumable_download(
-                self.oss_service,
+                bucket,
                 from_info.path,
                 to_file,
                 progress_callback=pbar.update_to,

dvc/fs/repo.py

@@ -31,13 +31,15 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         kwargs: Additional keyword arguments passed to the `DvcFileSystem()`.
     """
 
+    sep = os.sep
+
     scheme = "local"
     PARAM_CHECKSUM = "md5"
 
     def __init__(
-        self, repo, subrepos=False, repo_factory: RepoFactory = None,
+        self, repo=None, subrepos=False, repo_factory: RepoFactory = None
     ):
-        super().__init__(repo, {"url": repo.root_dir})
+        super().__init__()
 
         from dvc.utils.collections import PathStringTrie
 
@@ -49,6 +51,7 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             self.repo_factory = repo_factory
 
         self._main_repo = repo
+        self.hash_jobs = repo.fs.hash_jobs
         self.root_dir = repo.root_dir
         self._traverse_subrepos = subrepos
 
@@ -61,9 +64,15 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         """Keep a dvcfs instance of each repo."""
 
         if hasattr(repo, "dvc_dir"):
-            self._dvcfss[repo.root_dir] = DvcFileSystem(repo)
+            self._dvcfss[repo.root_dir] = DvcFileSystem(repo=repo)
 
-    def _get_repo(self, path) -> Optional["Repo"]:
+    @property
+    def repo_url(self):
+        if self._main_repo is None:
+            return None
+        return self._main_repo.url
+
+    def _get_repo(self, path: str) -> Optional["Repo"]:
         """Returns repo that the path falls in, using prefix.
 
         If the path is already tracked/collected, it just returns the repo.
@@ -93,11 +102,11 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             if self._is_dvc_repo(d):
                 repo = self.repo_factory(
                     d,
-                    scm=self.repo.scm,
-                    rev=self.repo.get_rev(),
+                    scm=self._main_repo.scm,
+                    rev=self._main_repo.get_rev(),
                     repo_factory=self.repo_factory,
                 )
-                self._dvcfss[repo.root_dir] = DvcFileSystem(repo)
+                self._dvcfss[repo.root_dir] = DvcFileSystem(repo=repo)
             self._subrepos_trie[d] = repo
 
     def _is_dvc_repo(self, dir_path):
@@ -108,8 +117,7 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         from dvc.repo import Repo
 
         repo_path = os.path.join(dir_path, Repo.DVC_DIR)
-        # dvcignore will ignore subrepos, therefore using `use_dvcignore=False`
-        return self._main_repo.fs.isdir(repo_path, use_dvcignore=False)
+        return self._main_repo.fs.isdir(repo_path)
 
     def _get_fs_pair(
         self, path
@@ -142,19 +150,20 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
         return dvc_fs.open(path_info, mode=mode, encoding=encoding, **kwargs)
 
-    def exists(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        fs, dvc_fs = self._get_fs_pair(path)
+    def exists(self, path_info) -> bool:
+        fs, dvc_fs = self._get_fs_pair(path_info)
 
         if not dvc_fs:
-            return fs.exists(path)
+            return fs.exists(path_info)
 
-        if fs.exists(path):
+        if dvc_fs.repo.dvcignore.is_ignored(fs, path_info):
+            return False
+
+        if fs.exists(path_info):
             return True
 
         try:
-            meta = dvc_fs.metadata(path)
+            meta = dvc_fs.metadata(path_info)
         except FileNotFoundError:
             return False
 
@@ -167,6 +176,9 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     def isdir(self, path):  # pylint: disable=arguments-differ
         fs, dvc_fs = self._get_fs_pair(path)
 
+        if dvc_fs and dvc_fs.repo.dvcignore.is_ignored_dir(path):
+            return False
+
         try:
             st = fs.stat(path)
             return stat.S_ISDIR(st.st_mode)
@@ -195,6 +207,9 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     def isfile(self, path):  # pylint: disable=arguments-differ
         fs, dvc_fs = self._get_fs_pair(path)
 
+        if dvc_fs and dvc_fs.repo.dvcignore.is_ignored_file(path):
+            return False
+
         try:
             st = fs.stat(path)
             return stat.S_ISREG(st.st_mode)
@@ -238,13 +253,11 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     def _subrepo_walk(self, dir_path, **kwargs):
         """Walk into a new repo.
 
-         NOTE: subrepo will only be discovered when walking if
-         ignore_subrepos is set to False.
+        NOTE: subrepo will only be discovered when walking if
+        ignore_subrepos is set to False.
         """
         fs, dvc_fs = self._get_fs_pair(dir_path)
-        fs_walk = fs.walk(
-            dir_path, topdown=True, ignore_subrepos=not self._traverse_subrepos
-        )
+        fs_walk = fs.walk(dir_path, topdown=True)
         if dvc_fs:
             dvc_walk = dvc_fs.walk(dir_path, topdown=True, **kwargs)
         else:
@@ -306,15 +319,7 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             elif dirname in repo_set:
                 yield from self._walk(repo_walk, None, dvcfiles=dvcfiles)
 
-    def walk(
-        self,
-        top,
-        topdown=True,
-        onerror=None,
-        dvcfiles=False,
-        follow_subrepos=None,
-        **kwargs
-    ):  # pylint: disable=arguments-differ
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
         """Walk and merge both DVC and repo fss.
 
         Args:
@@ -340,17 +345,14 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 onerror(NotADirectoryError(top))
             return
 
-        ignore_subrepos = not self._traverse_subrepos
-        if follow_subrepos is not None:
-            ignore_subrepos = not follow_subrepos
+        repo = self._get_repo(os.path.abspath(top))
+        dvcfiles = kwargs.pop("dvcfiles", False)
 
         fs, dvc_fs = self._get_fs_pair(top)
         repo_exists = fs.exists(top)
-        repo_walk = fs.walk(
-            top,
-            topdown=topdown,
-            onerror=onerror,
-            ignore_subrepos=ignore_subrepos,
+
+        repo_walk = repo.dvcignore.walk(
+            fs, top, topdown=topdown, onerror=onerror, **kwargs
         )
 
         if not dvc_fs or (repo_exists and dvc_fs.isdvc(top)):
@@ -358,16 +360,20 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             return
 
         if not repo_exists:
-            yield from dvc_fs.walk(top, topdown=topdown, **kwargs)
+            yield from dvc_fs.walk(
+                top, topdown=topdown, onerror=onerror, **kwargs
+            )
 
         dvc_walk = None
         if dvc_fs.exists(top):
-            dvc_walk = dvc_fs.walk(top, topdown=topdown, **kwargs)
+            dvc_walk = dvc_fs.walk(
+                top, topdown=topdown, onerror=onerror, **kwargs
+            )
 
         yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
 
-    def walk_files(self, top, **kwargs):  # pylint: disable=arguments-differ
-        for root, _, files in self.walk(top, **kwargs):
+    def walk_files(self, path_info, **kwargs):
+        for root, _, files in self.walk(path_info, **kwargs):
             for fname in files:
                 yield PathInfo(root) / fname
 
@@ -380,15 +386,11 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
         with open(to_file, "wb+") as to_fobj:
             with Tqdm.wrapattr(
-                to_fobj, "write", desc=name, disable=no_progress_bar,
+                to_fobj, "write", desc=name, disable=no_progress_bar
             ) as wrapped:
                 with self.open(from_info, "rb", **kwargs) as from_fobj:
                     shutil.copyfileobj(from_fobj, wrapped)
 
-    @property
-    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
-        return self._main_repo.fs.hash_jobs
-
     def metadata(self, path):
         abspath = os.path.abspath(path)
         path_info = PathInfo(abspath)

dvc/fs/s3.py

@@ -1,74 +1,34 @@
-import logging
+import functools
 import os
 import threading
-from contextlib import contextmanager
+from collections import defaultdict
 
 from funcy import cached_property, wrap_prop
 
-from dvc.config import ConfigError
-from dvc.exceptions import DvcException, ETagMismatchError
-from dvc.hash_info import HashInfo
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
-from dvc.utils import conversions, error_link
 
-from .base import BaseFileSystem
-
-logger = logging.getLogger(__name__)
+from .fsspec_wrapper import ObjectFSWrapper
 
 _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config")
 
 
-class S3FileSystem(BaseFileSystem):
+# pylint:disable=abstract-method
+class BaseS3FileSystem(ObjectFSWrapper):
     scheme = Schemes.S3
     PATH_CLS = CloudURLInfo
-    REQUIRES = {"boto3": "boto3"}
+    REQUIRES = {"s3fs": "s3fs", "boto3": "boto3"}
     PARAM_CHECKSUM = "etag"
     DETAIL_FIELDS = frozenset(("etag", "size"))
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    _GRANTS = {
+        "grant_full_control": "GrantFullControl",
+        "grant_read": "GrantRead",
+        "grant_read_acp": "GrantReadACP",
+        "grant_write_acp": "GrantWriteACP",
+    }
 
-        url = config.get("url", "s3://")
-        self.path_info = self.PATH_CLS(url)
-
-        self.region = config.get("region")
-        self.profile = config.get("profile")
-        self.endpoint_url = config.get("endpointurl")
-
-        self.use_ssl = config.get("use_ssl", True)
-
-        self.extra_args = {}
-
-        self.sse = config.get("sse")
-        if self.sse:
-            self.extra_args["ServerSideEncryption"] = self.sse
-
-        self.sse_kms_key_id = config.get("sse_kms_key_id")
-        if self.sse_kms_key_id:
-            self.extra_args["SSEKMSKeyId"] = self.sse_kms_key_id
-
-        self.acl = config.get("acl")
-        if self.acl:
-            self.extra_args["ACL"] = self.acl
-
-        self._append_aws_grants_to_extra_args(config)
-
-        self.access_key_id = config.get("access_key_id")
-        self.secret_access_key = config.get("secret_access_key")
-        self.session_token = config.get("session_token")
-
-        shared_creds = config.get("credentialpath")
-        if shared_creds:
-            os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
-
-        config_path = config.get("configpath")
-        if config_path:
-            os.environ.setdefault("AWS_CONFIG_FILE", config_path)
-        self._transfer_config = None
-
-    # https://github.com/aws/aws-cli/blob/0376c6262d6b15dc36c82e6da6e1aad10249cc8c/awscli/customizations/s3/transferconfig.py#L107-L113
     _TRANSFER_CONFIG_ALIASES = {
         "max_queue_size": "max_io_queue",
         "max_concurrent_requests": "max_concurrency",
@@ -76,11 +36,15 @@ class S3FileSystem(BaseFileSystem):
         "multipart_chunksize": "multipart_chunksize",
     }
 
-    def _transform_config(self, s3_config):
+    def _split_s3_config(self, s3_config):
         """Splits the general s3 config into 2 different config
         objects, one for transfer.TransferConfig and other is the
         general session config"""
 
+        from boto3.s3.transfer import TransferConfig
+
+        from dvc.utils import conversions
+
         config, transfer_config = {}, {}
         for key, value in s3_config.items():
             if key in self._TRANSFER_CONFIG_ALIASES:
@@ -93,370 +57,174 @@ class S3FileSystem(BaseFileSystem):
             else:
                 config[key] = value
 
-        return config, transfer_config
+        # pylint: disable=attribute-defined-outside-init
+        self._transfer_config = TransferConfig(**transfer_config)
+        return config
 
-    def _process_config(self):
-        from boto3.s3.transfer import TransferConfig
+    def _load_aws_config_file(self, profile):
         from botocore.configloader import load_config
 
+        # pylint: disable=attribute-defined-outside-init
+        self._transfer_config = None
         config_path = os.environ.get("AWS_CONFIG_FILE", _AWS_CONFIG_PATH)
         if not os.path.exists(config_path):
             return None
 
         config = load_config(config_path)
-        profile = config["profiles"].get(self.profile or "default")
-        if not profile:
+        profile_config = config["profiles"].get(profile or "default")
+        if not profile_config:
             return None
 
-        s3_config = profile.get("s3", {})
-        s3_config, transfer_config = self._transform_config(s3_config)
-        self._transfer_config = TransferConfig(**transfer_config)
-        return s3_config
+        s3_config = profile_config.get("s3", {})
+        return self._split_s3_config(s3_config)
 
+    def _prepare_credentials(self, **config):
+        from dvc.config import ConfigError
+        from dvc.utils.flatten import flatten, unflatten
+
+        login_info = defaultdict(dict)
+
+        # credentials
+        login_info["key"] = config.get("access_key_id")
+        login_info["secret"] = config.get("secret_access_key")
+        login_info["token"] = config.get("session_token")
+
+        # session configuration
+        login_info["profile"] = config.get("profile")
+        login_info["use_ssl"] = config.get("use_ssl", True)
+
+        # extra client configuration
+        client = login_info["client_kwargs"]
+        client["region_name"] = config.get("region")
+        client["endpoint_url"] = config.get("endpointurl")
+        client["verify"] = config.get("ssl_verify")
+
+        # encryptions
+        additional = login_info["s3_additional_kwargs"]
+        additional["ServerSideEncryption"] = config.get("sse")
+        additional["SSEKMSKeyId"] = config.get("sse_kms_key_id")
+        additional["ACL"] = config.get("acl")
+        for grant_option, grant_key in self._GRANTS.items():
+            if config.get(grant_option):
+                if additional["ACL"]:
+                    raise ConfigError(
+                        "`acl` and `grant_*` AWS S3 config options "
+                        "are mutually exclusive"
+                    )
+                additional[grant_key] = config[grant_option]
+
+        # config kwargs
+        session_config = login_info["config_kwargs"]
+        session_config["s3"] = self._load_aws_config_file(
+            login_info["profile"]
+        )
+
+        shared_creds = config.get("credentialpath")
+        if shared_creds:
+            os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
+
+        config_path = config.get("configpath")
+        if config_path:
+            os.environ.setdefault("AWS_CONFIG_FILE", config_path)
+
+        return unflatten(
+            {
+                key: value
+                for key, value in flatten(login_info).items()
+                if value is not None
+            }
+        )
+
+    def _entry_hook(self, entry):
+        entry = entry.copy()
+        if "ETag" in entry:
+            entry["etag"] = entry["ETag"].strip('"')
+        return entry
+
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def fs(self):
+        from s3fs import S3FileSystem as _S3FileSystem
+
+        return _S3FileSystem(**self.fs_args)
+
+
+def _translate_exceptions(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            func(*args, **kwargs)
+        except Exception as exc:
+            from s3fs.errors import translate_boto_error
+
+            raise translate_boto_error(exc)
+
+    return wrapper
+
+
+class S3FileSystem(BaseS3FileSystem):  # pylint:disable=abstract-method
     @wrap_prop(threading.Lock())
     @cached_property
     def s3(self):
         import boto3
 
+        login_info = self.fs_args
+        client_kwargs = login_info.get("client_kwargs", {})
         session_opts = {
-            "profile_name": self.profile,
-            "region_name": self.region,
+            "profile_name": login_info.get("profile"),
+            "region_name": client_kwargs.get("region_name"),
         }
 
-        if self.access_key_id:
-            session_opts["aws_access_key_id"] = self.access_key_id
-        if self.secret_access_key:
-            session_opts["aws_secret_access_key"] = self.secret_access_key
-        if self.session_token:
-            session_opts["aws_session_token"] = self.session_token
+        if "key" in login_info:
+            session_opts["aws_access_key_id"] = login_info["key"]
+        if "secret" in login_info:
+            session_opts["aws_secret_access_key"] = login_info["secret"]
+        if "token" in login_info:
+            session_opts["aws_session_token"] = login_info["token"]
 
         session = boto3.session.Session(**session_opts)
-        s3_config = self._process_config()
 
         return session.resource(
             "s3",
-            endpoint_url=self.endpoint_url,
-            use_ssl=self.use_ssl,
-            config=boto3.session.Config(
-                signature_version="s3v4", s3=s3_config
-            ),
+            endpoint_url=client_kwargs.get("endpoint_url"),
+            use_ssl=login_info["use_ssl"],
         )
 
-    @contextmanager
-    def _get_s3(self):
-        from botocore.exceptions import (
-            EndpointConnectionError,
-            NoCredentialsError,
-        )
-
-        try:
-            yield self.s3
-        except NoCredentialsError as exc:
-            link = error_link("no-credentials")
-            raise DvcException(
-                f"Unable to find AWS credentials. {link}"
-            ) from exc
-        except EndpointConnectionError as exc:
-            link = error_link("connection-error")
-            name = self.endpoint_url or "AWS S3"
-            raise DvcException(
-                f"Unable to connect to '{name}'. {link}"
-            ) from exc
-
-    @contextmanager
-    def _get_bucket(self, bucket):
-        with self._get_s3() as s3:
-            try:
-                yield s3.Bucket(bucket)
-            except s3.meta.client.exceptions.NoSuchBucket as exc:
-                link = error_link("no-bucket")
-                raise DvcException(
-                    f"Bucket '{bucket}' does not exist. {link}"
-                ) from exc
-
-    @contextmanager
     def _get_obj(self, path_info):
-        with self._get_bucket(path_info.bucket) as bucket:
-            try:
-                yield bucket.Object(path_info.path)
-            except bucket.meta.client.exceptions.NoSuchKey as exc:
-                raise DvcException(f"{path_info.url} does not exist") from exc
-
-    def _append_aws_grants_to_extra_args(self, config):
-        # Keys for extra_args can be one of the following list:
-        # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS
-        """
-          ALLOWED_UPLOAD_ARGS = [
-            'ACL', 'CacheControl', 'ContentDisposition', 'ContentEncoding',
-            'ContentLanguage', 'ContentType', 'Expires', 'GrantFullControl',
-            'GrantRead', 'GrantReadACP', 'GrantWriteACP', 'Metadata',
-            'RequestPayer', 'ServerSideEncryption', 'StorageClass',
-            'SSECustomerAlgorithm', 'SSECustomerKey', 'SSECustomerKeyMD5',
-            'SSEKMSKeyId', 'WebsiteRedirectLocation'
-          ]
-        """
-
-        grants = {
-            "grant_full_control": "GrantFullControl",
-            "grant_read": "GrantRead",
-            "grant_read_acp": "GrantReadACP",
-            "grant_write_acp": "GrantWriteACP",
-        }
-
-        for grant_option, extra_args_key in grants.items():
-            if config.get(grant_option):
-                if self.acl:
-                    raise ConfigError(
-                        "`acl` and `grant_*` AWS S3 config options "
-                        "are mutually exclusive"
-                    )
-
-                self.extra_args[extra_args_key] = config.get(grant_option)
-
-    def _generate_download_url(self, path_info, expires=3600):
-        params = {"Bucket": path_info.bucket, "Key": path_info.path}
-        with self._get_s3() as s3:
-            return s3.meta.client.generate_presigned_url(
-                ClientMethod="get_object",
-                Params=params,
-                ExpiresIn=int(expires),
-            )
-
-    def exists(self, path_info, use_dvcignore=True):
-        """Check if the blob exists. If it does not exist,
-        it could be a part of a directory path.
-
-        eg: if `data/file.txt` exists, check for `data` should return True
-        """
-        return self.isfile(path_info) or self.isdir(path_info)
-
-    def isdir(self, path_info):
-        # S3 doesn't have a concept for directories.
-        #
-        # Using `head_object` with a path pointing to a directory
-        # will throw a 404 error.
-        #
-        # A reliable way to know if a given path is a directory is by
-        # checking if there are more files sharing the same prefix
-        # with a `list_objects` call.
-        #
-        # We need to make sure that the path ends with a forward slash,
-        # since we can end with false-positives like the following example:
-        #
-        #       bucket
-        #       └── data
-        #          ├── alice
-        #          └── alpha
-        #
-        # Using `data/al` as prefix will return `[data/alice, data/alpha]`,
-        # While `data/al/` will return nothing.
-        #
-        dir_path = path_info / ""
-        return bool(list(self._list_paths(dir_path, max_items=1)))
-
-    def isfile(self, path_info):
-        if path_info.path.endswith("/"):
-            return False
-
-        return path_info.path in self._list_paths(path_info)
-
-    def info(self, path_info):
-        # temporary workaround, will be removed when migrating to s3fs
-        if self.isdir(path_info):
-            return {"type": "directory"}
-
-        with self._get_obj(path_info) as obj:
-            return {
-                "type": "file",
-                "etag": obj.e_tag.strip('"'),
-                "size": obj.content_length,
-            }
-
-    def _list_paths(self, path_info, max_items=None):
-        with self._get_bucket(path_info.bucket) as bucket:
-            obj_summaries = bucket.objects.filter(Prefix=path_info.path)
-            if max_items is not None:
-                obj_summaries = obj_summaries.page_size(max_items).limit(
-                    max_items
-                )
-            for obj_summary in obj_summaries:
-                yield obj_summary.key
-
-    def walk_files(self, path_info, **kwargs):
-        if not kwargs.pop("prefix", False):
-            path_info = path_info / ""
-        for fname in self._list_paths(path_info, **kwargs):
-            if fname.endswith("/"):
-                continue
-
-            yield path_info.replace(path=fname)
-
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
-        assert recursive
-
-        with self._get_bucket(path_info.bucket) as bucket:
-            for obj_summary in bucket.objects.filter(Prefix=path_info.path):
-                if detail:
-                    yield {
-                        "type": "file",
-                        "name": obj_summary.key,
-                        "size": obj_summary.size,
-                        "etag": obj_summary.e_tag.strip('"'),
-                    }
-                else:
-                    yield obj_summary.key
-
-    def remove(self, path_info):
-        if path_info.scheme != "s3":
-            raise NotImplementedError
-
-        logger.debug(f"Removing {path_info}")
-        with self._get_obj(path_info) as obj:
-            obj.delete()
-
-    def makedirs(self, path_info):
-        # We need to support creating empty directories, which means
-        # creating an object with an empty body and a trailing slash `/`.
-        #
-        # We are not creating directory objects for every parent prefix,
-        # as it is not required.
-        if not path_info.path:
-            return
-
-        dir_path = path_info / ""
-        with self._get_obj(dir_path) as obj:
-            obj.put(Body="")
-
-    def copy(self, from_info, to_info):
-        with self._get_s3() as s3:
-            self._copy(s3.meta.client, from_info, to_info, self.extra_args)
-
-    @classmethod
-    def _copy_multipart(
-        cls, s3, from_info, to_info, size, n_parts, extra_args
-    ):
-        mpu = s3.create_multipart_upload(
-            Bucket=to_info.bucket, Key=to_info.path, **extra_args
-        )
-        mpu_id = mpu["UploadId"]
-
-        parts = []
-        byte_position = 0
-        for i in range(1, n_parts + 1):
-            obj = s3.head_object(
-                Bucket=from_info.bucket, Key=from_info.path, PartNumber=i
-            )
-            part_size = obj["ContentLength"]
-            lastbyte = byte_position + part_size - 1
-            if lastbyte > size:
-                lastbyte = size - 1
-
-            srange = f"bytes={byte_position}-{lastbyte}"
-
-            part = s3.upload_part_copy(
-                Bucket=to_info.bucket,
-                Key=to_info.path,
-                PartNumber=i,
-                UploadId=mpu_id,
-                CopySourceRange=srange,
-                CopySource={"Bucket": from_info.bucket, "Key": from_info.path},
-            )
-            parts.append(
-                {"PartNumber": i, "ETag": part["CopyPartResult"]["ETag"]}
-            )
-            byte_position += part_size
-
-        assert n_parts == len(parts)
-
-        s3.complete_multipart_upload(
-            Bucket=to_info.bucket,
-            Key=to_info.path,
-            UploadId=mpu_id,
-            MultipartUpload={"Parts": parts},
-        )
-
-    @classmethod
-    def _copy(cls, s3, from_info, to_info, extra_args):
-        # NOTE: object's etag depends on the way it was uploaded to s3 or the
-        # way it was copied within the s3. More specifically, it depends on
-        # the chunk size that was used to transfer it, which would affect
-        # whether an object would be uploaded as a single part or as a
-        # multipart.
-        #
-        # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
-        # then it was transferred as a single part, which means that the chunk
-        # size used to transfer it was greater or equal to the ContentLength
-        # of that object. So to preserve that tag over the next transfer, we
-        # could use any value >= ContentLength.
-        #
-        # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
-        # then it was transferred as a multipart, which means that the chunk
-        # size used to transfer it was less than ContentLength of that object.
-        # Unfortunately, in general, it doesn't mean that the chunk size was
-        # the same throughout the transfer, so it means that in order to
-        # preserve etag, we need to transfer each part separately, so the
-        # object is transfered in the same chunks as it was originally.
-        from boto3.s3.transfer import TransferConfig
-
-        obj = s3.head_object(Bucket=from_info.bucket, Key=from_info.path)
-        etag = obj["ETag"].strip('"')
-        size = obj["ContentLength"]
-
-        _, _, parts_suffix = etag.partition("-")
-        if parts_suffix:
-            n_parts = int(parts_suffix)
-            cls._copy_multipart(
-                s3, from_info, to_info, size, n_parts, extra_args=extra_args
-            )
-        else:
-            source = {"Bucket": from_info.bucket, "Key": from_info.path}
-            s3.copy(
-                source,
-                to_info.bucket,
-                to_info.path,
-                ExtraArgs=extra_args,
-                Config=TransferConfig(multipart_threshold=size + 1),
-            )
-
-        cached_etag = s3.head_object(Bucket=to_info.bucket, Key=to_info.path)[
-            "ETag"
-        ].strip('"')
-        if etag != cached_etag:
-            raise ETagMismatchError(etag, cached_etag)
-
-    def etag(self, path_info):
-        with self._get_obj(path_info) as obj:
-            return HashInfo("etag", size=obj.content_length,)
-
-    def _upload_fobj(self, fobj, to_info):
-        with self._get_obj(to_info) as obj:
-            obj.upload_fileobj(fobj, Config=self._transfer_config)
+        bucket = self.s3.Bucket(path_info.bucket)
+        return bucket.Object(path_info.path)
 
+    @_translate_exceptions
     def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+        self, from_file, to_info, name=None, no_progress_bar=False, **pbar_args
     ):
-        with self._get_obj(to_info) as obj:
-            total = os.path.getsize(from_file)
-            with Tqdm(
-                disable=no_progress_bar, total=total, bytes=True, desc=name
-            ) as pbar:
-                obj.upload_file(
-                    from_file,
-                    Callback=pbar.update,
-                    ExtraArgs=self.extra_args,
-                    Config=self._transfer_config,
-                )
+        total = os.path.getsize(from_file)
+        with Tqdm(
+            disable=no_progress_bar,
+            total=total,
+            bytes=True,
+            desc=name,
+            **pbar_args,
+        ) as pbar:
+            obj = self._get_obj(to_info)
+            obj.upload_file(
+                from_file,
+                Callback=pbar.update,
+                ExtraArgs=self.fs_args.get("s3_additional_kwargs"),
+                Config=self._transfer_config,
+            )
+        self.fs.invalidate_cache(self._with_bucket(to_info.parent))
 
-    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        with self._get_obj(from_info) as obj:
-            with Tqdm(
-                disable=no_progress_bar,
-                total=obj.content_length,
-                bytes=True,
-                desc=name,
-            ) as pbar:
-                obj.download_file(
-                    to_file, Callback=pbar.update, Config=self._transfer_config
-                )
+    @_translate_exceptions
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **pbar_args
+    ):
+        obj = self._get_obj(from_info)
+        with Tqdm(
+            disable=no_progress_bar,
+            total=obj.content_length,
+            bytes=True,
+            desc=name,
+            **pbar_args,
+        ) as pbar:
+            obj.download_file(to_file, Callback=pbar.update)

dvc/fs/ssh/__init__.py

@@ -5,11 +5,10 @@ import os
 import shutil
 import threading
 from contextlib import closing, contextmanager
-from urllib.parse import urlparse
 
 from funcy import first, memoize, silent, wrap_with
 
-import dvc.prompt as prompt
+from dvc import prompt
 from dvc.hash_info import HashInfo
 from dvc.scheme import Schemes
 
@@ -30,10 +29,10 @@ def ask_password(host, user, port):
     )
 
 
-class SSHFileSystem(BaseFileSystem):
+class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     scheme = Schemes.SSH
     REQUIRES = {"paramiko": "paramiko"}
-    JOBS = 4
+    _JOBS = 4
 
     PARAM_CHECKSUM = "md5"
     # At any given time some of the connections will go over network and
@@ -46,36 +45,21 @@ class SSHFileSystem(BaseFileSystem):
     DEFAULT_PORT = 22
     TIMEOUT = 1800
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-        url = config.get("url")
-        if url:
-            parsed = urlparse(url)
-            user_ssh_config = self._load_user_ssh_config(parsed.hostname)
+    def __init__(self, **config):
+        super().__init__(**config)
+        user_ssh_config = self._load_user_ssh_config(config["host"])
 
-            host = user_ssh_config.get("hostname", parsed.hostname)
-            user = (
-                config.get("user")
-                or parsed.username
-                or user_ssh_config.get("user")
-                or getpass.getuser()
-            )
-            port = (
-                config.get("port")
-                or parsed.port
-                or self._try_get_ssh_config_port(user_ssh_config)
-                or self.DEFAULT_PORT
-            )
-            self.path_info = self.PATH_CLS.from_parts(
-                scheme=self.scheme,
-                host=host,
-                user=user,
-                port=port,
-                path=parsed.path,
-            )
-        else:
-            self.path_info = None
-            user_ssh_config = {}
+        self.host = user_ssh_config.get("hostname", config["host"])
+        self.user = (
+            config.get("user")
+            or user_ssh_config.get("user")
+            or getpass.getuser()
+        )
+        self.port = (
+            config.get("port")
+            or self._try_get_ssh_config_port(user_ssh_config)
+            or self.DEFAULT_PORT
+        )
 
         self.keyfile = config.get(
             "keyfile"
@@ -93,6 +77,16 @@ class SSHFileSystem(BaseFileSystem):
             self.sock = None
         self.allow_agent = config.get("allow_agent", True)
 
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):
+        from fsspec.implementations.sftp import SFTPFileSystem
+
+        # pylint:disable=protected-access
+        kwargs = SFTPFileSystem._get_kwargs_from_urls(urlpath)
+        if "username" in kwargs:
+            kwargs["user"] = kwargs.pop("username")
+        return kwargs
+
     @staticmethod
     def ssh_config_filename():
         return os.path.expanduser(os.path.join("~", ".ssh", "config"))
@@ -120,25 +114,21 @@ class SSHFileSystem(BaseFileSystem):
     def _try_get_ssh_config_keyfile(user_ssh_config):
         return first(user_ssh_config.get("identityfile") or ())
 
-    def ensure_credentials(self, path_info=None):
-        if path_info is None:
-            path_info = self.path_info
-
+    def ensure_credentials(self):
         # NOTE: we use the same password regardless of the server :(
         if self.ask_password and self.password is None:
-            host, user, port = path_info.host, path_info.user, path_info.port
-            self.password = ask_password(host, user, port)
+            self.password = ask_password(self.host, self.user, self.port)
 
-    def ssh(self, path_info):
-        self.ensure_credentials(path_info)
+    def ssh(self):
+        self.ensure_credentials()
 
         from .connection import SSHConnection
 
         return get_connection(
             SSHConnection,
-            path_info.host,
-            username=path_info.user,
-            port=path_info.port,
+            self.host,
+            username=self.user,
+            port=self.port,
             key_filename=self.keyfile,
             timeout=self.timeout,
             password=self.password,
@@ -151,7 +141,7 @@ class SSHFileSystem(BaseFileSystem):
     def open(self, path_info, mode="r", encoding=None, **kwargs):
         assert mode in {"r", "rt", "rb", "wb"}
 
-        with self.ssh(path_info) as ssh, closing(
+        with self.ssh() as ssh, closing(
             ssh.sftp.open(path_info.path, mode)
         ) as fd:
             if "b" in mode:
@@ -159,20 +149,20 @@ class SSHFileSystem(BaseFileSystem):
             else:
                 yield io.TextIOWrapper(fd, encoding=encoding)
 
-    def exists(self, path_info, use_dvcignore=True):
-        with self.ssh(path_info) as ssh:
+    def exists(self, path_info) -> bool:
+        with self.ssh() as ssh:
             return ssh.exists(path_info.path)
 
     def isdir(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return ssh.isdir(path_info.path)
 
     def isfile(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return ssh.isfile(path_info.path)
 
     def walk_files(self, path_info, **kwargs):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             for fname in ssh.walk_files(path_info.path):
                 yield path_info.replace(path=fname)
 
@@ -180,32 +170,32 @@ class SSHFileSystem(BaseFileSystem):
         if path_info.scheme != self.scheme:
             raise NotImplementedError
 
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             ssh.remove(path_info.path)
 
     def makedirs(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             ssh.makedirs(path_info.path)
 
     def move(self, from_info, to_info):
         if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
             raise NotImplementedError
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.move(from_info.path, to_info.path)
 
     def copy(self, from_info, to_info):
         if not from_info.scheme == to_info.scheme == self.scheme:
             raise NotImplementedError
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.atomic_copy(from_info.path, to_info.path)
 
     def symlink(self, from_info, to_info):
         if not from_info.scheme == to_info.scheme == self.scheme:
             raise NotImplementedError
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.symlink(from_info.path, to_info.path)
 
     def hardlink(self, from_info, to_info):
@@ -215,7 +205,7 @@ class SSHFileSystem(BaseFileSystem):
         # See dvc/remote/local/__init__.py - hardlink()
         if self.getsize(from_info) == 0:
 
-            with self.ssh(to_info) as ssh:
+            with self.ssh() as ssh:
                 ssh.sftp.open(to_info.path, "w").close()
 
             logger.debug(
@@ -225,18 +215,18 @@ class SSHFileSystem(BaseFileSystem):
             )
             return
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.hardlink(from_info.path, to_info.path)
 
     def reflink(self, from_info, to_info):
         if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
             raise NotImplementedError
 
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.reflink(from_info.path, to_info.path)
 
     def md5(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return HashInfo(
                 "md5",
                 ssh.md5(path_info.path),
@@ -244,16 +234,16 @@ class SSHFileSystem(BaseFileSystem):
             )
 
     def info(self, path_info):
-        with self.ssh(path_info) as ssh:
+        with self.ssh() as ssh:
             return ssh.info(path_info.path)
 
-    def _upload_fobj(self, fobj, to_info):
+    def _upload_fobj(self, fobj, to_info, **kwargs):
         self.makedirs(to_info.parent)
         with self.open(to_info, mode="wb") as fdest:
             shutil.copyfileobj(fobj, fdest)
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        with self.ssh(from_info) as ssh:
+        with self.ssh() as ssh:
             ssh.download(
                 from_info.path,
                 to_file,
@@ -264,7 +254,7 @@ class SSHFileSystem(BaseFileSystem):
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        with self.ssh(to_info) as ssh:
+        with self.ssh() as ssh:
             ssh.upload(
                 from_file,
                 to_info.path,